diff --git a/x-pack/plugins/integration_assistant/__jest__/fixtures/log_type_detection.ts b/x-pack/plugins/integration_assistant/__jest__/fixtures/log_type_detection.ts new file mode 100644 index 0000000000000..db570a6c181b5 --- /dev/null +++ b/x-pack/plugins/integration_assistant/__jest__/fixtures/log_type_detection.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SamplesFormatName } from '../../common/api/model/common_attributes'; + +export const logFormatDetectionTestState = { + lastExecutedChain: 'testchain', + logSamples: ['{"test1": "test1"}'], + exAnswer: 'testanswer', + packageName: 'testPackage', + dataStreamName: 'testDatastream', + finalized: false, + samplesFormat: { name: SamplesFormatName.Values.json }, + ecsVersion: 'testVersion', + results: { test1: 'test1' }, +}; diff --git a/x-pack/plugins/integration_assistant/common/api/analyze_logs/analyze_logs_route.schema.yaml b/x-pack/plugins/integration_assistant/common/api/analyze_logs/analyze_logs_route.schema.yaml new file mode 100644 index 0000000000000..944828e63f764 --- /dev/null +++ b/x-pack/plugins/integration_assistant/common/api/analyze_logs/analyze_logs_route.schema.yaml @@ -0,0 +1,36 @@ +openapi: 3.0.3 +info: + title: Auto Import Analyze Logs API endpoint + version: "1" +paths: + /api/integration_assistant/analyzelogs: + post: + summary: Analyzes log samples and processes them. + operationId: AnalyzeLogs + x-codegen-enabled: false + description: Analyzes log samples and processes them + tags: + - Analyze Logs API + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - logSamples + - connectorId + properties: + logSamples: + $ref: "../model/common_attributes.schema.yaml#/components/schemas/LogSamples" + connectorId: + $ref: "../model/common_attributes.schema.yaml#/components/schemas/Connector" + langSmithOptions: + $ref: "../model/common_attributes.schema.yaml#/components/schemas/LangSmithOptions" + responses: + 200: + description: Indicates a successful call. + content: + application/json: + schema: + $ref: "../model/response_schemas.schema.yaml#/components/schemas/AnalyzeLogsAPIResponse" diff --git a/x-pack/plugins/integration_assistant/common/api/analyze_logs/analyze_logs_route.ts b/x-pack/plugins/integration_assistant/common/api/analyze_logs/analyze_logs_route.ts new file mode 100644 index 0000000000000..d2f15525177d1 --- /dev/null +++ b/x-pack/plugins/integration_assistant/common/api/analyze_logs/analyze_logs_route.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* + * NOTICE: Do not edit this file manually. + * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. + * + * info: + * title: Auto Import Analyze Logs API endpoint + * version: 1 + */ + +import { z } from '@kbn/zod'; + +import { LogSamples, Connector, LangSmithOptions } from '../model/common_attributes'; +import { AnalyzeLogsAPIResponse } from '../model/response_schemas'; + +export type AnalyzeLogsRequestBody = z.infer; +export const AnalyzeLogsRequestBody = z.object({ + logSamples: LogSamples, + connectorId: Connector, + langSmithOptions: LangSmithOptions.optional(), +}); +export type AnalyzeLogsRequestBodyInput = z.input; + +export type AnalyzeLogsResponse = z.infer; +export const AnalyzeLogsResponse = AnalyzeLogsAPIResponse; diff --git a/x-pack/plugins/integration_assistant/common/api/model/common_attributes.schema.yaml b/x-pack/plugins/integration_assistant/common/api/model/common_attributes.schema.yaml index 7839a2dd3eaf7..eafd318b40c33 100644 --- a/x-pack/plugins/integration_assistant/common/api/model/common_attributes.schema.yaml +++ b/x-pack/plugins/integration_assistant/common/api/model/common_attributes.schema.yaml @@ -16,6 +16,12 @@ components: minLength: 1 description: DataStream name for the integration to be built. + LogSamples: + type: array + items: + type: string + description: String form of the input logsamples. + RawSamples: type: array items: @@ -42,6 +48,10 @@ components: enum: - ndjson - json + - csv + - structured + - unstructured + - unsupported SamplesFormat: type: object diff --git a/x-pack/plugins/integration_assistant/common/api/model/common_attributes.ts b/x-pack/plugins/integration_assistant/common/api/model/common_attributes.ts index 07d5323dc0969..4a82a3fd97534 100644 --- a/x-pack/plugins/integration_assistant/common/api/model/common_attributes.ts +++ b/x-pack/plugins/integration_assistant/common/api/model/common_attributes.ts @@ -21,6 +21,12 @@ export const PackageName = z.string().min(1); export type DataStreamName = z.infer; export const DataStreamName = z.string().min(1); +/** + * String form of the input logsamples. + */ +export type LogSamples = z.infer; +export const LogSamples = z.array(z.string()); + /** * String array containing the json raw samples that are used for ecs mapping. */ @@ -49,7 +55,14 @@ export const Docs = z.array(z.object({}).passthrough()); * The name of the log samples format. */ export type SamplesFormatName = z.infer; -export const SamplesFormatName = z.enum(['ndjson', 'json']); +export const SamplesFormatName = z.enum([ + 'ndjson', + 'json', + 'csv', + 'structured', + 'unstructured', + 'unsupported', +]); export type SamplesFormatNameEnum = typeof SamplesFormatName.enum; export const SamplesFormatNameEnum = SamplesFormatName.enum; diff --git a/x-pack/plugins/integration_assistant/common/api/model/response_schemas.schema.yaml b/x-pack/plugins/integration_assistant/common/api/model/response_schemas.schema.yaml index 8afbab533a6d3..039945fb7ba0b 100644 --- a/x-pack/plugins/integration_assistant/common/api/model/response_schemas.schema.yaml +++ b/x-pack/plugins/integration_assistant/common/api/model/response_schemas.schema.yaml @@ -66,3 +66,20 @@ components: properties: docs: $ref: "./common_attributes.schema.yaml#/components/schemas/Docs" + + AnalyzeLogsAPIResponse: + type: object + required: + - results + properties: + results: + type: object + required: + - parsedSamples + properties: + samplesFormat: + $ref: "./common_attributes.schema.yaml#/components/schemas/SamplesFormat" + parsedSamples: + type: array + items: + type: string diff --git a/x-pack/plugins/integration_assistant/common/api/model/response_schemas.ts b/x-pack/plugins/integration_assistant/common/api/model/response_schemas.ts index 7341d7aa0c287..de1b23ae5e8e3 100644 --- a/x-pack/plugins/integration_assistant/common/api/model/response_schemas.ts +++ b/x-pack/plugins/integration_assistant/common/api/model/response_schemas.ts @@ -16,7 +16,7 @@ import { z } from '@kbn/zod'; -import { Docs, Mapping, Pipeline } from './common_attributes'; +import { Docs, Mapping, Pipeline, SamplesFormat } from './common_attributes'; export type EcsMappingAPIResponse = z.infer; export const EcsMappingAPIResponse = z.object({ @@ -48,3 +48,11 @@ export const CheckPipelineAPIResponse = z.object({ docs: Docs, }), }); + +export type AnalyzeLogsAPIResponse = z.infer; +export const AnalyzeLogsAPIResponse = z.object({ + results: z.object({ + samplesFormat: SamplesFormat, + parsedSamples: z.array(z.string()), + }), +}); diff --git a/x-pack/plugins/integration_assistant/common/constants.ts b/x-pack/plugins/integration_assistant/common/constants.ts index 69b383d882869..296e38c01e71c 100644 --- a/x-pack/plugins/integration_assistant/common/constants.ts +++ b/x-pack/plugins/integration_assistant/common/constants.ts @@ -18,6 +18,7 @@ export const INTEGRATION_ASSISTANT_BASE_PATH = '/api/integration_assistant'; export const ECS_GRAPH_PATH = `${INTEGRATION_ASSISTANT_BASE_PATH}/ecs`; export const CATEGORIZATION_GRAPH_PATH = `${INTEGRATION_ASSISTANT_BASE_PATH}/categorization`; +export const ANALYZE_LOGS_PATH = `${INTEGRATION_ASSISTANT_BASE_PATH}/analyzelogs`; export const RELATED_GRAPH_PATH = `${INTEGRATION_ASSISTANT_BASE_PATH}/related`; export const CHECK_PIPELINE_PATH = `${INTEGRATION_ASSISTANT_BASE_PATH}/pipeline`; export const INTEGRATION_BUILDER_PATH = `${INTEGRATION_ASSISTANT_BASE_PATH}/build`; diff --git a/x-pack/plugins/integration_assistant/common/index.ts b/x-pack/plugins/integration_assistant/common/index.ts index 6a473d976fa88..5310fa67c8cac 100644 --- a/x-pack/plugins/integration_assistant/common/index.ts +++ b/x-pack/plugins/integration_assistant/common/index.ts @@ -15,6 +15,7 @@ export { } from './api/check_pipeline/check_pipeline'; export { EcsMappingRequestBody, EcsMappingResponse } from './api/ecs/ecs_route'; export { RelatedRequestBody, RelatedResponse } from './api/related/related_route'; +export { AnalyzeLogsRequestBody, AnalyzeLogsResponse } from './api/analyze_logs/analyze_logs_route'; export type { DataStream, @@ -35,4 +36,5 @@ export { PLUGIN_ID, RELATED_GRAPH_PATH, CHECK_PIPELINE_PATH, + ANALYZE_LOGS_PATH, } from './constants'; diff --git a/x-pack/plugins/integration_assistant/public/common/lib/api.ts b/x-pack/plugins/integration_assistant/public/common/lib/api.ts index 68c93f4f51a33..ea9fba83dc094 100644 --- a/x-pack/plugins/integration_assistant/public/common/lib/api.ts +++ b/x-pack/plugins/integration_assistant/public/common/lib/api.ts @@ -16,6 +16,8 @@ import type { CheckPipelineRequestBody, CheckPipelineResponse, BuildIntegrationRequestBody, + AnalyzeLogsRequestBody, + AnalyzeLogsResponse, } from '../../../common'; import { INTEGRATION_BUILDER_PATH, @@ -24,7 +26,7 @@ import { RELATED_GRAPH_PATH, CHECK_PIPELINE_PATH, } from '../../../common'; -import { FLEET_PACKAGES_PATH } from '../../../common/constants'; +import { ANALYZE_LOGS_PATH, FLEET_PACKAGES_PATH } from '../../../common/constants'; export interface EpmPackageResponse { response: [{ id: string; name: string }]; @@ -42,6 +44,16 @@ export interface RequestDeps { abortSignal: AbortSignal; } +export const runAnalyzeLogsGraph = async ( + body: AnalyzeLogsRequestBody, + { http, abortSignal }: RequestDeps +): Promise => + http.post(ANALYZE_LOGS_PATH, { + headers: defaultHeaders, + body: JSON.stringify(body), + signal: abortSignal, + }); + export const runEcsGraph = async ( body: EcsMappingRequestBody, { http, abortSignal }: RequestDeps diff --git a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/mocks/state.ts b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/mocks/state.ts index aa310f034290c..c842d097fa7c3 100644 --- a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/mocks/state.ts +++ b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/mocks/state.ts @@ -5,11 +5,11 @@ * 2.0. */ -import type { Pipeline, Docs } from '../../../../../common'; +import type { Pipeline, Docs, SamplesFormat } from '../../../../../common'; import type { Actions, State } from '../state'; import type { AIConnector } from '../types'; -const result: { pipeline: Pipeline; docs: Docs } = { +const result: { pipeline: Pipeline; docs: Docs; samplesFormat: SamplesFormat } = { pipeline: { description: 'Pipeline to process my_integration my_data_stream_title logs', processors: [ @@ -389,6 +389,7 @@ const result: { pipeline: Pipeline; docs: Docs } = { ], }, ], + samplesFormat: { name: 'json' }, }; const rawSamples = [ @@ -419,8 +420,7 @@ export const mockState: State = { dataStreamName: 'mocked_datastream_name', dataStreamDescription: 'Mocked Data Stream Description', inputTypes: ['filestream'], - logsSampleParsed: rawSamples, - samplesFormat: { name: 'ndjson', multiline: false }, + logSamples: rawSamples, }, isGenerating: false, result, diff --git a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/state.ts b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/state.ts index 161d1b0646541..bef5b35624df4 100644 --- a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/state.ts +++ b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/state.ts @@ -5,7 +5,7 @@ * 2.0. */ import { createContext, useContext } from 'react'; -import type { Pipeline, Docs } from '../../../../common'; +import type { Pipeline, Docs, SamplesFormat } from '../../../../common'; import type { AIConnector, IntegrationSettings } from './types'; export interface State { @@ -16,6 +16,7 @@ export interface State { result?: { pipeline: Pipeline; docs: Docs; + samplesFormat?: SamplesFormat; }; } diff --git a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/generation_modal.test.tsx b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/generation_modal.test.tsx index 1c1cf3c881cdf..4d0f6eee1c407 100644 --- a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/generation_modal.test.tsx +++ b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/generation_modal.test.tsx @@ -17,9 +17,14 @@ import { TelemetryEventType } from '../../../../../services/telemetry/types'; const integrationSettings = mockState.integrationSettings!; const connector = mockState.connector!; +const mockAnalyzeLogsResults = { + parsedSamples: [{ test: 'analyzeLogsResponse' }], + sampleLogsFormat: { name: 'json' }, +}; const mockEcsMappingResults = { pipeline: { test: 'ecsMappingResponse' }, docs: [] }; const mockCategorizationResults = { pipeline: { test: 'categorizationResponse' }, docs: [] }; const mockRelatedResults = { pipeline: { test: 'relatedResponse' }, docs: [] }; +const mockRunAnalyzeLogsGraph = jest.fn((_: unknown) => ({ results: mockAnalyzeLogsResults })); const mockRunEcsGraph = jest.fn((_: unknown) => ({ results: mockEcsMappingResults })); const mockRunCategorizationGraph = jest.fn((_: unknown) => ({ results: mockCategorizationResults, @@ -27,13 +32,12 @@ const mockRunCategorizationGraph = jest.fn((_: unknown) => ({ const mockRunRelatedGraph = jest.fn((_: unknown) => ({ results: mockRelatedResults })); const defaultRequest = { - packageName: integrationSettings.name ?? '', - dataStreamName: integrationSettings.dataStreamName ?? '', - rawSamples: integrationSettings.logsSampleParsed ?? [], connectorId: connector.id, + LangSmithOptions: undefined, }; jest.mock('../../../../../common/lib/api', () => ({ + runAnalyzeLogsGraph: (params: unknown) => mockRunAnalyzeLogsGraph(params), runEcsGraph: (params: unknown) => mockRunEcsGraph(params), runCategorizationGraph: (params: unknown) => mockRunCategorizationGraph(params), runRelatedGraph: (params: unknown) => mockRunRelatedGraph(params), @@ -74,14 +78,29 @@ describe('GenerationModal', () => { expect(result.queryByTestId('generationModal')).toBeInTheDocument(); }); + it('should call runAnalyzeLogsGraph with correct parameters', () => { + expect(mockRunAnalyzeLogsGraph).toHaveBeenCalledWith({ + ...defaultRequest, + logSamples: integrationSettings.logSamples ?? [], + }); + }); + it('should call runEcsGraph with correct parameters', () => { - expect(mockRunEcsGraph).toHaveBeenCalledWith(defaultRequest); + expect(mockRunEcsGraph).toHaveBeenCalledWith({ + ...defaultRequest, + rawSamples: mockAnalyzeLogsResults.parsedSamples, + packageName: integrationSettings.name ?? '', + dataStreamName: integrationSettings.dataStreamName ?? '', + }); }); it('should call runCategorizationGraph with correct parameters', () => { expect(mockRunCategorizationGraph).toHaveBeenCalledWith({ ...defaultRequest, currentPipeline: mockEcsMappingResults.pipeline, + rawSamples: mockAnalyzeLogsResults.parsedSamples, + packageName: integrationSettings.name ?? '', + dataStreamName: integrationSettings.dataStreamName ?? '', }); }); @@ -89,6 +108,9 @@ describe('GenerationModal', () => { expect(mockRunRelatedGraph).toHaveBeenCalledWith({ ...defaultRequest, currentPipeline: mockCategorizationResults.pipeline, + rawSamples: mockAnalyzeLogsResults.parsedSamples, + packageName: integrationSettings.name ?? '', + dataStreamName: integrationSettings.dataStreamName ?? '', }); }); @@ -101,7 +123,7 @@ describe('GenerationModal', () => { TelemetryEventType.IntegrationAssistantGenerationComplete, { sessionId: expect.any(String), - sampleRows: integrationSettings.logsSampleParsed?.length ?? 0, + sampleRows: integrationSettings.logSamples?.length ?? 0, actionTypeId: connector.actionTypeId, model: expect.anything(), provider: connector.apiProvider ?? 'unknown', @@ -147,7 +169,7 @@ describe('GenerationModal', () => { TelemetryEventType.IntegrationAssistantGenerationComplete, { sessionId: expect.any(String), - sampleRows: integrationSettings.logsSampleParsed?.length ?? 0, + sampleRows: integrationSettings.logSamples?.length ?? 0, actionTypeId: connector.actionTypeId, model: expect.anything(), provider: connector.apiProvider ?? 'unknown', diff --git a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/generation_modal.tsx b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/generation_modal.tsx index b83c5315bd619..f25423390fb6d 100644 --- a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/generation_modal.tsx +++ b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/generation_modal.tsx @@ -25,15 +25,17 @@ import { isEmpty } from 'lodash/fp'; import React, { useCallback, useEffect, useMemo, useState } from 'react'; import { css } from '@emotion/react'; import { getLangSmithOptions } from '../../../../../common/lib/lang_smith'; -import type { - CategorizationRequestBody, - EcsMappingRequestBody, - RelatedRequestBody, +import { + type AnalyzeLogsRequestBody, + type CategorizationRequestBody, + type EcsMappingRequestBody, + type RelatedRequestBody, } from '../../../../../../common'; import { runCategorizationGraph, runEcsGraph, runRelatedGraph, + runAnalyzeLogsGraph, } from '../../../../../common/lib/api'; import { useKibana } from '../../../../../common/hooks/use_kibana'; import type { State } from '../../state'; @@ -46,6 +48,7 @@ const ProgressOrder = ['ecs', 'categorization', 'related']; type ProgressItem = (typeof ProgressOrder)[number]; const progressText: Record = { + analyzeLogs: i18n.PROGRESS_ANALYZE_LOGS, ecs: i18n.PROGRESS_ECS_MAPPING, categorization: i18n.PROGRESS_CATEGORIZATION, related: i18n.PROGRESS_RELATED_GRAPH, @@ -83,10 +86,31 @@ export const useGeneration = ({ (async () => { try { + let logSamples = integrationSettings.logSamples; + let samplesFormat = integrationSettings.samplesFormat; + + if (integrationSettings.samplesFormat === undefined) { + const analyzeLogsRequest: AnalyzeLogsRequestBody = { + logSamples: integrationSettings.logSamples ?? [], + connectorId: connector.id, + langSmithOptions: getLangSmithOptions(), + }; + + setProgress('analyzeLogs'); + const analyzeLogsResult = await runAnalyzeLogsGraph(analyzeLogsRequest, deps); + if (abortController.signal.aborted) return; + if (isEmpty(analyzeLogsResult?.results)) { + setError('No results from Analyze Logs Graph'); + return; + } + logSamples = analyzeLogsResult.results.parsedSamples; + samplesFormat = analyzeLogsResult.results.samplesFormat; + } + const ecsRequest: EcsMappingRequestBody = { packageName: integrationSettings.name ?? '', dataStreamName: integrationSettings.dataStreamName ?? '', - rawSamples: integrationSettings.logsSampleParsed ?? [], + rawSamples: logSamples ?? [], connectorId: connector.id, langSmithOptions: getLangSmithOptions(), }; @@ -125,7 +149,13 @@ export const useGeneration = ({ durationMs: Date.now() - generationStartedAt, }); - onComplete(relatedGraphResult.results); + const result = { + pipeline: relatedGraphResult.results.pipeline, + docs: relatedGraphResult.results.docs, + samplesFormat, + }; + + onComplete(result); } catch (e) { if (abortController.signal.aborted) return; const errorMessage = `${e.message}${ diff --git a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/is_step_ready.ts b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/is_step_ready.ts index 43e0006275fa9..4a40334a72ab2 100644 --- a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/is_step_ready.ts +++ b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/is_step_ready.ts @@ -12,5 +12,5 @@ export const isDataStreamStepReady = ({ integrationSettings }: State) => integrationSettings?.dataStreamTitle && integrationSettings?.dataStreamDescription && integrationSettings?.dataStreamName && - integrationSettings?.logsSampleParsed + integrationSettings?.logSamples ); diff --git a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/sample_logs_input.test.tsx b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/sample_logs_input.test.tsx index 4c15aa8a4785c..2d994f9030764 100644 --- a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/sample_logs_input.test.tsx +++ b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/sample_logs_input.test.tsx @@ -161,7 +161,7 @@ describe('SampleLogsInput', () => { it('should set the integrationSetting correctly', () => { expect(mockActions.setIntegrationSettings).toBeCalledWith({ - logsSampleParsed: logsSampleRaw.split(','), + logSamples: logsSampleRaw.split(','), samplesFormat: { name: 'json', json_path: [] }, }); }); @@ -174,7 +174,7 @@ describe('SampleLogsInput', () => { it('should truncate the logs sample', () => { expect(mockActions.setIntegrationSettings).toBeCalledWith({ - logsSampleParsed: tooLargeLogsSample.split(',').slice(0, 10), + logSamples: tooLargeLogsSample.split(',').slice(0, 10), samplesFormat: { name: 'json', json_path: [] }, }); }); @@ -193,7 +193,7 @@ describe('SampleLogsInput', () => { it('should set the integrationSetting correctly', () => { expect(mockActions.setIntegrationSettings).toBeCalledWith({ - logsSampleParsed: splitNDJSON, + logSamples: splitNDJSON, samplesFormat: { name: 'json', json_path: ['events'] }, }); }); @@ -201,10 +201,6 @@ describe('SampleLogsInput', () => { describe('when the file is invalid', () => { describe.each([ - [ - '[{"message":"test message 1"}', - 'Cannot parse the logs sample file as either a JSON or NDJSON file', - ], ['["test message 1"]', 'The logs sample file contains non-object entries'], ['[]', 'The logs sample file is empty'], ])('with logs content %s', (logsSample, errorMessage) => { @@ -218,7 +214,7 @@ describe('SampleLogsInput', () => { it('should set the integrationSetting correctly', () => { expect(mockActions.setIntegrationSettings).toBeCalledWith({ - logsSampleParsed: undefined, + logSamples: undefined, samplesFormat: undefined, }); }); @@ -236,7 +232,7 @@ describe('SampleLogsInput', () => { it('should set the integrationSetting correctly', () => { expect(mockActions.setIntegrationSettings).toBeCalledWith({ - logsSampleParsed: splitNDJSON, + logSamples: splitNDJSON, samplesFormat: { name: 'ndjson', multiline: false }, }); }); @@ -249,7 +245,7 @@ describe('SampleLogsInput', () => { it('should truncate the logs sample', () => { expect(mockActions.setIntegrationSettings).toBeCalledWith({ - logsSampleParsed: tooLargeLogsSample.split('\n').slice(0, 10), + logSamples: tooLargeLogsSample.split('\n').slice(0, 10), samplesFormat: { name: 'ndjson', multiline: false }, }); }); @@ -268,7 +264,7 @@ describe('SampleLogsInput', () => { it('should set the integrationSetting correctly', () => { expect(mockActions.setIntegrationSettings).toBeCalledWith({ - logsSampleParsed: [splitNDJSON[0]], + logSamples: [splitNDJSON[0]], samplesFormat: { name: 'ndjson', multiline: false }, }); }); @@ -281,7 +277,7 @@ describe('SampleLogsInput', () => { it('should set the integrationSetting correctly', () => { expect(mockActions.setIntegrationSettings).toBeCalledWith({ - logsSampleParsed: splitNDJSON, + logSamples: splitNDJSON, samplesFormat: { name: 'ndjson', multiline: true }, }); }); @@ -289,10 +285,6 @@ describe('SampleLogsInput', () => { describe('when the file is invalid', () => { describe.each([ - [ - '{"message":"test message 1"}\n{"message": }', - 'Cannot parse the logs sample file as either a JSON or NDJSON file', - ], ['"test message 1"', 'The logs sample file contains non-object entries'], ['', 'The logs sample file is empty'], ])('with logs content %s', (logsSample, errorMessage) => { @@ -306,7 +298,7 @@ describe('SampleLogsInput', () => { it('should set the integrationSetting correctly', () => { expect(mockActions.setIntegrationSettings).toBeCalledWith({ - logsSampleParsed: undefined, + logSamples: undefined, samplesFormat: undefined, }); }); diff --git a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/sample_logs_input.tsx b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/sample_logs_input.tsx index fd9c2e3f8c362..51ad4543bb30a 100644 --- a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/sample_logs_input.tsx +++ b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/sample_logs_input.tsx @@ -7,8 +7,8 @@ import React, { useCallback, useState } from 'react'; import { EuiCallOut, EuiFilePicker, EuiFormRow, EuiSpacer, EuiText } from '@elastic/eui'; -import { useKibana } from '@kbn/kibana-react-plugin/public'; import { isPlainObject } from 'lodash/fp'; +import { useKibana } from '@kbn/kibana-react-plugin/public'; import type { IntegrationSettings } from '../../types'; import * as i18n from './translations'; import { useActions } from '../../state'; @@ -68,16 +68,12 @@ export const parseJSONArray = ( * Parse the logs sample file content (json or ndjson) and return the parsed logs sample */ const parseLogsContent = ( - fileContent: string | undefined + fileContent: string ): { error?: string; - isTruncated?: boolean; - logsSampleParsed?: string[]; + logSamples: string[]; samplesFormat?: SamplesFormat; } => { - if (fileContent == null) { - return { error: i18n.LOGS_SAMPLE_ERROR.CAN_NOT_READ }; - } let parsedContent: unknown[]; let samplesFormat: SamplesFormat; @@ -97,7 +93,7 @@ const parseLogsContent = ( try { const { entries, pathToEntries, errorNoArrayFound } = parseJSONArray(fileContent); if (errorNoArrayFound) { - return { error: i18n.LOGS_SAMPLE_ERROR.NOT_ARRAY }; + return { error: i18n.LOGS_SAMPLE_ERROR.NOT_ARRAY, logSamples: [] }; } parsedContent = entries; samplesFormat = { name: 'json', json_path: pathToEntries }; @@ -106,32 +102,29 @@ const parseLogsContent = ( parsedContent = parseNDJSON(fileContent, true); samplesFormat = { name: 'ndjson', multiline: true }; } catch (parseMultilineNDJSONError) { - return { error: i18n.LOGS_SAMPLE_ERROR.CAN_NOT_PARSE }; + return { + logSamples: fileContent.split('\n').filter((line) => line.trim() !== ''), + }; } } } if (parsedContent.length === 0) { - return { error: i18n.LOGS_SAMPLE_ERROR.EMPTY }; - } - - let isTruncated = false; - if (parsedContent.length > MaxLogsSampleRows) { - parsedContent = parsedContent.slice(0, MaxLogsSampleRows); - isTruncated = true; + return { error: i18n.LOGS_SAMPLE_ERROR.EMPTY, logSamples: [] }; } if (parsedContent.some((log) => !isPlainObject(log))) { - return { error: i18n.LOGS_SAMPLE_ERROR.NOT_OBJECT }; + return { error: i18n.LOGS_SAMPLE_ERROR.NOT_OBJECT, logSamples: [] }; } - const logsSampleParsed = parsedContent.map((log) => JSON.stringify(log)); - return { isTruncated, logsSampleParsed, samplesFormat }; + const logSamples = parsedContent.map((log) => JSON.stringify(log)); + return { logSamples, samplesFormat }; }; interface SampleLogsInputProps { integrationSettings: IntegrationSettings | undefined; } + export const SampleLogsInput = React.memo(({ integrationSettings }) => { const { notifications } = useKibana().services; const { setIntegrationSettings } = useActions(); @@ -145,7 +138,7 @@ export const SampleLogsInput = React.memo(({ integrationSe setSampleFileError(undefined); setIntegrationSettings({ ...integrationSettings, - logsSampleParsed: undefined, + logSamples: undefined, samplesFormat: undefined, }); return; @@ -154,26 +147,32 @@ export const SampleLogsInput = React.memo(({ integrationSe const reader = new FileReader(); reader.onload = function (e) { const fileContent = e.target?.result as string | undefined; // We can safely cast to string since we call `readAsText` to load the file. - const { error, isTruncated, logsSampleParsed, samplesFormat } = - parseLogsContent(fileContent); + if (fileContent == null) { + return { error: i18n.LOGS_SAMPLE_ERROR.CAN_NOT_READ }; + } + let samples; + const { error, logSamples, samplesFormat } = parseLogsContent(fileContent); setIsParsing(false); - setSampleFileError(error); + samples = logSamples; + if (error) { + setSampleFileError(error); setIntegrationSettings({ ...integrationSettings, - logsSampleParsed: undefined, + logSamples: undefined, samplesFormat: undefined, }); return; } - if (isTruncated) { + if (samples.length > MaxLogsSampleRows) { + samples = samples.slice(0, MaxLogsSampleRows); notifications?.toasts.addInfo(i18n.LOGS_SAMPLE_TRUNCATED(MaxLogsSampleRows)); } setIntegrationSettings({ ...integrationSettings, - logsSampleParsed, + logSamples: samples, samplesFormat, }); }; diff --git a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/translations.ts b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/translations.ts index 0af9f803f71fc..bbc7000073f52 100644 --- a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/translations.ts +++ b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/translations.ts @@ -149,6 +149,12 @@ export const LOGS_SAMPLE_ERROR = { export const ANALYZING = i18n.translate('xpack.integrationAssistant.step.dataStream.analyzing', { defaultMessage: 'Analyzing', }); +export const PROGRESS_ANALYZE_LOGS = i18n.translate( + 'xpack.integrationAssistant.step.dataStream.progress.analyzeLogs', + { + defaultMessage: 'Analyzing Sample logs', + } +); export const PROGRESS_ECS_MAPPING = i18n.translate( 'xpack.integrationAssistant.step.dataStream.progress.ecsMapping', { diff --git a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/deploy_step/deploy_step.test.tsx b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/deploy_step/deploy_step.test.tsx index d4920ba927d20..1a35c9e4f02c9 100644 --- a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/deploy_step/deploy_step.test.tsx +++ b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/deploy_step/deploy_step.test.tsx @@ -31,10 +31,10 @@ const parameters: BuildIntegrationRequestBody = { description: integrationSettings.dataStreamDescription!, name: integrationSettings.dataStreamName!, inputTypes: integrationSettings.inputTypes!, - rawSamples: integrationSettings.logsSampleParsed!, + rawSamples: integrationSettings.logSamples!, docs: results.docs!, pipeline: results.pipeline, - samplesFormat: integrationSettings.samplesFormat!, + samplesFormat: results.samplesFormat!, }, ], }, diff --git a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/deploy_step/use_deploy_integration.ts b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/deploy_step/use_deploy_integration.ts index c1451a9d81a9d..5ec27b4e6de65 100644 --- a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/deploy_step/use_deploy_integration.ts +++ b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/deploy_step/use_deploy_integration.ts @@ -37,7 +37,8 @@ export const useDeployIntegration = ({ connector == null || integrationSettings == null || notifications?.toasts == null || - result?.pipeline == null + result?.pipeline == null || + result?.samplesFormat == null ) { return; } @@ -46,12 +47,6 @@ export const useDeployIntegration = ({ (async () => { try { - if (integrationSettings.samplesFormat == null) { - throw new Error( - 'Logic error: samplesFormat is required and cannot be null or undefined when creating integration.' - ); - } - const parameters: BuildIntegrationRequestBody = { integration: { title: integrationSettings.title ?? '', @@ -64,10 +59,10 @@ export const useDeployIntegration = ({ description: integrationSettings.dataStreamDescription ?? '', name: integrationSettings.dataStreamName ?? '', inputTypes: integrationSettings.inputTypes ?? [], - rawSamples: integrationSettings.logsSampleParsed ?? [], + rawSamples: integrationSettings.logSamples ?? [], docs: result.docs ?? [], + samplesFormat: result.samplesFormat ?? { name: 'json' }, pipeline: result.pipeline, - samplesFormat: integrationSettings.samplesFormat, }, ], }, @@ -123,6 +118,7 @@ export const useDeployIntegration = ({ notifications?.toasts, result?.docs, result?.pipeline, + result?.samplesFormat, telemetry, ]); diff --git a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/review_step/review_step.test.tsx b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/review_step/review_step.test.tsx index 2c8fd73f16998..8a9da9295a8f0 100644 --- a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/review_step/review_step.test.tsx +++ b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/review_step/review_step.test.tsx @@ -25,7 +25,7 @@ const customPipeline = { }; const defaultRequest = { pipeline: customPipeline, - rawSamples: integrationSettings.logsSampleParsed!, + rawSamples: integrationSettings.logSamples!, }; const mockRunCheckPipelineResults = jest.fn((_: unknown) => ({ results: mockResults })); jest.mock('../../../../../common/lib/api', () => ({ diff --git a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/review_step/use_check_pipeline.ts b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/review_step/use_check_pipeline.ts index 953b8c442abb0..a166e7c73fcc3 100644 --- a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/review_step/use_check_pipeline.ts +++ b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/review_step/use_check_pipeline.ts @@ -38,7 +38,7 @@ export const useCheckPipeline = ({ integrationSettings, customPipeline }: CheckP try { const parameters: CheckPipelineRequestBody = { pipeline: customPipeline, - rawSamples: integrationSettings.logsSampleParsed ?? [], + rawSamples: integrationSettings.logSamples ?? [], }; setIsGenerating(true); diff --git a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/types.ts b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/types.ts index c924415ec53e1..6ba7b2945b7a8 100644 --- a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/types.ts +++ b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/types.ts @@ -33,6 +33,6 @@ export interface IntegrationSettings { dataStreamDescription?: string; dataStreamName?: string; inputTypes?: InputType[]; - logsSampleParsed?: string[]; + logSamples?: string[]; samplesFormat?: SamplesFormat; } diff --git a/x-pack/plugins/integration_assistant/public/components/create_integration/telemetry.tsx b/x-pack/plugins/integration_assistant/public/components/create_integration/telemetry.tsx index 54988e238bd4d..f4dd6d7d436be 100644 --- a/x-pack/plugins/integration_assistant/public/components/create_integration/telemetry.tsx +++ b/x-pack/plugins/integration_assistant/public/components/create_integration/telemetry.tsx @@ -102,7 +102,7 @@ export const TelemetryContextProvider = React.memo>(({ chi ({ connector, integrationSettings, durationMs, error }) => { telemetry.reportEvent(TelemetryEventType.IntegrationAssistantGenerationComplete, { sessionId: sessionData.current.sessionId, - sampleRows: integrationSettings?.logsSampleParsed?.length ?? 0, + sampleRows: integrationSettings?.logSamples?.length ?? 0, actionTypeId: connector.actionTypeId, model: getConnectorModel(connector), provider: connector.apiProvider ?? 'unknown', diff --git a/x-pack/plugins/integration_assistant/server/graphs/log_type_detection/constants.ts b/x-pack/plugins/integration_assistant/server/graphs/log_type_detection/constants.ts new file mode 100644 index 0000000000000..b7814b390f8ac --- /dev/null +++ b/x-pack/plugins/integration_assistant/server/graphs/log_type_detection/constants.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const EX_ANSWER_LOG_TYPE = { + log_type: 'structured', +}; diff --git a/x-pack/plugins/integration_assistant/server/graphs/log_type_detection/detection.test.ts b/x-pack/plugins/integration_assistant/server/graphs/log_type_detection/detection.test.ts new file mode 100644 index 0000000000000..5008f5fa3ef34 --- /dev/null +++ b/x-pack/plugins/integration_assistant/server/graphs/log_type_detection/detection.test.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FakeLLM } from '@langchain/core/utils/testing'; +import { handleLogFormatDetection } from './detection'; +import type { LogFormatDetectionState } from '../../types'; +import { logFormatDetectionTestState } from '../../../__jest__/fixtures/log_type_detection'; +import { + ActionsClientChatOpenAI, + ActionsClientSimpleChatModel, +} from '@kbn/langchain/server/language_models'; + +const mockLLM = new FakeLLM({ + response: '{ "log_type": "structured"}', +}) as unknown as ActionsClientChatOpenAI | ActionsClientSimpleChatModel; + +const testState: LogFormatDetectionState = logFormatDetectionTestState; + +describe('Testing log type detection handler', () => { + it('handleLogFormatDetection()', async () => { + const response = await handleLogFormatDetection(testState, mockLLM); + expect(response.logFormat).toStrictEqual('structured'); + expect(response.lastExecutedChain).toBe('logFormatDetection'); + }); +}); diff --git a/x-pack/plugins/integration_assistant/server/graphs/log_type_detection/detection.ts b/x-pack/plugins/integration_assistant/server/graphs/log_type_detection/detection.ts new file mode 100644 index 0000000000000..c41b66263c7f4 --- /dev/null +++ b/x-pack/plugins/integration_assistant/server/graphs/log_type_detection/detection.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { + ActionsClientChatOpenAI, + ActionsClientSimpleChatModel, +} from '@kbn/langchain/server/language_models'; +import { JsonOutputParser } from '@langchain/core/output_parsers'; +import type { LogFormatDetectionState } from '../../types'; +import { LOG_FORMAT_DETECTION_PROMPT } from './prompts'; + +const MaxLogSamplesInPrompt = 5; + +export async function handleLogFormatDetection( + state: LogFormatDetectionState, + model: ActionsClientChatOpenAI | ActionsClientSimpleChatModel +) { + const outputParser = new JsonOutputParser(); + const logFormatDetectionNode = LOG_FORMAT_DETECTION_PROMPT.pipe(model).pipe(outputParser); + + const samples = + state.logSamples.length > MaxLogSamplesInPrompt + ? state.logSamples.slice(0, MaxLogSamplesInPrompt) + : state.logSamples; + + const detectedLogFormatAnswer = await logFormatDetectionNode.invoke({ + ex_answer: state.exAnswer, + log_samples: samples, + }); + const logFormat = detectedLogFormatAnswer.log_type; + + return { logFormat, lastExecutedChain: 'logFormatDetection' }; +} diff --git a/x-pack/plugins/integration_assistant/server/graphs/log_type_detection/graph.test.ts b/x-pack/plugins/integration_assistant/server/graphs/log_type_detection/graph.test.ts new file mode 100644 index 0000000000000..3a14239a1c8f2 --- /dev/null +++ b/x-pack/plugins/integration_assistant/server/graphs/log_type_detection/graph.test.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FakeLLM } from '@langchain/core/utils/testing'; +import { getLogFormatDetectionGraph } from './graph'; +import { + ActionsClientChatOpenAI, + ActionsClientSimpleChatModel, +} from '@kbn/langchain/server/language_models'; + +const mockLLM = new FakeLLM({ + response: '{"log_type": "structured"}', +}) as unknown as ActionsClientChatOpenAI | ActionsClientSimpleChatModel; + +describe('LogFormatDetectionGraph', () => { + describe('Compiling and Running', () => { + it('Ensures that the graph compiles', async () => { + // When getLogFormatDetectionGraph runs, langgraph compiles the graph it will error if the graph has any issues. + // Common issues for example detecting a node has no next step, or there is a infinite loop between them. + try { + await getLogFormatDetectionGraph(mockLLM); + } catch (error) { + fail(`getLogFormatDetectionGraph threw an error: ${error}`); + } + }); + }); +}); diff --git a/x-pack/plugins/integration_assistant/server/graphs/log_type_detection/graph.ts b/x-pack/plugins/integration_assistant/server/graphs/log_type_detection/graph.ts new file mode 100644 index 0000000000000..e0773b556e844 --- /dev/null +++ b/x-pack/plugins/integration_assistant/server/graphs/log_type_detection/graph.ts @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + ActionsClientChatOpenAI, + ActionsClientSimpleChatModel, +} from '@kbn/langchain/server/language_models'; +import type { StateGraphArgs } from '@langchain/langgraph'; +import { END, START, StateGraph } from '@langchain/langgraph'; +import type { LogFormatDetectionState } from '../../types'; +import { EX_ANSWER_LOG_TYPE } from './constants'; +import { handleLogFormatDetection } from './detection'; +import { SamplesFormat } from '../../../common'; + +const graphState: StateGraphArgs['channels'] = { + lastExecutedChain: { + value: (x: string, y?: string) => y ?? x, + default: () => '', + }, + logSamples: { + value: (x: string[], y?: string[]) => y ?? x, + default: () => [], + }, + exAnswer: { + value: (x: string, y?: string) => y ?? x, + default: () => '', + }, + finalized: { + value: (x: boolean, y?: boolean) => y ?? x, + default: () => false, + }, + samplesFormat: { + value: (x: SamplesFormat, y?: SamplesFormat) => y ?? x, + default: () => ({ name: 'unsupported' }), + }, + ecsVersion: { + value: (x: string, y?: string) => y ?? x, + default: () => '8.11.0', + }, + results: { + value: (x: object, y?: object) => y ?? x, + default: () => ({}), + }, +}; + +function modelInput(state: LogFormatDetectionState): Partial { + return { + exAnswer: JSON.stringify(EX_ANSWER_LOG_TYPE, null, 2), + finalized: false, + lastExecutedChain: 'modelInput', + }; +} + +function modelOutput(state: LogFormatDetectionState): Partial { + return { + finalized: true, + lastExecutedChain: 'modelOutput', + results: { + samplesFormat: state.samplesFormat, + parsedSamples: state.logSamples, // TODO: Add parsed samples + }, + }; +} + +function logFormatRouter(state: LogFormatDetectionState): string { + // if (state.samplesFormat === LogFormat.STRUCTURED) { + // return 'structured'; + // } + // if (state.samplesFormat === LogFormat.UNSTRUCTURED) { + // return 'unstructured'; + // } + // if (state.samplesFormat === LogFormat.CSV) { + // return 'csv'; + // } + return 'unsupported'; +} + +export async function getLogFormatDetectionGraph( + model: ActionsClientChatOpenAI | ActionsClientSimpleChatModel +) { + const workflow = new StateGraph({ + channels: graphState, + }) + .addNode('modelInput', modelInput) + .addNode('modelOutput', modelOutput) + .addNode('handleLogFormatDetection', (state: LogFormatDetectionState) => + handleLogFormatDetection(state, model) + ) + // .addNode('handleKVGraph', (state: LogFormatDetectionState) => getCompiledKvGraph(state, model)) + // .addNode('handleUnstructuredGraph', (state: LogFormatDetectionState) => getCompiledUnstructuredGraph(state, model)) + // .addNode('handleCsvGraph', (state: LogFormatDetectionState) => getCompiledCsvGraph(state, model)) + .addEdge(START, 'modelInput') + .addEdge('modelInput', 'handleLogFormatDetection') + .addEdge('modelOutput', END) + .addConditionalEdges('handleLogFormatDetection', logFormatRouter, { + // TODO: Add structured, unstructured, csv nodes + // structured: 'handleKVGraph', + // unstructured: 'handleUnstructuredGraph', + // csv: 'handleCsvGraph', + unsupported: 'modelOutput', + }); + + const compiledLogFormatDetectionGraph = workflow.compile(); + + return compiledLogFormatDetectionGraph; +} diff --git a/x-pack/plugins/integration_assistant/server/graphs/log_type_detection/prompts.ts b/x-pack/plugins/integration_assistant/server/graphs/log_type_detection/prompts.ts new file mode 100644 index 0000000000000..9ad06d7592c0f --- /dev/null +++ b/x-pack/plugins/integration_assistant/server/graphs/log_type_detection/prompts.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { ChatPromptTemplate } from '@langchain/core/prompts'; +export const LOG_FORMAT_DETECTION_PROMPT = ChatPromptTemplate.fromMessages([ + [ + 'system', + `You are a helpful, expert assistant in identifying different log types based on the format. + +Here is some context for you to reference for your task, read it carefully as you will get questions about it later: + + +{log_samples} + +`, + ], + [ + 'human', + `Looking at the log samples , our goal is to identify the syslog type based on the guidelines below. + +- Go through each log sample and identify the log format type. +- If the syslog samples have header and structured body then classify it as "structured". +- If the syslog samples have header and unstructured body then classify it as "unstructured". +- If the syslog samples follow a csv format then classify it as "csv". +- If you do not find the log format in any of the above categories then classify it as "unsupported". +- Do not respond with anything except the updated current mapping JSON object enclosed with 3 backticks (\`). See example response below. + + +Example response format: + +A: Please find the JSON object below: +\`\`\`json +{ex_answer} +\`\`\` +`, + ], + ['ai', 'Please find the JSON object below:'], +]); diff --git a/x-pack/plugins/integration_assistant/server/routes/analyze_logs_routes.ts b/x-pack/plugins/integration_assistant/server/routes/analyze_logs_routes.ts new file mode 100644 index 0000000000000..85826e56c2004 --- /dev/null +++ b/x-pack/plugins/integration_assistant/server/routes/analyze_logs_routes.ts @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, IRouter } from '@kbn/core/server'; +import { getRequestAbortedSignal } from '@kbn/data-plugin/server'; +import { + ActionsClientChatOpenAI, + ActionsClientSimpleChatModel, +} from '@kbn/langchain/server/language_models'; +import { APMTracer } from '@kbn/langchain/server/tracers/apm'; +import { getLangSmithTracer } from '@kbn/langchain/server/tracers/langsmith'; +import { ANALYZE_LOGS_PATH, AnalyzeLogsRequestBody, AnalyzeLogsResponse } from '../../common'; +import { ROUTE_HANDLER_TIMEOUT } from '../constants'; +import type { IntegrationAssistantRouteHandlerContext } from '../plugin'; +import { buildRouteValidationWithZod } from '../util/route_validation'; +import { withAvailability } from './with_availability'; +import { getLogFormatDetectionGraph } from '../graphs/log_type_detection/graph'; + +export function registerAnalyzeLogsRoutes( + router: IRouter +) { + router.versioned + .post({ + path: ANALYZE_LOGS_PATH, + access: 'internal', + options: { + timeout: { + idleSocket: ROUTE_HANDLER_TIMEOUT, + }, + }, + }) + .addVersion( + { + version: '1', + validate: { + request: { + body: buildRouteValidationWithZod(AnalyzeLogsRequestBody), + }, + }, + }, + withAvailability(async (context, req, res): Promise> => { + const { logSamples, langSmithOptions } = req.body; + const { getStartServices, logger } = await context.integrationAssistant; + const [, { actions: actionsPlugin }] = await getStartServices(); + try { + const actionsClient = await actionsPlugin.getActionsClientWithRequest(req); + const connector = req.body.connectorId + ? await actionsClient.get({ id: req.body.connectorId }) + : (await actionsClient.getAll()).filter( + (connectorItem) => connectorItem.actionTypeId === '.bedrock' + )[0]; + const abortSignal = getRequestAbortedSignal(req.events.aborted$); + const isOpenAI = connector.actionTypeId === '.gen-ai'; + const llmClass = isOpenAI ? ActionsClientChatOpenAI : ActionsClientSimpleChatModel; + const model = new llmClass({ + actionsClient, + connectorId: connector.id, + logger, + llmType: isOpenAI ? 'openai' : 'bedrock', + model: connector.config?.defaultModel, + temperature: 0.05, + maxTokens: 4096, + signal: abortSignal, + streaming: false, + }); + const options = { + callbacks: [ + new APMTracer({ projectName: langSmithOptions?.projectName ?? 'default' }, logger), + ...getLangSmithTracer({ ...langSmithOptions, logger }), + ], + }; + + const logFormatParameters = { + logSamples, + }; + const graph = await getLogFormatDetectionGraph(model); + const graphResults = await graph.invoke(logFormatParameters, options); + const graphLogFormat = graphResults.results.samplesFormat.name; + if (graphLogFormat === 'unsupported') { + return res.customError({ + statusCode: 501, + body: { message: `Unsupported log samples format` }, + }); + } + return res.ok({ body: AnalyzeLogsResponse.parse(graphResults) }); + } catch (e) { + return res.badRequest({ body: e }); + } + }) + ); +} diff --git a/x-pack/plugins/integration_assistant/server/routes/register_routes.ts b/x-pack/plugins/integration_assistant/server/routes/register_routes.ts index a8ccc39ff2a0f..781010972ddcb 100644 --- a/x-pack/plugins/integration_assistant/server/routes/register_routes.ts +++ b/x-pack/plugins/integration_assistant/server/routes/register_routes.ts @@ -12,8 +12,10 @@ import { registerCategorizationRoutes } from './categorization_routes'; import { registerRelatedRoutes } from './related_routes'; import { registerPipelineRoutes } from './pipeline_routes'; import type { IntegrationAssistantRouteHandlerContext } from '../plugin'; +import { registerAnalyzeLogsRoutes } from './analyze_logs_routes'; export function registerRoutes(router: IRouter) { + registerAnalyzeLogsRoutes(router); registerEcsRoutes(router); registerIntegrationBuilderRoutes(router); registerCategorizationRoutes(router); diff --git a/x-pack/plugins/integration_assistant/server/types.ts b/x-pack/plugins/integration_assistant/server/types.ts index 2f5d9f1237870..d6fa5652c44b8 100644 --- a/x-pack/plugins/integration_assistant/server/types.ts +++ b/x-pack/plugins/integration_assistant/server/types.ts @@ -12,6 +12,7 @@ import { ActionsClientSimpleChatModel, ActionsClientGeminiChatModel, } from '@kbn/langchain/server'; +import { SamplesFormat } from '../common'; export interface IntegrationAssistantPluginSetup { setIsAvailable: (isAvailable: boolean) => void; @@ -84,6 +85,16 @@ export interface EcsMappingState { ecsVersion: string; } +export interface LogFormatDetectionState { + lastExecutedChain: string; + logSamples: string[]; + exAnswer: string; + finalized: boolean; + samplesFormat: SamplesFormat; + ecsVersion: string; + results: object; +} + export interface RelatedState { rawSamples: string[]; samples: string[];