diff --git a/src/platform/packages/shared/kbn-utility-types/index.ts b/src/platform/packages/shared/kbn-utility-types/index.ts index 78626bc7fdc55..d7e44ab1048e5 100644 --- a/src/platform/packages/shared/kbn-utility-types/index.ts +++ b/src/platform/packages/shared/kbn-utility-types/index.ts @@ -172,3 +172,8 @@ export type RecursivePartial = { : RecursivePartial; }; type NonAny = number | boolean | string | symbol | null; + +/** + * Utility type for making all properties of an object nullable. + */ +export type Nullable = { [K in keyof T]: T[K] | null }; diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/constants.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/constants.ts index 5210f7087f7b7..04103f333e94d 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/constants.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/constants.ts @@ -8,6 +8,8 @@ export const SIEM_MIGRATIONS_ASSISTANT_USER = 'assistant'; export const SIEM_MIGRATIONS_PATH = '/internal/siem_migrations' as const; + +// TODO: Move `SIEM_RULE_MIGRATIONS_PATH` and composed paths to rules/constants.ts export const SIEM_RULE_MIGRATIONS_PATH = `${SIEM_MIGRATIONS_PATH}/rules` as const; export const SIEM_RULE_MIGRATIONS_ALL_STATS_PATH = `${SIEM_RULE_MIGRATIONS_PATH}/stats` as const; @@ -61,11 +63,17 @@ export enum SiemMigrationRetryFilter { NOT_FULLY_TRANSLATED = 'not_fully_translated', } +// TODO: Refactor all uses of `RuleTranslationResult` -> `MigrationTranslationResult` export enum RuleTranslationResult { FULL = 'full', PARTIAL = 'partial', UNTRANSLATABLE = 'untranslatable', } +export enum MigrationTranslationResult { + FULL = 'full', + PARTIAL = 'partial', + UNTRANSLATABLE = 'untranslatable', +} export const DEFAULT_TRANSLATION_FIELDS = { from: 'now-360s', diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/dashboards/constants.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/dashboards/constants.ts index fbc3c9bd3fbba..1bcdea3bc99d8 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/dashboards/constants.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/dashboards/constants.ts @@ -9,11 +9,23 @@ import { SIEM_MIGRATIONS_PATH } from '../constants'; export const SIEM_DASHBOARD_MIGRATIONS_PATH = `${SIEM_MIGRATIONS_PATH}/dashboards` as const; +export const SIEM_DASHBOARD_MIGRATION_EVALUATE_PATH = + `${SIEM_DASHBOARD_MIGRATIONS_PATH}/evaluate` as const; + +// Migration ID specific routes + export const SIEM_DASHBOARD_MIGRATION_PATH = `${SIEM_DASHBOARD_MIGRATIONS_PATH}/{migration_id}` as const; +export const SIEM_DASHBOARD_MIGRATION_DASHBOARDS_PATH = + `${SIEM_DASHBOARD_MIGRATION_PATH}/dashboards` as const; + +export const SIEM_DASHBOARD_MIGRATION_START_PATH = + `${SIEM_DASHBOARD_MIGRATION_PATH}/start` as const; +export const SIEM_DASHBOARD_MIGRATION_STOP_PATH = `${SIEM_DASHBOARD_MIGRATION_PATH}/stop` as const; + export const SIEM_DASHBOARD_MIGRATION_STATS_PATH = `${SIEM_DASHBOARD_MIGRATION_PATH}/stats` as const; -export const SIEM_DASHBOARD_MIGRATION_DASHBOARDS_PATH = - `${SIEM_DASHBOARD_MIGRATION_PATH}/dashboards` as const; +export const SIEM_DASHBOARD_MIGRATION_TRANSLATION_STATS_PATH = + `${SIEM_DASHBOARD_MIGRATION_PATH}/translation_stats` as const; diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/dashboards/resources/index.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/dashboards/resources/index.ts new file mode 100644 index 0000000000000..d636cecbed06f --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/dashboards/resources/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +export * from './resource_identifier'; diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/dashboards/resources/resource_identifier.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/dashboards/resources/resource_identifier.ts new file mode 100644 index 0000000000000..1b3612d8c4410 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/dashboards/resources/resource_identifier.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +// TODO: move resource related types to migration.gen.ts +import type { RuleMigrationResourceBase } from '../../model/rule_migration.gen'; +import type { DashboardMigrationDashboard } from '../../model/dashboard_migration.gen'; +import { ResourceIdentifier } from '../../resources/resource_identifier'; +import type { SiemMigrationVendor } from '../../types'; + +export class DashboardResourceIdentifier extends ResourceIdentifier { + protected getVendor(): SiemMigrationVendor { + return this.item.original_dashboard.vendor; + } + + public fromOriginal(rule?: DashboardMigrationDashboard): RuleMigrationResourceBase[] { + const originalDashboard = rule?.original_dashboard ?? this.item.original_dashboard; + const queries: string[] = []; // TODO: Parse the originalDashboard to extract the queries + return queries.flatMap((query) => this.identifier(query)); + } +} diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/dashboards/types.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/dashboards/types.ts new file mode 100644 index 0000000000000..6756a42cdb834 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/dashboards/types.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SiemMigrationFilters } from '../types'; + +export interface DashboardMigrationFilters extends SiemMigrationFilters { + searchTerm?: string; + installed?: boolean; + installable?: boolean; +} diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/api/dashboards/dashboard_migration.gen.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/api/dashboards/dashboard_migration.gen.ts index 5bb821fab0580..e51d25f311425 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/api/dashboards/dashboard_migration.gen.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/api/dashboards/dashboard_migration.gen.ts @@ -17,8 +17,14 @@ import { z } from '@kbn/zod'; import { NonEmptyString } from '../../../../api/model/primitives.gen'; -import { DashboardMigration, DashboardMigrationTaskStats } from '../../dashboard_migration.gen'; +import { + DashboardMigration, + DashboardMigrationTaskExecutionSettings, + DashboardMigrationRetryFilter, + DashboardMigrationTaskStats, +} from '../../dashboard_migration.gen'; import { SplunkOriginalDashboardExport } from '../../vendor/dashboards/splunk.gen'; +import { LangSmithOptions } from '../../common.gen'; export type CreateDashboardMigrationRequestBody = z.infer< typeof CreateDashboardMigrationRequestBody @@ -82,3 +88,57 @@ export type GetDashboardMigrationStatsRequestParamsInput = z.input< export type GetDashboardMigrationStatsResponse = z.infer; export const GetDashboardMigrationStatsResponse = DashboardMigrationTaskStats; + +export type StartDashboardsMigrationRequestParams = z.infer< + typeof StartDashboardsMigrationRequestParams +>; +export const StartDashboardsMigrationRequestParams = z.object({ + migration_id: NonEmptyString, +}); +export type StartDashboardsMigrationRequestParamsInput = z.input< + typeof StartDashboardsMigrationRequestParams +>; + +export type StartDashboardsMigrationRequestBody = z.infer< + typeof StartDashboardsMigrationRequestBody +>; +export const StartDashboardsMigrationRequestBody = z.object({ + /** + * Settings applicable to current dashboard migration task execution. + */ + settings: DashboardMigrationTaskExecutionSettings, + langsmith_options: LangSmithOptions.optional(), + /** + * The optional indicator to retry the dashboard translation based on this filter criteria. + */ + retry: DashboardMigrationRetryFilter.optional(), +}); +export type StartDashboardsMigrationRequestBodyInput = z.input< + typeof StartDashboardsMigrationRequestBody +>; + +export type StartDashboardsMigrationResponse = z.infer; +export const StartDashboardsMigrationResponse = z.object({ + /** + * Indicates the migration has been started. `false` means the migration does not need to be started. + */ + started: z.boolean(), +}); + +export type StopDashboardsMigrationRequestParams = z.infer< + typeof StopDashboardsMigrationRequestParams +>; +export const StopDashboardsMigrationRequestParams = z.object({ + migration_id: NonEmptyString, +}); +export type StopDashboardsMigrationRequestParamsInput = z.input< + typeof StopDashboardsMigrationRequestParams +>; + +export type StopDashboardsMigrationResponse = z.infer; +export const StopDashboardsMigrationResponse = z.object({ + /** + * Indicates the migration has been stopped. + */ + stopped: z.boolean(), +}); diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/api/dashboards/dashboard_migration.schema.yaml b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/api/dashboards/dashboard_migration.schema.yaml index cf1adfb04b938..998aa0ae1951a 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/api/dashboards/dashboard_migration.schema.yaml +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/api/dashboards/dashboard_migration.schema.yaml @@ -94,6 +94,86 @@ paths: 200: description: Indicates dashboards have been added to the migration successfully. + /internal/siem_migrations/dashboards/{migration_id}/start: + post: + summary: Starts a dashboard migration + operationId: StartDashboardsMigration + x-codegen-enabled: true + x-internal: true + description: Starts a SIEM dashboards migration using the migration id provided + tags: + - SIEM Dashboard Migrations + parameters: + - name: migration_id + in: path + required: true + schema: + description: The migration id to start + $ref: '../../../../../common/api/model/primitives.schema.yaml#/components/schemas/NonEmptyString' + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - settings + properties: + settings: + $ref: '../../dashboard_migration.schema.yaml#/components/schemas/DashboardMigrationTaskExecutionSettings' + description: Settings applicable to current dashboard migration task execution. + langsmith_options: + $ref: '../../common.schema.yaml#/components/schemas/LangSmithOptions' + retry: + $ref: '../../dashboard_migration.schema.yaml#/components/schemas/DashboardMigrationRetryFilter' + description: The optional indicator to retry the dashboard translation based on this filter criteria. + responses: + 200: + description: Indicates the migration start request has been processed successfully. + content: + application/json: + schema: + type: object + required: + - started + properties: + started: + type: boolean + description: Indicates the migration has been started. `false` means the migration does not need to be started. + 204: + description: Indicates the migration id was not found. + + /internal/siem_migrations/dashboards/{migration_id}/stop: + post: + summary: Stops an existing dashboard migration + operationId: StopDashboardsMigration + x-codegen-enabled: true + x-internal: true + description: Stops a running SIEM dashboards migration using the migration id provided + tags: + - SIEM Dashboard Migrations + parameters: + - name: migration_id + in: path + required: true + schema: + description: The migration id to stop + $ref: '../../../../../common/api/model/primitives.schema.yaml#/components/schemas/NonEmptyString' + responses: + 200: + description: Indicates migration task stop has been processed successfully. + content: + application/json: + schema: + type: object + required: + - stopped + properties: + stopped: + type: boolean + description: Indicates the migration has been stopped. + 204: + description: Indicates the migration id was not found running. /internal/siem_migrations/dashboards/{migration_id}/stats: get: diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/common.gen.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/common.gen.ts index 8cfd45d5386fe..ac7ab0886cb2e 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/common.gen.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/common.gen.ts @@ -49,52 +49,3 @@ export const LangSmithEvaluationOptions = LangSmithOptions.merge( dataset: z.string(), }) ); - -/** - * The status of migration. - */ -export type MigrationStatus = z.infer; -export const MigrationStatus = z.enum(['pending', 'processing', 'completed', 'failed']); -export type MigrationStatusEnum = typeof MigrationStatus.enum; -export const MigrationStatusEnum = MigrationStatus.enum; - -/** - * The status of the migration task. - */ -export type MigrationTaskStatus = z.infer; -export const MigrationTaskStatus = z.enum([ - 'ready', - 'running', - 'stopped', - 'finished', - 'interrupted', -]); -export type MigrationTaskStatusEnum = typeof MigrationTaskStatus.enum; -export const MigrationTaskStatusEnum = MigrationTaskStatus.enum; - -/** - * The last execution of a migration task. - */ -export type MigrationLastExecution = z.infer; -export const MigrationLastExecution = z.object({ - /** - * The moment the last execution started. - */ - started_at: z.string().optional(), - /** - * The moment the last execution finished. - */ - finished_at: z.string().nullable().optional(), - /** - * The connector ID used for the last execution. - */ - connector_id: z.string().optional(), - /** - * The error message if the last execution failed. - */ - error: z.string().nullable().optional(), - /** - * Indicates if the last execution was stopped by the user. - */ - is_stopped: z.boolean().optional(), -}); diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/common.schema.yaml b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/common.schema.yaml index 3c07a7b3a0a78..4251aae40ac55 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/common.schema.yaml +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/common.schema.yaml @@ -35,45 +35,3 @@ components: dataset: type: string description: The dataset name to use for evaluations. - - MigrationStatus: - type: string - description: The status of migration. - enum: # should match SiemMigrationsStatus enum at ../constants.ts - - pending - - processing - - completed - - failed - - MigrationTaskStatus: - type: string - description: The status of the migration task. - enum: # should match SiemMigrationTaskStatus enum at ../constants.ts - - ready - - running - - stopped - - finished - - interrupted - - MigrationLastExecution: - type: object - description: The last execution of a migration task. - properties: - started_at: - type: string - description: The moment the last execution started. - finished_at: - type: string - nullable: true - description: The moment the last execution finished. - connector_id: - type: string - description: The connector ID used for the last execution. - error: - type: string - nullable: true - description: The error message if the last execution failed. - is_stopped: - type: boolean - description: Indicates if the last execution was stopped by the user. - diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/dashboard_migration.gen.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/dashboard_migration.gen.ts index 524af13a9b463..247fc1022a279 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/dashboard_migration.gen.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/dashboard_migration.gen.ts @@ -17,9 +17,21 @@ import { z } from '@kbn/zod'; import { NonEmptyString } from '../../api/model/primitives.gen'; -import { MigrationLastExecution, MigrationStatus, MigrationTaskStatus } from './common.gen'; +import { + MigrationLastExecution, + MigrationTranslationResult, + MigrationStatus, + MigrationComments, + MigrationTaskStats, +} from './migration.gen'; import { SplunkOriginalDashboardProperties } from './vendor/dashboards/splunk.gen'; +/** + * The original dashboard vendor identifier. + */ +export type OriginalDashboardVendor = z.infer; +export const OriginalDashboardVendor = z.literal('splunk'); + /** * The dashboard migration object ( without Id ) with its settings. */ @@ -65,10 +77,7 @@ export const OriginalDashboard = z.object({ * The unique identifier for the dashboard */ id: z.string(), - /** - * The vendor of the dashboard (e.g., 'splunk') - */ - vendor: z.string(), + vendor: OriginalDashboardVendor, /** * The title of the dashboard */ @@ -78,15 +87,15 @@ export const OriginalDashboard = z.object({ */ description: z.string(), /** - * The data of the dashboard, typically in JSON format + * The data of the dashboard in the specified format */ - data: z.object({}), + data: z.string(), /** * The last updated timestamp of the dashboard */ - last_updated: z.string(), + last_updated: z.string().optional(), /** - * The format of the dashboard (e.g., 'json', 'xml') + * The format of the dashboard data (e.g., 'json', 'xml') */ format: z.string(), /** @@ -95,6 +104,29 @@ export const OriginalDashboard = z.object({ splunk_properties: SplunkOriginalDashboardProperties.optional(), }); +/** + * The elastic dashboard translation. + */ +export type ElasticDashboard = z.infer; +export const ElasticDashboard = z.object({ + /** + * The unique identifier for the dashboard installed Saved Object + */ + id: z.string().optional(), + /** + * The title of the dashboard + */ + title: z.string(), + /** + * The description of the dashboard + */ + description: z.string().optional(), + /** + * The data of the dashboard Saved Object + */ + data: z.object({}).optional(), +}); + /** * The dashboard migration document object. */ @@ -116,10 +148,22 @@ export const DashboardMigrationDashboardData = z.object({ * The original dashboard to migrate. */ original_dashboard: OriginalDashboard, + /** + * The translated elastic dashboard. + */ + elastic_dashboard: ElasticDashboard.optional(), + /** + * The rule translation result. + */ + translation_result: MigrationTranslationResult.optional(), /** * The status of the dashboard migration process. */ status: MigrationStatus.default('pending'), + /** + * The comments for the migration including a summary from the LLM in markdown. + */ + comments: MigrationComments.optional(), /** * The moment of the last update */ @@ -147,52 +191,25 @@ export const DashboardMigrationDashboard = z * The dashboard migration task stats object. */ export type DashboardMigrationTaskStats = z.infer; -export const DashboardMigrationTaskStats = z.object({ - /** - * The migration id - */ - id: NonEmptyString, +export const DashboardMigrationTaskStats = MigrationTaskStats; + +/** + * The dashboard migration task execution settings. + */ +export type DashboardMigrationTaskExecutionSettings = z.infer< + typeof DashboardMigrationTaskExecutionSettings +>; +export const DashboardMigrationTaskExecutionSettings = z.object({ /** - * The migration name + * The connector ID used in the last execution. */ - name: NonEmptyString, - /** - * Indicates if the migration task status. - */ - status: MigrationTaskStatus, - /** - * The dashboards migration stats. - */ - dashboards: z - .object({ - /** - * The total number of dashboards to migrate. - */ - total: z.number().int(), - /** - * The number of dashboards that are pending migration. - */ - pending: z.number().int(), - /** - * The number of dashboards that are being migrated. - */ - processing: z.number().int(), - /** - * The number of dashboards that have been migrated successfully. - */ - completed: z.number().int(), - /** - * The number of dashboards that have failed migration. - */ - failed: z.number().int(), - }) - .optional(), - /** - * The moment the migration was created. - */ - created_at: z.string(), - /** - * The moment of the last update. - */ - last_updated_at: z.string(), + connector_id: z.string(), }); + +/** + * Indicates the filter to retry the migrations dashboards translation + */ +export type DashboardMigrationRetryFilter = z.infer; +export const DashboardMigrationRetryFilter = z.enum(['failed', 'not_fully_translated']); +export type DashboardMigrationRetryFilterEnum = typeof DashboardMigrationRetryFilter.enum; +export const DashboardMigrationRetryFilterEnum = DashboardMigrationRetryFilter.enum; diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/dashboard_migration.schema.yaml b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/dashboard_migration.schema.yaml index 830571684cebf..f77363cbcfaa9 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/dashboard_migration.schema.yaml +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/dashboard_migration.schema.yaml @@ -6,6 +6,11 @@ paths: {} components: x-codegen-enabled: true schemas: + OriginalDashboardVendor: + type: string + description: The original dashboard vendor identifier. + enum: + - splunk DashboardMigration: description: The dashboard migration object with its settings. @@ -34,12 +39,11 @@ components: description: The user profile ID of the user who created the migration. $ref: '../../../common/api/model/primitives.schema.yaml#/components/schemas/NonEmptyString' created_at: - description: The moment migration was created + description: The moment migration was created $ref: '../../../common/api/model/primitives.schema.yaml#/components/schemas/NonEmptyString' last_execution: description: The last execution of the dashboard migration task. - $ref: './common.schema.yaml#/components/schemas/MigrationLastExecution' - + $ref: './migration.schema.yaml#/components/schemas/MigrationLastExecution' DashboardMigrationDashboard: description: The dashboard migration document object. @@ -75,10 +79,19 @@ components: original_dashboard: description: The original dashboard to migrate. $ref: '#/components/schemas/OriginalDashboard' + elastic_dashboard: + description: The translated elastic dashboard. + $ref: '#/components/schemas/ElasticDashboard' + translation_result: + description: The rule translation result. + $ref: './migration.schema.yaml#/components/schemas/MigrationTranslationResult' status: description: The status of the dashboard migration process. - $ref: './common.schema.yaml#/components/schemas/MigrationStatus' + $ref: './migration.schema.yaml#/components/schemas/MigrationStatus' default: pending + comments: + description: The comments for the migration including a summary from the LLM in markdown. + $ref: './migration.schema.yaml#/components/schemas/MigrationComments' updated_at: type: string description: The moment of the last update @@ -94,7 +107,6 @@ components: - title - description - data - - last_updated - format description: The raw dashboard object from different vendors properties: @@ -102,8 +114,7 @@ components: type: string description: The unique identifier for the dashboard vendor: - type: string - description: The vendor of the dashboard (e.g., 'splunk') + $ref: '#/components/schemas/OriginalDashboardVendor' title: type: string description: The title of the dashboard @@ -111,66 +122,54 @@ components: type: string description: The description of the dashboard data: - type: object - description: The data of the dashboard, typically in JSON format + type: string + description: The data of the dashboard in the specified format last_updated: type: string description: The last updated timestamp of the dashboard format: type: string - description: The format of the dashboard (e.g., 'json', 'xml') + description: The format of the dashboard data (e.g., 'json', 'xml') splunk_properties: description: Additional properties specific to the splunk $ref: './vendor/dashboards/splunk.schema.yaml#/components/schemas/SplunkOriginalDashboardProperties' - DashboardMigrationTaskStats: + ElasticDashboard: type: object - description: The dashboard migration task stats object. + description: The elastic dashboard translation. required: - - id - - name - - status - - dashboard - - created_at - - last_updated_at + - title properties: id: - description: The migration id - $ref: '../../../common/api/model/primitives.schema.yaml#/components/schemas/NonEmptyString' - name: - description: The migration name - $ref: '../../../common/api/model/primitives.schema.yaml#/components/schemas/NonEmptyString' - status: - description: Indicates if the migration task status. - $ref: './common.schema.yaml#/components/schemas/MigrationTaskStatus' - dashboards: - type: object - description: The dashboards migration stats. - required: - - total - - pending - - processing - - completed - - failed - properties: - total: - type: integer - description: The total number of dashboards to migrate. - pending: - type: integer - description: The number of dashboards that are pending migration. - processing: - type: integer - description: The number of dashboards that are being migrated. - completed: - type: integer - description: The number of dashboards that have been migrated successfully. - failed: - type: integer - description: The number of dashboards that have failed migration. - created_at: type: string - description: The moment the migration was created. - last_updated_at: + description: The unique identifier for the dashboard installed Saved Object + title: + type: string + description: The title of the dashboard + description: type: string - description: The moment of the last update. + description: The description of the dashboard + data: + type: object + description: The data of the dashboard Saved Object + + DashboardMigrationTaskStats: + description: The dashboard migration task stats object. + $ref: './migration.schema.yaml#/components/schemas/MigrationTaskStats' + + DashboardMigrationTaskExecutionSettings: + type: object + description: The dashboard migration task execution settings. + required: + - connector_id + properties: + connector_id: + type: string + description: The connector ID used in the last execution. + + DashboardMigrationRetryFilter: + type: string + description: Indicates the filter to retry the migrations dashboards translation + enum: # should match SiemMigrationRetryFilter enum at ../constants.ts + - failed + - not_fully_translated diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/migration.gen.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/migration.gen.ts new file mode 100644 index 0000000000000..07cea07a74499 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/migration.gen.ts @@ -0,0 +1,169 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* + * NOTICE: Do not edit this file manually. + * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. + * + * info: + * title: SIEM Rule Migration migrations components + * version: not applicable + */ + +import { z } from '@kbn/zod'; + +import { NonEmptyString } from '../../api/model/primitives.gen'; + +/** + * The vendor identifier. + */ +export type Vendor = z.infer; +export const Vendor = z.literal('splunk'); + +/** + * A comment for the migration process + */ +export type MigrationComment = z.infer; +export const MigrationComment = z.object({ + /** + * The message of the migration comment + */ + message: z.string(), + /** + * The moment of creation of the comment + */ + created_at: z.string(), + /** + * The user profile ID of the user who created the comment, or `assistant` if it was generated by the LLM + */ + created_by: z.string(), +}); + +/** + * The comments for the migration including a summary from the LLM in markdown. + */ +export type MigrationComments = z.infer; +export const MigrationComments = z.array(MigrationComment); + +/** + * The status of migration. + */ +export type MigrationStatus = z.infer; +export const MigrationStatus = z.enum(['pending', 'processing', 'completed', 'failed']); +export type MigrationStatusEnum = typeof MigrationStatus.enum; +export const MigrationStatusEnum = MigrationStatus.enum; + +/** + * The status of the migration task. + */ +export type MigrationTaskStatus = z.infer; +export const MigrationTaskStatus = z.enum([ + 'ready', + 'running', + 'stopped', + 'finished', + 'interrupted', +]); +export type MigrationTaskStatusEnum = typeof MigrationTaskStatus.enum; +export const MigrationTaskStatusEnum = MigrationTaskStatus.enum; + +/** + * The last execution of a migration task. + */ +export type MigrationLastExecution = z.infer; +export const MigrationLastExecution = z.object({ + /** + * The moment the last execution started. + */ + started_at: z.string().optional(), + /** + * The moment the last execution finished. + */ + finished_at: z.string().nullable().optional(), + /** + * The connector ID used for the last execution. + */ + connector_id: z.string().optional(), + /** + * The error message if the last execution failed. + */ + error: z.string().nullable().optional(), + /** + * Indicates if the last execution was stopped by the user. + */ + is_stopped: z.boolean().optional(), +}); + +/** + * The migration translation result. + */ +export type MigrationTranslationResult = z.infer; +export const MigrationTranslationResult = z.enum(['full', 'partial', 'untranslatable']); +export type MigrationTranslationResultEnum = typeof MigrationTranslationResult.enum; +export const MigrationTranslationResultEnum = MigrationTranslationResult.enum; + +/** + * The migration items stats. + */ +export type MigrationTaskItemsStats = z.infer; +export const MigrationTaskItemsStats = z.object({ + /** + * The total number of items to migrate. + */ + total: z.number().int(), + /** + * The number of items that are pending migration. + */ + pending: z.number().int(), + /** + * The number of items that are being migrated. + */ + processing: z.number().int(), + /** + * The number of items that have been migrated successfully. + */ + completed: z.number().int(), + /** + * The number of items that have failed migration. + */ + failed: z.number().int(), +}); + +/** + * The migration task stats object. + */ +export type MigrationTaskStats = z.infer; +export const MigrationTaskStats = z.object({ + /** + * The migration id + */ + id: NonEmptyString, + /** + * The migration name + */ + name: NonEmptyString, + /** + * Indicates if the migration task status. + */ + status: MigrationTaskStatus, + /** + * The migration items stats. + */ + items: MigrationTaskItemsStats, + /** + * The moment the migration was created. + */ + created_at: z.string(), + /** + * The moment of the last update. + */ + last_updated_at: z.string(), + /** + * The last execution of the migration task. + */ + last_execution: MigrationLastExecution.optional(), +}); diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/migration.schema.yaml b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/migration.schema.yaml new file mode 100644 index 0000000000000..c8e4a4897aea9 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/migration.schema.yaml @@ -0,0 +1,146 @@ +openapi: 3.0.3 +info: + title: SIEM Rule Migration migrations components + version: 'not applicable' +paths: {} +components: + x-codegen-enabled: true + schemas: + Vendor: + type: string + description: The vendor identifier. + enum: + - splunk + + MigrationComment: + type: object + description: A comment for the migration process + required: + - message + - created_at + - created_by + properties: + message: + type: string + description: The message of the migration comment + created_at: + type: string + description: The moment of creation of the comment + created_by: + type: string + description: The user profile ID of the user who created the comment, or `assistant` if it was generated by the LLM + + MigrationComments: + type: array + description: The comments for the migration including a summary from the LLM in markdown. + items: + description: The comments for the migration process + $ref: '#/components/schemas/MigrationComment' + + MigrationStatus: + type: string + description: The status of migration. + enum: # should match SiemMigrationStatus enum at ../constants.ts + - pending + - processing + - completed + - failed + + MigrationTaskStatus: + type: string + description: The status of the migration task. + enum: # should match SiemMigrationTaskStatus enum at ../constants.ts + - ready + - running + - stopped + - finished + - interrupted + + MigrationLastExecution: + type: object + description: The last execution of a migration task. + properties: + started_at: + type: string + description: The moment the last execution started. + finished_at: + type: string + nullable: true + description: The moment the last execution finished. + connector_id: + type: string + description: The connector ID used for the last execution. + error: + type: string + nullable: true + description: The error message if the last execution failed. + is_stopped: + type: boolean + description: Indicates if the last execution was stopped by the user. + + MigrationTranslationResult: + type: string + description: The migration translation result. + enum: # should match RuleTranslationResult enum at ../constants.ts TODO: refactor enum to MigrationTranslationResult + - full + - partial + - untranslatable + + MigrationTaskStats: + type: object + description: The migration task stats object. + required: + - id + - name + - status + - items + - created_at + - last_updated_at + properties: + id: + description: The migration id + $ref: '../../../common/api/model/primitives.schema.yaml#/components/schemas/NonEmptyString' + name: + description: The migration name + $ref: '../../../common/api/model/primitives.schema.yaml#/components/schemas/NonEmptyString' + status: + description: Indicates if the migration task status. + $ref: '#/components/schemas/MigrationTaskStatus' + items: + description: The migration items stats. + $ref: '#/components/schemas/MigrationTaskItemsStats' + created_at: + type: string + description: The moment the migration was created. + last_updated_at: + type: string + description: The moment of the last update. + last_execution: + description: The last execution of the migration task. + $ref: '#/components/schemas/MigrationLastExecution' + + MigrationTaskItemsStats: + type: object + description: The migration items stats. + required: + - total + - pending + - processing + - completed + - failed + properties: + total: + type: integer + description: The total number of items to migrate. + pending: + type: integer + description: The number of items that are pending migration. + processing: + type: integer + description: The number of items that are being migrated. + completed: + type: integer + description: The number of items that have been migrated successfully. + failed: + type: integer + description: The number of items that have failed migration. diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts index 468302f09f447..512b58afdbd66 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts @@ -18,7 +18,13 @@ import { z } from '@kbn/zod'; import { NonEmptyString } from '../../api/model/primitives.gen'; import { RuleResponse } from '../../api/detection_engine/model/rule_schema/rule_schemas.gen'; -import { MigrationStatus, MigrationTaskStatus, MigrationLastExecution } from './common.gen'; +import { + MigrationTranslationResult, + MigrationStatus, + MigrationComments, + MigrationTaskStats, + MigrationLastExecution, +} from './migration.gen'; /** * The original rule vendor identifier. @@ -146,14 +152,14 @@ export const PrebuiltRuleVersion = z.object({ * The last execution of the rule migration task. */ export type RuleMigrationLastExecution = z.infer; -export const RuleMigrationLastExecution = z - .object({ +export const RuleMigrationLastExecution = MigrationLastExecution.merge( + z.object({ /** * Indicates if the last execution skipped matching prebuilt rules. */ skip_prebuilt_rules_matching: z.boolean().optional(), }) - .merge(MigrationLastExecution); +); /** * The rule migration object ( without Id ) with its settings. @@ -191,39 +197,6 @@ export const RuleMigration = z }) .merge(RuleMigrationData); -/** - * The rule translation result. - */ -export type RuleMigrationTranslationResult = z.infer; -export const RuleMigrationTranslationResult = z.enum(['full', 'partial', 'untranslatable']); -export type RuleMigrationTranslationResultEnum = typeof RuleMigrationTranslationResult.enum; -export const RuleMigrationTranslationResultEnum = RuleMigrationTranslationResult.enum; - -/** - * The comment for the migration - */ -export type RuleMigrationComment = z.infer; -export const RuleMigrationComment = z.object({ - /** - * The comment for the migration - */ - message: z.string(), - /** - * The moment of creation - */ - created_at: z.string(), - /** - * The user profile ID of the user who created the comment or `assistant` if it was generated by the LLM - */ - created_by: z.string(), -}); - -/** - * The comments for the migration including a summary from the LLM in markdown. - */ -export type RuleMigrationComments = z.infer; -export const RuleMigrationComments = z.array(RuleMigrationComment); - /** * The rule migration document object. */ @@ -252,7 +225,7 @@ export const RuleMigrationRuleData = z.object({ /** * The rule translation result. */ - translation_result: RuleMigrationTranslationResult.optional(), + translation_result: MigrationTranslationResult.optional(), /** * The status of the rule migration process. */ @@ -260,7 +233,7 @@ export const RuleMigrationRuleData = z.object({ /** * The comments for the migration including a summary from the LLM in markdown. */ - comments: RuleMigrationComments.optional(), + comments: MigrationComments.optional(), /** * The moment of the last update */ @@ -288,57 +261,14 @@ export const RuleMigrationRule = z * The rule migration task stats object. */ export type RuleMigrationTaskStats = z.infer; -export const RuleMigrationTaskStats = z.object({ - /** - * The migration id - */ - id: NonEmptyString, - /** - * The migration name - */ - name: NonEmptyString, - /** - * Indicates if the migration task status. - */ - status: MigrationTaskStatus, - /** - * The rules migration stats. - */ - rules: z.object({ - /** - * The total number of rules to migrate. - */ - total: z.number().int(), - /** - * The number of rules that are pending migration. - */ - pending: z.number().int(), - /** - * The number of rules that are being migrated. - */ - processing: z.number().int(), - /** - * The number of rules that have been migrated successfully. - */ - completed: z.number().int(), +export const RuleMigrationTaskStats = MigrationTaskStats.merge( + z.object({ /** - * The number of rules that have failed migration. + * The last execution of the rule migration task. */ - failed: z.number().int(), - }), - /** - * The moment the migration was created. - */ - created_at: z.string(), - /** - * The moment of the last update. - */ - last_updated_at: z.string(), - /** - * The last execution of the migration task. - */ - last_execution: RuleMigrationLastExecution.optional(), -}); + last_execution: RuleMigrationLastExecution.optional(), + }) +); /** * The rule migration translation stats object. @@ -414,7 +344,7 @@ export const UpdateRuleMigrationRule = z.object({ /** * The comments for the migration including a summary from the LLM in markdown. */ - comments: RuleMigrationComments.optional(), + comments: MigrationComments.optional(), }); /** diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml index 29c0b02493664..3f63321babf8b 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml @@ -145,13 +145,12 @@ components: description: The user profile ID of the user who created the migration. $ref: '../../../common/api/model/primitives.schema.yaml#/components/schemas/NonEmptyString' created_at: - description: The moment migration was created + description: The moment migration was created $ref: '../../../common/api/model/primitives.schema.yaml#/components/schemas/NonEmptyString' last_execution: description: The last execution details of a rule migration task. $ref: '#/components/schemas/RuleMigrationLastExecution' - RuleMigrationRule: description: The rule migration document object. allOf: @@ -191,14 +190,14 @@ components: $ref: '#/components/schemas/ElasticRule' translation_result: description: The rule translation result. - $ref: '#/components/schemas/RuleMigrationTranslationResult' + $ref: './migration.schema.yaml#/components/schemas/MigrationTranslationResult' status: description: The status of the rule migration process. - $ref: './common.schema.yaml#/components/schemas/MigrationStatus' + $ref: './migration.schema.yaml#/components/schemas/MigrationStatus' default: pending comments: description: The comments for the migration including a summary from the LLM in markdown. - $ref: '#/components/schemas/RuleMigrationComments' + $ref: './migration.schema.yaml#/components/schemas/MigrationComments' updated_at: type: string description: The moment of the last update @@ -207,59 +206,14 @@ components: description: The user who last updated the migration RuleMigrationTaskStats: - type: object description: The rule migration task stats object. - required: - - id - - name - - status - - rules - - created_at - - last_updated_at - properties: - id: - description: The migration id - $ref: '../../../common/api/model/primitives.schema.yaml#/components/schemas/NonEmptyString' - name: - description: The migration name - $ref: '../../../common/api/model/primitives.schema.yaml#/components/schemas/NonEmptyString' - status: - description: Indicates if the migration task status. - $ref: './common.schema.yaml#/components/schemas/MigrationTaskStatus' - rules: - type: object - description: The rules migration stats. - required: - - total - - pending - - processing - - completed - - failed + allOf: + - $ref: './migration.schema.yaml#/components/schemas/MigrationTaskStats' + - type: object properties: - total: - type: integer - description: The total number of rules to migrate. - pending: - type: integer - description: The number of rules that are pending migration. - processing: - type: integer - description: The number of rules that are being migrated. - completed: - type: integer - description: The number of rules that have been migrated successfully. - failed: - type: integer - description: The number of rules that have failed migration. - created_at: - type: string - description: The moment the migration was created. - last_updated_at: - type: string - description: The moment of the last update. - last_execution: - description: The last execution of the migration task. - $ref: '#/components/schemas/RuleMigrationLastExecution' + last_execution: + description: The last execution of the rule migration task. + $ref: '#/components/schemas/RuleMigrationLastExecution' RuleMigrationTranslationStats: type: object @@ -320,38 +274,6 @@ components: failed: type: integer description: The number of rules that have failed translation. - RuleMigrationTranslationResult: - type: string - description: The rule translation result. - enum: # should match SiemMigrationRuleTranslationResult enum at ../constants.ts - - full - - partial - - untranslatable - - RuleMigrationComment: - type: object - description: The comment for the migration - required: - - message - - created_at - - created_by - properties: - message: - type: string - description: The comment for the migration - created_at: - type: string - description: The moment of creation - created_by: - type: string - description: The user profile ID of the user who created the comment or `assistant` if it was generated by the LLM - - RuleMigrationComments: - type: array - description: The comments for the migration including a summary from the LLM in markdown. - items: - description: The comments for the migration - $ref: '#/components/schemas/RuleMigrationComment' UpdateRuleMigrationRule: type: object @@ -367,7 +289,7 @@ components: $ref: '#/components/schemas/ElasticRulePartial' comments: description: The comments for the migration including a summary from the LLM in markdown. - $ref: '#/components/schemas/RuleMigrationComments' + $ref: './migration.schema.yaml#/components/schemas/MigrationComments' RuleMigrationRetryFilter: type: string @@ -463,14 +385,12 @@ components: RuleMigrationLastExecution: description: The last execution of the rule migration task. allOf: + - $ref: './migration.schema.yaml#/components/schemas/MigrationLastExecution' - type: object - required: - - connector_id properties: skip_prebuilt_rules_matching: type: boolean description: Indicates if the last execution skipped matching prebuilt rules. - - $ref: './common.schema.yaml#/components/schemas/MigrationLastExecution' RuleMigrationTaskExecutionSettings: type: object diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/vendor/dashboards/splunk.gen.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/vendor/dashboards/splunk.gen.ts index a2b1ba684c0ef..d8115fcc480cc 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/vendor/dashboards/splunk.gen.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/vendor/dashboards/splunk.gen.ts @@ -26,7 +26,7 @@ export const SplunkOriginalDashboardExportProperties = z.object({ /** * The unique identifier for the dashboard */ - id: z.string().optional(), + id: z.string(), /** * The label of the dashboard */ @@ -34,7 +34,7 @@ export const SplunkOriginalDashboardExportProperties = z.object({ /** * The title of the dashboard */ - title: z.string().optional(), + title: z.string(), /** * The description of the dashboard */ @@ -42,7 +42,7 @@ export const SplunkOriginalDashboardExportProperties = z.object({ /** * The EAI data of the dashboard, typically in XML format */ - 'eai:data': z.string().optional(), + 'eai:data': z.string(), /** * The application associated with the EAI ACL */ diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/vendor/dashboards/splunk.schema.yaml b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/vendor/dashboards/splunk.schema.yaml index f4e89dd4150de..f2be5ee3b515e 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/vendor/dashboards/splunk.schema.yaml +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/vendor/dashboards/splunk.schema.yaml @@ -21,6 +21,10 @@ components: SplunkOriginalDashboardExportProperties: type: object description: Properties of the original dashboard + required: + - id + - title + - eai:data properties: id: type: string diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/resources/resource_identifier.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/resources/resource_identifier.ts new file mode 100644 index 0000000000000..9b6f51cf8b15a --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/resources/resource_identifier.ts @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +// TODO: move resource related types to migration.gen.ts +import type { + RuleMigrationResourceData, + RuleMigrationResourceBase, +} from '../model/rule_migration.gen'; +import type { VendorResourceIdentifier } from './types'; +import { splResourceIdentifier } from './splunk'; +import type { ItemDocument, SiemMigrationVendor } from '../types'; + +export const identifiers: Record = { + splunk: splResourceIdentifier, +}; + +// Type for a class that extends the ResourceIdentifier abstract class +export type ResourceIdentifierClass = new ( + item: I +) => ResourceIdentifier; + +export abstract class ResourceIdentifier { + protected identifier: VendorResourceIdentifier; + + constructor(protected readonly item: I) { + this.identifier = identifiers[this.getVendor()]; + } + + protected abstract getVendor(): SiemMigrationVendor; + public abstract fromOriginal(item?: I): RuleMigrationResourceBase[]; + + public fromOriginals(item: I[]): RuleMigrationResourceBase[] { + const lookups = new Set(); + const macros = new Set(); + item.forEach((rule) => { + const resources = this.fromOriginal(rule); + resources.forEach((resource) => { + if (resource.type === 'macro') { + macros.add(resource.name); + } else if (resource.type === 'lookup') { + lookups.add(resource.name); + } + }); + }); + return [ + ...Array.from(macros).map((name) => ({ type: 'macro', name })), + ...Array.from(lookups).map((name) => ({ type: 'lookup', name })), + ]; + } + + public fromResource(resource: RuleMigrationResourceData): RuleMigrationResourceBase[] { + if (resource.type === 'macro' && resource.content) { + return this.identifier(resource.content); + } + return []; + } + + public fromResources(resources: RuleMigrationResourceData[]): RuleMigrationResourceBase[] { + const lookups = new Set(); + const macros = new Set(); + resources.forEach((resource) => { + this.fromResource(resource).forEach((identifiedResource) => { + if (identifiedResource.type === 'macro') { + macros.add(identifiedResource.name); + } else if (identifiedResource.type === 'lookup') { + lookups.add(identifiedResource.name); + } + }); + }); + return [ + ...Array.from(macros).map((name) => ({ type: 'macro', name })), + ...Array.from(lookups).map((name) => ({ type: 'lookup', name })), + ]; + } +} diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/resources/splunk/index.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/resources/splunk/index.ts new file mode 100644 index 0000000000000..9bb513df812c0 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/resources/splunk/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './splunk_identifier'; diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.test.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/resources/splunk/splunk_identifier.test.ts similarity index 100% rename from x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.test.ts rename to x-pack/solutions/security/plugins/security_solution/common/siem_migrations/resources/splunk/splunk_identifier.test.ts diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/resources/splunk/splunk_identifier.ts similarity index 89% rename from x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.ts rename to x-pack/solutions/security/plugins/security_solution/common/siem_migrations/resources/splunk/splunk_identifier.ts index 90856896193dc..3ed9210d2dac5 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/splunk/splunk_identifier.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/resources/splunk/splunk_identifier.ts @@ -11,13 +11,14 @@ * Please make sure to test all regular expressions them before using them. * At the time of writing, this tool can be used to test it: https://devina.io/redos-checker */ -import type { RuleMigrationResourceBase } from '../../../model/rule_migration.gen'; -import type { ResourceIdentifier } from '../types'; +// TODO: move resource related types to migration.gen.ts +import type { RuleMigrationResourceBase } from '../../model/rule_migration.gen'; +import type { VendorResourceIdentifier } from '../types'; const lookupRegex = /\b(?:lookup)\s+([\w-]+)\b/g; // Captures only the lookup name const macrosRegex = /`([\w-]+)(?:\(([^`]*?)\))?`/g; // Captures only the macro name and arguments -export const splResourceIdentifier: ResourceIdentifier = (input) => { +export const splResourceIdentifier: VendorResourceIdentifier = (input) => { // sanitize the query to avoid mismatching macro and lookup names inside comments or literal strings const sanitizedInput = sanitizeInput(input); diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/types.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/resources/types.ts similarity index 67% rename from x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/types.ts rename to x-pack/solutions/security/plugins/security_solution/common/siem_migrations/resources/types.ts index db4df43cc3df6..3e5c61e042e04 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/types.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/resources/types.ts @@ -4,16 +4,14 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - +// TODO: move resource related types to migration.gen.ts import type { - OriginalRule, RuleMigrationResourceBase, RuleMigrationResourceData, -} from '../../model/rule_migration.gen'; +} from '../model/rule_migration.gen'; -export type ResourceIdentifier = (input: string) => RuleMigrationResourceBase[]; +export type VendorResourceIdentifier = (input: string) => RuleMigrationResourceBase[]; export interface ResourceIdentifiers { - fromOriginalRule: (originalRule: OriginalRule) => RuleMigrationResourceBase[]; fromResource: (resource: RuleMigrationResourceData) => RuleMigrationResourceBase[]; } diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/index.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/index.ts index 4b6472eb0e55c..d636cecbed06f 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/index.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/index.ts @@ -4,74 +4,4 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - -import type { - OriginalRule, - OriginalRuleVendor, - RuleMigrationResourceData, - RuleMigrationResourceBase, -} from '../../model/rule_migration.gen'; -import type { ResourceIdentifiers } from './types'; -import { splResourceIdentifiers } from './splunk'; - -const ruleResourceIdentifiers: Record = { - splunk: splResourceIdentifiers, -}; - -export const getRuleResourceIdentifier = (vendor: OriginalRuleVendor): ResourceIdentifiers => { - return ruleResourceIdentifiers[vendor]; -}; - -export class ResourceIdentifier { - private identifiers: ResourceIdentifiers; - - constructor(vendor: OriginalRuleVendor) { - // The constructor may need query_language as an argument for other vendors - this.identifiers = ruleResourceIdentifiers[vendor]; - } - - public fromOriginalRule(originalRule: OriginalRule): RuleMigrationResourceBase[] { - return this.identifiers.fromOriginalRule(originalRule); - } - - public fromResource(resource: RuleMigrationResourceData): RuleMigrationResourceBase[] { - return this.identifiers.fromResource(resource); - } - - public fromOriginalRules(originalRules: OriginalRule[]): RuleMigrationResourceBase[] { - const lookups = new Set(); - const macros = new Set(); - originalRules.forEach((rule) => { - const resources = this.identifiers.fromOriginalRule(rule); - resources.forEach((resource) => { - if (resource.type === 'macro') { - macros.add(resource.name); - } else if (resource.type === 'lookup') { - lookups.add(resource.name); - } - }); - }); - return [ - ...Array.from(macros).map((name) => ({ type: 'macro', name })), - ...Array.from(lookups).map((name) => ({ type: 'lookup', name })), - ]; - } - - public fromResources(resources: RuleMigrationResourceData[]): RuleMigrationResourceBase[] { - const lookups = new Set(); - const macros = new Set(); - resources.forEach((resource) => { - this.identifiers.fromResource(resource).forEach((identifiedResource) => { - if (identifiedResource.type === 'macro') { - macros.add(identifiedResource.name); - } else if (identifiedResource.type === 'lookup') { - lookups.add(identifiedResource.name); - } - }); - }); - return [ - ...Array.from(macros).map((name) => ({ type: 'macro', name })), - ...Array.from(lookups).map((name) => ({ type: 'lookup', name })), - ]; - } -} +export * from './resource_identifier'; diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/resource_identifier.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/resource_identifier.ts new file mode 100644 index 0000000000000..bf376fb58c301 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/resource_identifier.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +// TODO: move resource related types to migration.gen.ts +import type { RuleMigrationResourceBase, RuleMigrationRule } from '../../model/rule_migration.gen'; +import { ResourceIdentifier } from '../../resources/resource_identifier'; +import type { SiemMigrationVendor } from '../../types'; + +export class RuleResourceIdentifier extends ResourceIdentifier { + protected getVendor(): SiemMigrationVendor { + return this.item.original_rule.vendor; + } + + public fromOriginal(rule?: RuleMigrationRule): RuleMigrationResourceBase[] { + const originalRule = rule?.original_rule ?? this.item.original_rule; + return this.identifier(originalRule.query); + } +} diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/splunk/index.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/splunk/index.ts deleted file mode 100644 index a16c328da947a..0000000000000 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/resources/splunk/index.ts +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { ResourceIdentifiers } from '../types'; -import { splResourceIdentifier } from './splunk_identifier'; - -export const splResourceIdentifiers: ResourceIdentifiers = { - fromOriginalRule: (originalRule) => splResourceIdentifier(originalRule.query), - fromResource: (resource) => { - if (resource.type === 'macro' && resource.content) { - return splResourceIdentifier(resource.content); - } - return []; - }, -}; diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/types.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/types.ts new file mode 100644 index 0000000000000..f3bcaac24bfc4 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/types.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SiemMigrationFilters } from '../types'; + +export interface RuleMigrationFilters extends SiemMigrationFilters { + searchTerm?: string; + installed?: boolean; + installable?: boolean; + prebuilt?: boolean; +} diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/types.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/types.ts index 7b275975695c7..4ac9cbb2c6ddc 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/types.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/types.ts @@ -5,17 +5,28 @@ * 2.0. */ +import type { + DashboardMigration, + DashboardMigrationDashboard, + OriginalDashboardVendor, +} from './model/dashboard_migration.gen'; +import type { + OriginalRuleVendor, + RuleMigration, + RuleMigrationRule, +} from './model/rule_migration.gen'; import type { SiemMigrationStatus } from './constants'; -export interface RuleMigrationFilters { +export interface SiemMigrationFilters { status?: SiemMigrationStatus | SiemMigrationStatus[]; ids?: string[]; - installed?: boolean; - installable?: boolean; - prebuilt?: boolean; failed?: boolean; fullyTranslated?: boolean; partiallyTranslated?: boolean; untranslatable?: boolean; - searchTerm?: string; } + +export type SiemMigrationVendor = OriginalRuleVendor | OriginalDashboardVendor; + +export type MigrationDocument = RuleMigration | DashboardMigration; +export type ItemDocument = RuleMigrationRule | DashboardMigrationDashboard; diff --git a/x-pack/solutions/security/plugins/security_solution/docs/generate_esql/img/generate_esql_graph.png b/x-pack/solutions/security/plugins/security_solution/docs/generate_esql/img/generate_esql_graph.png index 140c640c783d7..ea1f41358d315 100644 Binary files a/x-pack/solutions/security/plugins/security_solution/docs/generate_esql/img/generate_esql_graph.png and b/x-pack/solutions/security/plugins/security_solution/docs/generate_esql/img/generate_esql_graph.png differ diff --git a/x-pack/solutions/security/plugins/security_solution/docs/siem_migration/img/agent_graph.png b/x-pack/solutions/security/plugins/security_solution/docs/siem_migration/img/agent_graph.png deleted file mode 100644 index 373055b9c5621..0000000000000 Binary files a/x-pack/solutions/security/plugins/security_solution/docs/siem_migration/img/agent_graph.png and /dev/null differ diff --git a/x-pack/solutions/security/plugins/security_solution/docs/siem_migration/img/dashboard_migration_agent_graph.png b/x-pack/solutions/security/plugins/security_solution/docs/siem_migration/img/dashboard_migration_agent_graph.png new file mode 100644 index 0000000000000..f26bcfdab8a1a Binary files /dev/null and b/x-pack/solutions/security/plugins/security_solution/docs/siem_migration/img/dashboard_migration_agent_graph.png differ diff --git a/x-pack/solutions/security/plugins/security_solution/docs/siem_migration/img/rule_migration_agent_graph.png b/x-pack/solutions/security/plugins/security_solution/docs/siem_migration/img/rule_migration_agent_graph.png new file mode 100644 index 0000000000000..90978ed577288 Binary files /dev/null and b/x-pack/solutions/security/plugins/security_solution/docs/siem_migration/img/rule_migration_agent_graph.png differ diff --git a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/common/mocks/migration_result.data.ts b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/common/mocks/migration_result.data.ts index ac2ad35d71b24..53d9cce15e7bc 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/common/mocks/migration_result.data.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/common/mocks/migration_result.data.ts @@ -64,7 +64,7 @@ export const mockedMigrationLatestStatsData: RuleMigrationStats[] = [ { id: '1', status: SiemMigrationTaskStatus.FINISHED, - rules: { + items: { total: 1, pending: 0, processing: 0, @@ -78,7 +78,7 @@ export const mockedMigrationLatestStatsData: RuleMigrationStats[] = [ { id: '2', status: SiemMigrationTaskStatus.FINISHED, - rules: { + items: { total: 2, pending: 0, processing: 0, diff --git a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/api/index.ts b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/api/index.ts index c4fcbb22f2c00..0e7a83ccdcae6 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/api/index.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/api/index.ts @@ -8,7 +8,7 @@ import { replaceParams } from '@kbn/openapi-common/shared'; import type { UpdateRuleMigrationRule } from '../../../../common/siem_migrations/model/rule_migration.gen'; -import type { RuleMigrationFilters } from '../../../../common/siem_migrations/types'; +import type { RuleMigrationFilters } from '../../../../common/siem_migrations/rules/types'; import type { LangSmithOptions } from '../../../../common/siem_migrations/model/common.gen'; import { KibanaServices } from '../../../common/lib/kibana'; diff --git a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/migration_panel_title.test.tsx b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/migration_panel_title.test.tsx index 9cfbc839ddb56..dc0047e4d3332 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/migration_panel_title.test.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/migration_panel_title.test.tsx @@ -29,7 +29,7 @@ const mockMigrationStatsReady: RuleMigrationStats = { id: 'test-migration-id', name: 'Test Migration', status: SiemMigrationTaskStatus.READY, - rules: { total: 6, pending: 6, processing: 0, completed: 0, failed: 0 }, + items: { total: 6, pending: 6, processing: 0, completed: 0, failed: 0 }, created_at: '2025-05-27T12:12:17.563Z', last_updated_at: '2025-05-27T12:12:17.563Z', }; diff --git a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/migration_progress_panel.test.tsx b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/migration_progress_panel.test.tsx index 49ff0bf66d9c7..39db06efe6e52 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/migration_progress_panel.test.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/migration_progress_panel.test.tsx @@ -23,7 +23,7 @@ const inProgressMigrationStats: RuleMigrationStats = { name: 'test-migration', status: SiemMigrationTaskStatus.RUNNING, id: 'c44d2c7d-0de1-4231-8b82-0dcfd67a9fe3', - rules: { total: 26, pending: 6, processing: 10, completed: 9, failed: 1 }, + items: { total: 26, pending: 6, processing: 10, completed: 9, failed: 1 }, created_at: '2025-05-27T12:12:17.563Z', last_updated_at: '2025-05-27T12:12:17.563Z', }; @@ -31,7 +31,7 @@ const preparingMigrationStats: RuleMigrationStats = { ...inProgressMigrationStats, // status RUNNING and the same number of total and pending rules, means the migration is still preparing the environment status: SiemMigrationTaskStatus.RUNNING, - rules: { total: 6, pending: 6, processing: 0, completed: 0, failed: 0 }, + items: { total: 6, pending: 6, processing: 0, completed: 0, failed: 0 }, }; const renderMigrationProgressPanel = (migrationStats: RuleMigrationStats) => { diff --git a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/migration_progress_panel.tsx b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/migration_progress_panel.tsx index 46df0cdf6cc3e..14a0ae25f95bd 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/migration_progress_panel.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/migration_progress_panel.tsx @@ -39,10 +39,11 @@ export const MigrationProgressPanel = React.memo( stopMigration(migrationStats.id); }, [migrationStats.id, stopMigration]); - const finishedCount = migrationStats.rules.completed + migrationStats.rules.failed; - const progressValue = (finishedCount / migrationStats.rules.total) * 100; + const { items } = migrationStats; + const finishedCount = items.completed + items.failed; + const progressValue = (finishedCount / items.total) * 100; - const preparing = migrationStats.rules.pending === migrationStats.rules.total; + const preparing = items.pending === items.total; return ( @@ -53,9 +54,7 @@ export const MigrationProgressPanel = React.memo( - - {i18n.RULE_MIGRATION_PROGRESS_DESCRIPTION(migrationStats.rules.total)} - + {i18n.RULE_MIGRATION_PROGRESS_DESCRIPTION(items.total)} diff --git a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/migration_ready_panel.test.tsx b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/migration_ready_panel.test.tsx index e52538a1b831e..7093b7fe4a1e9 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/migration_ready_panel.test.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/migration_ready_panel.test.tsx @@ -27,7 +27,7 @@ jest.mock('../../service/hooks/use_start_migration'); const useStartMigrationMock = useStartMigration as jest.Mock; const mockStartMigration = jest.fn(); -const mockMigrationStateWithError = { +const mockMigrationStateWithError: RuleMigrationStats = { status: SiemMigrationTaskStatus.READY, last_execution: { error: @@ -35,16 +35,16 @@ const mockMigrationStateWithError = { }, id: 'c44d2c7d-0de1-4231-8b82-0dcfd67a9fe3', name: 'Migration 1', - rules: { total: 6, pending: 6, processing: 0, completed: 0, failed: 0 }, + items: { total: 6, pending: 6, processing: 0, completed: 0, failed: 0 }, created_at: '2025-05-27T12:12:17.563Z', last_updated_at: '2025-05-27T12:12:17.563Z', }; -const mockMigrationStatsStopped = { +const mockMigrationStatsStopped: RuleMigrationStats = { status: SiemMigrationTaskStatus.STOPPED, id: 'c44d2c7d-0de1-4231-8b82-0dcfd67a9fe3', name: 'Migration 1', - rules: { total: 6, pending: 6, processing: 0, completed: 0, failed: 0 }, + items: { total: 6, pending: 6, processing: 0, completed: 0, failed: 0 }, created_at: '2025-05-27T12:12:17.563Z', last_updated_at: '2025-05-27T12:12:17.563Z', }; @@ -53,7 +53,7 @@ const mockMigrationStatsReady: RuleMigrationStats = { status: SiemMigrationTaskStatus.READY, id: 'c44d2c7d-0de1-4231-8b82-0dcfd67a9fe3', name: 'Migration 1', - rules: { total: 6, pending: 6, processing: 0, completed: 0, failed: 0 }, + items: { total: 6, pending: 6, processing: 0, completed: 0, failed: 0 }, created_at: '2025-05-27T12:12:17.563Z', last_updated_at: '2025-05-27T12:12:17.563Z', }; diff --git a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/migration_ready_panel.tsx b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/migration_ready_panel.tsx index 5569f3a098293..59d80756817a7 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/migration_ready_panel.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/migration_ready_panel.tsx @@ -56,14 +56,14 @@ export const MigrationReadyPanel = React.memo(({ migra const migrationPanelDescription = useMemo(() => { if (migrationStats.last_execution?.error) { - return i18n.RULE_MIGRATION_ERROR_DESCRIPTION(migrationStats.rules.total); + return i18n.RULE_MIGRATION_ERROR_DESCRIPTION(migrationStats.items.total); } if (isStopped) { - return i18n.RULE_MIGRATION_STOPPED_DESCRIPTION(migrationStats.rules.total); + return i18n.RULE_MIGRATION_STOPPED_DESCRIPTION(migrationStats.items.total); } - return i18n.RULE_MIGRATION_READY_DESCRIPTION(migrationStats.rules.total); - }, [migrationStats.last_execution?.error, migrationStats.rules.total, isStopped]); + return i18n.RULE_MIGRATION_READY_DESCRIPTION(migrationStats.items.total); + }, [migrationStats.last_execution?.error, migrationStats.items.total, isStopped]); return ( diff --git a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/rule_details_flyout/tabs/translation/callout.tsx b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/rule_details_flyout/tabs/translation/callout.tsx index 1c2106dbe48fd..3c5416a809f0c 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/rule_details_flyout/tabs/translation/callout.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/rule_details_flyout/tabs/translation/callout.tsx @@ -9,13 +9,11 @@ import type { FC } from 'react'; import React from 'react'; import type { IconType } from '@elastic/eui'; import { EuiCallOut } from '@elastic/eui'; -import { - type RuleMigrationTranslationResult, - type RuleMigrationRule, -} from '../../../../../../../common/siem_migrations/model/rule_migration.gen'; +import { type MigrationTranslationResult } from '../../../../../../../common/siem_migrations/model/migration.gen'; +import { type RuleMigrationRule } from '../../../../../../../common/siem_migrations/model/rule_migration.gen'; import * as i18n from './translations'; -type RuleMigrationTranslationCallOutMode = RuleMigrationTranslationResult | 'mapped'; +type RuleMigrationTranslationCallOutMode = MigrationTranslationResult | 'mapped'; const getCallOutInfo = ( mode: RuleMigrationTranslationCallOutMode diff --git a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/rules_table/index.tsx b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/rules_table/index.tsx index fdfd55547aa11..1dc1313da5af7 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/rules_table/index.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/rules_table/index.tsx @@ -18,7 +18,7 @@ import { } from '@elastic/eui'; import React, { useCallback, useMemo, useState } from 'react'; -import type { RuleMigrationFilters } from '../../../../../common/siem_migrations/types'; +import type { RuleMigrationFilters } from '../../../../../common/siem_migrations/rules/types'; import { useIsOpenState } from '../../../../common/hooks/use_is_open_state'; import type { RelatedIntegration, RuleResponse } from '../../../../../common/api/detection_engine'; import { isMigrationPrebuiltRule } from '../../../../../common/siem_migrations/rules/utils'; diff --git a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/rules_table/utils/filters.ts b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/rules_table/utils/filters.ts index 885b6085bf110..8e49eb5144f68 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/rules_table/utils/filters.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/rules_table/utils/filters.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { RuleMigrationFilters } from '../../../../../../common/siem_migrations/types'; +import type { RuleMigrationFilters } from '../../../../../../common/siem_migrations/rules/types'; import type { FilterOptions } from '../../../types'; import { AuthorFilter, StatusFilter } from '../../../types'; diff --git a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/status_badge/index.tsx b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/status_badge/index.tsx index 44361ccf672c5..eb8d0d0bf9599 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/status_badge/index.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/status_badge/index.tsx @@ -9,8 +9,10 @@ import React from 'react'; import { EuiFlexGroup, EuiFlexItem, EuiHealth, EuiIcon, EuiToolTip } from '@elastic/eui'; import { css } from '@emotion/css'; -import { MigrationStatusEnum } from '../../../../../common/siem_migrations/model/common.gen'; -import { RuleTranslationResult } from '../../../../../common/siem_migrations/constants'; +import { + RuleTranslationResult, + SiemMigrationStatus, +} from '../../../../../common/siem_migrations/constants'; import { convertTranslationResultIntoText, useResultVisColors, @@ -48,7 +50,7 @@ export const StatusBadge: React.FC = React.memo( } // Failed - if (migrationRule.status === MigrationStatusEnum.failed) { + if (migrationRule.status === SiemMigrationStatus.FAILED) { const tooltipMessage = migrationRule.comments?.length ? migrationRule.comments[0].message : i18n.RULE_STATUS_FAILED; diff --git a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/logic/use_get_migration_rules.ts b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/logic/use_get_migration_rules.ts index 117ddd83d3e70..97717aa4439e1 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/logic/use_get_migration_rules.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/logic/use_get_migration_rules.ts @@ -8,7 +8,7 @@ import { useQuery, useQueryClient } from '@tanstack/react-query'; import { replaceParams } from '@kbn/openapi-common/shared'; import { useCallback } from 'react'; -import type { RuleMigrationFilters } from '../../../../common/siem_migrations/types'; +import type { RuleMigrationFilters } from '../../../../common/siem_migrations/rules/types'; import { SIEM_RULE_MIGRATION_RULES_PATH } from '../../../../common/siem_migrations/constants'; import { useAppToasts } from '../../../common/hooks/use_app_toasts'; import * as i18n from './translations'; diff --git a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/utils/translation_results/index.ts b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/utils/translation_results/index.ts index 29c99a11dfa0d..ce52794b22b21 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/utils/translation_results/index.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/utils/translation_results/index.ts @@ -7,7 +7,7 @@ import { useEuiTheme } from '@elastic/eui'; import { RuleTranslationResult } from '../../../../../common/siem_migrations/constants'; -import type { RuleMigrationTranslationResult } from '../../../../../common/siem_migrations/model/rule_migration.gen'; +import type { MigrationTranslationResult } from '../../../../../common/siem_migrations/model/migration.gen'; import * as i18n from './translations'; const COLORS = { @@ -31,7 +31,7 @@ export const useResultVisColors = () => { return COLORS; }; -export const convertTranslationResultIntoColor = (status?: RuleMigrationTranslationResult) => { +export const convertTranslationResultIntoColor = (status?: MigrationTranslationResult) => { switch (status) { case RuleTranslationResult.FULL: return COLORS[RuleTranslationResult.FULL]; @@ -44,7 +44,7 @@ export const convertTranslationResultIntoColor = (status?: RuleMigrationTranslat } }; -export const convertTranslationResultIntoText = (status?: RuleMigrationTranslationResult) => { +export const convertTranslationResultIntoText = (status?: MigrationTranslationResult) => { switch (status) { case RuleTranslationResult.FULL: return i18n.SIEM_TRANSLATION_RESULT_FULL_LABEL; diff --git a/x-pack/solutions/security/plugins/security_solution/scripts/langgraph/draw_graphs_script.ts b/x-pack/solutions/security/plugins/security_solution/scripts/langgraph/draw_graphs_script.ts index 95878ae1a96cc..5eff27f947967 100644 --- a/x-pack/solutions/security/plugins/security_solution/scripts/langgraph/draw_graphs_script.ts +++ b/x-pack/solutions/security/plugins/security_solution/scripts/langgraph/draw_graphs_script.ts @@ -14,13 +14,17 @@ import { ToolingLog } from '@kbn/tooling-log'; import { FakeLLM } from '@langchain/core/utils/testing'; import fs from 'fs/promises'; import path from 'path'; -import type { ElasticsearchClient, KibanaRequest } from '@kbn/core/server'; +import type { ElasticsearchClient, IScopedClusterClient, KibanaRequest } from '@kbn/core/server'; import type { InferenceServerStart } from '@kbn/inference-plugin/server'; +import type { DashboardMigrationsRetriever } from '../../server/lib/siem_migrations/dashboards/task/retrievers'; +import { getDashboardMigrationAgent } from '../../server/lib/siem_migrations/dashboards/task/agent'; +import type { DashboardMigrationTelemetryClient } from '../../server/lib/siem_migrations/dashboards/task/dashboard_migrations_telemetry_client'; +import type { ChatModel } from '../../server/lib/siem_migrations/common/task/util/actions_client_chat'; import { getGenerateEsqlGraph as getGenerateEsqlAgent } from '../../server/assistant/tools/esql/graphs/generate_esql/generate_esql'; import { getRuleMigrationAgent } from '../../server/lib/siem_migrations/rules/task/agent'; import type { RuleMigrationsRetriever } from '../../server/lib/siem_migrations/rules/task/retrievers'; -import type { EsqlKnowledgeBase } from '../../server/lib/siem_migrations/rules/task/util/esql_knowledge_base'; -import type { SiemMigrationTelemetryClient } from '../../server/lib/siem_migrations/rules/task/rule_migrations_telemetry_client'; +import type { EsqlKnowledgeBase } from '../../server/lib/siem_migrations/common/task/util/esql_knowledge_base'; +import type { RuleMigrationTelemetryClient } from '../../server/lib/siem_migrations/rules/task/rule_migrations_telemetry_client'; import type { CreateLlmInstance } from '../../server/assistant/tools/esql/utils/common'; interface Drawable { @@ -33,14 +37,15 @@ const mockLlm = new FakeLLM({ const esqlKnowledgeBase = {} as EsqlKnowledgeBase; const ruleMigrationsRetriever = {} as RuleMigrationsRetriever; +const dashboardMigrationsRetriever = {} as DashboardMigrationsRetriever; const createLlmInstance = () => { return mockLlm; }; -async function getSiemMigrationGraph(logger: Logger): Promise { - const model = createLlmInstance(); - const telemetryClient = {} as SiemMigrationTelemetryClient; +async function getSiemRuleMigrationGraph(logger: Logger): Promise { + const model = createLlmInstance() as ChatModel; + const telemetryClient = {} as RuleMigrationTelemetryClient; const graph = getRuleMigrationAgent({ model, esqlKnowledgeBase, @@ -51,6 +56,21 @@ async function getSiemMigrationGraph(logger: Logger): Promise { return graph.getGraphAsync({ xray: true }); } +async function getSiemDashboardMigrationGraph(logger: Logger): Promise { + const model = { bindTools: () => null } as unknown as ChatModel; + const telemetryClient = {} as DashboardMigrationTelemetryClient; + const esScopedClient = {} as IScopedClusterClient; + const graph = getDashboardMigrationAgent({ + model, + esScopedClient, + esqlKnowledgeBase, + dashboardMigrationsRetriever, + logger, + telemetryClient, + }); + return graph.getGraphAsync({ xray: true }); +} + async function getGenerateEsqlGraph(logger: Logger): Promise { const graph = await getGenerateEsqlAgent({ esClient: {} as unknown as ElasticsearchClient, @@ -90,7 +110,11 @@ export const draw = async () => { outputFilename: '../../docs/generate_esql/img/generate_esql_graph.png', }); await drawGraph({ - getGraphAsync: getSiemMigrationGraph, - outputFilename: '../../docs/siem_migration/img/agent_graph.png', + getGraphAsync: getSiemRuleMigrationGraph, + outputFilename: '../../docs/siem_migration/img/rule_migration_agent_graph.png', + }); + await drawGraph({ + getGraphAsync: getSiemDashboardMigrationGraph, + outputFilename: '../../docs/siem_migration/img/dashboard_migration_agent_graph.png', }); }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/utils/audit.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/api/util/audit.ts similarity index 99% rename from x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/utils/audit.ts rename to x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/api/util/audit.ts index aedefb472b4f7..7ac2f297d61ed 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/utils/audit.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/api/util/audit.ts @@ -7,7 +7,7 @@ import type { AuditLogger, EcsEvent } from '@kbn/core/server'; import type { ArrayElement } from '@kbn/utility-types'; -import type { SecuritySolutionApiRequestHandlerContext } from '../../../..'; +import type { SecuritySolutionApiRequestHandlerContext } from '../../../../..'; export enum SiemMigrationsAuditActions { SIEM_MIGRATION_CREATED = 'siem_migration_created', diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/utils/authz.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/api/util/authz.ts similarity index 100% rename from x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/utils/authz.ts rename to x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/api/util/authz.ts diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/utils/is_not_found_error.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/api/util/is_not_found_error.ts similarity index 100% rename from x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/utils/is_not_found_error.ts rename to x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/api/util/is_not_found_error.ts diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/util/retry.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/api/util/retry.ts similarity index 95% rename from x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/util/retry.ts rename to x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/api/util/retry.ts index 3f2ce8dafd97e..d43be51bd0067 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/util/retry.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/api/util/retry.ts @@ -6,7 +6,7 @@ */ import type { RuleMigrationRetryFilter } from '../../../../../../common/siem_migrations/model/rule_migration.gen'; -import type { RuleMigrationFilters } from '../../../../../../common/siem_migrations/types'; +import type { RuleMigrationFilters } from '../../../../../../common/siem_migrations/rules/types'; const RETRY_FILTERS: Record = { failed: { failed: true }, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/util/tracing.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/api/util/tracing.ts similarity index 100% rename from x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/util/tracing.ts rename to x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/api/util/tracing.ts diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/utils/utils.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/api/util/utils.test.ts similarity index 100% rename from x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/utils/utils.test.ts rename to x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/api/util/utils.test.ts diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/utils/with_license.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/api/util/with_license.ts similarity index 98% rename from x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/utils/with_license.ts rename to x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/api/util/with_license.ts index bcba543e2537a..1cacee0bbae71 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/utils/with_license.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/api/util/with_license.ts @@ -7,7 +7,7 @@ import type { RequestHandler, RouteMethod } from '@kbn/core/server'; import { i18n } from '@kbn/i18n'; -import type { SecuritySolutionRequestHandlerContext } from '../../../../types'; +import type { SecuritySolutionRequestHandlerContext } from '../../../../../types'; const LICENSE_ERROR_MESSAGE = i18n.translate('xpack.securitySolution.api.licenseError', { defaultMessage: 'Your license does not support this feature.', diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/__mocks__/mocks.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/__mocks__/mocks.ts new file mode 100644 index 0000000000000..17ff2627f79b3 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/__mocks__/mocks.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SiemMigrationsDataClient } from '../siem_migrations_data_client'; +import type { SiemMigrationsDataMigrationClient } from '../siem_migrations_data_migration_client'; +import type { SiemMigrationsDataResourcesClient } from '../siem_migrations_data_resources_client'; +import type { SiemMigrationsDataItemClient } from '../siem_migrations_data_item_client'; + +// Rule migrations data items client +export const mockSiemMigrationsDataItemClient = { + create: jest.fn().mockResolvedValue(undefined), + get: jest.fn().mockResolvedValue({ data: [], total: 0 }), + searchBatches: jest.fn().mockReturnValue({ + next: jest.fn().mockResolvedValue([]), + all: jest.fn().mockResolvedValue([]), + }), + saveProcessing: jest.fn().mockResolvedValue(undefined), + saveCompleted: jest.fn().mockResolvedValue(undefined), + saveError: jest.fn().mockResolvedValue(undefined), + releaseProcessing: jest.fn().mockResolvedValue(undefined), + updateStatus: jest.fn().mockResolvedValue(undefined), + getStats: jest.fn().mockResolvedValue(undefined), + getAllStats: jest.fn().mockResolvedValue([]), +} as unknown as jest.Mocked; +export const MockSiemMigrationsDataItemClient = jest + .fn() + .mockImplementation(() => mockSiemMigrationsDataItemClient); + +// Rule migrations data resources client +export const mockSiemMigrationsDataResourcesClient = { + upsert: jest.fn().mockResolvedValue(undefined), + get: jest.fn().mockResolvedValue(undefined), + searchBatches: jest.fn().mockReturnValue({ + next: jest.fn().mockResolvedValue([]), + all: jest.fn().mockResolvedValue([]), + }), +} as unknown as jest.Mocked; +export const MockSiemMigrationsDataResourcesClient = jest + .fn() + .mockImplementation(() => mockSiemMigrationsDataResourcesClient); + +export const mockSiemMigrationsDataMigrationsClient = { + create: jest.fn().mockResolvedValue(undefined), + get: jest.fn().mockResolvedValue(undefined), + getAll: jest.fn().mockResolvedValue([]), + saveAsStarted: jest.fn().mockResolvedValue(undefined), + saveAsFinished: jest.fn().mockResolvedValue(undefined), + saveAsFailed: jest.fn().mockResolvedValue(undefined), + setIsStopped: jest.fn().mockResolvedValue(undefined), + updateLastExecution: jest.fn().mockResolvedValue(undefined), +} as unknown as jest.Mocked; + +export const mockDeleteMigration = jest.fn().mockResolvedValue(undefined); + +// Rule migrations data client +export const createSiemMigrationsDataClientMock = () => + ({ + items: mockSiemMigrationsDataItemClient, + resources: mockSiemMigrationsDataResourcesClient, + migrations: mockSiemMigrationsDataMigrationsClient, + deleteMigration: mockDeleteMigration, + } as unknown as jest.MockedObjectDeep); + +export const MockSiemMigrationsDataClient = jest + .fn() + .mockImplementation(() => createSiemMigrationsDataClientMock()); + +// Rule migrations data service +export const mockIndexName = 'mocked_siem_siem_migrations_index_name'; +export const mockInstall = jest.fn().mockResolvedValue(undefined); +export const mockCreateClient = jest.fn(() => createSiemMigrationsDataClientMock()); +export const mockSetup = jest.fn().mockResolvedValue(undefined); + +export const MockSiemMigrationsDataService = jest.fn().mockImplementation(() => ({ + createAdapter: jest.fn(), + install: mockInstall, + createClient: mockCreateClient, + createIndexNameProvider: jest.fn().mockResolvedValue(mockIndexName), + setup: mockSetup, +})); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/__mocks__/siem_migrations_data_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/__mocks__/siem_migrations_data_client.ts new file mode 100644 index 0000000000000..c8e0057f3cfe1 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/__mocks__/siem_migrations_data_client.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MockSiemMigrationsDataClient } from './mocks'; +export const SiemMigrationsDataClient = MockSiemMigrationsDataClient; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/__mocks__/siem_migrations_data_prebuilt_rules_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/__mocks__/siem_migrations_data_prebuilt_rules_client.ts new file mode 100644 index 0000000000000..1061adc52eced --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/__mocks__/siem_migrations_data_prebuilt_rules_client.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { mockRuleMigrationsDataPrebuiltRulesClient } from './mocks'; +export const RuleMigrationsDataPrebuiltRulesClient = mockRuleMigrationsDataPrebuiltRulesClient; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/__mocks__/siem_migrations_data_resources_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/__mocks__/siem_migrations_data_resources_client.ts new file mode 100644 index 0000000000000..96fc5b47fb1cc --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/__mocks__/siem_migrations_data_resources_client.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MockRuleMigrationsDataResourcesClient } from './mocks'; +export const RuleMigrationsDataResourcesClient = MockRuleMigrationsDataResourcesClient; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/__mocks__/siem_migrations_data_rules_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/__mocks__/siem_migrations_data_rules_client.ts new file mode 100644 index 0000000000000..a7a6a29c17cbe --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/__mocks__/siem_migrations_data_rules_client.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MockRuleMigrationsDataRulesClient } from './mocks'; +export const RuleMigrationsDataRulesClient = MockRuleMigrationsDataRulesClient; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/__mocks__/siem_migrations_data_service.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/__mocks__/siem_migrations_data_service.ts new file mode 100644 index 0000000000000..e53eec629e3f1 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/__mocks__/siem_migrations_data_service.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MockSiemMigrationsDataService } from './mocks'; +export const SiemMigrationsDataService = MockSiemMigrationsDataService; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/constants.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/constants.ts new file mode 100644 index 0000000000000..89f0faf683ad0 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/constants.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** Maximum size for searches, aggregations and terms queries */ +export const MAX_ES_SEARCH_SIZE = 10_000 as const; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/search.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/dsl_queries.ts similarity index 51% rename from x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/search.ts rename to x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/dsl_queries.ts index 40281c77da412..e042cf880ae42 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/search.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/dsl_queries.ts @@ -7,54 +7,34 @@ import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; import { + // TODO: RuleTranslationResult -> TranslationResult RuleTranslationResult, SiemMigrationStatus, } from '../../../../../common/siem_migrations/constants'; -export const conditions = { +export const dsl = { isFullyTranslated(): QueryDslQueryContainer { return { term: { translation_result: RuleTranslationResult.FULL } }; }, isNotFullyTranslated(): QueryDslQueryContainer { - return { bool: { must_not: conditions.isFullyTranslated() } }; + return { bool: { must_not: dsl.isFullyTranslated() } }; }, isPartiallyTranslated(): QueryDslQueryContainer { return { term: { translation_result: RuleTranslationResult.PARTIAL } }; }, isNotPartiallyTranslated(): QueryDslQueryContainer { - return { bool: { must_not: conditions.isPartiallyTranslated() } }; + return { bool: { must_not: dsl.isPartiallyTranslated() } }; }, isUntranslatable(): QueryDslQueryContainer { return { term: { translation_result: RuleTranslationResult.UNTRANSLATABLE } }; }, isNotUntranslatable(): QueryDslQueryContainer { - return { bool: { must_not: conditions.isUntranslatable() } }; - }, - isInstalled(): QueryDslQueryContainer { - return { exists: { field: 'elastic_rule.id' } }; - }, - isNotInstalled(): QueryDslQueryContainer { - return { bool: { must_not: conditions.isInstalled() } }; - }, - isPrebuilt(): QueryDslQueryContainer { - return { exists: { field: 'elastic_rule.prebuilt_rule_id' } }; - }, - isCustom(): QueryDslQueryContainer { - return { bool: { must_not: conditions.isPrebuilt() } }; - }, - matchTitle(title: string): QueryDslQueryContainer { - return { match: { 'elastic_rule.title': title } }; - }, - isInstallable(): QueryDslQueryContainer[] { - return [conditions.isFullyTranslated(), conditions.isNotInstalled()]; - }, - isNotInstallable(): QueryDslQueryContainer[] { - return [conditions.isNotFullyTranslated(), conditions.isInstalled()]; + return { bool: { must_not: dsl.isUntranslatable() } }; }, isFailed(): QueryDslQueryContainer { return { term: { status: SiemMigrationStatus.FAILED } }; }, isNotFailed(): QueryDslQueryContainer { - return { bool: { must_not: conditions.isFailed() } }; + return { bool: { must_not: dsl.isFailed() } }; }, }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/field_maps.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/field_maps.ts new file mode 100644 index 0000000000000..fdc051882d949 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/field_maps.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { FieldMap, SchemaFieldMapKeys } from '@kbn/data-stream-adapter'; + +// TODO: Extract RuleMigrationResource -> MigrationResource schema to the generic migration.schema +import type { RuleMigrationResource } from '../../../../../common/siem_migrations/model/rule_migration.gen'; +import type { MigrationDocument } from '../types'; + +export const migrationsFieldMaps: FieldMap< + SchemaFieldMapKeys> +> = { + name: { type: 'keyword', required: true }, + created_at: { type: 'date', required: true }, + created_by: { type: 'keyword', required: true }, +}; + +export const migrationResourcesFieldMap: FieldMap< + SchemaFieldMapKeys> +> = { + migration_id: { type: 'keyword', required: true }, + type: { type: 'keyword', required: true }, + name: { type: 'keyword', required: true }, + content: { type: 'text', required: false }, + metadata: { type: 'object', required: false }, + updated_at: { type: 'date', required: false }, + updated_by: { type: 'keyword', required: false }, +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/siem_migrations_data_base_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/siem_migrations_data_base_client.ts index c0319bea69b1c..432661434b14e 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/siem_migrations_data_base_client.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/siem_migrations_data_base_client.ts @@ -23,7 +23,9 @@ import type { Stored } from '../../types'; const DEFAULT_PIT_KEEP_ALIVE: Duration = '30s' as const; -export class SiemMigrationsDataBaseClient { +export class SiemMigrationsDataBaseClient< + D extends SiemMigrationsClientDependencies = SiemMigrationsClientDependencies +> { protected esClient: ElasticsearchClient; constructor( @@ -31,7 +33,7 @@ export class SiemMigrationsDataBaseClient { protected currentUser: AuthenticatedUser, protected esScopedClient: IScopedClusterClient, protected logger: Logger, - protected dependencies: SiemMigrationsClientDependencies + protected dependencies: D ) { this.esClient = esScopedClient.asInternalUser; } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/siem_migrations_data_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/siem_migrations_data_client.ts new file mode 100644 index 0000000000000..d08b099b0f67d --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/siem_migrations_data_client.ts @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IScopedClusterClient, Logger } from '@kbn/core/server'; +import type { SiemMigrationsDataResourcesClient } from './siem_migrations_data_resources_client'; +import type { SiemMigrationsDataItemClient } from './siem_migrations_data_item_client'; +import type { ItemDocument, MigrationDocument } from '../types'; +import type { SiemMigrationsDataMigrationClient } from './siem_migrations_data_migration_client'; + +export abstract class SiemMigrationsDataClient< + M extends MigrationDocument = MigrationDocument, + I extends ItemDocument = ItemDocument +> { + // Data clients use the ES client `asInternalUser` by default. + // We may want to use `asCurrentUser` instead in the future if the APIs are made public. + protected readonly esClient: IScopedClusterClient['asInternalUser']; + + public abstract readonly migrations: SiemMigrationsDataMigrationClient; + public abstract readonly items: SiemMigrationsDataItemClient; + public abstract readonly resources: SiemMigrationsDataResourcesClient; + + constructor( + public readonly esScopedClient: IScopedClusterClient, + protected readonly logger: Logger + ) { + this.esClient = esScopedClient.asInternalUser; + } + + /** Deletes a migration and all its associated items and resources. */ + public async deleteMigration(migrationId: string) { + const [ + migrationDeleteOperations, + migrationItemsDeleteOperations, + migrationResourcesDeleteOperations, + ] = await Promise.all([ + this.migrations.prepareDelete(migrationId), + this.items.prepareDelete(migrationId), + this.resources.prepareDelete(migrationId), + ]); + + return this.esClient + .bulk({ + refresh: 'wait_for', + operations: [ + ...migrationDeleteOperations, + ...migrationItemsDeleteOperations, + ...migrationResourcesDeleteOperations, + ], + }) + .then(() => { + this.logger.info(`Deleted migration ${migrationId}`); + }) + .catch((error) => { + this.logger.error(`Error deleting migration ${migrationId}: ${error}`); + throw error; + }); + } +} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/siem_migrations_data_item_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/siem_migrations_data_item_client.ts new file mode 100644 index 0000000000000..572ce9d533982 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/siem_migrations_data_item_client.ts @@ -0,0 +1,375 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { + AggregationsAggregationContainer, + AggregationsMaxAggregate, + AggregationsMinAggregate, + AggregationsStringTermsAggregate, + AggregationsStringTermsBucket, + BulkOperationContainer, + Duration, + QueryDslQueryContainer, +} from '@elastic/elasticsearch/lib/api/types'; +import type { estypes } from '@elastic/elasticsearch'; +import type { ItemDocument, Stored } from '../types'; +import { + SiemMigrationStatus, + RuleTranslationResult, +} from '../../../../../common/siem_migrations/constants'; +import { SiemMigrationsDataBaseClient } from './siem_migrations_data_base_client'; +import { MAX_ES_SEARCH_SIZE } from './constants'; +import type { + MigrationType, + SiemMigrationAllDataStats, + SiemMigrationDataStats, + SiemMigrationFilters, +} from './types'; +import { dsl } from './dsl_queries'; + +export type CreateMigrationItemInput = Omit< + I, + '@timestamp' | 'id' | 'status' | 'created_by' | 'updated_by' | 'updated_at' +>; + +export interface SiemMigrationItemSort { + sortField?: string; + sortDirection?: estypes.SortOrder; +} + +export interface SiemMigrationGetItemsOptions { + filters?: F; + sort?: SiemMigrationItemSort; + from?: number; + size?: number; +} + +/* BULK_MAX_SIZE defines the number to break down the bulk operations by. + * The 500 number was chosen as a reasonable number to avoid large payloads. It can be adjusted if needed. */ +const BULK_MAX_SIZE = 500 as const; +/* DEFAULT_SEARCH_BATCH_SIZE defines the default number of documents to retrieve per search operation + * when retrieving search results in batches. */ +const DEFAULT_SEARCH_BATCH_SIZE = 500 as const; + +export abstract class SiemMigrationsDataItemClient< + I extends ItemDocument = ItemDocument +> extends SiemMigrationsDataBaseClient { + protected abstract type: MigrationType; + + /** Indexes an array of migration items in pending status */ + async create(items: CreateMigrationItemInput[]): Promise { + const index = await this.getIndexName(); + const profileId = await this.getProfileUid(); + + let itemsSlice: CreateMigrationItemInput[]; + const createdAt = new Date().toISOString(); + while ((itemsSlice = items.splice(0, BULK_MAX_SIZE)).length) { + await this.esClient + .bulk({ + refresh: 'wait_for', + operations: itemsSlice.flatMap((item) => [ + { create: { _index: index } }, + { + ...item, + '@timestamp': createdAt, + status: SiemMigrationStatus.PENDING, + created_by: profileId, + updated_by: profileId, + updated_at: createdAt, + }, + ]), + }) + .catch((error) => { + this.logger.error(`Error creating migration ${this.type}: ${error.message}`); + throw error; + }); + } + } + + /** Updates an array of migration items */ + async update>(itemsUpdate: U[]): Promise { + const index = await this.getIndexName(); + const profileId = await this.getProfileUid(); + + let itemsUpdateSlice: U[]; + const updatedAt = new Date().toISOString(); + while ((itemsUpdateSlice = itemsUpdate.splice(0, BULK_MAX_SIZE)).length) { + await this.esClient + .bulk({ + refresh: 'wait_for', + operations: itemsUpdateSlice.flatMap((item) => { + const { id, ...rest } = item; + return [ + { update: { _index: index, _id: id } }, + { + doc: { + ...rest, + updated_by: profileId, + updated_at: updatedAt, + }, + }, + ]; + }), + }) + .catch((error) => { + this.logger.error(`Error updating migration ${this.type}: ${error.message}`); + throw error; + }); + } + } + + /** Retrieves an array of migration items of a specific migration */ + async get( + migrationId: string, + { filters = {}, sort: sortParam = {}, from, size }: SiemMigrationGetItemsOptions = {} + ): Promise<{ total: number; data: Stored[] }> { + const index = await this.getIndexName(); + const query = this.getFilterQuery(migrationId, filters); + const sort = this.getSortOptions(sortParam); + + const result = await this.esClient + .search({ index, query, sort, from, size }) + .catch((error) => { + this.logger.error(`Error searching migration ${this.type}: ${error.message}`); + throw error; + }); + return { + total: this.getTotalHits(result), + data: this.processResponseHits(result), + }; + } + + /** Prepares bulk ES delete operations for the migration items based on migrationId. */ + public async prepareDelete(migrationId: string): Promise { + const index = await this.getIndexName(); + const itemsToBeDeleted = await this.get(migrationId, { size: MAX_ES_SEARCH_SIZE }); + const itemsToBeDeletedDocIds = itemsToBeDeleted.data.map((item) => item.id); + + return itemsToBeDeletedDocIds.map((docId) => ({ + delete: { _index: index, _id: docId }, + })); + } + + /** Returns batching functions to traverse all the migration items search results */ + public searchBatches( + migrationId: string, + options: { scroll?: Duration; size?: number; filters?: object } = {} + ) { + const { size = DEFAULT_SEARCH_BATCH_SIZE, filters = {}, scroll } = options; + const query = this.getFilterQuery(migrationId, filters); + const search = { query, sort: '_doc', scroll, size }; // sort by _doc to ensure consistent order + try { + return this.getSearchBatches(search); + } catch (error) { + this.logger.error(`Error scrolling migration ${this.type}: ${error.message}`); + throw error; + } + } + + /** Retrieves the stats for the migrations items with the provided id */ + public async getStats(migrationId: string): Promise { + const index = await this.getIndexName(); + const query = this.getFilterQuery(migrationId); + const aggregations = { + status: { terms: { field: 'status' } }, + createdAt: { min: { field: '@timestamp' } }, + lastUpdatedAt: { max: { field: 'updated_at' } }, + }; + const result = await this.esClient + .search({ index, query, aggregations, _source: false }) + .catch((error) => { + this.logger.error(`Error getting migration ${this.type} stats: ${error.message}`); + throw error; + }); + + const aggs = result.aggregations ?? {}; + return { + id: migrationId, + items: { + total: this.getTotalHits(result), + ...this.statusAggCounts(aggs.status as AggregationsStringTermsAggregate), + }, + created_at: (aggs.createdAt as AggregationsMinAggregate)?.value_as_string ?? '', + last_updated_at: (aggs.lastUpdatedAt as AggregationsMaxAggregate)?.value_as_string ?? '', + }; + } + + /** Retrieves the stats for all the migration items aggregated by migration id, in creation order */ + async getAllStats(): Promise { + const index = await this.getIndexName(); + const aggregations: { migrationIds: AggregationsAggregationContainer } = { + migrationIds: { + terms: { field: 'migration_id', order: { createdAt: 'asc' }, size: MAX_ES_SEARCH_SIZE }, + aggregations: { + status: { terms: { field: 'status' } }, + createdAt: { min: { field: '@timestamp' } }, + lastUpdatedAt: { max: { field: 'updated_at' } }, + }, + }, + }; + const result = await this.esClient + .search({ index, aggregations, _source: false }) + .catch((error) => { + this.logger.error(`Error getting all migration ${this.type} stats: ${error.message}`); + throw error; + }); + + const migrationsAgg = result.aggregations?.migrationIds as AggregationsStringTermsAggregate; + const buckets = (migrationsAgg?.buckets as AggregationsStringTermsBucket[]) ?? []; + return buckets.map((bucket) => ({ + id: `${bucket.key}`, + items: { + total: bucket.doc_count, + ...this.statusAggCounts(bucket.status as AggregationsStringTermsAggregate), + }, + created_at: (bucket.createdAt as AggregationsMinAggregate | undefined) + ?.value_as_string as string, + last_updated_at: (bucket.lastUpdatedAt as AggregationsMaxAggregate | undefined) + ?.value_as_string as string, + })); + } + + /** Updates one migration item status to `processing` */ + public async saveProcessing(id: string): Promise { + const index = await this.getIndexName(); + const profileId = await this.getProfileUid(); + const doc = { + status: SiemMigrationStatus.PROCESSING, + updated_by: profileId, + updated_at: new Date().toISOString(), + }; + await this.esClient.update({ index, id, doc, refresh: 'wait_for' }).catch((error) => { + this.logger.error( + `Error updating migration ${this.type} status to processing: ${error.message}` + ); + throw error; + }); + } + + /** Updates one migration item with the provided data and sets the status to `completed` */ + public async saveCompleted({ id, ...migrationItem }: Stored): Promise { + const index = await this.getIndexName(); + const profileId = await this.getProfileUid(); + const doc = { + ...migrationItem, + status: SiemMigrationStatus.COMPLETED, + updated_by: profileId, + updated_at: new Date().toISOString(), + }; + await this.esClient.update({ index, id, doc, refresh: 'wait_for' }).catch((error) => { + this.logger.error( + `Error updating migration ${this.type} status to completed: ${error.message}` + ); + throw error; + }); + } + + /** Updates one migration item with the provided data and sets the status to `failed` */ + public async saveError({ id, ...migrationItem }: Stored): Promise { + const index = await this.getIndexName(); + const profileId = await this.getProfileUid(); + const doc = { + ...migrationItem, + status: SiemMigrationStatus.FAILED, + updated_by: profileId, + updated_at: new Date().toISOString(), + }; + await this.esClient.update({ index, id, doc, refresh: 'wait_for' }).catch((error) => { + this.logger.error(`Error updating migration ${this.type} status to failed: ${error.message}`); + throw error; + }); + } + + /** Updates all the migration items with the provided id with status `processing` back to `pending` */ + public async releaseProcessing(migrationId: string): Promise { + return this.updateStatus( + migrationId, + { status: SiemMigrationStatus.PROCESSING }, + SiemMigrationStatus.PENDING + ); + } + + /** Updates all the migration items with the provided id and with status `statusToQuery` to `statusToUpdate` */ + public async updateStatus( + migrationId: string, + filter: object, + statusToUpdate: SiemMigrationStatus, + { refresh = false }: { refresh?: boolean } = {} + ): Promise { + const index = await this.getIndexName(); + const query = this.getFilterQuery(migrationId, filter); + const script = { source: `ctx._source['status'] = '${statusToUpdate}'` }; + await this.esClient.updateByQuery({ index, query, script, refresh }).catch((error) => { + this.logger.error(`Error updating migration ${this.type} status: ${error.message}`); + throw error; + }); + } + + protected statusAggCounts( + statusAgg: AggregationsStringTermsAggregate + ): Record { + const buckets = statusAgg.buckets as AggregationsStringTermsBucket[]; + return { + [SiemMigrationStatus.PENDING]: + buckets.find(({ key }) => key === SiemMigrationStatus.PENDING)?.doc_count ?? 0, + [SiemMigrationStatus.PROCESSING]: + buckets.find(({ key }) => key === SiemMigrationStatus.PROCESSING)?.doc_count ?? 0, + [SiemMigrationStatus.COMPLETED]: + buckets.find(({ key }) => key === SiemMigrationStatus.COMPLETED)?.doc_count ?? 0, + [SiemMigrationStatus.FAILED]: + buckets.find(({ key }) => key === SiemMigrationStatus.FAILED)?.doc_count ?? 0, + }; + } + + protected translationResultAggCount( + resultAgg: AggregationsStringTermsAggregate + ): Record { + const buckets = resultAgg.buckets as AggregationsStringTermsBucket[]; + return { + [RuleTranslationResult.FULL]: + buckets.find(({ key }) => key === RuleTranslationResult.FULL)?.doc_count ?? 0, + [RuleTranslationResult.PARTIAL]: + buckets.find(({ key }) => key === RuleTranslationResult.PARTIAL)?.doc_count ?? 0, + [RuleTranslationResult.UNTRANSLATABLE]: + buckets.find(({ key }) => key === RuleTranslationResult.UNTRANSLATABLE)?.doc_count ?? 0, + }; + } + + protected getFilterQuery( + migrationId: string, + filters: SiemMigrationFilters = {} + ): { bool: { filter: QueryDslQueryContainer[] } } { + const filter: QueryDslQueryContainer[] = [{ term: { migration_id: migrationId } }]; + + if (filters.status) { + if (Array.isArray(filters.status)) { + filter.push({ terms: { status: filters.status } }); + } else { + filter.push({ term: { status: filters.status } }); + } + } + if (filters.ids) { + filter.push({ terms: { _id: filters.ids } }); + } + if (filters.failed != null) { + filter.push(filters.failed ? dsl.isFailed() : dsl.isNotFailed()); + } + if (filters.fullyTranslated != null) { + filter.push(filters.fullyTranslated ? dsl.isFullyTranslated() : dsl.isNotFullyTranslated()); + } + if (filters.partiallyTranslated != null) { + filter.push( + filters.partiallyTranslated ? dsl.isPartiallyTranslated() : dsl.isNotPartiallyTranslated() + ); + } + if (filters.untranslatable != null) { + filter.push(filters.untranslatable ? dsl.isUntranslatable() : dsl.isNotUntranslatable()); + } + return { bool: { filter } }; + } + + protected abstract getSortOptions(sort?: SiemMigrationItemSort): estypes.Sort; +} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/siem_migrations_data_migration_client.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/siem_migrations_data_migration_client.test.ts new file mode 100644 index 0000000000000..2f3bac761dbac --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/siem_migrations_data_migration_client.test.ts @@ -0,0 +1,277 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IScopedClusterClient } from '@kbn/core/server'; +import { SiemMigrationsDataMigrationClient } from './siem_migrations_data_migration_client'; +import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks'; +import type { AuthenticatedUser } from '@kbn/security-plugin-types-common'; +import type IndexApi from '@elastic/elasticsearch/lib/api/api'; +import type GetApi from '@elastic/elasticsearch/lib/api/api/get'; +import type SearchApi from '@elastic/elasticsearch/lib/api/api/search'; +import type { SiemMigrationsClientDependencies } from '../types'; + +describe('SiemMigrationsDataMigrationClient', () => { + let siemMigrationsDataMigrationClient: SiemMigrationsDataMigrationClient; + const esClient = + elasticsearchServiceMock.createCustomClusterClient() as unknown as IScopedClusterClient; + + const logger = loggingSystemMock.createLogger(); + const indexNameProvider = jest.fn().mockReturnValue('.kibana-siem-rule-migrations'); + const currentUser = { + userName: 'testUser', + profile_uid: 'testProfileUid', + } as unknown as AuthenticatedUser; + const dependencies = {} as unknown as SiemMigrationsClientDependencies; + + beforeEach(() => { + siemMigrationsDataMigrationClient = new SiemMigrationsDataMigrationClient( + indexNameProvider, + currentUser, + esClient, + logger, + dependencies + ); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + describe('create', () => { + test('should create a new migration', async () => { + const index = '.kibana-siem-rule-migrations'; + const name = 'test name'; + + const result = await siemMigrationsDataMigrationClient.create(name); + + expect(result).not.toBeFalsy(); + expect(esClient.asInternalUser.create).toHaveBeenCalledWith({ + refresh: 'wait_for', + id: result, + index, + document: { + created_by: currentUser.profile_uid, + created_at: expect.any(String), + name, + }, + }); + }); + + test('should throw an error if an error occurs', async () => { + ( + esClient.asInternalUser.create as unknown as jest.MockedFn + ).mockRejectedValueOnce(new Error('Test error')); + + await expect(siemMigrationsDataMigrationClient.create('test')).rejects.toThrow('Test error'); + + expect(esClient.asInternalUser.create).toHaveBeenCalled(); + expect(logger.error).toHaveBeenCalled(); + }); + }); + + describe('get', () => { + test('should get a migration', async () => { + const index = '.kibana-siem-rule-migrations'; + const id = 'testId'; + const response = { + _index: index, + found: true, + _source: { + created_by: currentUser.profile_uid, + created_at: new Date().toISOString(), + }, + _id: id, + }; + + ( + esClient.asInternalUser.get as unknown as jest.MockedFn + ).mockResolvedValueOnce(response); + + const result = await siemMigrationsDataMigrationClient.get({ id }); + + expect(result).toEqual({ + ...response._source, + id: response._id, + }); + }); + + test('should return undefined if the migration is not found', async () => { + const id = 'testId'; + const response = { + _index: '.kibana-siem-rule-migrations', + found: false, + }; + + ( + esClient.asInternalUser.get as unknown as jest.MockedFn + ).mockRejectedValueOnce({ + message: JSON.stringify(response), + }); + + const result = await siemMigrationsDataMigrationClient.get({ id }); + + expect(result).toBeUndefined(); + }); + + test('should throw an error if an error occurs', async () => { + const id = 'testId'; + ( + esClient.asInternalUser.get as unknown as jest.MockedFn + ).mockRejectedValueOnce(new Error('Test error')); + + await expect(siemMigrationsDataMigrationClient.get({ id })).rejects.toThrow('Test error'); + + expect(esClient.asInternalUser.get).toHaveBeenCalled(); + expect(logger.error).toHaveBeenCalledWith(`Error getting migration ${id}: Error: Test error`); + }); + }); + + describe('prepareDelete', () => { + beforeEach(() => jest.clearAllMocks()); + + it('should delete the migration and associated rules and resources', async () => { + const migrationId = 'testId'; + const index = '.kibana-siem-rule-migrations'; + + const operations = await siemMigrationsDataMigrationClient.prepareDelete(migrationId); + + expect(operations).toMatchObject([ + { + delete: { + _index: index, + _id: migrationId, + }, + }, + ]); + }); + }); + + describe('getAll', () => { + it('should return all migrations', async () => { + const response = { + hits: { + hits: [ + { + _index: '.kibana-siem-rule-migrations', + _id: '1', + _source: { + created_by: currentUser.profile_uid, + created_at: new Date().toISOString(), + }, + }, + { + _index: '.kibana-siem-rule-migrations', + _id: '2', + _source: { + created_by: currentUser.profile_uid, + created_at: new Date().toISOString(), + }, + }, + ], + }, + } as unknown as ReturnType; + + ( + esClient.asInternalUser.search as unknown as jest.MockedFn + ).mockResolvedValueOnce(response); + + await siemMigrationsDataMigrationClient.getAll(); + expect(esClient.asInternalUser.search).toHaveBeenCalledWith({ + index: '.kibana-siem-rule-migrations', + size: 10000, + query: { + match_all: {}, + }, + _source: true, + }); + }); + }); + + describe('updateLastExecution', () => { + const connectorId = 'testConnector'; + it('should update `started_at` & `connector_id` when called saveAsStarted', async () => { + const migrationId = 'testId'; + + await siemMigrationsDataMigrationClient.saveAsStarted({ id: migrationId, connectorId }); + + expect(esClient.asInternalUser.update).toHaveBeenCalledWith({ + index: '.kibana-siem-rule-migrations', + id: migrationId, + refresh: 'wait_for', + doc: { + last_execution: { + started_at: expect.stringMatching(/^\d{4}-\d{2}-\d{2}T/), + is_stopped: false, + error: null, + finished_at: null, + connector_id: connectorId, + skip_prebuilt_rules_matching: false, + }, + }, + retry_on_conflict: 1, + }); + }); + + it('should update `finished_at` when called saveAsEnded', async () => { + const migrationId = 'testId'; + + await siemMigrationsDataMigrationClient.saveAsFinished({ id: migrationId }); + + expect(esClient.asInternalUser.update).toHaveBeenCalledWith({ + index: '.kibana-siem-rule-migrations', + id: migrationId, + refresh: 'wait_for', + doc: { + last_execution: { + finished_at: expect.stringMatching(/^\d{4}-\d{2}-\d{2}T/), + }, + }, + retry_on_conflict: 1, + }); + }); + + it('should update `is_stopped` correctly when called setIsStopped', async () => { + const migrationId = 'testId'; + + await siemMigrationsDataMigrationClient.setIsStopped({ id: migrationId }); + + expect(esClient.asInternalUser.update).toHaveBeenCalledWith({ + index: '.kibana-siem-rule-migrations', + id: migrationId, + refresh: 'wait_for', + doc: { + last_execution: { + is_stopped: true, + }, + }, + retry_on_conflict: 1, + }); + }); + + it('should update `error` params correctly when called saveAsFailed', async () => { + const migrationId = 'testId'; + + await siemMigrationsDataMigrationClient.saveAsFailed({ + id: migrationId, + error: 'Test error', + }); + + expect(esClient.asInternalUser.update).toHaveBeenCalledWith({ + index: '.kibana-siem-rule-migrations', + id: migrationId, + refresh: 'wait_for', + doc: { + last_execution: { + error: 'Test error', + finished_at: expect.stringMatching(/^\d{4}-\d{2}-\d{2}T/), + }, + }, + retry_on_conflict: 1, + }); + }); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/siem_migrations_data_migration_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/siem_migrations_data_migration_client.ts new file mode 100644 index 0000000000000..35b5bc8a43b8f --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/siem_migrations_data_migration_client.ts @@ -0,0 +1,158 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { v4 as uuidV4 } from 'uuid'; +import type { BulkOperationContainer } from '@elastic/elasticsearch/lib/api/types'; +import type { RuleMigrationLastExecution } from '../../../../../common/siem_migrations/model/rule_migration.gen'; +import type { MigrationDocument, Stored } from '../types'; +import { SiemMigrationsDataBaseClient } from './siem_migrations_data_base_client'; +import { isNotFoundError } from '../api/util/is_not_found_error'; +import { MAX_ES_SEARCH_SIZE } from './constants'; + +export class SiemMigrationsDataMigrationClient< + M extends MigrationDocument = MigrationDocument +> extends SiemMigrationsDataBaseClient { + async create(name: string): Promise { + const migrationId = uuidV4(); + const index = await this.getIndexName(); + const profileUid = await this.getProfileUid(); + const createdAt = new Date().toISOString(); + + await this.esClient + .create({ + refresh: 'wait_for', + id: migrationId, + index, + document: { name, created_by: profileUid, created_at: createdAt }, + }) + .catch((error) => { + this.logger.error(`Error creating migration ${migrationId}: ${error}`); + throw error; + }); + + return migrationId; + } + + /** + * + * Gets the migration document by id or returns undefined if it does not exist. + * + * */ + async get(migrationId: string): Promise | undefined> { + const index = await this.getIndexName(); + return this.esClient + .get>({ index, id: migrationId }) + .then(this.processHit) + .catch((error) => { + if (isNotFoundError(error)) { + return undefined; + } + this.logger.error(`Error getting migration ${migrationId}: ${error}`); + throw error; + }); + } + + /** + * Gets all migrations from the index. + */ + async getAll(): Promise[]> { + const index = await this.getIndexName(); + return this.esClient + .search>({ + index, + size: MAX_ES_SEARCH_SIZE, // Adjust size as needed + query: { match_all: {} }, + _source: true, + }) + .then((result) => this.processResponseHits(result)) + .catch((error) => { + this.logger.error(`Error getting all migrations:- ${error}`); + throw error; + }); + } + + /** + * + * Prepares bulk ES delete operation for a migration document based on its id. + * + */ + async prepareDelete(migrationId: string): Promise { + const index = await this.getIndexName(); + const migrationDeleteOperation = { + delete: { _index: index, _id: migrationId }, + }; + return [migrationDeleteOperation]; + } + + /** + * Saves a migration as started, updating the last execution parameters with the current timestamp. + */ + async saveAsStarted({ + id, + connectorId, + }: { id: string; connectorId: string } & Record): Promise { + await this.updateLastExecution(id, { + started_at: new Date().toISOString(), + connector_id: connectorId, + is_stopped: false, + error: null, + finished_at: null, + }); + } + + /** + * Saves a migration as ended, updating the last execution parameters with the current timestamp. + */ + async saveAsFinished({ id }: { id: string }): Promise { + await this.updateLastExecution(id, { finished_at: new Date().toISOString() }); + } + + /** + * Saves a migration as failed, updating the last execution parameters with the provided error message. + */ + async saveAsFailed({ id, error }: { id: string; error: string }): Promise { + await this.updateLastExecution(id, { error, finished_at: new Date().toISOString() }); + } + + /** + * Sets `is_stopped` flag for migration document. + * It does not update `finished_at` timestamp, `saveAsFinished` or `saveAsFailed` should be called separately. + */ + async setIsStopped({ id }: { id: string }): Promise { + await this.updateLastExecution(id, { is_stopped: true }); + } + + /** + * Updates the last execution parameters for a migration document. + */ + protected async updateLastExecution( + id: string, + lastExecutionParams: RuleMigrationLastExecution + ): Promise { + const index = await this.getIndexName(); + const doc = { last_execution: lastExecutionParams }; + await this.esClient + .update({ index, id, refresh: 'wait_for', doc, retry_on_conflict: 1 }) + .catch((error) => { + this.logger.error(`Error updating last execution for migration ${id}: ${error}`); + throw error; + }); + } + + /** + * Updates the migration document with the provided values. + */ + async update(id: string, doc: Partial>): Promise { + const index = await this.getIndexName(); + await this.esClient + .update({ index, id, doc, refresh: 'wait_for', retry_on_conflict: 1 }) + .catch((error) => { + this.logger.error(`Error updating migration: ${error}`); + throw error; + }); + } +} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_resources_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/siem_migrations_data_resources_client.ts similarity index 75% rename from x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_resources_client.ts rename to x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/siem_migrations_data_resources_client.ts index d8a2931b6315b..25aa537de3e86 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_resources_client.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/siem_migrations_data_resources_client.ts @@ -11,25 +11,28 @@ import type { Duration, BulkOperationContainer, } from '@elastic/elasticsearch/lib/api/types'; +// TODO: Use a common schema to define the common types import type { - RuleMigrationResource, - RuleMigrationResourceType, + RuleMigrationResource as SiemMigrationResource, + RuleMigrationResourceType as SiemMigrationResourceType, } from '../../../../../common/siem_migrations/model/rule_migration.gen'; -import type { StoredRuleMigrationResource } from '../types'; -import { SiemMigrationsDataBaseClient } from '../../common/data/siem_migrations_data_base_client'; -import { MAX_ES_SEARCH_SIZE } from '../constants'; +import type { Stored } from '../types'; +import { SiemMigrationsDataBaseClient } from './siem_migrations_data_base_client'; +import { MAX_ES_SEARCH_SIZE } from './constants'; -export type CreateRuleMigrationResourceInput = Pick< - RuleMigrationResource, +export type StoredSiemMigrationResource = Stored; + +export type CreateSiemMigrationResourceInput = Pick< + SiemMigrationResource, 'migration_id' | 'type' | 'name' | 'content' | 'metadata' >; -export interface RuleMigrationResourceFilters { - type?: RuleMigrationResourceType; +export interface SiemMigrationResourceFilters { + type?: SiemMigrationResourceType; names?: string[]; hasContent?: boolean; } -export interface RuleMigrationResourceGetOptions { - filters?: RuleMigrationResourceFilters; +export interface SiemMigrationResourceGetOptions { + filters?: SiemMigrationResourceFilters; size?: number; from?: number; } @@ -42,12 +45,12 @@ const BULK_MAX_SIZE = 500 as const; * when retrieving search results in batches. */ const DEFAULT_SEARCH_BATCH_SIZE = 500 as const; -export class RuleMigrationsDataResourcesClient extends SiemMigrationsDataBaseClient { - public async upsert(resources: CreateRuleMigrationResourceInput[]): Promise { +export class SiemMigrationsDataResourcesClient extends SiemMigrationsDataBaseClient { + public async upsert(resources: CreateSiemMigrationResourceInput[]): Promise { const index = await this.getIndexName(); const profileId = await this.getProfileUid(); - let resourcesSlice: CreateRuleMigrationResourceInput[]; + let resourcesSlice: CreateSiemMigrationResourceInput[]; const createdAt = new Date().toISOString(); while ((resourcesSlice = resources.splice(0, BULK_MAX_SIZE)).length > 0) { @@ -75,11 +78,11 @@ export class RuleMigrationsDataResourcesClient extends SiemMigrationsDataBaseCli } /** Creates the resources in the index only if they do not exist */ - public async create(resources: CreateRuleMigrationResourceInput[]): Promise { + public async create(resources: CreateSiemMigrationResourceInput[]): Promise { const index = await this.getIndexName(); const profileId = await this.getProfileUid(); - let resourcesSlice: CreateRuleMigrationResourceInput[]; + let resourcesSlice: CreateSiemMigrationResourceInput[]; const createdAt = new Date().toISOString(); while ((resourcesSlice = resources.splice(0, BULK_MAX_SIZE)).length > 0) { await this.esClient @@ -104,14 +107,14 @@ export class RuleMigrationsDataResourcesClient extends SiemMigrationsDataBaseCli public async get( migrationId: string, - options: RuleMigrationResourceGetOptions = {} - ): Promise { + options: SiemMigrationResourceGetOptions = {} + ): Promise { const { filters, size, from } = options; const index = await this.getIndexName(); const query = this.getFilterQuery(migrationId, filters); return this.esClient - .search({ index, query, size, from }) + .search({ index, query, size, from }) .then(this.processResponseHits.bind(this)) .catch((error) => { this.logger.error(`Error searching resources: ${error.message}`); @@ -120,9 +123,9 @@ export class RuleMigrationsDataResourcesClient extends SiemMigrationsDataBaseCli } /** Returns batching functions to traverse all the migration resources search results */ - searchBatches( + public searchBatches( migrationId: string, - options: { scroll?: Duration; size?: number; filters?: RuleMigrationResourceFilters } = {} + options: { scroll?: Duration; size?: number; filters?: SiemMigrationResourceFilters } = {} ) { const { size = DEFAULT_SEARCH_BATCH_SIZE, filters = {}, scroll } = options; const query = this.getFilterQuery(migrationId, filters); @@ -135,14 +138,24 @@ export class RuleMigrationsDataResourcesClient extends SiemMigrationsDataBaseCli } } - private createId(resource: CreateRuleMigrationResourceInput): string { + /** Prepares bulk ES delete operations for the resources of a given migrationId. */ + public async prepareDelete(migrationId: string): Promise { + const index = await this.getIndexName(); + const resourcesToBeDeleted = await this.get(migrationId, { size: MAX_ES_SEARCH_SIZE }); + const resourcesToBeDeletedDocIds = resourcesToBeDeleted.map((resource) => resource.id); + return resourcesToBeDeletedDocIds.map((docId) => ({ + delete: { _id: docId, _index: index }, + })); + } + + private createId(resource: CreateSiemMigrationResourceInput): string { const key = `${resource.migration_id}-${resource.type}-${resource.name}`; return sha256.create().update(key).hex(); } private getFilterQuery( migrationId: string, - filters: RuleMigrationResourceFilters = {} + filters: SiemMigrationResourceFilters = {} ): QueryDslQueryContainer { const filter: QueryDslQueryContainer[] = [{ term: { migration_id: migrationId } }]; if (filters.type) { @@ -161,21 +174,4 @@ export class RuleMigrationsDataResourcesClient extends SiemMigrationsDataBaseCli } return { bool: { filter } }; } - - /** - * - * Prepares bulk ES delete operations for the resources of a given migrationId. - * - */ - async prepareDelete(migrationId: string): Promise { - const index = await this.getIndexName(); - const resourcesToBeDeleted = await this.get(migrationId, { size: MAX_ES_SEARCH_SIZE }); - const resourcesToBeDeletedDocIds = resourcesToBeDeleted.map((resource) => resource.id); - return resourcesToBeDeletedDocIds.map((docId) => ({ - delete: { - _id: docId, - _index: index, - }, - })); - } } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/types.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/types.ts new file mode 100644 index 0000000000000..6a05babd54c0e --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/types.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SiemMigrationStatus } from '../../../../../common/siem_migrations/constants'; +import type { DashboardMigrationTaskStats } from '../../../../../common/siem_migrations/model/dashboard_migration.gen'; +import type { RuleMigrationTaskStats } from '../../../../../common/siem_migrations/model/rule_migration.gen'; + +export type MigrationType = 'rule' | 'dashboard'; + +export type SiemMigrationTaskStats = RuleMigrationTaskStats | DashboardMigrationTaskStats; +export type SiemMigrationDataStats = Omit; +export type SiemMigrationAllDataStats = SiemMigrationDataStats[]; + +export interface SiemMigrationFilters { + status?: SiemMigrationStatus | SiemMigrationStatus[]; + ids?: string[]; + installed?: boolean; + installable?: boolean; + failed?: boolean; + fullyTranslated?: boolean; + partiallyTranslated?: boolean; + untranslatable?: boolean; + searchTerm?: string; +} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/siem_migrations_base_service.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/siem_migrations_base_service.ts index 8572f54005498..4bf1c752ef51b 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/siem_migrations_base_service.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/siem_migrations_base_service.ts @@ -5,21 +5,28 @@ * 2.0. */ -import type { FieldMap } from '@kbn/index-adapter'; +import type { FieldMap, InstallParams } from '@kbn/index-adapter'; import { IndexAdapter, IndexPatternAdapter } from '@kbn/index-adapter'; +import type { ElasticsearchClient } from '@kbn/core/server'; import type { SiemMigrationsIndexNameProvider } from './types'; const TOTAL_FIELDS_LIMIT = 2500; +export interface SetupParams extends Omit { + esClient: ElasticsearchClient; +} + interface CreateAdapterParams { name: string; fieldMap: FieldMap; } -export class SiemMigrationsBaseDataService { +export abstract class SiemMigrationsBaseDataService { + protected abstract readonly baseIndexName: string; + constructor(protected kibanaVersion: string) {} - public createIndexPatternAdapter({ name, fieldMap }: CreateAdapterParams) { + protected createIndexPatternAdapter({ name, fieldMap }: CreateAdapterParams) { const adapter = new IndexPatternAdapter(name, { kibanaVersion: this.kibanaVersion, totalFieldsLimit: TOTAL_FIELDS_LIMIT, @@ -29,7 +36,7 @@ export class SiemMigrationsBaseDataService { return adapter; } - public createIndexAdapter({ name, fieldMap }: CreateAdapterParams) { + protected createIndexAdapter({ name, fieldMap }: CreateAdapterParams) { const adapter = new IndexAdapter(name, { kibanaVersion: this.kibanaVersion, totalFieldsLimit: TOTAL_FIELDS_LIMIT, @@ -39,7 +46,11 @@ export class SiemMigrationsBaseDataService { return adapter; } - public createIndexNameProvider( + protected getAdapterIndexName(adapterId: string) { + return `${this.baseIndexName}-${adapterId}`; + } + + protected createIndexNameProvider( adapter: IndexPatternAdapter, spaceId: string ): SiemMigrationsIndexNameProvider { diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/__mocks__/mocks.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/__mocks__/mocks.ts new file mode 100644 index 0000000000000..a9f3daa65a1d5 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/__mocks__/mocks.ts @@ -0,0 +1,125 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { PublicMethodsOf } from '@kbn/utility-types'; +import { FakeLLM } from '@langchain/core/utils/testing'; +import { AsyncLocalStorageProviderSingleton } from '@langchain/core/singletons'; +import type { SiemMigrationTelemetryClient } from '../siem_migrations_telemetry_client'; +import type { BaseLLMParams } from '@langchain/core/language_models/llms'; + +export const createSiemMigrationTelemetryClientMock = () => { + // Mock for the object returned by startSiemMigrationTask + const mockStartItemTranslationReturn = { + success: jest.fn(), + failure: jest.fn(), + }; + + // Mock for the function returned by startSiemMigrationTask + const mockStartItemTranslation = jest.fn().mockReturnValue(mockStartItemTranslationReturn); + + // Mock for startSiemMigrationTask return value + const mockStartSiemMigrationTaskReturn = { + startItemTranslation: mockStartItemTranslation, + success: jest.fn(), + failure: jest.fn(), + aborted: jest.fn(), + }; + + return { + startSiemMigrationTask: jest.fn().mockReturnValue(mockStartSiemMigrationTaskReturn), + } as jest.Mocked>; +}; + +// Factory function for the mock class +export const MockSiemMigrationTelemetryClient = jest + .fn() + .mockImplementation(() => createSiemMigrationTelemetryClientMock()); + +export const createSiemMigrationsTaskClientMock = () => ({ + start: jest.fn().mockResolvedValue({ started: true }), + stop: jest.fn().mockResolvedValue({ stopped: true }), + getStats: jest.fn().mockResolvedValue({ + status: 'done', + rules: { + total: 1, + finished: 1, + processing: 0, + pending: 0, + failed: 0, + }, + }), + getAllStats: jest.fn().mockResolvedValue([]), +}); + +export const MockSiemMigrationsTaskClient = jest + .fn() + .mockImplementation(() => createSiemMigrationsTaskClientMock()); + +// Siem migrations task service +export const mockStopAll = jest.fn(); +export const mockCreateClient = jest.fn(() => createSiemMigrationsTaskClientMock()); + +export const MockSiemMigrationsTaskService = jest.fn().mockImplementation(() => ({ + createClient: mockCreateClient, + stopAll: mockStopAll, +})); + +export interface NodeResponse { + nodeId: string; + response: string; +} + +interface SiemMigrationFakeLLMParams extends BaseLLMParams { + nodeResponses: NodeResponse[]; +} + +export class SiemMigrationFakeLLM extends FakeLLM { + private nodeResponses: NodeResponse[]; + private defaultResponse: string; + private callCount: Map; + private totalCount: number; + + constructor(fields: SiemMigrationFakeLLMParams) { + super({ + response: 'unexpected node call', + ...fields, + }); + this.nodeResponses = fields.nodeResponses; + this.defaultResponse = 'unexpected node call'; + this.callCount = new Map(); + this.totalCount = 0; + } + + _llmType(): string { + return 'fake'; + } + + async _call(prompt: string, _options: this['ParsedCallOptions']): Promise { + // Get the current runnable config metadata + const item = AsyncLocalStorageProviderSingleton.getRunnableConfig(); + for (const nodeResponse of this.nodeResponses) { + if (item.metadata.langgraph_node === nodeResponse.nodeId) { + const currentCount = this.callCount.get(nodeResponse.nodeId) || 0; + this.callCount.set(nodeResponse.nodeId, currentCount + 1); + this.totalCount += 1; + return nodeResponse.response; + } + } + return this.defaultResponse; + } + + getNodeCallCount(nodeId: string): number { + return this.callCount.get(nodeId) || 0; + } + + getTotalCallCount(): number { + return this.totalCount; + } + + resetCallCounts(): void { + this.callCount.clear(); + } +} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/__mocks__/siem_migrations_task_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/__mocks__/siem_migrations_task_client.ts new file mode 100644 index 0000000000000..b900fe857e099 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/__mocks__/siem_migrations_task_client.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MockSiemMigrationsTaskClient } from './mocks'; +export const SiemMigrationsTaskClient = MockSiemMigrationsTaskClient; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/__mocks__/siem_migrations_task_service.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/__mocks__/siem_migrations_task_service.ts new file mode 100644 index 0000000000000..6dd8ea2f6d62a --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/__mocks__/siem_migrations_task_service.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MockSiemMigrationsTaskService } from './mocks'; +export const SiemMigrationsTaskService = MockSiemMigrationsTaskService; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/__mocks__/siem_migrations_telemetry_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/__mocks__/siem_migrations_telemetry_client.ts new file mode 100644 index 0000000000000..199e630d2f1de --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/__mocks__/siem_migrations_telemetry_client.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MockSiemMigrationTelemetryClient } from './mocks'; +export const SiemMigrationTelemetryClient = MockSiemMigrationTelemetryClient; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/cim_ecs_map.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/convert_esql_schema_cim_to_ecs/cim_ecs_map.ts similarity index 99% rename from x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/cim_ecs_map.ts rename to x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/convert_esql_schema_cim_to_ecs/cim_ecs_map.ts index 3bafaf2fc6518..8514b6ab393d9 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/cim_ecs_map.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/convert_esql_schema_cim_to_ecs/cim_ecs_map.ts @@ -5,7 +5,7 @@ * 2.0. */ -export const SIEM_RULE_MIGRATION_CIM_ECS_MAP = ` +export const CIM_TO_ECS_MAP = ` datamodel,object,source_field,ecs_field,data_type Application_State,All_Application_State,dest,service.node.name,string Application_State,All_Application_State,process,process.title,string diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/convert_esql_schema_cim_to_ecs/convert_esql_schema_cim_to_ecs.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/convert_esql_schema_cim_to_ecs/convert_esql_schema_cim_to_ecs.ts new file mode 100644 index 0000000000000..8f20c3de2f77d --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/convert_esql_schema_cim_to_ecs/convert_esql_schema_cim_to_ecs.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; +import type { MigrationComments } from '../../../../../../../../common/siem_migrations/model/migration.gen'; +import type { EsqlKnowledgeBase } from '../../../util/esql_knowledge_base'; +import { CIM_TO_ECS_MAP } from './cim_ecs_map'; +import { ESQL_CONVERT_CIM_TO_ECS_PROMPT } from './prompts'; +import { cleanMarkdown, generateAssistantComment } from '../../../util/comments'; +import type { NodeToolCreator } from '../types'; + +export interface GetConvertEsqlSchemaCisToEcsParams { + esqlKnowledgeBase: EsqlKnowledgeBase; + logger: Logger; +} +export interface ConvertEsqlSchemaCisToEcsInput { + title: string; + description: string; + query: string; + originalQuery: string; +} +export interface ConvertEsqlSchemaCisToEcsOutput { + query: string | undefined; + comments: MigrationComments; +} + +export const getConvertEsqlSchemaCisToEcs: NodeToolCreator< + GetConvertEsqlSchemaCisToEcsParams, + ConvertEsqlSchemaCisToEcsInput, + ConvertEsqlSchemaCisToEcsOutput +> = ({ esqlKnowledgeBase, logger }) => { + return async (input) => { + const esqlQuery = { + title: input.title, + description: input.description, + query: input.query, + }; + + const prompt = await ESQL_CONVERT_CIM_TO_ECS_PROMPT.format({ + field_mapping: CIM_TO_ECS_MAP, + spl_query: input.originalQuery, + esql_query: JSON.stringify(esqlQuery, null, 2), + }); + + const response = await esqlKnowledgeBase.translate(prompt); + + const updatedQuery = response.match(/```esql\n([\s\S]*?)\n```/)?.[1] ?? ''; + if (!updatedQuery) { + logger.warn('Failed to apply ECS mapping to the query'); + const summary = '## Field Mapping Summary\n\nFailed to apply ECS mapping to the query'; + return { + query: undefined, // No updated query if ECS mapping failed + comments: [generateAssistantComment(summary)], + }; + } + + const ecsSummary = response.match(/## Field Mapping Summary[\s\S]*$/)?.[0] ?? ''; + + // We set success to true to indicate that the ecs mapping has been applied. + // This is to ensure that the node only runs once + return { + comments: [generateAssistantComment(cleanMarkdown(ecsSummary))], + query: updatedQuery, + }; + }; +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/convert_esql_schema_cim_to_ecs/index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/convert_esql_schema_cim_to_ecs/index.ts new file mode 100644 index 0000000000000..3d85289bf2b21 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/convert_esql_schema_cim_to_ecs/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +export * from './convert_esql_schema_cim_to_ecs'; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/ecs_mapping/prompts.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/convert_esql_schema_cim_to_ecs/prompts.ts similarity index 97% rename from x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/ecs_mapping/prompts.ts rename to x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/convert_esql_schema_cim_to_ecs/prompts.ts index 1e89cda884ca0..41a8f3954fde8 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/ecs_mapping/prompts.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/convert_esql_schema_cim_to_ecs/prompts.ts @@ -7,7 +7,7 @@ import { ChatPromptTemplate } from '@langchain/core/prompts'; -export const ESQL_TRANSLATE_ECS_MAPPING_PROMPT = +export const ESQL_CONVERT_CIM_TO_ECS_PROMPT = ChatPromptTemplate.fromTemplate(`You are a helpful cybersecurity (SIEM) expert agent. Your task is to migrate "detection rules" from Splunk SPL to Elasticsearch ESQL. Your task is to look at the new ESQL query already generated from its initial Splunk SPL query and translate the Splunk CIM field names to the Elastic Common Schema (ECS) fields. Below is the relevant context used when deciding which Elastic Common Schema field to use when translating from Splunk CIM fields: @@ -17,10 +17,10 @@ Below is the relevant context used when deciding which Elastic Common Schema fie {field_mapping} -{splunk_query} +{spl_query} -{elastic_rule} +{esql_query} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/fix_esql_query_errors/fix_esql_query_errors.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/fix_esql_query_errors/fix_esql_query_errors.ts new file mode 100644 index 0000000000000..7e32a2ed71ddb --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/fix_esql_query_errors/fix_esql_query_errors.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; +import type { NodeToolCreator } from '../types'; +import type { EsqlKnowledgeBase } from '../../../util/esql_knowledge_base'; +import { RESOLVE_ESQL_ERRORS_TEMPLATE } from './prompts'; + +export interface GetFixEsqlQueryErrorsParams { + esqlKnowledgeBase: EsqlKnowledgeBase; + logger: Logger; +} +export interface FixEsqlQueryErrorsInput { + invalidQuery?: string; + validationErrors?: string; +} +export interface FixEsqlQueryErrorsOutput { + query?: string; +} + +export const getFixEsqlQueryErrors: NodeToolCreator< + GetFixEsqlQueryErrorsParams, + FixEsqlQueryErrorsInput, + FixEsqlQueryErrorsOutput +> = ({ esqlKnowledgeBase, logger }) => { + return async (input) => { + if (!input.validationErrors) { + logger.debug('Trying to fix errors without validationErrors'); + return {}; + } + if (!input.invalidQuery) { + logger.debug('Trying to fix errors without invalidQuery'); + return {}; + } + + const prompt = await RESOLVE_ESQL_ERRORS_TEMPLATE.format({ + esql_errors: input.validationErrors, + esql_query: input.invalidQuery, + }); + const response = await esqlKnowledgeBase.translate(prompt); + + const query = response.match(/```esql\n([\s\S]*?)\n```/)?.[1]; + return { query }; + }; +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/fix_esql_query_errors/index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/fix_esql_query_errors/index.ts new file mode 100644 index 0000000000000..361b44bb22b99 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/fix_esql_query_errors/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +export * from './fix_esql_query_errors'; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/fix_query_errors/prompts.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/fix_esql_query_errors/prompts.ts similarity index 100% rename from x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/fix_query_errors/prompts.ts rename to x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/fix_esql_query_errors/prompts.ts diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/inline_spl_query/index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/inline_spl_query/index.ts new file mode 100644 index 0000000000000..a0b2989ff407e --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/inline_spl_query/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +export * from './inline_spl_query'; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/inline_spl_query/inline_spl_query.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/inline_spl_query/inline_spl_query.ts new file mode 100644 index 0000000000000..0f4e9d6998a2c --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/inline_spl_query/inline_spl_query.ts @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; +import { StringOutputParser } from '@langchain/core/output_parsers'; +import { isEmpty } from 'lodash/fp'; +import type { MigrationComments } from '../../../../../../../../common/siem_migrations/model/migration.gen'; +import type { ChatModel } from '../../../util/actions_client_chat'; +import { cleanMarkdown, generateAssistantComment } from '../../../util/comments'; +import type { MigrationResources } from '../../../retrievers/resource_retriever'; +import type { NodeToolCreator } from '../types'; +import { REPLACE_QUERY_RESOURCE_PROMPT, getResourcesContext } from './prompts'; + +export interface GetInlineSplQueryParams { + model: ChatModel; + logger: Logger; +} +export interface InlineSplQueryInput { + query: string; + resources: MigrationResources; +} +export interface InlineSplQueryOutput { + inlineQuery?: string | undefined; + isUnsupported?: boolean; + comments: MigrationComments; +} + +export const getInlineSplQuery: NodeToolCreator< + GetInlineSplQueryParams, + InlineSplQueryInput, + InlineSplQueryOutput +> = ({ model, logger }) => { + return async ({ query, resources }) => { + // Early check to avoid unnecessary LLM calls + let unsupportedComment = getUnsupportedComment(query); + if (unsupportedComment) { + return { + isUnsupported: true, + comments: [generateAssistantComment(unsupportedComment)], + }; + } + + if (isEmpty(resources)) { + // No resources identified in the query, no need to replace anything + const summary = '## Inlining Summary\n\nNo macro or lookup found in the query.'; + return { inlineQuery: query, comments: [generateAssistantComment(summary)] }; + } + + const replaceQueryParser = new StringOutputParser(); + const replaceQueryResourcePrompt = + REPLACE_QUERY_RESOURCE_PROMPT.pipe(model).pipe(replaceQueryParser); + const resourceContext = getResourcesContext(resources); + const response = await replaceQueryResourcePrompt.invoke({ + query, + macros: resourceContext.macros, + lookups: resourceContext.lookups, + }); + + const inlineQuery = response.match(/```spl\n([\s\S]*?)\n```/)?.[1].trim() ?? ''; + if (!inlineQuery) { + logger.warn('Failed to retrieve inline query'); + const summary = '## Inlining Summary\n\nFailed to retrieve inline query'; + return { comments: [generateAssistantComment(summary)] }; + } + + // Check after replacing in case the inlining made it untranslatable + unsupportedComment = getUnsupportedComment(inlineQuery); + if (unsupportedComment) { + return { + isUnsupported: true, + comments: [generateAssistantComment(unsupportedComment)], + }; + } + + const inliningSummary = response.match(/## Inlining Summary[\s\S]*$/)?.[0] ?? ''; + return { + inlineQuery, + comments: [generateAssistantComment(cleanMarkdown(inliningSummary))], + }; + }; +}; + +const getUnsupportedComment = (query: string): string | undefined => { + const unsupportedText = '## Translation Summary\nCan not create query translation.\n'; + if (query.includes(' inputlookup')) { + return `${unsupportedText}Reason: \`inputlookup\` command is not supported.`; + } +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/prompts.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/inline_spl_query/prompts.ts similarity index 96% rename from x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/prompts.ts rename to x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/inline_spl_query/prompts.ts index 290a2d4e0e05e..d2581edd9c2f9 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/prompts.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/inline_spl_query/prompts.ts @@ -6,14 +6,14 @@ */ import { ChatPromptTemplate } from '@langchain/core/prompts'; -import type { RuleMigrationResources } from '../../../../../retrievers/rule_resource_retriever'; +import type { MigrationResources } from '../../../retrievers/resource_retriever'; interface ResourceContext { macros: string; lookups: string; } -export const getResourcesContext = (resources: RuleMigrationResources): ResourceContext => { +export const getResourcesContext = (resources: MigrationResources): ResourceContext => { const context: ResourceContext = { macros: '', lookups: '' }; // Process macros diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/translate_spl_to_esql/index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/translate_spl_to_esql/index.ts new file mode 100644 index 0000000000000..2fd911b4b6a53 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/translate_spl_to_esql/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +export * from './translate_rule'; +export { TASK_DESCRIPTION } from './prompts'; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/prompts.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/translate_spl_to_esql/prompts.ts similarity index 58% rename from x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/prompts.ts rename to x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/translate_spl_to_esql/prompts.ts index 23a6829e6235d..d5aea1c5cf833 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/prompts.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/translate_spl_to_esql/prompts.ts @@ -7,15 +7,29 @@ import { ChatPromptTemplate } from '@langchain/core/prompts'; +export const TASK_DESCRIPTION = { + migrate_rule: `Your task is to migrate a "detection rule" SPL search from Splunk to an Elasticsearch ES|QL query.`, + migrate_dashboard: `Your task is to migrate a "dashboard" SPL search from Splunk to an Elasticsearch ES|QL query.`, +}; + export const ESQL_SYNTAX_TRANSLATION_PROMPT = - ChatPromptTemplate.fromTemplate(`You are a helpful cybersecurity (SIEM) expert agent. Your task is to migrate "detection rules" from Splunk SPL to Elasticsearch ES|QL. -Your goal is to translate the SPL query syntax into an equivalent Elastic Search Query Language (ES|QL) query without changing any of the field names except lookup lists and macros when relevant and focusing only on translating the syntax and structure. + ChatPromptTemplate.fromTemplate(`You are a helpful cybersecurity (SIEM) expert agent. {task_description} +Your goal is to translate the SPL query syntax into an equivalent Elastic Search Query Language (ES|QL) query without changing any of the field names, except for lookup lists when relevant, and focusing only on translating the syntax and structure. +Also you'll need to write a summary at the end in markdown language. Here are some context for you to reference for your task, read it carefully as you will get questions about it later: - -{splunk_rule} - + +{splunk_query} + + +If you encounter any placeholders for macros or lookups in the SPL query, leave them as-is in the ES|QL query output. They are markers that need to be preserved. +They are wrapped in brackets ("[]") and always start with "macro:" or "lookup:". Mention all placeholders you left in the final summary. +Examples of macros and lookups placeholders: +- [macro:someMacroName(3)] +- [macro:another_macro] +- [lookup:someLookup_name] + If in an SPL query you identify a lookup call, it should be translated the following way: \`\`\`spl @@ -30,11 +44,13 @@ However in the ES|QL query, some of the information is removed and should be use ... | LOOKUP JOIN 'index_name' ON 'field_to_match' \`\`\` We do not define OUTPUTNEW or which fields is returned, only the index name and the field to match. + +Mention all translated lookups in the final summary. -Go through each step and part of the splunk rule and query while following the below guide to produce the resulting ES|QL query: -- Analyze all the information about the related splunk rule and try to determine the intent of the rule, in order to translate into an equivalent ES|QL rule. +Go through each step and part of the splunk_query while following the below guide to produce the resulting ES|QL query: +- Analyze all the information about the related splunk query and try to determine the intent of the query, in order to translate into an equivalent ES|QL query. - Go through each part of the SPL query and determine the steps required to produce the same end results using ES|QL. Only focus on translating the structure without modifying any of the field names. - Do NOT map any of the fields to the Elastic Common Schema (ECS), this will happen in a later step. - Always remember to translate any lookup list using the lookup_syntax above @@ -42,9 +58,9 @@ Go through each step and part of the splunk rule and query while following the b - Analyze the SPL query and identify the key components. - Do NOT translate the field names of the SPL query. -- Always start the resulting ES|QL query by filtering using FROM and with these index pattern: {indexPatterns}. -- Always remember to translate any lookup list using the lookup_syntax above -- Always remember to replace macro call with the appropriate placeholder as defined in the macro info. +- Always start the resulting ES|QL query by filtering using FROM and with the following index pattern: {index_pattern} +- Always remember to leave placeholders defined in the placeholders_syntax context as they are, don't replace them. +- Always remember to translate any lookup (that are not inside a placeholder) using the lookup_syntax rules above. diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/translate_spl_to_esql/translate_rule.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/translate_spl_to_esql/translate_rule.ts new file mode 100644 index 0000000000000..cc09c9f78e4fc --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/translate_spl_to_esql/translate_rule.ts @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; +import type { MigrationComments } from '../../../../../../../../common/siem_migrations/model/migration.gen'; +import { cleanMarkdown, generateAssistantComment } from '../../../util/comments'; +import type { EsqlKnowledgeBase } from '../../../util/esql_knowledge_base'; +import { ESQL_SYNTAX_TRANSLATION_PROMPT } from './prompts'; +import type { NodeToolCreator } from '../types'; + +export interface GetTranslateSplToEsqlParams { + esqlKnowledgeBase: EsqlKnowledgeBase; + logger: Logger; +} + +export interface TranslateSplToEsqlInput { + title: string; + taskDescription: string; + description: string; + inlineQuery: string; + indexPattern: string; +} +export interface TranslateSplToEsqlOutput { + esqlQuery?: string; + comments: MigrationComments; +} + +export const getTranslateSplToEsql: NodeToolCreator< + GetTranslateSplToEsqlParams, + TranslateSplToEsqlInput, + TranslateSplToEsqlOutput +> = ({ esqlKnowledgeBase, logger }) => { + return async (input) => { + const splunkQuery = { + title: input.title, + description: input.description, + inline_query: input.inlineQuery, + }; + + const prompt = await ESQL_SYNTAX_TRANSLATION_PROMPT.format({ + splunk_query: JSON.stringify(splunkQuery, null, 2), + index_pattern: input.indexPattern, + task_description: input.taskDescription, + }); + const response = await esqlKnowledgeBase.translate(prompt); + + const esqlQuery = response.match(/```esql\n([\s\S]*?)\n```/)?.[1].trim() ?? ''; + if (!esqlQuery) { + logger.warn('Failed to extract ESQL query from translation response'); + const comment = + '## Translation Summary\n\nFailed to extract ESQL query from translation response'; + return { + comments: [generateAssistantComment(comment)], + }; + } + + const translationSummary = response.match(/## Translation Summary[\s\S]*$/)?.[0] ?? ''; + + return { + esqlQuery, + comments: [generateAssistantComment(cleanMarkdown(translationSummary))], + }; + }; +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/types.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/types.ts new file mode 100644 index 0000000000000..aee3340345d70 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/types.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export type NodeTool = (input: I) => Promise; +export type NodeToolCreator

= ( + params: P +) => NodeTool; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/validate_esql/index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/validate_esql/index.ts new file mode 100644 index 0000000000000..a8c0f55191e42 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/validate_esql/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +export { getValidationNode } from './validation'; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/validate_esql/validation.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/validate_esql/validation.ts new file mode 100644 index 0000000000000..ea40e07de0135 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/validate_esql/validation.ts @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; +import { isEmpty } from 'lodash/fp'; +import { parseEsqlQuery } from '@kbn/securitysolution-utils'; +import type { NodeToolCreator } from '../types'; + +export interface GetValidateEsqlParams { + logger: Logger; +} + +export interface ValidateEsqlInput { + query: string; +} + +export interface ValidateEsqlOutput { + error?: string; +} + +export const getValidateEsql: NodeToolCreator< + GetValidateEsqlParams, + ValidateEsqlInput, + ValidateEsqlOutput +> = ({ logger }) => { + return async (input) => { + // We want to prevent infinite loops, so we increment the iterations counter for each validation run. + let error: string = ''; + try { + const sanitizedQuery = input.query ? sanitizeQuery(input.query) : ''; + if (!isEmpty(sanitizedQuery)) { + const { errors, isEsqlQueryAggregating, hasMetadataOperator } = + parseEsqlQuery(sanitizedQuery); + if (!isEmpty(errors)) { + error = JSON.stringify(errors); + } else if (!isEsqlQueryAggregating && !hasMetadataOperator) { + error = `Queries that do't use the STATS...BY function (non-aggregating queries) must include the "metadata _id, _version, _index" operator after the source command. For example: FROM logs* metadata _id, _version, _index.`; + } + } + if (error) { + logger.debug(`ESQL query validation failed: ${error}`); + } + } catch (err) { + error = err.message.toString(); + logger.info(`Error parsing ESQL query: ${error}`); + } + return { error }; + }; +}; + +function sanitizeQuery(query: string): string { + return query + .replace('FROM [indexPattern]', 'FROM *') // Replace the index pattern placeholder with a wildcard + .replaceAll(/\[(macro|lookup):.*?\]/g, '') // Removes any macro or lookup placeholders + .replaceAll(/\n(\s*?\|\s*?\n)*/g, '\n'); // Removes any empty lines with | (pipe) alone after removing the placeholders +} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/retrievers/resource_retriever.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/retrievers/resource_retriever.test.ts new file mode 100644 index 0000000000000..7602ccfe03d64 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/retrievers/resource_retriever.test.ts @@ -0,0 +1,138 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ResourceRetriever } from './resource_retriever'; // Adjust path as needed +import type { ResourceIdentifierClass } from '../../../../../../common/siem_migrations/resources/resource_identifier'; +import { ResourceIdentifier } from '../../../../../../common/siem_migrations/resources/resource_identifier'; +import type { SiemMigrationsDataResourcesClient } from '../../data/siem_migrations_data_resources_client'; +import type { ItemDocument } from '../../../../../../common/siem_migrations/types'; + +jest.mock('../../data/siem_migrations_data_resources_client'); +jest.mock('../../../../../../common/siem_migrations/resources/resource_identifier'); + +const migrationItem = {} as unknown as ItemDocument; + +const MockResourceIdentifier = ResourceIdentifier as jest.MockedClass; + +class TestResourceRetriever extends ResourceRetriever { + protected ResourceIdentifierClass = MockResourceIdentifier; +} + +const defaultResourceIdentifier = () => + ({ + getVendor: jest.fn().mockReturnValue('splunk'), + fromOriginal: jest.fn().mockReturnValue([]), + fromResources: jest.fn().mockReturnValue([]), + } as unknown as jest.Mocked); + +describe('ResourceRetriever', () => { + let retriever: ResourceRetriever; + let mockDataClient: jest.Mocked; + let mockResourceIdentifier: jest.Mocked; + + beforeEach(() => { + mockDataClient = { + searchBatches: jest.fn().mockReturnValue({ next: jest.fn(() => []) }), + } as unknown as jest.Mocked; + + retriever = new TestResourceRetriever('mockMigrationId', mockDataClient); + + MockResourceIdentifier.mockImplementation(defaultResourceIdentifier); + mockResourceIdentifier = new MockResourceIdentifier( + migrationItem + ) as jest.Mocked; + }); + + it('throws an error if initialize is not called before getResources', async () => { + await expect(retriever.getResources(migrationItem)).rejects.toThrow( + 'initialize must be called before calling getResources' + ); + }); + + it('returns an empty object if no matching resources are found', async () => { + // Mock the resource identifier to return no resources + mockResourceIdentifier.fromOriginal.mockReturnValue([]); + await retriever.initialize(); // Pretend initialize has been called + + const result = await retriever.getResources(migrationItem); + expect(result).toEqual({}); + }); + + it('returns matching macro and lookup resources', async () => { + const mockExistingResources = { + macro: { macro1: { name: 'macro1', type: 'macro' } }, + lookup: { lookup1: { name: 'lookup1', type: 'lookup' } }, + }; + // Inject existing resources manually + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (retriever as any).existingResources = mockExistingResources; + + const mockResourcesIdentified = [ + { name: 'macro1', type: 'macro' as const }, + { name: 'lookup1', type: 'lookup' as const }, + ]; + MockResourceIdentifier.mockImplementation( + () => + ({ + ...defaultResourceIdentifier(), + fromOriginal: jest.fn().mockReturnValue(mockResourcesIdentified), + } as unknown as jest.Mocked) + ); + + const result = await retriever.getResources(migrationItem); + expect(result).toEqual({ + macro: [{ name: 'macro1', type: 'macro' }], + lookup: [{ name: 'lookup1', type: 'lookup' }], + }); + }); + + it('handles nested resources properly', async () => { + const mockExistingResources = { + macro: { + macro1: { name: 'macro1', type: 'macro' }, + macro2: { name: 'macro2', type: 'macro' }, + }, + lookup: { + lookup1: { name: 'lookup1', type: 'lookup' }, + lookup2: { name: 'lookup2', type: 'lookup' }, + }, + }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (retriever as any).existingResources = mockExistingResources; + + const mockResourcesIdentifiedFromRule = [ + { name: 'macro1', type: 'macro' as const }, + { name: 'lookup1', type: 'lookup' as const }, + ]; + + const mockNestedResources = [ + { name: 'macro2', type: 'macro' as const }, + { name: 'lookup2', type: 'lookup' as const }, + ]; + + MockResourceIdentifier.mockImplementation( + () => + ({ + ...defaultResourceIdentifier(), + fromOriginal: jest.fn().mockReturnValue(mockResourcesIdentifiedFromRule), + fromResources: jest.fn().mockReturnValue([]).mockReturnValueOnce(mockNestedResources), + } as unknown as jest.Mocked) + ); + + const result = await retriever.getResources(migrationItem); + expect(result).toEqual({ + macro: [ + { name: 'macro1', type: 'macro' }, + { name: 'macro2', type: 'macro' }, + ], + lookup: [ + { name: 'lookup1', type: 'lookup' }, + { name: 'lookup2', type: 'lookup' }, + ], + }); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/retrievers/resource_retriever.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/retrievers/resource_retriever.ts new file mode 100644 index 0000000000000..989f89e1467f3 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/retrievers/resource_retriever.ts @@ -0,0 +1,113 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ResourceIdentifierClass } from '../../../../../../common/siem_migrations/resources/resource_identifier'; +// TODO: move resource related types to migration.gen.ts +import type { + RuleMigrationResource as MigrationResource, + RuleMigrationResourceType as MigrationResourceType, +} from '../../../../../../common/siem_migrations/model/rule_migration.gen'; +import type { SiemMigrationsDataResourcesClient } from '../../data/siem_migrations_data_resources_client'; +import type { ItemDocument } from '../../types'; + +export interface MigrationDefinedResource extends MigrationResource { + content: string; // ensures content exists +} +export type MigrationResourcesData = Pick; +export type MigrationResources = Partial>; +interface ExistingResources { + macro: Record; + lookup: Record; +} + +export abstract class ResourceRetriever { + protected abstract ResourceIdentifierClass: ResourceIdentifierClass; + + private existingResources?: ExistingResources; + + constructor( + private readonly migrationId: string, + private readonly resourcesDataClient: SiemMigrationsDataResourcesClient + ) {} + + public async initialize(): Promise { + const batches = this.resourcesDataClient.searchBatches( + this.migrationId, + { filters: { hasContent: true } } // filters out missing (undefined) content resources, empty strings content will be included + ); + + const existingResources: ExistingResources = { macro: {}, lookup: {} }; + let resources; + do { + resources = await batches.next(); + resources.forEach((resource) => { + existingResources[resource.type][resource.name] = resource; + }); + } while (resources.length > 0); + + this.existingResources = existingResources; + } + + public async getResources(migrationItem: I): Promise { + const existingResources = this.existingResources; + if (!existingResources) { + throw new Error('initialize must be called before calling getResources'); + } + + const resourceIdentifier = new this.ResourceIdentifierClass(migrationItem); + const resourcesIdentifiedFromRule = resourceIdentifier.fromOriginal(); + + const macrosFound = new Map(); + const lookupsFound = new Map(); + resourcesIdentifiedFromRule.forEach((resource) => { + const existingResource = existingResources[resource.type][resource.name]; + if (existingResource) { + if (resource.type === 'macro') { + macrosFound.set(resource.name, existingResource); + } else if (resource.type === 'lookup') { + lookupsFound.set(resource.name, existingResource); + } + } + }); + + const resourcesFound = [...macrosFound.values(), ...lookupsFound.values()]; + if (!resourcesFound.length) { + return {}; + } + + let nestedResourcesFound = resourcesFound; + do { + const nestedResourcesIdentified = resourceIdentifier.fromResources(nestedResourcesFound); + + nestedResourcesFound = []; + nestedResourcesIdentified.forEach((resource) => { + const existingResource = existingResources[resource.type][resource.name]; + if (existingResource) { + nestedResourcesFound.push(existingResource); + if (resource.type === 'macro') { + macrosFound.set(resource.name, existingResource); + } else if (resource.type === 'lookup') { + lookupsFound.set(resource.name, existingResource); + } + } + }); + } while (nestedResourcesFound.length > 0); + + return { + ...(macrosFound.size > 0 ? { macro: this.formatOutput(macrosFound) } : {}), + ...(lookupsFound.size > 0 ? { lookup: this.formatOutput(lookupsFound) } : {}), + }; + } + + private formatOutput(resources: Map): MigrationResourcesData[] { + return Array.from(resources.values()).map(({ name, content, type }) => ({ + name, + content, + type, + })); + } +} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/siem_migrations_task_client.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/siem_migrations_task_client.test.ts new file mode 100644 index 0000000000000..f63e1f6d240ae --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/siem_migrations_task_client.test.ts @@ -0,0 +1,513 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { AuthenticatedUser } from '@kbn/core/server'; +import type { MigrationsRunning } from './siem_migrations_task_client'; +import { SiemMigrationsTaskClient } from './siem_migrations_task_client'; +import { + SiemMigrationStatus, + SiemMigrationTaskStatus, +} from '../../../../../common/siem_migrations/constants'; +import { RuleMigrationTaskRunner } from './siem_migrations_task_runner'; +import type { MockedLogger } from '@kbn/logging-mocks'; +import { loggerMock } from '@kbn/logging-mocks'; +import type { StoredSiemMigration } from '../types'; +import type { SiemMigrationTaskStartParams } from './types'; +import { createRuleMigrationsDataClientMock } from '../data/__mocks__/mocks'; +import type { SiemMigrationDataStats } from '../data/siem_migrations_data_item_client'; +import type { RuleMigrationFilters } from '../../../../../common/siem_migrations/rules/types'; +import type { SiemMigrationsClientDependencies } from '../types'; + +jest.mock('./rule_migrations_task_runner', () => { + return { + RuleMigrationTaskRunner: jest.fn().mockImplementation(() => { + return { + setup: jest.fn().mockResolvedValue(undefined), + run: jest.fn().mockResolvedValue(undefined), + abortController: { abort: jest.fn() }, + }; + }), + }; +}); + +const currentUser = {} as AuthenticatedUser; +const dependencies = {} as SiemMigrationsClientDependencies; +const migrationId = 'migration1'; + +describe('RuleMigrationsTaskClient', () => { + let migrationsRunning: MigrationsRunning; + let logger: MockedLogger; + let data: ReturnType; + const params: SiemMigrationTaskStartParams = { + migrationId, + connectorId: 'connector1', + invocationConfig: {}, + }; + + beforeEach(() => { + migrationsRunning = new Map(); + logger = loggerMock.create(); + + data = createRuleMigrationsDataClientMock(); + // @ts-expect-error resetting private property for each test. + SiemMigrationsTaskClient.migrationsLastError = new Map(); + jest.clearAllMocks(); + }); + + describe('start', () => { + it('should not start if migration is already running', async () => { + // Pre-populate with the migration id. + migrationsRunning.set(migrationId, {} as RuleMigrationTaskRunner); + const client = new SiemMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + const result = await client.start(params); + expect(result).toEqual({ exists: true, started: false }); + expect(data.rules.updateStatus).not.toHaveBeenCalled(); + }); + + it('should not start if there are no rules to migrate (total = 0)', async () => { + data.rules.getStats.mockResolvedValue({ + rules: { total: 0, pending: 0, completed: 0, failed: 0 }, + } as SiemMigrationDataStats); + const client = new SiemMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + const result = await client.start(params); + expect(data.rules.updateStatus).toHaveBeenCalledWith( + migrationId, + { status: SiemMigrationStatus.PROCESSING }, + SiemMigrationStatus.PENDING, + { refresh: true } + ); + expect(result).toEqual({ exists: false, started: false }); + }); + + it('should not start if there are no pending rules', async () => { + data.rules.getStats.mockResolvedValue({ + rules: { total: 10, pending: 0, completed: 10, failed: 0 }, + } as SiemMigrationDataStats); + const client = new SiemMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + const result = await client.start(params); + expect(result).toEqual({ exists: true, started: false }); + }); + + it('should start migration successfully', async () => { + data.rules.getStats.mockResolvedValue({ + rules: { total: 10, pending: 5, completed: 0, failed: 0 }, + } as SiemMigrationDataStats); + const mockedRunnerInstance = { + setup: jest.fn().mockResolvedValue(undefined), + run: jest.fn().mockResolvedValue(undefined), + abortController: { abort: jest.fn() }, + }; + // Use our custom mock for this test. + (RuleMigrationTaskRunner as jest.Mock).mockImplementationOnce(() => mockedRunnerInstance); + + const client = new SiemMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + const result = await client.start(params); + expect(result).toEqual({ exists: true, started: true }); + expect(logger.get).toHaveBeenCalledWith(migrationId); + expect(mockedRunnerInstance.setup).toHaveBeenCalledWith(params.connectorId); + expect(logger.get(migrationId).info).toHaveBeenCalledWith('Starting migration'); + expect(migrationsRunning.has(migrationId)).toBe(true); + + // Allow the asynchronous run() call to complete its finally callback. + await new Promise(process.nextTick); + expect(migrationsRunning.has(migrationId)).toBe(false); + // @ts-expect-error check private property + expect(SiemMigrationsTaskClient.migrationsLastError.has(migrationId)).toBe(false); + }); + + it('should throw error if a race condition occurs after setup', async () => { + data.rules.getStats.mockResolvedValue({ + rules: { total: 10, pending: 5, completed: 0, failed: 0 }, + } as SiemMigrationDataStats); + const mockedRunnerInstance = { + setup: jest.fn().mockImplementationOnce(() => { + // Simulate a race condition by setting the migration as running during setup. + migrationsRunning.set(migrationId, {} as RuleMigrationTaskRunner); + return Promise.resolve(); + }), + run: jest.fn().mockResolvedValue(undefined), + abortController: { abort: jest.fn() }, + }; + (RuleMigrationTaskRunner as jest.Mock).mockImplementation(() => mockedRunnerInstance); + + const client = new SiemMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + await expect(client.start(params)).rejects.toThrow('Task already running for this migration'); + }); + + it('should mark migration as started by calling saveAsStarted', async () => { + data.rules.getStats.mockResolvedValue({ + rules: { total: 10, pending: 5, completed: 0, failed: 0 }, + } as SiemMigrationDataStats); + + const client = new SiemMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + + await client.start(params); + expect(data.migrations.saveAsStarted).toHaveBeenCalledWith({ + id: migrationId, + connectorId: params.connectorId, + }); + }); + + it('should mark migration as ended by calling saveAsEnded if run completes successfully', async () => { + migrationsRunning = new Map(); + data.rules.getStats.mockResolvedValue({ + rules: { total: 10, pending: 5, completed: 0, failed: 0 }, + } as SiemMigrationDataStats); + + const client = new SiemMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + + await client.start(params); + // Allow the asynchronous run() call to complete its finally callback. + await new Promise(process.nextTick); + expect(data.migrations.saveAsFinished).toHaveBeenCalledWith({ id: migrationId }); + }); + }); + + describe('updateToRetry', () => { + it('should not update if migration is currently running', async () => { + migrationsRunning.set(migrationId, {} as RuleMigrationTaskRunner); + const client = new SiemMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + const filter: RuleMigrationFilters = { fullyTranslated: true }; + const result = await client.updateToRetry(migrationId, filter); + expect(result).toEqual({ updated: false }); + expect(data.rules.updateStatus).not.toHaveBeenCalled(); + }); + + it('should update to retry if migration is not running', async () => { + const client = new SiemMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + const filter: RuleMigrationFilters = { fullyTranslated: true }; + const result = await client.updateToRetry(migrationId, filter); + expect(filter.installed).toBe(false); + expect(data.rules.updateStatus).toHaveBeenCalledWith( + migrationId, + { fullyTranslated: true, installed: false }, + SiemMigrationStatus.PENDING, + { refresh: true } + ); + expect(result).toEqual({ updated: true }); + }); + }); + + describe('getStats', () => { + it('should return RUNNING status if migration is running', async () => { + migrationsRunning.set(migrationId, {} as RuleMigrationTaskRunner); // migration is running + data.rules.getStats.mockResolvedValue({ + rules: { total: 10, pending: 5, completed: 3, failed: 2 }, + } as SiemMigrationDataStats); + + data.migrations.get.mockResolvedValue({ + id: migrationId, + } as unknown as StoredSiemMigration); + + const client = new SiemMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + const stats = await client.getStats(migrationId); + expect(stats.status).toEqual(SiemMigrationTaskStatus.RUNNING); + }); + + it('should return READY status if pending equals total', async () => { + data.rules.getStats.mockResolvedValue({ + rules: { total: 10, pending: 10, completed: 0, failed: 0 }, + } as SiemMigrationDataStats); + data.migrations.get.mockResolvedValue({ + id: migrationId, + } as unknown as StoredSiemMigration); + + const client = new SiemMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + const stats = await client.getStats(migrationId); + expect(stats.status).toEqual(SiemMigrationTaskStatus.READY); + }); + + it('should return FINISHED status if completed+failed equals total', async () => { + data.rules.getStats.mockResolvedValue({ + rules: { total: 10, pending: 0, completed: 5, failed: 5 }, + } as SiemMigrationDataStats); + + data.migrations.get.mockResolvedValue({ + id: migrationId, + } as unknown as StoredSiemMigration); + const client = new SiemMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + const stats = await client.getStats(migrationId); + expect(stats.status).toEqual(SiemMigrationTaskStatus.FINISHED); + }); + + it('should return STOPPED status for other cases', async () => { + data.rules.getStats.mockResolvedValue({ + rules: { total: 10, pending: 2, completed: 3, failed: 2 }, + } as SiemMigrationDataStats); + const client = new SiemMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + const stats = await client.getStats(migrationId); + expect(stats.status).toEqual(SiemMigrationTaskStatus.INTERRUPTED); + }); + + it('should include error if one exists', async () => { + const errorMessage = 'Test error'; + data.rules.getStats.mockResolvedValue({ + id: 'migration-1', + rules: { total: 10, pending: 2, completed: 3, failed: 2 }, + } as SiemMigrationDataStats); + + data.migrations.get.mockResolvedValue({ + id: 'migration-1', + name: 'Test Migration', + created_at: new Date().toISOString(), + created_by: 'test-user', + last_execution: { + error: errorMessage, + }, + }); + + data.migrations.get.mockResolvedValue({ + id: migrationId, + last_execution: { + error: 'Test error', + }, + } as unknown as StoredSiemMigration); + + const client = new SiemMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + const stats = await client.getStats(migrationId); + expect(stats.last_execution?.error).toEqual('Test error'); + }); + }); + + describe('getAllStats', () => { + it('should return combined stats for all migrations', async () => { + const statsArray = [ + { + id: 'm1', + rules: { total: 10, pending: 10, completed: 0, failed: 0 }, + } as SiemMigrationDataStats, + { + id: 'm2', + rules: { total: 10, pending: 2, completed: 3, failed: 2 }, + } as SiemMigrationDataStats, + ]; + const migrations = [{ id: 'm1' }, { id: 'm2' }] as unknown as StoredSiemMigration[]; + data.rules.getAllStats.mockResolvedValue(statsArray); + data.migrations.getAll.mockResolvedValue(migrations); + // Mark migration m1 as running. + migrationsRunning.set('m1', {} as RuleMigrationTaskRunner); + const client = new SiemMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + const allStats = await client.getAllStats(); + const m1Stats = allStats.find((s) => s.id === 'm1'); + const m2Stats = allStats.find((s) => s.id === 'm2'); + expect(m1Stats?.status).toEqual(SiemMigrationTaskStatus.RUNNING); + expect(m2Stats?.status).toEqual(SiemMigrationTaskStatus.INTERRUPTED); + }); + }); + + describe('stop', () => { + it('should stop a running migration', async () => { + const abortMock = jest.fn(); + const migrationRunner = { + abortController: { abort: abortMock }, + } as unknown as RuleMigrationTaskRunner; + migrationsRunning.set(migrationId, migrationRunner); + const client = new SiemMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + const result = await client.stop(migrationId); + expect(result).toEqual({ exists: true, stopped: true }); + expect(abortMock).toHaveBeenCalled(); + }); + + it('should return stopped even if migration is already stopped', async () => { + data.rules.getStats.mockResolvedValue({ + rules: { total: 10, pending: 10, completed: 0, failed: 0 }, + } as SiemMigrationDataStats); + const client = new SiemMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + const result = await client.stop(migrationId); + expect(result).toEqual({ exists: true, stopped: true }); + }); + + it('should return exists false if migration is not running and total equals 0', async () => { + data.rules.getStats.mockResolvedValue({ + rules: { total: 0, pending: 0, completed: 0, failed: 0 }, + } as SiemMigrationDataStats); + const client = new SiemMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + const result = await client.stop(migrationId); + expect(result).toEqual({ exists: false, stopped: true }); + }); + + it('should catch errors and return exists true, stopped false', async () => { + const error = new Error('Stop error'); + data.rules.getStats.mockRejectedValue(error); + const client = new SiemMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + const result = await client.stop(migrationId); + expect(result).toEqual({ exists: true, stopped: false }); + expect(logger.error).toHaveBeenCalledWith( + `Error stopping migration ID:${migrationId}`, + error + ); + }); + + it('should mark migration task as stopped when manually stopping a running migration', async () => { + const abortMock = jest.fn(); + const migrationRunner = { + abortController: { abort: abortMock }, + } as unknown as RuleMigrationTaskRunner; + migrationsRunning.set(migrationId, migrationRunner); + data.migrations.setIsStopped.mockResolvedValue(undefined); + + const client = new SiemMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + await client.stop(migrationId); + expect(data.migrations.setIsStopped).toHaveBeenCalledWith({ id: migrationId }); + }); + }); + describe('task error', () => { + it('should call saveAsFailed when there has been an error during the migration', async () => { + data.rules.getStats.mockResolvedValue({ + rules: { total: 10, pending: 10, completed: 0, failed: 0 }, + } as SiemMigrationDataStats); + const error = new Error('Migration error'); + + const mockedRunnerInstance = { + setup: jest.fn().mockResolvedValue(undefined), + run: jest.fn().mockRejectedValue(error), + } as unknown as RuleMigrationTaskRunner; + + (RuleMigrationTaskRunner as jest.Mock).mockImplementation(() => mockedRunnerInstance); + + const client = new SiemMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + + const response = await client.start(params); + + // Allow the asynchronous run() call to complete its finally callback. + await new Promise(process.nextTick); + + expect(response).toEqual({ exists: true, started: true }); + + expect(data.migrations.saveAsFailed).toHaveBeenCalledWith({ + id: migrationId, + error: error.message, + }); + }); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/siem_migrations_task_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/siem_migrations_task_client.ts new file mode 100644 index 0000000000000..bfdad05d01bb7 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/siem_migrations_task_client.ts @@ -0,0 +1,259 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { AuthenticatedUser, Logger } from '@kbn/core/server'; +import type { RunnableConfig } from '@langchain/core/runnables'; +import type { MigrationTaskItemsStats } from '../../../../../common/siem_migrations/model/migration.gen'; +import { + SiemMigrationStatus, + SiemMigrationTaskStatus, +} from '../../../../../common/siem_migrations/constants'; +import type { RuleMigrationFilters } from '../../../../../common/siem_migrations/rules/types'; +import type { SiemMigrationsDataClient } from '../data/siem_migrations_data_client'; +import type { SiemMigrationTaskStats } from '../data/types'; +import type { + StoredSiemMigration, + SiemMigrationsClientDependencies, + MigrationDocument, + ItemDocument, +} from '../types'; +import type { + SiemMigrationTaskEvaluateParams, + SiemMigrationTaskStartParams, + SiemMigrationTaskStartResult, + SiemMigrationTaskStopResult, +} from './types'; +import type { SiemMigrationTaskRunner } from './siem_migrations_task_runner'; +import type { SiemMigrationTaskEvaluatorClass } from './siem_migrations_task_evaluator'; + +export abstract class SiemMigrationsTaskClient< + M extends MigrationDocument = StoredSiemMigration, + I extends ItemDocument = ItemDocument, + P extends object = {}, // The migration task input parameters schema + C extends object = {}, // The migration task config schema + O extends object = {} // The migration task output schema +> { + protected abstract readonly TaskRunnerClass: typeof SiemMigrationTaskRunner; + protected abstract readonly EvaluatorClass?: SiemMigrationTaskEvaluatorClass; + + constructor( + protected migrationsRunning: Map>, + private logger: Logger, + private data: SiemMigrationsDataClient, + private currentUser: AuthenticatedUser, + private dependencies: SiemMigrationsClientDependencies + ) {} + + /** Starts a rule migration task */ + async start(params: SiemMigrationTaskStartParams): Promise { + const { migrationId, connectorId, invocationConfig } = params; + if (this.migrationsRunning.has(migrationId)) { + return { exists: true, started: false }; + } + // Just in case some previous execution was interrupted without cleaning up + await this.data.items.updateStatus( + migrationId, + { status: SiemMigrationStatus.PROCESSING }, + SiemMigrationStatus.PENDING, + { refresh: true } + ); + + const { items } = await this.data.items.getStats(migrationId); + if (items.total === 0) { + return { exists: false, started: false }; + } + if (items.pending === 0) { + return { exists: true, started: false }; + } + + const migrationLogger = this.logger.get(migrationId); + const abortController = new AbortController(); + const migrationTaskRunner = new this.TaskRunnerClass( + migrationId, + this.currentUser, + abortController, + this.data, + migrationLogger, + this.dependencies + ); + + await migrationTaskRunner.setup(connectorId); + + if (this.migrationsRunning.has(migrationId)) { + // Just to prevent a race condition in the setup + throw new Error('Task already running for this migration'); + } + + migrationLogger.info('Starting migration'); + + this.migrationsRunning.set(migrationId, migrationTaskRunner); + + await this.data.migrations.saveAsStarted({ + id: migrationId, + connectorId, + ...this.getLastExecutionConfig(invocationConfig), + // skipPrebuiltRulesMatching: invocationConfig.configurable?.skipPrebuiltRulesMatching, + }); + + // run the migration in the background without awaiting and resolve the `start` promise + migrationTaskRunner + .run(invocationConfig) + .then(() => { + // The task runner has finished normally. Abort errors are also handled here, it's an expected finish scenario, nothing special should be done. + migrationLogger.debug('Migration execution task finished'); + this.data.migrations.saveAsFinished({ id: migrationId }).catch((error) => { + migrationLogger.error(`Error saving migration as finished: ${error}`); + }); + }) + .catch((error) => { + // Unexpected errors, no use in throwing them since the `start` promise is long gone. Just log and store the error message + migrationLogger.error(`Error executing migration task: ${error}`); + this.data.migrations + .saveAsFailed({ id: migrationId, error: error.message }) + .catch((saveError) => { + migrationLogger.error(`Error saving migration as failed: ${saveError}`); + }); + }) + .finally(() => { + this.migrationsRunning.delete(migrationId); + }); + + return { exists: true, started: true }; + } + + /** Updates all the rules in a migration to be re-executed */ + public async updateToRetry( + migrationId: string, + filter: RuleMigrationFilters + ): Promise<{ updated: boolean }> { + if (this.migrationsRunning.has(migrationId)) { + // not update migrations that are currently running + return { updated: false }; + } + filter.installed = false; // only retry rules that are not installed + await this.data.items.updateStatus(migrationId, filter, SiemMigrationStatus.PENDING, { + refresh: true, + }); + return { updated: true }; + } + + /** Returns the stats of a migration */ + public async getStats(migrationId: string): Promise { + const migration = await this.data.migrations.get(migrationId); + if (!migration) { + throw new Error(`Migration with ID ${migrationId} not found`); + } + const dataStats = await this.data.items.getStats(migrationId); + const taskStats = this.getTaskStats(migration, dataStats.items); + return { ...taskStats, ...dataStats, name: migration.name }; + } + + /** Returns the stats of all migrations */ + async getAllStats(): Promise { + const allDataStats = await this.data.items.getAllStats(); + const allMigrations = await this.data.migrations.getAll(); + const allMigrationsMap = new Map( + allMigrations.map((migration) => [migration.id, migration]) + ); + + const allStats: SiemMigrationTaskStats[] = []; + + for (const dataStats of allDataStats) { + const migration = allMigrationsMap.get(dataStats.id); + if (migration) { + const tasksStats = this.getTaskStats(migration, dataStats.items); + allStats.push({ name: migration.name, ...tasksStats, ...dataStats }); + } + } + return allStats; + } + + private getTaskStats( + migration: StoredSiemMigration, + dataStats: MigrationTaskItemsStats + ): Pick { + return { + status: this.getTaskStatus(migration, dataStats), + last_execution: migration.last_execution, + }; + } + + private getTaskStatus( + migration: StoredSiemMigration, + dataStats: MigrationTaskItemsStats + ): SiemMigrationTaskStatus { + const { id: migrationId, last_execution: lastExecution } = migration; + if (this.migrationsRunning.has(migrationId)) { + return SiemMigrationTaskStatus.RUNNING; + } + if (dataStats.completed + dataStats.failed === dataStats.total) { + return SiemMigrationTaskStatus.FINISHED; + } + if (lastExecution?.is_stopped) { + return SiemMigrationTaskStatus.STOPPED; + } + if (dataStats.pending === dataStats.total) { + return SiemMigrationTaskStatus.READY; + } + return SiemMigrationTaskStatus.INTERRUPTED; + } + + // Overridable method to get the last execution config + protected getLastExecutionConfig(_invocationConfig: RunnableConfig): Record { + return {}; + } + + /** Stops one running migration */ + async stop(migrationId: string): Promise { + try { + const migrationRunning = this.migrationsRunning.get(migrationId); + if (migrationRunning) { + migrationRunning.abortController.abort('Stopped by user'); + await this.data.migrations.setIsStopped({ id: migrationId }); + return { exists: true, stopped: true }; + } + + const { items } = await this.data.items.getStats(migrationId); + if (items.total > 0) { + return { exists: true, stopped: true }; + } + return { exists: false, stopped: true }; + } catch (err) { + this.logger.error(`Error stopping migration ID:${migrationId}`, err); + return { exists: true, stopped: false }; + } + } + + /** Creates a new evaluator for the rule migration task */ + async evaluate(params: SiemMigrationTaskEvaluateParams): Promise { + if (!this.EvaluatorClass) { + throw new Error('Evaluator class needs to be defined to use evaluate method'); + } + + const { evaluationId, langsmithOptions, connectorId, invocationConfig, abortController } = + params; + const migrationLogger = this.logger.get('evaluate'); + const migrationTaskEvaluator = new this.EvaluatorClass( + evaluationId, + this.currentUser, + abortController, + this.data, + migrationLogger, + this.dependencies + ); + await migrationTaskEvaluator.evaluate({ + connectorId, + langsmithOptions, + invocationConfig, + }); + } + + /** Returns if a migration is running or not */ + isMigrationRunning(migrationId: string): boolean { + return this.migrationsRunning.has(migrationId); + } +} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/siem_migrations_task_evaluator.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/siem_migrations_task_evaluator.test.ts new file mode 100644 index 0000000000000..29449c7dc39a3 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/siem_migrations_task_evaluator.test.ts @@ -0,0 +1,135 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { CustomEvaluator } from './siem_migrations_task_evaluator'; +import { SiemMigrationTaskEvaluable } from './siem_migrations_task_evaluator'; +import type { Run, Example } from 'langsmith/schemas'; +import { createSiemMigrationsDataClientMock } from '../data/__mocks__/mocks'; +import { loggerMock } from '@kbn/logging-mocks'; +import type { AuthenticatedUser } from '@kbn/core/server'; +import type { SiemMigrationsClientDependencies } from '../types'; +import { SiemMigrationTaskRunner } from './siem_migrations_task_runner'; + +// Mock dependencies +jest.mock('langsmith/evaluation', () => ({ + evaluate: jest.fn(() => Promise.resolve()), +})); + +jest.mock('@kbn/langchain/server/tracers/langsmith', () => ({ + isLangSmithEnabled: jest.fn(() => true), +})); + +jest.mock('langsmith', () => ({ + Client: jest.fn().mockImplementation(() => ({ + listExamples: jest.fn(() => [{ id: 'example-1' }, { id: 'example-2' }]), + })), +})); + +// Create generic task evaluator class using the generic task runner +class SiemMigrationTaskEvaluator extends SiemMigrationTaskEvaluable(SiemMigrationTaskRunner) { + public async setup(_: string) {} + protected evaluators: Record = {}; +} + +describe('SiemMigrationTaskEvaluator', () => { + let taskEvaluator: SiemMigrationTaskEvaluator; + let mockRuleMigrationsDataClient: ReturnType; + let abortController: AbortController; + + const mockLogger = loggerMock.create(); + const mockDependencies: jest.Mocked = { + rulesClient: {}, + savedObjectsClient: {}, + inferenceClient: {}, + actionsClient: { + get: jest.fn().mockResolvedValue({ id: 'test-connector-id', name: 'Test Connector' }), + }, + telemetry: {}, + } as unknown as SiemMigrationsClientDependencies; + + const mockUser = {} as unknown as AuthenticatedUser; + + beforeAll(() => { + mockRuleMigrationsDataClient = createSiemMigrationsDataClientMock(); + abortController = new AbortController(); + + taskEvaluator = new SiemMigrationTaskEvaluator( + 'test-migration-id', + mockUser, + abortController, + mockRuleMigrationsDataClient, + mockLogger, + mockDependencies + ); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + describe('evaluators', () => { + let evaluator: CustomEvaluator; + // Helper to access private evaluator methods + const setEvaluator = (name: string) => { + // @ts-expect-error accessing protected property + evaluator = taskEvaluator.genericEvaluators[name]; + }; + + describe('translation_result evaluator', () => { + beforeAll(() => { + setEvaluator('translation_result'); + }); + + it('should return true score when translation results match', () => { + const mockRun = { outputs: { translation_result: 'full' } } as unknown as Run; + const mockExample = { outputs: { translation_result: 'full' } } as unknown as Example; + + const result = evaluator({ run: mockRun, example: mockExample }); + + expect(result).toEqual({ + score: true, + comment: 'Correct', + }); + }); + + it('should return false score when translation results do not match', () => { + const mockRun = { outputs: { translation_result: 'full' } } as unknown as Run; + const mockExample = { outputs: { translation_result: 'partial' } } as unknown as Example; + + const result = evaluator({ run: mockRun, example: mockExample }); + + expect(result).toEqual({ + score: false, + comment: 'Incorrect, expected "partial" but got "full"', + }); + }); + + it('should ignore score when expected result is missing', () => { + const mockRun = { outputs: { translation_result: 'full' } } as unknown as Run; + const mockExample = { outputs: {} } as unknown as Example; + + const result = evaluator({ run: mockRun, example: mockExample }); + + expect(result).toEqual({ + comment: 'No translation result expected', + }); + }); + + it('should return false score when run result is missing', () => { + const mockRun = { outputs: {} } as unknown as Run; + const mockExample = { outputs: { translation_result: 'full' } } as unknown as Example; + + const result = evaluator({ run: mockRun, example: mockExample }); + + expect(result).toEqual({ + score: false, + comment: 'No translation result received', + }); + }); + }); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/siem_migrations_task_evaluator.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/siem_migrations_task_evaluator.ts new file mode 100644 index 0000000000000..c025ef39305a1 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/siem_migrations_task_evaluator.ts @@ -0,0 +1,135 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import assert from 'assert'; +import type { EvaluationResult } from 'langsmith/evaluation'; +import type { RunnableConfig } from '@langchain/core/runnables'; +import type { Run, Example } from 'langsmith/schemas'; +import { evaluate } from 'langsmith/evaluation'; +import { isLangSmithEnabled } from '@kbn/langchain/server/tracers/langsmith'; +import { Client } from 'langsmith'; +import type { LangSmithEvaluationOptions } from '../../../../../common/siem_migrations/model/common.gen'; +import type { SiemMigrationTaskRunner } from './siem_migrations_task_runner'; +import type { MigrationDocument, ItemDocument } from '../types'; + +export interface EvaluateParams { + connectorId: string; + langsmithOptions: LangSmithEvaluationOptions; + invocationConfig?: RunnableConfig; +} + +export type Evaluator = (args: { run: Run; example: Example }) => EvaluationResult; +type CustomEvaluatorResult = Omit; +export type CustomEvaluator = (args: { run: Run; example: Example }) => CustomEvaluatorResult; + +export type SiemMigrationTaskEvaluatorClass< + M extends MigrationDocument = MigrationDocument, + I extends ItemDocument = ItemDocument, + P extends object = {}, + C extends object = {}, + O extends object = {} +> = ReturnType>; + +/** + * Mixin to create a task evaluator based on a concrete implementation of a SiemMigrationTaskRunner. + * @param TaskRunnerConcreteClass: The concrete class that extends SiemMigrationTaskRunner. + * @returns the class that extends the TaskRunnerConcreteClass with evaluation capabilities. + */ +export function SiemMigrationTaskEvaluable< + M extends MigrationDocument = MigrationDocument, + I extends ItemDocument = ItemDocument, + P extends object = {}, // The migration task input parameters schema + C extends object = {}, // The migration task config schema + O extends object = {} // The migration task output schema +>(TaskRunnerConcreteClass: typeof SiemMigrationTaskRunner) { + return class extends TaskRunnerConcreteClass { + protected evaluators!: Record; + + public async evaluate({ connectorId, langsmithOptions, invocationConfig }: EvaluateParams) { + if (!isLangSmithEnabled()) { + throw Error('LangSmith is not enabled'); + } + + const client = new Client({ apiKey: langsmithOptions.api_key }); + + // Make sure the dataset exists + const dataset: Example[] = []; + for await (const example of client.listExamples({ datasetName: langsmithOptions.dataset })) { + dataset.push(example); + } + if (dataset.length === 0) { + throw Error(`LangSmith dataset not found: ${langsmithOptions.dataset}`); + } + + // Initialize the task runner first, this may take some time + await this.initialize(); + + // Check if the connector exists and user has privileges to read it + const connector = await this.dependencies.actionsClient.get({ id: connectorId }); + if (!connector) { + throw Error(`Connector with id ${connectorId} not found`); + } + + // for each connector, setup the evaluator + await this.setup(connectorId); + + // create the migration task after setup + const migrateItemTask = (params: P) => { + assert(this.task, 'Task is not defined'); + return this.task(params, invocationConfig); + }; + const evaluators = this.getEvaluators(); + + evaluate(migrateItemTask, { + data: langsmithOptions.dataset, + experimentPrefix: connector.name, + evaluators, + client, + maxConcurrency: 3, + }) + .then(() => { + this.logger.info('Evaluation finished'); + }) + .catch((err) => { + this.logger.error(`Evaluation error:\n ${JSON.stringify(err, null, 2)}`); + }); + } + + private genericEvaluators: Record = { + translation_result: ({ run, example }) => { + const runResult = (run?.outputs as ItemDocument)?.translation_result; + const expectedResult = (example?.outputs as ItemDocument)?.translation_result; + + if (!expectedResult) { + return { comment: 'No translation result expected' }; + } + if (!runResult) { + return { score: false, comment: 'No translation result received' }; + } + + if (runResult === expectedResult) { + return { score: true, comment: 'Correct' }; + } + + return { + score: false, + comment: `Incorrect, expected "${expectedResult}" but got "${runResult}"`, + }; + }, + }; + + private getEvaluators(): Evaluator[] { + return Object.entries({ ...this.genericEvaluators, ...this.evaluators }).map( + ([key, evaluator]) => { + return (args) => { + const result = evaluator(args); + return { key, ...result }; + }; + } + ); + } + }; +} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/siem_migrations_task_runner.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/siem_migrations_task_runner.test.ts new file mode 100644 index 0000000000000..2bd54f41ea5ac --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/siem_migrations_task_runner.test.ts @@ -0,0 +1,397 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SiemMigrationTaskRunner } from './siem_migrations_task_runner'; +import { SiemMigrationStatus } from '../../../../../common/siem_migrations/constants'; +import type { AuthenticatedUser } from '@kbn/core/server'; +import type { StoredSiemMigrationItem, SiemMigrationsClientDependencies } from '../types'; +import { createSiemMigrationsDataClientMock } from '../data/__mocks__/mocks'; +import { loggerMock } from '@kbn/logging-mocks'; +import { SiemMigrationTelemetryClient } from './__mocks__/siem_migrations_telemetry_client'; +import { TELEMETRY_SIEM_MIGRATION_ID } from './util/constants'; + +jest.mock('./siem_migrations_telemetry_client'); + +// Mock dependencies +const mockLogger = loggerMock.create(); + +const mockDependencies: jest.Mocked = { + itemsClient: {}, + savedObjectsClient: {}, + inferenceClient: {}, + actionsClient: {}, + telemetry: {}, +} as unknown as SiemMigrationsClientDependencies; + +const mockUser = {} as unknown as AuthenticatedUser; +const itemId = 'test-item-id'; + +jest.useFakeTimers(); +jest.spyOn(global, 'setTimeout'); +const mockTimeout = setTimeout as unknown as jest.Mock; +mockTimeout.mockImplementation((cb) => { + // never actually wait, we'll check the calls manually + cb(); +}); + +const mockSetup = jest.fn().mockResolvedValue(undefined); +const mockInvoke = jest.fn().mockResolvedValue(undefined); +const mockPrepareTaskInput = jest.fn().mockResolvedValue({}); +const mockProcessTaskOutput = jest.fn().mockResolvedValue({}); +const mockInitialize = jest.fn().mockResolvedValue(undefined); + +class TestMigrationTaskRunner extends SiemMigrationTaskRunner { + protected TaskRunnerClass = SiemMigrationTaskRunner; + protected EvaluatorClass = undefined; + + public async setup(connectorId: string): Promise { + await mockSetup(); + this.task = mockInvoke; + this.telemetry = new SiemMigrationTelemetryClient( + this.dependencies.telemetry, + this.logger, + this.migrationId, + TELEMETRY_SIEM_MIGRATION_ID + ); + } + + prepareTaskInput = mockPrepareTaskInput; + processTaskOutput = mockProcessTaskOutput; + + public async initialize(): Promise { + await mockInitialize(); + } +} + +describe('SiemMigrationTaskRunner', () => { + let taskRunner: SiemMigrationTaskRunner; + let abortController: AbortController; + let mockSiemMigrationsDataClient: ReturnType; + + beforeEach(() => { + mockSetup.mockResolvedValue(undefined); // Reset the mock + mockInitialize.mockResolvedValue(undefined); // Reset the mock + mockInvoke.mockResolvedValue({}); // Reset the mock + mockSiemMigrationsDataClient = createSiemMigrationsDataClientMock(); + jest.clearAllMocks(); + + abortController = new AbortController(); + taskRunner = new TestMigrationTaskRunner( + 'test-migration-id', + mockUser, + abortController, + mockSiemMigrationsDataClient, + mockLogger, + mockDependencies + ); + }); + + describe('setup', () => { + it('should create the task and tools', async () => { + await expect(taskRunner.setup('test-connector-id')).resolves.toBeUndefined(); + // @ts-expect-error (checking private properties) + expect(taskRunner.task).toBeDefined(); + // @ts-expect-error (checking private properties) + expect(taskRunner.telemetry).toBeDefined(); + }); + + it('should throw if an error occurs', async () => { + const errorMessage = 'Test error'; + mockSetup.mockImplementationOnce(() => { + throw new Error(errorMessage); + }); + + await expect(taskRunner.setup('test-connector-id')).rejects.toThrowError(errorMessage); + }); + }); + + describe('run', () => { + let runPromise: Promise; + beforeEach(async () => { + await taskRunner.setup('test-connector-id'); + }); + + it('should handle the migration successfully', async () => { + mockSiemMigrationsDataClient.items.get.mockResolvedValue({ total: 0, data: [] }); + mockSiemMigrationsDataClient.items.get.mockResolvedValueOnce({ + total: 1, + data: [{ id: itemId, status: SiemMigrationStatus.PENDING }] as StoredSiemMigrationItem[], + }); + + await taskRunner.setup('test-connector-id'); + await expect(taskRunner.run({})).resolves.toBeUndefined(); + + expect(mockSiemMigrationsDataClient.items.saveProcessing).toHaveBeenCalled(); + expect(mockTimeout).toHaveBeenCalledTimes(1); // random execution sleep + expect(mockTimeout).toHaveBeenNthCalledWith(1, expect.any(Function), expect.any(Number)); + + expect(mockPrepareTaskInput).toHaveBeenCalledTimes(1); + expect(mockInvoke).toHaveBeenCalledTimes(1); + expect(mockProcessTaskOutput).toHaveBeenCalledTimes(1); + expect(mockSiemMigrationsDataClient.items.saveCompleted).toHaveBeenCalled(); + expect(mockSiemMigrationsDataClient.items.get).toHaveBeenCalledTimes(2); // One with data, one without + expect(mockLogger.info).toHaveBeenCalledWith('Migration completed successfully'); + }); + + describe('when error occurs', () => { + const errorMessage = 'Test error message'; + + describe('during initialization', () => { + it('should handle abort error correctly', async () => { + abortController.abort(); + + runPromise = taskRunner.run({}); + await expect(runPromise).resolves.toBeUndefined(); // Ensure the function handles abort gracefully + + expect(mockLogger.info).toHaveBeenCalledWith( + 'Abort signal received, stopping initialization' + ); + }); + + it('should handle other errors correctly', async () => { + mockInitialize.mockRejectedValueOnce(new Error(errorMessage)); + + runPromise = taskRunner.run({}); + await expect(runPromise).rejects.toEqual( + Error('Migration initialization failed. Error: Test error message') + ); + }); + }); + + describe('during migration', () => { + beforeEach(() => { + mockSiemMigrationsDataClient.items.get.mockRestore(); + mockSiemMigrationsDataClient.items.get + .mockResolvedValue({ total: 0, data: [] }) + .mockResolvedValueOnce({ + total: 1, + data: [ + { id: itemId, status: SiemMigrationStatus.PENDING }, + ] as StoredSiemMigrationItem[], + }); + }); + + it('should handle abort error correctly', async () => { + runPromise = taskRunner.run({}); + // Wait for the initialization to complete, needs 2 ticks + await Promise.resolve(); + await Promise.resolve(); + + abortController.abort(); // Trigger the abort signal + + await expect(runPromise).resolves.toBeUndefined(); // Ensure the function handles abort gracefully + expect(mockLogger.info).toHaveBeenCalledWith('Abort signal received, stopping migration'); + expect(mockSiemMigrationsDataClient.items.releaseProcessing).toHaveBeenCalled(); + }); + + it('should handle other errors correctly', async () => { + mockInvoke.mockRejectedValue(new Error(errorMessage)); + + runPromise = taskRunner.run({}); + await expect(runPromise).resolves.toBeUndefined(); + + expect(mockLogger.error).toHaveBeenCalledWith( + `Error translating document \"${itemId}\" with error: ${errorMessage}` + ); + expect(mockSiemMigrationsDataClient.items.saveError).toHaveBeenCalled(); + }); + + describe('during rate limit errors', () => { + const item2Id = 'test-item-id-2'; + const error = new Error('429. You did way too many requests to this random LLM API bud'); + + beforeEach(async () => { + mockSiemMigrationsDataClient.items.get.mockRestore(); + mockSiemMigrationsDataClient.items.get + .mockResolvedValue({ total: 0, data: [] }) + .mockResolvedValueOnce({ + total: 2, + data: [ + { id: itemId, status: SiemMigrationStatus.PENDING }, + { id: item2Id, status: SiemMigrationStatus.PENDING }, + ] as StoredSiemMigrationItem[], + }); + }); + + it('should retry with exponential backoff', async () => { + mockInvoke + .mockResolvedValue({}) // Successful calls from here on + .mockRejectedValueOnce(error) // First failed call for item 1 + .mockRejectedValueOnce(error) // First failed call for item 2 + .mockRejectedValueOnce(error) // Second failed call for item 1 + .mockRejectedValueOnce(error); // Third failed call for item 1 + + await expect(taskRunner.run({})).resolves.toBeUndefined(); // success + + expect(mockPrepareTaskInput).toHaveBeenCalledTimes(2); // 2 items + /** + * Invoke calls: + * item 1 -> failure -> start backoff retries + * item 2 -> failure -> await for item 1 backoff + * then: + * item 1 retry 1 -> failure + * item 1 retry 2 -> failure + * item 1 retry 3 -> success + * then: + * item 2 -> success + */ + expect(mockInvoke).toHaveBeenCalledTimes(6); + expect(mockTimeout).toHaveBeenCalledTimes(6); // 2 execution sleeps + 3 backoff sleeps + 1 execution sleep + expect(mockTimeout).toHaveBeenNthCalledWith( + 1, + expect.any(Function), + expect.any(Number) // exec random sleep + ); + expect(mockTimeout).toHaveBeenNthCalledWith( + 2, + expect.any(Function), + expect.any(Number) // exec random sleep + ); + expect(mockTimeout).toHaveBeenNthCalledWith(3, expect.any(Function), 1000); + expect(mockTimeout).toHaveBeenNthCalledWith(4, expect.any(Function), 2000); + expect(mockTimeout).toHaveBeenNthCalledWith(5, expect.any(Function), 4000); + expect(mockTimeout).toHaveBeenNthCalledWith( + 6, + expect.any(Function), + expect.any(Number) // exec random sleep + ); + + expect(mockLogger.debug).toHaveBeenCalledWith( + `Awaiting backoff task for document "${item2Id}"` + ); + expect(mockPrepareTaskInput).toHaveBeenCalledTimes(2); // 2 items + expect(mockInvoke).toHaveBeenCalledTimes(6); // 3 retries + 3 executions + expect(mockSiemMigrationsDataClient.items.saveCompleted).toHaveBeenCalledTimes(2); // 2 items + }); + + it('should fail when reached maxRetries', async () => { + mockInvoke.mockRejectedValue(error); + + await expect(taskRunner.run({})).resolves.toBeUndefined(); // success + + expect(mockPrepareTaskInput).toHaveBeenCalledTimes(2); // 2 items + // maxRetries = 8 + expect(mockInvoke).toHaveBeenCalledTimes(10); // 8 retries + 2 executions + expect(mockTimeout).toHaveBeenCalledTimes(10); // 2 execution sleeps + 8 backoff sleeps + + expect(mockSiemMigrationsDataClient.items.saveError).toHaveBeenCalledTimes(2); // 2 items + }); + + it('should fail when reached max recovery attempts', async () => { + const item3Id = 'test-item-id-3'; + const item4Id = 'test-item-id-4'; + mockSiemMigrationsDataClient.items.get.mockRestore(); + mockSiemMigrationsDataClient.items.get + .mockResolvedValue({ total: 0, data: [] }) + .mockResolvedValueOnce({ + total: 4, + data: [ + { id: itemId, status: SiemMigrationStatus.PENDING }, + { id: item2Id, status: SiemMigrationStatus.PENDING }, + { id: item3Id, status: SiemMigrationStatus.PENDING }, + { id: item4Id, status: SiemMigrationStatus.PENDING }, + ] as StoredSiemMigrationItem[], + }); + + // max recovery attempts = 3 + mockInvoke + .mockResolvedValue({}) // should never reach this + .mockRejectedValueOnce(error) // 1st failed call for item 1 + .mockRejectedValueOnce(error) // 1st failed call for item 2 + .mockRejectedValueOnce(error) // 1st failed call for item 3 + .mockRejectedValueOnce(error) // 1st failed call for item 4 + .mockResolvedValueOnce({}) // Successful call for the item 1 backoff + .mockRejectedValueOnce(error) // 2nd failed call for the item 2 recover + .mockRejectedValueOnce(error) // 2nd failed call for the item 3 recover + .mockRejectedValueOnce(error) // 2nd failed call for the item 4 recover + .mockResolvedValueOnce({}) // Successful call for the item 2 backoff + .mockRejectedValueOnce(error) // 3rd failed call for the item 3 recover + .mockRejectedValueOnce(error) // 3rd failed call for the item 4 recover + .mockResolvedValueOnce({}) // Successful call for the item 3 backoff + .mockRejectedValueOnce(error); // 4th failed call for the item 4 recover (max attempts failure) + + await expect(taskRunner.run({})).resolves.toBeUndefined(); // success + + expect(mockSiemMigrationsDataClient.items.saveCompleted).toHaveBeenCalledTimes(3); // items 1, 2 and 3 + expect(mockSiemMigrationsDataClient.items.saveError).toHaveBeenCalledTimes(1); // item 4 + }); + + it('should increase the executor sleep time when rate limited', async () => { + const getResponse = { + total: 1, + data: [ + { id: itemId, status: SiemMigrationStatus.PENDING }, + ] as StoredSiemMigrationItem[], + }; + mockSiemMigrationsDataClient.items.get.mockRestore(); + mockSiemMigrationsDataClient.items.get + .mockResolvedValue({ total: 0, data: [] }) + .mockResolvedValueOnce(getResponse) + .mockResolvedValueOnce({ total: 0, data: [] }) + .mockResolvedValueOnce(getResponse) + .mockResolvedValueOnce({ total: 0, data: [] }) + .mockResolvedValueOnce(getResponse) + .mockResolvedValueOnce({ total: 0, data: [] }) + .mockResolvedValueOnce(getResponse) + .mockResolvedValueOnce({ total: 0, data: [] }) + .mockResolvedValueOnce(getResponse) + .mockResolvedValueOnce({ total: 0, data: [] }) + .mockResolvedValueOnce(getResponse) + .mockResolvedValueOnce({ total: 0, data: [] }); + + /** + * Current EXECUTOR_SLEEP settings: + * initialValueSeconds: 3, multiplier: 2, limitSeconds: 96, // 1m36s (5 increases) + */ + + // @ts-expect-error (checking private properties) + expect(taskRunner.executorSleepMultiplier).toBe(3); + + mockInvoke.mockResolvedValue({}).mockRejectedValueOnce(error); // rate limit and recovery + await expect(taskRunner.run({})).resolves.toBeUndefined(); // success + + // @ts-expect-error (checking private properties) + expect(taskRunner.executorSleepMultiplier).toBe(6); + + mockInvoke.mockResolvedValue({}).mockRejectedValueOnce(error); // rate limit and recovery + await expect(taskRunner.run({})).resolves.toBeUndefined(); // success + + // @ts-expect-error (checking private properties) + expect(taskRunner.executorSleepMultiplier).toBe(12); + + mockInvoke.mockResolvedValue({}).mockRejectedValueOnce(error); // rate limit and recovery + await expect(taskRunner.run({})).resolves.toBeUndefined(); // success + + // @ts-expect-error (checking private properties) + expect(taskRunner.executorSleepMultiplier).toBe(24); + + mockInvoke.mockResolvedValue({}).mockRejectedValueOnce(error); // rate limit and recovery + await expect(taskRunner.run({})).resolves.toBeUndefined(); // success + + // @ts-expect-error (checking private properties) + expect(taskRunner.executorSleepMultiplier).toBe(48); + + mockInvoke.mockResolvedValue({}).mockRejectedValueOnce(error); // rate limit and recovery + await expect(taskRunner.run({})).resolves.toBeUndefined(); // success + + // @ts-expect-error (checking private properties) + expect(taskRunner.executorSleepMultiplier).toBe(96); + + mockInvoke.mockResolvedValue({}).mockRejectedValueOnce(error); // rate limit and recovery + await expect(taskRunner.run({})).resolves.toBeUndefined(); // success + + // @ts-expect-error (checking private properties) + expect(taskRunner.executorSleepMultiplier).toBe(96); // limit reached + expect(mockLogger.warn).toHaveBeenCalledWith( + 'Executor sleep reached the maximum value' + ); + }); + }); + }); + }); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/siem_migrations_task_runner.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/siem_migrations_task_runner.ts new file mode 100644 index 0000000000000..69cab860fc54f --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/siem_migrations_task_runner.ts @@ -0,0 +1,323 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import assert from 'assert'; +import type { AuthenticatedUser, Logger } from '@kbn/core/server'; +import { abortSignalToPromise, AbortError } from '@kbn/kibana-utils-plugin/server'; +import type { RunnableConfig } from '@langchain/core/runnables'; +import { SiemMigrationStatus } from '../../../../../common/siem_migrations/constants'; +import { initPromisePool } from '../../../../utils/promise_pool'; +import type { SiemMigrationsDataClient } from '../data/siem_migrations_data_client'; +import type { Invocation, Invoke, MigrationTask } from './types'; +import { generateAssistantComment } from './util/comments'; +import type { + ItemDocument, + MigrationDocument, + SiemMigrationsClientDependencies, + Stored, +} from '../types'; +import { ActionsClientChat } from './util/actions_client_chat'; +import type { SiemMigrationTelemetryClient } from './siem_migrations_telemetry_client'; + +/** Number of concurrent item translations in the pool */ +const TASK_CONCURRENCY = 10 as const; +/** Number of items loaded in memory to be translated in the pool */ +const TASK_BATCH_SIZE = 100 as const; +/** The timeout of each individual agent invocation in minutes */ +const AGENT_INVOKE_TIMEOUT_MIN = 3 as const; + +/** Exponential backoff configuration to handle rate limit errors */ +const RETRY_CONFIG = { + initialRetryDelaySeconds: 1, + backoffMultiplier: 2, + maxRetries: 8, + // max waiting time 4m15s (1*2^8 = 256s) +} as const; + +/** Executor sleep configuration + * A sleep time applied at the beginning of each single item translation in the execution pool, + * The objective of this sleep is to spread the load of concurrent translations, and prevent hitting the rate limit repeatedly. + * The sleep time applied is a random number between [0-value]. Every time we hit rate limit the value is increased by the multiplier, up to the limit. + */ +const EXECUTOR_SLEEP = { + initialValueSeconds: 3, + multiplier: 2, + limitSeconds: 96, // 1m36s (5 increases) +} as const; + +/** This limit should never be reached, it's a safety net to prevent infinite loops. + * It represents the max number of consecutive rate limit recovery & failure attempts. + * This can only happen when the API can not process TASK_CONCURRENCY translations at a time, + * even after the executor sleep is increased on every attempt. + **/ +const EXECUTOR_RECOVER_MAX_ATTEMPTS = 3 as const; + +export class SiemMigrationTaskRunner< + M extends MigrationDocument = MigrationDocument, // The migration document type (rule migrations and dashboard migrations very similar but have differences) + I extends ItemDocument = ItemDocument, // The rule or dashboard document type + P extends object = {}, // The migration task input parameters schema + C extends object = {}, // The migration task config schema + O extends object = {} // The migration task output schema +> { + protected telemetry?: SiemMigrationTelemetryClient; + protected task?: MigrationTask; + declare actionsClientChat: ActionsClientChat; + private abort: ReturnType; + private executorSleepMultiplier: number = EXECUTOR_SLEEP.initialValueSeconds; + public isWaiting: boolean = false; + + constructor( + public readonly migrationId: string, + public readonly startedBy: AuthenticatedUser, + public readonly abortController: AbortController, + protected readonly data: SiemMigrationsDataClient, + protected readonly logger: Logger, + protected readonly dependencies: SiemMigrationsClientDependencies + ) { + this.actionsClientChat = new ActionsClientChat(this.dependencies.actionsClient, this.logger); + this.abort = abortSignalToPromise(this.abortController.signal); + } + + /** Receives the connectorId and creates the `this.task` and `this.telemetry` attributes */ + public async setup(connectorId: string): Promise { + throw new Error('setup method must be implemented in the subclass'); + } + + /** Prepares the migration item for the task execution */ + protected async prepareTaskInput(item: Stored): Promise

{ + throw new Error('prepareTaskInput method must be implemented in the subclass'); + } + + /** Processes the output of the migration task and returns the item to save */ + protected processTaskOutput(item: Stored, output: O): Stored { + throw new Error('processTaskOutput method must be implemented in the subclass'); + } + + /** Optional initialization logic */ + protected async initialize() {} + + public async run(invocationConfig: RunnableConfig): Promise { + assert(this.telemetry, 'telemetry is missing please call setup() first'); + const { telemetry, migrationId } = this; + + const migrationTaskTelemetry = telemetry.startSiemMigrationTask(); + + try { + this.logger.debug('Initializing migration'); + await this.withAbort(this.initialize()); + } catch (error) { + migrationTaskTelemetry.failure(error); + if (error instanceof AbortError) { + this.logger.info('Abort signal received, stopping initialization'); + return; + } else { + throw new Error(`Migration initialization failed. ${error}`); + } + } + + const migrateItemTask = this.createMigrateItemTask(invocationConfig); + this.logger.debug(`Started translations. Concurrency is: ${TASK_CONCURRENCY}`); + + try { + do { + const { data: migrationItems } = await this.data.items.get(migrationId, { + filters: { status: SiemMigrationStatus.PENDING }, + size: TASK_BATCH_SIZE, // keep these items in memory and process them in the promise pool with concurrency limit + }); + if (migrationItems.length === 0) { + break; + } + + this.logger.debug(`Start processing batch of ${migrationItems.length} items`); + + const { errors } = await initPromisePool, void, Error>({ + concurrency: TASK_CONCURRENCY, + abortSignal: this.abortController.signal, + items: migrationItems, + executor: async (migrationItem) => { + const itemTranslationTelemetry = migrationTaskTelemetry.startItemTranslation(); + try { + await this.saveItemProcessing(migrationItem); + + const migratedItem = await migrateItemTask(migrationItem); + + await this.saveItemCompleted(migratedItem); + itemTranslationTelemetry.success(migratedItem); + } catch (error) { + if (this.abortController.signal.aborted) { + throw new AbortError(); + } + itemTranslationTelemetry.failure(error); + await this.saveItemFailed(migrationItem, error); + } + }, + }); + + if (errors.length > 0) { + throw errors[0].error; // Only AbortError is thrown from the pool. The task was aborted + } + + this.logger.debug('Batch processed successfully'); + } while (true); + + migrationTaskTelemetry.success(); + this.logger.info('Migration completed successfully'); + } catch (error) { + await this.data.items.releaseProcessing(migrationId); + + if (error instanceof AbortError) { + migrationTaskTelemetry.aborted(error); + this.logger.info('Abort signal received, stopping migration'); + } else { + migrationTaskTelemetry.failure(error); + throw new Error(`Error processing migration: ${error}`); + } + } finally { + this.abort.cleanup(); + } + } + + /** Creates the task invoke function, the input is prepared and the output is processed as a migrationItem */ + private createTaskInvoke = async ( + migrationItem: I, + config: RunnableConfig + ): Promise> => { + const input = await this.prepareTaskInput(migrationItem); + return async () => { + assert(this.task, 'Migration task is not defined'); + const output = await this.task(input, config); + return this.processTaskOutput(migrationItem, output); + }; + }; + + protected createMigrateItemTask(invocationConfig?: RunnableConfig) { + const config: RunnableConfig = { + timeout: AGENT_INVOKE_TIMEOUT_MIN * 60 * 1000, // milliseconds timeout + ...invocationConfig, + signal: this.abortController.signal, + }; + + // Invokes the item translation with exponential backoff, should be called only when the rate limit has been hit + const invokeWithBackoff = async (invoke: Invoke): Invocation => { + this.logger.debug('Rate limit backoff started'); + let retriesLeft: number = RETRY_CONFIG.maxRetries; + while (true) { + try { + await this.sleepRetry(retriesLeft); + retriesLeft--; + const result = await invoke(); + this.logger.info( + `Rate limit backoff completed successfully after ${ + RETRY_CONFIG.maxRetries - retriesLeft + } retries` + ); + return result; + } catch (error) { + if (!this.isRateLimitError(error) || retriesLeft === 0) { + const logMessage = + retriesLeft === 0 + ? `Rate limit backoff completed unsuccessfully` + : `Rate limit backoff interrupted. ${error} `; + this.logger.debug(logMessage); + throw error; + } + this.logger.debug(`Rate limit backoff not completed, retries left: ${retriesLeft}`); + } + } + }; + + let backoffPromise: Invocation | undefined; + // Migrates one item, this function will be called concurrently by the promise pool. + // Handles rate limit errors and ensures only one task is executing the backoff retries at a time, the rest of translation will await. + const migrateItem = async (migrationItem: I): Invocation => { + const invoke = await this.createTaskInvoke(migrationItem, config); + + let recoverAttemptsLeft: number = EXECUTOR_RECOVER_MAX_ATTEMPTS; + while (true) { + try { + await this.executorSleep(); // Random sleep, increased every time we hit the rate limit. + return await invoke(); + } catch (error) { + if (!this.isRateLimitError(error) || recoverAttemptsLeft === 0) { + throw error; + } + if (!backoffPromise) { + // only one translation handles the rate limit backoff retries, the rest will await it and try again when it's resolved + backoffPromise = invokeWithBackoff(invoke); + this.isWaiting = true; + return backoffPromise.finally(() => { + backoffPromise = undefined; + this.increaseExecutorSleep(); + this.isWaiting = false; + }); + } + this.logger.debug(`Awaiting backoff task for document "${migrationItem.id}"`); + await backoffPromise.catch(() => { + throw error; // throw the original error + }); + recoverAttemptsLeft--; + } + } + }; + + return migrateItem; + } + + private isRateLimitError(error: Error) { + return error.message.match(/\b429\b/); // "429" (whole word in the error message): Too Many Requests. + } + + private async withAbort(promise: Promise): Promise { + return Promise.race([promise, this.abort.promise]); + } + + private async sleep(seconds: number) { + await this.withAbort(new Promise((resolve) => setTimeout(resolve, seconds * 1000))); + } + + // Exponential backoff implementation + private async sleepRetry(retriesLeft: number) { + const seconds = + RETRY_CONFIG.initialRetryDelaySeconds * + Math.pow(RETRY_CONFIG.backoffMultiplier, RETRY_CONFIG.maxRetries - retriesLeft); + this.logger.debug(`Retry sleep: ${seconds}s`); + await this.sleep(seconds); + } + + private executorSleep = async () => { + const seconds = Math.random() * this.executorSleepMultiplier; + this.logger.debug(`Executor sleep: ${seconds.toFixed(3)}s`); + await this.sleep(seconds); + }; + + private increaseExecutorSleep = () => { + const increasedMultiplier = this.executorSleepMultiplier * EXECUTOR_SLEEP.multiplier; + if (increasedMultiplier > EXECUTOR_SLEEP.limitSeconds) { + this.logger.warn('Executor sleep reached the maximum value'); + return; + } + this.executorSleepMultiplier = increasedMultiplier; + }; + + protected async saveItemProcessing(migrationItem: Stored) { + this.logger.debug(`Starting translation of document "${migrationItem.id}"`); + return this.data.items.saveProcessing(migrationItem.id); + } + + protected async saveItemCompleted(migrationItem: Stored) { + this.logger.debug(`Translation of document "${migrationItem.id}" succeeded`); + return this.data.items.saveCompleted(migrationItem); + } + + protected async saveItemFailed(migrationItem: Stored, error: Error) { + this.logger.error( + `Error translating document "${migrationItem.id}" with error: ${error.message}` + ); + const comments = [generateAssistantComment(`Error migrating document: ${error.message}`)]; + return this.data.items.saveError({ ...migrationItem, comments }); + } +} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/siem_migrations_telemetry_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/siem_migrations_telemetry_client.ts new file mode 100644 index 0000000000000..a7a20ba1d88e3 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/siem_migrations_telemetry_client.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ItemDocument } from '../types'; +import type { MigrationState } from './types'; + +interface StartMigrationTaskTelemetry { + startItemTranslation: () => { + success: (migrationResult: MigrationState) => void; + failure: (error: Error) => void; + }; + success: () => void; + failure: (error: Error) => void; + aborted: (error: Error) => void; +} + +export abstract class SiemMigrationTelemetryClient { + public abstract startSiemMigrationTask(): StartMigrationTaskTelemetry; +} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/types.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/types.ts new file mode 100644 index 0000000000000..440999c9e994d --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/types.ts @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { AuthenticatedUser } from '@kbn/core/server'; +import type { RunnableConfig } from '@langchain/core/runnables'; +import type { LangSmithEvaluationOptions } from '../../../../../common/siem_migrations/model/common.gen'; +import type { SiemMigrationsDataClient } from '../data/siem_migrations_data_client'; +import type { ItemDocument, SiemMigrationsClientDependencies } from '../types'; +import type { Stored } from '../../types'; + +export interface SiemMigrationTaskCreateClientParams { + currentUser: AuthenticatedUser; + dataClient: SiemMigrationsDataClient; + dependencies: SiemMigrationsClientDependencies; +} + +export interface SiemMigrationTaskStartParams { + migrationId: string; + connectorId: string; + invocationConfig: RunnableConfig; +} + +export interface SiemMigrationTaskStartResult { + started: boolean; + exists: boolean; +} + +export interface SiemMigrationTaskStopResult { + stopped: boolean; + exists: boolean; +} + +export interface SiemMigrationTaskEvaluateParams { + evaluationId: string; + connectorId: string; + langsmithOptions: LangSmithEvaluationOptions; + invocationConfig: RunnableConfig; + abortController: AbortController; +} + +export type Invocation = Promise>; +export type Invoke = () => Invocation; + +export type MigrationTask

= ( + params: P, + config?: RunnableConfig +) => Promise; + +export interface RuleMigrationAgentRunOptions { + skipPrebuiltRulesMatching: boolean; +} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/util/__mocks__/esql_knowledge_base.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/util/__mocks__/esql_knowledge_base.ts similarity index 100% rename from x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/util/__mocks__/esql_knowledge_base.ts rename to x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/util/__mocks__/esql_knowledge_base.ts diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/util/__mocks__/mocks.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/util/__mocks__/mocks.ts similarity index 100% rename from x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/util/__mocks__/mocks.ts rename to x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/util/__mocks__/mocks.ts diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/util/actions_client_chat.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/util/actions_client_chat.ts similarity index 63% rename from x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/util/actions_client_chat.ts rename to x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/util/actions_client_chat.ts index 4cea0b06655d6..fb21f343484f5 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/util/actions_client_chat.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/util/actions_client_chat.ts @@ -7,28 +7,32 @@ import type { ActionsClient } from '@kbn/actions-plugin/server'; import type { Logger } from '@kbn/core/server'; -import type { ActionsClientSimpleChatModel } from '@kbn/langchain/server'; import { - ActionsClientBedrockChatModel, + ActionsClientChatBedrockConverse, ActionsClientChatOpenAI, ActionsClientChatVertexAI, } from '@kbn/langchain/server'; import type { CustomChatModelInput as ActionsClientBedrockChatModelParams } from '@kbn/langchain/server/language_models/bedrock_chat'; import type { ActionsClientChatOpenAIParams } from '@kbn/langchain/server/language_models/chat_openai'; -import type { CustomChatModelInput as ActionsClientChatVertexAIParams } from '@kbn/langchain/server/language_models/gemini_chat'; +import type { + CustomChatModelInput as ActionsClientChatVertexAIParams, + ActionsClientGeminiChatModel, +} from '@kbn/langchain/server/language_models/gemini_chat'; import type { CustomChatModelInput as ActionsClientSimpleChatModelParams } from '@kbn/langchain/server/language_models/simple_chat_model'; +import { InferenceChatModel } from '@kbn/inference-langchain'; import { TELEMETRY_SIEM_MIGRATION_ID } from './constants'; export type ChatModel = - | ActionsClientSimpleChatModel + | ActionsClientChatBedrockConverse | ActionsClientChatOpenAI - | ActionsClientBedrockChatModel - | ActionsClientChatVertexAI; + | ActionsClientGeminiChatModel + | ActionsClientChatVertexAI + | InferenceChatModel; export type ActionsClientChatModelClass = - | typeof ActionsClientSimpleChatModel + | typeof ActionsClientChatBedrockConverse | typeof ActionsClientChatOpenAI - | typeof ActionsClientBedrockChatModel + | typeof ActionsClientGeminiChatModel | typeof ActionsClientChatVertexAI; export type ChatModelParams = Partial & @@ -36,11 +40,17 @@ export type ChatModelParams = Partial & Partial & Partial; -const llmTypeDictionary: Record = { - [`.gen-ai`]: `openai`, - [`.bedrock`]: `bedrock`, - [`.gemini`]: `gemini`, - [`.inference`]: `inference`, +const llmTypeDictionary = { + '.gen-ai': 'openai', + '.bedrock': 'bedrock', + '.gemini': 'gemini', + '.inference': 'inference', +} as const; +type SupportedActionTypeId = keyof typeof llmTypeDictionary; +type LlmType = (typeof llmTypeDictionary)[SupportedActionTypeId]; + +const isSupportedActionTypeId = (actionTypeId: string): actionTypeId is SupportedActionTypeId => { + return actionTypeId in llmTypeDictionary; }; interface CreateModelParams { @@ -61,8 +71,16 @@ export class ActionsClientChat { if (!connector) { throw new Error(`Connector not found: ${connectorId}`); } + if (!isSupportedActionTypeId(connector.actionTypeId)) { + throw new Error(`Connector type not supported: ${connector.actionTypeId}`); + } const llmType = this.getLLMType(connector.actionTypeId); + if (llmType === 'inference') { + // TODO: instantiate from inferenceService + throw new Error('Inference model creation not implemented yet'); + } + const ChatModelClass = this.getLLMClass(llmType); const model = new ChatModelClass({ @@ -81,21 +99,28 @@ export class ActionsClientChat { return model; } - private getLLMType(actionTypeId: string): string | undefined { + public getModelName(model: ChatModel): string { + if (model instanceof InferenceChatModel) { + const modelName = model.identifyingParams().model_name; + return `inference${modelName ? ` (${modelName})` : ''}`; + } + return model.model; + } + + private getLLMType(actionTypeId: SupportedActionTypeId): LlmType { if (llmTypeDictionary[actionTypeId]) { return llmTypeDictionary[actionTypeId]; } throw new Error(`Unknown LLM type for action type ID: ${actionTypeId}`); } - private getLLMClass(llmType?: string): ActionsClientChatModelClass { + private getLLMClass(llmType: Omit): ActionsClientChatModelClass { switch (llmType) { case 'bedrock': - return ActionsClientBedrockChatModel; + return ActionsClientChatBedrockConverse; case 'gemini': return ActionsClientChatVertexAI; case 'openai': - case 'inference': default: return ActionsClientChatOpenAI; } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/util/comments.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/util/comments.ts similarity index 77% rename from x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/util/comments.ts rename to x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/util/comments.ts index 291e8c9bcf094..65fe046d48c03 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/util/comments.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/util/comments.ts @@ -6,14 +6,14 @@ */ import { SIEM_MIGRATIONS_ASSISTANT_USER } from '../../../../../../common/siem_migrations/constants'; -import type { RuleMigrationComment } from '../../../../../../common/siem_migrations/model/rule_migration.gen'; +import type { MigrationComment } from '../../../../../../common/siem_migrations/model/migration.gen'; export const cleanMarkdown = (markdown: string): string => { // Use languages known by the code block plugin return markdown.replaceAll('```esql', '```sql').replaceAll('```spl', '```splunk-spl'); }; -export const generateAssistantComment = (message: string): RuleMigrationComment => { +export const generateAssistantComment = (message: string): MigrationComment => { return { message, created_at: new Date().toISOString(), diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/util/constants.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/util/constants.ts similarity index 100% rename from x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/util/constants.ts rename to x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/util/constants.ts diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/util/esql_knowledge_base.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/util/esql_knowledge_base.ts similarity index 100% rename from x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/util/esql_knowledge_base.ts rename to x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/util/esql_knowledge_base.ts diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/util/nullify_missing_properties.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/util/nullify_missing_properties.test.ts new file mode 100644 index 0000000000000..8e285db2fa440 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/util/nullify_missing_properties.test.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { nullifyMissingProperties } from './nullify_missing_properties'; + +interface TestObject { + title?: string; + name?: string; + boolean?: boolean; + missing?: string | undefined; + emptyText?: string; + counter?: number; +} + +describe('nullifyMissingProperties', () => { + it('should return an object with nullified empty values', () => { + const source: TestObject = { + title: 'Some Title', + boolean: true, + missing: 'defined', + emptyText: 'defined', + counter: 1, + }; + const target: TestObject = { + name: 'Some Name', + boolean: false, + emptyText: '', + counter: 0, + missing: undefined, + }; + + const result = nullifyMissingProperties({ + source, + target, + }); + + expect(result).toMatchObject({ + title: null, + name: 'Some Name', + boolean: false, + emptyText: '', + counter: 0, + missing: null, + }); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/util/nullify_missing_properties.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/util/nullify_missing_properties.ts new file mode 100644 index 0000000000000..926338fce50bf --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/util/nullify_missing_properties.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +interface NullifyMissingProperties { + source?: T; + target: T; +} +export const nullifyMissingProperties = ( + params: NullifyMissingProperties +): T => { + const { source: stored, target: output } = params; + if (!stored) { + return output; + } + const result: T = { ...stored, ...output }; + (Object.keys(stored) as Array).forEach((key) => { + if (output[key] == null) { + result[key] = null as T[keyof T]; + } + }); + return result; +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/types.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/types.ts index 3482e8ff156e6..7b0d74680f1bd 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/types.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/types.ts @@ -14,6 +14,14 @@ import type { } from '@kbn/core/server'; import type { PackageService } from '@kbn/fleet-plugin/server'; import type { InferenceClient } from '@kbn/inference-common'; +import type { + DashboardMigration, + DashboardMigrationDashboard, +} from '../../../../common/siem_migrations/model/dashboard_migration.gen'; +import type { + RuleMigration, + RuleMigrationRule, +} from '../../../../common/siem_migrations/model/rule_migration.gen'; export interface SiemMigrationsClientDependencies { inferenceClient: InferenceClient; @@ -32,3 +40,13 @@ export interface SiemMigrationsCreateClientParams { } export type SiemMigrationsIndexNameProvider = () => Promise; + +export type Stored = T & { id: string }; + +// TODO: replace these with the schemas for the common properties of the migrations and items +// TODO: move these to the security_solution/common/siem_migrations/types.ts +export type MigrationDocument = RuleMigration | DashboardMigration; +export type ItemDocument = RuleMigrationRule | DashboardMigrationDashboard; + +export type StoredSiemMigration = Stored; +export type StoredSiemMigrationItem = Stored; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/__mocks__/original_dashboard_example.xml b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/__mocks__/original_dashboard_example.xml new file mode 100644 index 0000000000000..c802f65601318 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/__mocks__/original_dashboard_example.xml @@ -0,0 +1,53 @@ + + + + + Number of dashboards by user | Search app only + + + | rest /servicesNS/-/-/data/ui/views + | search eai:acl.app = "search" ```eai:acl.owner!="nobody"``` + | stats count by eai:acl.owner + -24h@h + now + + + + + + + + Number of custom dashboards in Search app + + + | rest /servicesNS/-/-/data/ui/views + | search eai:acl.app = "search" eai:acl.owner!="nobody" + | stats count + -24h@h + now + + + + + + + + + Number of dashboards by app + + + | rest /servicesNS/-/-/data/ui/views + ```| search eai:acl.app = "search" ``` + | stats count by eai:acl.app | sort - count + -24h@h + now + + + + + + + + + + \ No newline at end of file diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/api/create.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/api/create.ts index c0b99bce900e9..a3bf7bb8a7bbe 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/api/create.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/api/create.ts @@ -10,11 +10,11 @@ import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; import type { IKibanaResponse } from '@kbn/core/server'; import { SIEM_DASHBOARD_MIGRATIONS_PATH } from '../../../../../common/siem_migrations/dashboards/constants'; import type { SecuritySolutionPluginRouter } from '../../../../types'; -import { authz } from '../../common/utils/authz'; +import { authz } from '../../common/api/util/authz'; import type { CreateDashboardMigrationResponse } from '../../../../../common/siem_migrations/model/api/dashboards/dashboard_migration.gen'; import { CreateDashboardMigrationRequestBody } from '../../../../../common/siem_migrations/model/api/dashboards/dashboard_migration.gen'; -import { withLicense } from '../../common/utils/with_license'; -import { SiemMigrationAuditLogger } from '../../common/utils/audit'; +import { withLicense } from '../../common/api/util/with_license'; +import { SiemMigrationAuditLogger } from '../../common/api/util/audit'; export const registerSiemDashboardMigrationsCreateRoute = ( router: SecuritySolutionPluginRouter, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/api/dashboards/create.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/api/dashboards/create.ts index 82216fbefede7..a841bb8dcf0d5 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/api/dashboards/create.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/api/dashboards/create.ts @@ -8,14 +8,18 @@ import type { Logger } from '@kbn/logging'; import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; import type { IKibanaResponse } from '@kbn/core/server'; +import type { DashboardMigrationDashboard } from '../../../../../../common/siem_migrations/model/dashboard_migration.gen'; import { CreateDashboardMigrationDashboardsRequestBody, CreateDashboardMigrationDashboardsRequestParams, } from '../../../../../../common/siem_migrations/model/api/dashboards/dashboard_migration.gen'; import { SIEM_DASHBOARD_MIGRATION_DASHBOARDS_PATH } from '../../../../../../common/siem_migrations/dashboards/constants'; import type { SecuritySolutionPluginRouter } from '../../../../../types'; -import { authz } from '../../../common/utils/authz'; -import { withLicense } from '../../../common/utils/with_license'; +import { authz } from '../../../common/api/util/authz'; +import { withLicense } from '../../../common/api/util/with_license'; +import type { CreateMigrationItemInput } from '../../../common/data/siem_migrations_data_item_client'; + +type CreateMigrationDashboardInput = CreateMigrationItemInput; export const registerSiemDashboardMigrationsCreateDashboardsRoute = ( router: SecuritySolutionPluginRouter, @@ -50,10 +54,30 @@ export const registerSiemDashboardMigrationsCreateDashboardsRoute = ( const ctx = await context.resolve(['securitySolution']); const dashboardMigrationsClient = ctx.securitySolution.siemMigrations.getDashboardsClient(); - await dashboardMigrationsClient.data.dashboards.create( - migrationId, - originalDashboardsExport + + // Convert the original splunk dashboards format to the migration dashboard item document format + const items = originalDashboardsExport.map( + ({ result: { ...originalDashboard } }) => ({ + migration_id: migrationId, + original_dashboard: { + id: originalDashboard.id, + title: originalDashboard.label ?? originalDashboard.title, + description: originalDashboard.description ?? '', + data: originalDashboard['eai:data'], + format: 'xml', + vendor: 'splunk', + last_updated: originalDashboard.updated, + splunk_properties: { + app: originalDashboard['eai:acl.app'], + owner: originalDashboard['eai:acl.owner'], + sharing: originalDashboard['eai:acl.sharing'], + }, + }, + }) ); + + await dashboardMigrationsClient.data.items.create(items); + return res.ok(); } catch (error) { logger.error(`Error creating dashboards for migration ID ${migrationId}: ${error}`); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/api/evaluation/evaluate.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/api/evaluation/evaluate.ts new file mode 100644 index 0000000000000..c9a623bb61609 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/api/evaluation/evaluate.ts @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, Logger } from '@kbn/core/server'; +import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; +import { v4 as uuidV4 } from 'uuid'; +import { z } from '@kbn/zod'; +import { DashboardMigrationTaskExecutionSettings } from '../../../../../../common/siem_migrations/model/dashboard_migration.gen'; +import { LangSmithEvaluationOptions } from '../../../../../../common/siem_migrations/model/common.gen'; +import { SIEM_DASHBOARD_MIGRATION_EVALUATE_PATH } from '../../../../../../common/siem_migrations/dashboards/constants'; +import { createTracersCallbacks } from '../../../common/api/util/tracing'; +import type { SecuritySolutionPluginRouter } from '../../../../../types'; +import { authz } from '../../../common/api/util/authz'; +import { withLicense } from '../../../common/api/util/with_license'; +import type { MigrateDashboardConfig } from '../../task/agent/types'; + +const REQUEST_TIMEOUT = 10 * 60 * 1000; // 10 minutes + +const requestBodyValidation = buildRouteValidationWithZod( + z.object({ + settings: DashboardMigrationTaskExecutionSettings, + langsmith_options: LangSmithEvaluationOptions, + }) +); + +interface EvaluateDashboardMigrationResponse { + evaluationId: string; +} + +export const registerSiemDashboardMigrationsEvaluateRoute = ( + router: SecuritySolutionPluginRouter, + logger: Logger +) => { + router.versioned + .post({ + path: SIEM_DASHBOARD_MIGRATION_EVALUATE_PATH, + access: 'internal', + security: { authz }, + options: { timeout: { idleSocket: REQUEST_TIMEOUT } }, + }) + .addVersion( + { version: '1', validate: { request: { body: requestBodyValidation } } }, + withLicense( + async (context, req, res): Promise> => { + const { + settings: { connector_id: connectorId }, + langsmith_options: langsmithOptions, + } = req.body; + + try { + const evaluationId = uuidV4(); + const abortController = new AbortController(); + req.events.aborted$.subscribe(() => abortController.abort()); + + const securitySolutionContext = await context.securitySolution; + const dashboardMigrationsClient = + securitySolutionContext.siemMigrations.getDashboardsClient(); + + const invocationConfig: MigrateDashboardConfig = { + callbacks: createTracersCallbacks(langsmithOptions, logger), + }; + + await dashboardMigrationsClient.task.evaluate({ + evaluationId, + connectorId, + langsmithOptions, + abortController, + invocationConfig, + }); + + return res.ok({ body: { evaluationId } }); + } catch (err) { + logger.error(err); + return res.customError({ body: err.message, statusCode: err.statusCode ?? 500 }); + } + } + ) + ); +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/api/get.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/api/get.ts index 058231eaec6dc..330bf5f8a320e 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/api/get.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/api/get.ts @@ -11,9 +11,9 @@ import type { GetDashboardMigrationResponse } from '../../../../../common/siem_m import { GetDashboardMigrationRequestParams } from '../../../../../common/siem_migrations/model/api/dashboards/dashboard_migration.gen'; import { SIEM_DASHBOARD_MIGRATION_PATH } from '../../../../../common/siem_migrations/dashboards/constants'; import type { SecuritySolutionPluginRouter } from '../../../../types'; -import { authz } from '../../common/utils/authz'; -import { SiemMigrationAuditLogger } from '../../common/utils/audit'; -import { withLicense } from '../../common/utils/with_license'; +import { authz } from '../../common/api/util/authz'; +import { SiemMigrationAuditLogger } from '../../common/api/util/audit'; +import { withLicense } from '../../common/api/util/with_license'; import { MIGRATION_ID_NOT_FOUND } from '../../common/translations'; export const registerSiemDashboardMigrationsGetRoute = ( diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/api/index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/api/index.ts index 6cd9b01a1918b..afae1edf7de32 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/api/index.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/api/index.ts @@ -5,14 +5,19 @@ * 2.0. */ import type { Logger } from '@kbn/logging'; +import type { ConfigType } from '../../../../config'; import type { SecuritySolutionPluginRouter } from '../../../../types'; import { registerSiemDashboardMigrationsCreateRoute } from './create'; import { registerSiemDashboardMigrationsCreateDashboardsRoute } from './dashboards/create'; import { registerSiemDashboardMigrationsStatsRoute } from './stats'; import { registerSiemDashboardMigrationsGetRoute } from './get'; +import { registerSiemDashboardMigrationsStartRoute } from './start'; +import { registerSiemDashboardMigrationsStopRoute } from './stop'; +import { registerSiemDashboardMigrationsEvaluateRoute } from './evaluation/evaluate'; export const registerSiemDashboardMigrationsRoutes = ( router: SecuritySolutionPluginRouter, + config: ConfigType, logger: Logger ) => { // ===== Dashboard Migrations ====== @@ -22,6 +27,16 @@ export const registerSiemDashboardMigrationsRoutes = ( // ===== Stats ======== registerSiemDashboardMigrationsStatsRoute(router, logger); + // ===== Task ======== + registerSiemDashboardMigrationsStartRoute(router, logger); + registerSiemDashboardMigrationsStopRoute(router, logger); + // ===== Dashboards ====== registerSiemDashboardMigrationsCreateDashboardsRoute(router, logger); + + if (config.experimentalFeatures.assistantModelEvaluation) { + // Use the same experimental feature flag as the assistant model evaluation. + // This route is not intended to be used by the end user, but rather for internal purposes. + registerSiemDashboardMigrationsEvaluateRoute(router, logger); + } }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/api/start.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/api/start.ts new file mode 100644 index 0000000000000..dc710412e62dd --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/api/start.ts @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, Logger } from '@kbn/core/server'; +import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; +import { SIEM_DASHBOARD_MIGRATION_START_PATH } from '../../../../../common/siem_migrations/dashboards/constants'; +import { + StartDashboardsMigrationRequestBody, + StartDashboardsMigrationRequestParams, + type StartDashboardsMigrationResponse, +} from '../../../../../common/siem_migrations/model/api/dashboards/dashboard_migration.gen'; +import type { SecuritySolutionPluginRouter } from '../../../../types'; +import { SiemMigrationAuditLogger } from '../../common/api/util/audit'; +import { authz } from '../../common/api/util/authz'; +import { getRetryFilter } from '../../common/api/util/retry'; +import { withLicense } from '../../common/api/util/with_license'; +import { createTracersCallbacks } from '../../common/api/util/tracing'; +import { withExistingDashboardMigration } from './util/with_existing_dashboard_migration'; + +export const registerSiemDashboardMigrationsStartRoute = ( + router: SecuritySolutionPluginRouter, + logger: Logger +) => { + router.versioned + .post({ + path: SIEM_DASHBOARD_MIGRATION_START_PATH, + access: 'internal', + security: { authz }, + }) + .addVersion( + { + version: '1', + validate: { + request: { + params: buildRouteValidationWithZod(StartDashboardsMigrationRequestParams), + body: buildRouteValidationWithZod(StartDashboardsMigrationRequestBody), + }, + }, + }, + withLicense( + withExistingDashboardMigration( + async (context, req, res): Promise> => { + const migrationId = req.params.migration_id; + const { + langsmith_options: langsmithOptions, + settings: { connector_id: connectorId }, + retry, + } = req.body; + + const siemMigrationAuditLogger = new SiemMigrationAuditLogger(context.securitySolution); + try { + const ctx = await context.resolve(['actions', 'securitySolution']); + + // Check if the connector exists and user has permissions to read it + const connector = await ctx.actions.getActionsClient().get({ id: connectorId }); + if (!connector) { + return res.badRequest({ body: `Connector with id ${connectorId} not found` }); + } + + const dashboardMigrationsClient = + ctx.securitySolution.siemMigrations.getDashboardsClient(); + if (retry) { + const { updated } = await dashboardMigrationsClient.task.updateToRetry( + migrationId, + getRetryFilter(retry) + ); + if (!updated) { + return res.ok({ body: { started: false } }); + } + } + + const callbacks = createTracersCallbacks(langsmithOptions, logger); + + const { exists, started } = await dashboardMigrationsClient.task.start({ + migrationId, + connectorId, + invocationConfig: { callbacks }, + }); + + if (!exists) { + return res.notFound(); + } + + await siemMigrationAuditLogger.logStart({ migrationId }); + + return res.ok({ body: { started } }); + } catch (error) { + logger.error(error); + await siemMigrationAuditLogger.logStart({ migrationId, error }); + return res.customError({ statusCode: 500, body: error.message }); + } + } + ) + ) + ); +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/api/stats.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/api/stats.ts index 3585c44e64f61..e29554d91509d 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/api/stats.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/api/stats.ts @@ -11,10 +11,9 @@ import type { GetDashboardMigrationStatsResponse } from '../../../../../common/s import { GetDashboardMigrationStatsRequestParams } from '../../../../../common/siem_migrations/model/api/dashboards/dashboard_migration.gen'; import { SIEM_DASHBOARD_MIGRATION_STATS_PATH } from '../../../../../common/siem_migrations/dashboards/constants'; import type { SecuritySolutionPluginRouter } from '../../../../types'; -import { withLicense } from '../../common/utils/with_license'; -import { authz } from '../../common/utils/authz'; -import { MIGRATION_ID_NOT_FOUND } from '../../common/translations'; -import { withExistingDashboardMigration } from './utils/use_existing_dashboard_migration'; +import { withLicense } from '../../common/api/util/with_license'; +import { authz } from '../../common/api/util/authz'; +import { withExistingDashboardMigration } from './util/with_existing_dashboard_migration'; export const registerSiemDashboardMigrationsStatsRoute = ( router: SecuritySolutionPluginRouter, @@ -46,26 +45,12 @@ export const registerSiemDashboardMigrationsStatsRoute = ( const dashboardMigrationClient = ctx.securitySolution.siemMigrations.getDashboardsClient(); - const [stats, migration] = await Promise.all([ - dashboardMigrationClient.data.dashboards.getStats(migrationId), - dashboardMigrationClient.data.migrations.get(migrationId), - ]); + const stats = await dashboardMigrationClient.task.getStats(migrationId); - if (!migration) { - return res.notFound({ - body: MIGRATION_ID_NOT_FOUND(migrationId), - }); - } - - if (stats.dashboards?.total === 0) { + if (stats.items.total === 0) { return res.noContent(); } - return res.ok({ - body: { - ...stats, - name: migration.name, - }, - }); + return res.ok({ body: stats }); } catch (err) { logger.error(err); return res.badRequest({ body: err.message }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/api/stop.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/api/stop.ts new file mode 100644 index 0000000000000..8502fbd0ccab0 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/api/stop.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, Logger } from '@kbn/core/server'; +import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; +import { SIEM_DASHBOARD_MIGRATION_STOP_PATH } from '../../../../../common/siem_migrations/dashboards/constants'; +import { + StopDashboardsMigrationRequestParams, + type StopDashboardsMigrationResponse, +} from '../../../../../common/siem_migrations/model/api/dashboards/dashboard_migration.gen'; +import type { SecuritySolutionPluginRouter } from '../../../../types'; +import { SiemMigrationAuditLogger } from '../../common/api/util/audit'; +import { authz } from '../../common/api/util/authz'; +import { withLicense } from '../../common/api/util/with_license'; +import { withExistingDashboardMigration } from './util/with_existing_dashboard_migration'; + +export const registerSiemDashboardMigrationsStopRoute = ( + router: SecuritySolutionPluginRouter, + logger: Logger +) => { + router.versioned + .post({ + path: SIEM_DASHBOARD_MIGRATION_STOP_PATH, + access: 'internal', + security: { authz }, + }) + .addVersion( + { + version: '1', + validate: { + request: { params: buildRouteValidationWithZod(StopDashboardsMigrationRequestParams) }, + }, + }, + withLicense( + withExistingDashboardMigration( + async (context, req, res): Promise> => { + const migrationId = req.params.migration_id; + const siemMigrationAuditLogger = new SiemMigrationAuditLogger(context.securitySolution); + try { + const ctx = await context.resolve(['securitySolution']); + const dashboardMigrationsClient = + ctx.securitySolution.siemMigrations.getDashboardsClient(); + + const { exists, stopped } = await dashboardMigrationsClient.task.stop(migrationId); + + if (!exists) { + return res.notFound(); + } + await siemMigrationAuditLogger.logStop({ migrationId }); + + return res.ok({ body: { stopped } }); + } catch (error) { + logger.error(error); + await siemMigrationAuditLogger.logStop({ migrationId, error }); + return res.badRequest({ body: error.message }); + } + } + ) + ) + ); +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/api/utils/use_existing_dashboard_migration.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/api/util/with_existing_dashboard_migration.ts similarity index 100% rename from x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/api/utils/use_existing_dashboard_migration.ts rename to x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/api/util/with_existing_dashboard_migration.ts diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/data/__mocks__/mocks.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/data/__mocks__/mocks.ts index cc1d94268894e..4b9eb50c85636 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/data/__mocks__/mocks.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/data/__mocks__/mocks.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { DashboardMigrationsDataDashboardsClient } from '../dashboard_migrations_dashboards_client'; +import type { DashboardMigrationsDataDashboardsClient } from '../dashboard_migrations_data_dashboards_client'; import type { DashboardMigrationsDataMigrationClient } from '../dashboard_migrations_migration_client'; export const mockDashboardMigrationDataMigrationClient = { diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/data/dashboard_migrations_dashboards_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/data/dashboard_migrations_dashboards_client.ts deleted file mode 100644 index 143fe211d6cd7..0000000000000 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/data/dashboard_migrations_dashboards_client.ts +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { - AggregationsMaxAggregate, - AggregationsMinAggregate, - AggregationsStringTermsAggregate, - AggregationsStringTermsBucket, - QueryDslQueryContainer, -} from '@elastic/elasticsearch/lib/api/types'; -import { MigrationTaskStatusEnum } from '../../../../../common/siem_migrations/model/common.gen'; -import type { SplunkOriginalDashboardExport } from '../../../../../common/siem_migrations/model/vendor/dashboards/splunk.gen'; -import { SiemMigrationStatus } from '../../../../../common/siem_migrations/constants'; -import { SiemMigrationsDataBaseClient } from '../../common/data/siem_migrations_data_base_client'; -import { - type DashboardMigrationDashboard, - type DashboardMigrationTaskStats, -} from '../../../../../common/siem_migrations/model/dashboard_migration.gen'; - -/* BULK_MAX_SIZE defines the number to break down the bulk operations by. - * The 500 number was chosen as a reasonable number to avoid large payloads. It can be adjusted if needed. */ -const BULK_MAX_SIZE = 500 as const; - -export class DashboardMigrationsDataDashboardsClient extends SiemMigrationsDataBaseClient { - /** Indexes an array of dashboards to be processed as a part of single migration */ - async create( - migrationId: string, - originalDashboards: SplunkOriginalDashboardExport[] - ): Promise { - const index = await this.getIndexName(); - const profileId = await this.getProfileUid(); - - let originalDashboardsMaxBatch: SplunkOriginalDashboardExport[]; - const createdAt = new Date().toISOString(); - while ((originalDashboardsMaxBatch = originalDashboards.splice(0, BULK_MAX_SIZE)).length) { - await this.esClient - .bulk({ - refresh: 'wait_for', - operations: originalDashboardsMaxBatch.flatMap(({ result: { ...originalDashboard } }) => [ - { create: { _index: index } }, - { - migration_id: migrationId, - '@timestamp': createdAt, - status: SiemMigrationStatus.PENDING, - created_by: profileId, - updated_by: profileId, - updated_at: createdAt, - original_dashboard: { - id: originalDashboard.id, - title: originalDashboard.label ?? originalDashboard.title, - description: originalDashboard.description ?? '', - data: originalDashboard?.['eai:data'], - format: 'xml', - vendor: 'splunk', - last_updated: originalDashboard.updated, - splunk_properties: { - app: originalDashboard['eai:acl.app'], - owner: originalDashboard['eai:acl.owner'], - sharing: originalDashboard['eai:acl.sharing'], - }, - }, - }, - ]), - }) - .catch((error) => { - this.logger.error( - `Error adding dashboards to migration (id:${migrationId}) : ${error.message}` - ); - throw error; - }); - } - } - - async getStats(migrationId: string): Promise> { - const index = await this.getIndexName(); - - const migrationIdFilter: QueryDslQueryContainer = { term: { migration_id: migrationId } }; - const query = { - bool: { - filter: migrationIdFilter, - }, - }; - const aggregations = { - status: { terms: { field: 'status' } }, - createdAt: { min: { field: '@timestamp' } }, - lastUpdatedAt: { max: { field: 'updated_at' } }, - }; - const result = await this.esClient - .search({ index, query, aggregations, _source: false }) - .catch((error) => { - this.logger.error(`Error getting dashboard migration stats: ${error.message}`); - throw error; - }); - - const aggs = result.aggregations ?? {}; - - return { - id: migrationId, - dashboards: { - total: this.getTotalHits(result), - ...this.statusAggCounts(aggs.status as AggregationsStringTermsAggregate), - }, - created_at: (aggs.createdAt as AggregationsMinAggregate)?.value_as_string ?? '', - last_updated_at: (aggs.lastUpdatedAt as AggregationsMaxAggregate)?.value_as_string ?? '', - status: MigrationTaskStatusEnum.ready, - }; - } - - private statusAggCounts( - statusAgg: AggregationsStringTermsAggregate - ): Record { - const buckets = statusAgg.buckets as AggregationsStringTermsBucket[]; - return { - [SiemMigrationStatus.PENDING]: - buckets.find(({ key }) => key === SiemMigrationStatus.PENDING)?.doc_count ?? 0, - [SiemMigrationStatus.PROCESSING]: - buckets.find(({ key }) => key === SiemMigrationStatus.PROCESSING)?.doc_count ?? 0, - [SiemMigrationStatus.COMPLETED]: - buckets.find(({ key }) => key === SiemMigrationStatus.COMPLETED)?.doc_count ?? 0, - [SiemMigrationStatus.FAILED]: - buckets.find(({ key }) => key === SiemMigrationStatus.FAILED)?.doc_count ?? 0, - }; - } -} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/data/dashboard_migrations_data_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/data/dashboard_migrations_data_client.ts index 3fec05f0e8f99..6c69f3aaa9dcb 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/data/dashboard_migrations_data_client.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/data/dashboard_migrations_data_client.ts @@ -7,14 +7,18 @@ import type { Logger } from '@kbn/logging'; import type { AuthenticatedUser, IScopedClusterClient } from '@kbn/core/server'; -import { DashboardMigrationsDataDashboardsClient } from './dashboard_migrations_dashboards_client'; -import { DashboardMigrationsDataMigrationClient } from './dashboard_migrations_migration_client'; +import type { DashboardMigration } from '../../../../../common/siem_migrations/model/dashboard_migration.gen'; +import { SiemMigrationsDataMigrationClient } from '../../common/data/siem_migrations_data_migration_client'; +import { DashboardMigrationsDataDashboardsClient } from './dashboard_migrations_data_dashboards_client'; import type { DashboardMigrationIndexNameProviders } from '../types'; import type { SiemMigrationsClientDependencies } from '../../common/types'; +import { SiemMigrationsDataClient } from '../../common/data/siem_migrations_data_client'; +import { SiemMigrationsDataResourcesClient } from '../../common/data/siem_migrations_data_resources_client'; -export class DashboardMigrationsDataClient { - public readonly migrations: DashboardMigrationsDataMigrationClient; - public readonly dashboards: DashboardMigrationsDataDashboardsClient; +export class DashboardMigrationsDataClient extends SiemMigrationsDataClient { + public readonly migrations: SiemMigrationsDataMigrationClient; + public readonly items: DashboardMigrationsDataDashboardsClient; + public readonly resources: SiemMigrationsDataResourcesClient; constructor( indexNameProviders: DashboardMigrationIndexNameProviders, @@ -24,19 +28,28 @@ export class DashboardMigrationsDataClient { spaceId: string, dependencies: SiemMigrationsClientDependencies ) { - this.migrations = new DashboardMigrationsDataMigrationClient( + super(esScopedClient, logger); + + this.migrations = new SiemMigrationsDataMigrationClient( indexNameProviders.migrations, currentUser, esScopedClient, logger, dependencies ); - this.dashboards = new DashboardMigrationsDataDashboardsClient( + this.items = new DashboardMigrationsDataDashboardsClient( indexNameProviders.dashboards, currentUser, esScopedClient, logger, dependencies ); + this.resources = new SiemMigrationsDataResourcesClient( + indexNameProviders.resources, + currentUser, + esScopedClient, + logger, + dependencies + ); } } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/data/dashboard_migrations_dashboards_client.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/data/dashboard_migrations_data_dashboards_client.test.ts similarity index 99% rename from x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/data/dashboard_migrations_dashboards_client.test.ts rename to x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/data/dashboard_migrations_data_dashboards_client.test.ts index cfb69a6127779..b0bf66d0c7207 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/data/dashboard_migrations_dashboards_client.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/data/dashboard_migrations_data_dashboards_client.test.ts @@ -7,7 +7,7 @@ import { v4 as uuidv4 } from 'uuid'; import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks'; -import { DashboardMigrationsDataDashboardsClient } from './dashboard_migrations_dashboards_client'; +import { DashboardMigrationsDataDashboardsClient } from './dashboard_migrations_data_dashboards_client'; import type { AuthenticatedUser, IScopedClusterClient } from '@kbn/core/server'; import type { SiemMigrationsClientDependencies } from '../../common/types'; import type { SplunkOriginalDashboardExport } from '../../../../../common/siem_migrations/model/vendor/dashboards/splunk.gen'; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/data/dashboard_migrations_data_dashboards_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/data/dashboard_migrations_data_dashboards_client.ts new file mode 100644 index 0000000000000..5537d1f1e9d7e --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/data/dashboard_migrations_data_dashboards_client.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { estypes } from '@elastic/elasticsearch'; +import type { DashboardMigrationDashboard } from '../../../../../common/siem_migrations/model/dashboard_migration.gen'; +import type { SiemMigrationItemSort } from '../../common/data/siem_migrations_data_item_client'; +import { SiemMigrationsDataItemClient } from '../../common/data/siem_migrations_data_item_client'; + +export class DashboardMigrationsDataDashboardsClient extends SiemMigrationsDataItemClient { + protected type = 'dashboard' as const; + + protected getSortOptions(sort: SiemMigrationItemSort = {}): estypes.Sort { + // TODO: implement sorting logic similar to getSortOptions in the rules client + return []; + } +} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/data/dashboard_migrations_data_service.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/data/dashboard_migrations_data_service.ts index 7cfd5093f7dc3..6cb649e9d4d42 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/data/dashboard_migrations_data_service.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/data/dashboard_migrations_data_service.ts @@ -21,7 +21,7 @@ import { SiemMigrationsBaseDataService } from '../../common/siem_migrations_base import { dashboardMigrationsDashboardsFieldMap, dashboardMigrationsFieldMap } from './field_maps'; import { DashboardMigrationsDataClient } from './dashboard_migrations_data_client'; import type { SiemMigrationsClientDependencies } from '../../common/types'; -export const INDEX_PATTERN = '.kibana-siem-dashboard-migrations'; +import { migrationResourcesFieldMap } from '../../common/data/field_maps'; interface CreateClientParams { spaceId: string; @@ -39,6 +39,8 @@ export interface SetupParams extends Omit { } export class DashboardMigrationsDataService extends SiemMigrationsBaseDataService { + protected readonly baseIndexName = '.kibana-siem-dashboard-migrations'; + private readonly adapters: DashboardMigrationAdapters; constructor(private logger: Logger, protected kibanaVersion: string) { @@ -52,13 +54,13 @@ export class DashboardMigrationsDataService extends SiemMigrationsBaseDataServic adapterId: 'dashboards', fieldMap: dashboardMigrationsDashboardsFieldMap, }), + resources: this.createDashboardIndexPatternAdapter({ + adapterId: 'resources', + fieldMap: migrationResourcesFieldMap, + }), }; } - private getAdapterIndexName(adapterId: DashboardMigrationAdapterId) { - return `${INDEX_PATTERN}-${adapterId}`; - } - private createDashboardIndexPatternAdapter({ adapterId, fieldMap, @@ -71,6 +73,7 @@ export class DashboardMigrationsDataService extends SiemMigrationsBaseDataServic await Promise.all([ this.adapters.dashboards.install({ ...params, logger: this.logger }), this.adapters.migrations.install({ ...params, logger: this.logger }), + this.adapters.resources.install({ ...params, logger: this.logger }), ]); } @@ -82,6 +85,7 @@ export class DashboardMigrationsDataService extends SiemMigrationsBaseDataServic const indexNameProviders: DashboardMigrationIndexNameProviders = { dashboards: this.createIndexNameProvider(this.adapters.dashboards, spaceId), migrations: this.createIndexNameProvider(this.adapters.migrations, spaceId), + resources: this.createIndexNameProvider(this.adapters.resources, spaceId), }; return new DashboardMigrationsDataClient( diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/data/dashboard_migrations_migration_client.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/data/dashboard_migrations_migration_client.test.ts deleted file mode 100644 index 2bda2f54ef672..0000000000000 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/data/dashboard_migrations_migration_client.test.ts +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks'; -import { DashboardMigrationsDataMigrationClient } from './dashboard_migrations_migration_client'; -import type { AuthenticatedUser, IScopedClusterClient } from '@kbn/core/server'; -import type { SiemMigrationsClientDependencies } from '../../common/types'; -import type IndexApi from '@elastic/elasticsearch/lib/api/api'; -import expect from 'expect'; -import type GetApi from '@elastic/elasticsearch/lib/api/api/get'; -import type { GetGetResult } from '@elastic/elasticsearch/lib/api/types'; -import type { DashboardMigration } from '../../../../../common/siem_migrations/model/dashboard_migration.gen'; - -const INDEX_NAME = '.kibana-siem-dashboard-migrations'; - -describe('Dashboard Migrations Client', () => { - let dashboardMigrationsDataMigrationClient: DashboardMigrationsDataMigrationClient; - - const esClientMock = - elasticsearchServiceMock.createCustomClusterClient() as unknown as IScopedClusterClient; - const logger = loggingSystemMock.createLogger(); - const indexNameProvider = jest.fn().mockResolvedValue(INDEX_NAME); - const currentUser = { - userName: 'testUser', - profile_uid: 'testProfileUid', - } as unknown as AuthenticatedUser; - - const dependencies = {} as unknown as SiemMigrationsClientDependencies; - - beforeEach(() => { - dashboardMigrationsDataMigrationClient = new DashboardMigrationsDataMigrationClient( - indexNameProvider, - currentUser, - esClientMock, - logger, - dependencies - ); - }); - - describe('create', () => { - it('should be able to create a migration doc', async () => { - const name = 'test name'; - - const result = await dashboardMigrationsDataMigrationClient.create(name); - - expect(result).not.toBeFalsy(); - expect(esClientMock.asInternalUser.create).toHaveBeenCalledWith({ - refresh: 'wait_for', - id: result, - index: INDEX_NAME, - document: { - created_by: currentUser.profile_uid, - created_at: expect.any(String), - name, - }, - }); - }); - - it('should throw an error if create fails', async () => { - ( - esClientMock.asInternalUser.create as unknown as jest.MockedFn - ).mockRejectedValue(new Error('Create failed')); - - await expect(dashboardMigrationsDataMigrationClient.create('test name')).rejects.toThrow( - 'Create failed' - ); - }); - }); - - describe('get', () => { - it('should be able to get a migration', async () => { - const id = 'test-id'; - const mockResponse = { - _id: id, - _index: INDEX_NAME, - _source: { - created_by: currentUser.profile_uid, - created_at: '2023-10-01T00:00:00Z', - name: 'Test Migration', - }, - } as GetGetResult; - - ( - esClientMock.asInternalUser.get as unknown as jest.MockedFn - ).mockResolvedValue(mockResponse); - - const result = await dashboardMigrationsDataMigrationClient.get(id); - expect(result).toEqual({ - id, - created_by: currentUser.profile_uid, - created_at: '2023-10-01T00:00:00Z', - name: 'Test Migration', - }); - - expect(esClientMock.asInternalUser.get).toHaveBeenCalledWith({ - index: INDEX_NAME, - id, - }); - }); - - it('should return undefined if the migration does not exist', async () => { - const mockResponse = { - _index: INDEX_NAME, - found: false, - }; - - ( - esClientMock.asInternalUser.get as unknown as jest.MockedFn - ).mockRejectedValue({ - message: JSON.stringify(mockResponse), - }); - - const result = await dashboardMigrationsDataMigrationClient.get('non-existent-id'); - expect(result).toBeUndefined(); - }); - - it('should throw an error if get fails', async () => { - ( - esClientMock.asInternalUser.get as unknown as jest.MockedFn - ).mockRejectedValue(new Error('Get failed')); - - await expect(dashboardMigrationsDataMigrationClient.get('test-id')).rejects.toThrow( - 'Get failed' - ); - }); - }); -}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/data/dashboard_migrations_migration_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/data/dashboard_migrations_migration_client.ts deleted file mode 100644 index e2be6f489f4d6..0000000000000 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/data/dashboard_migrations_migration_client.ts +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { v4 as uuidV4 } from 'uuid'; -import type { DashboardMigration } from '../../../../../common/siem_migrations/model/dashboard_migration.gen'; -import { SiemMigrationsDataBaseClient } from '../../common/data/siem_migrations_data_base_client'; -import { isNotFoundError } from '../../common/utils/is_not_found_error'; - -export class DashboardMigrationsDataMigrationClient extends SiemMigrationsDataBaseClient { - async create(name: string): Promise { - const migrationId = uuidV4(); - const index = await this.getIndexName(); - const profileUid = await this.getProfileUid(); - const createdAt = new Date().toISOString(); - - await this.esClient - .create({ - refresh: 'wait_for', - id: migrationId, - index, - document: { - created_by: profileUid, - created_at: createdAt, - name, - }, - }) - .catch((error) => { - this.logger.error(`Error creating migration ${migrationId}: ${error}`); - throw error; - }); - - return migrationId; - } - - /** Gets the migration document by id or returns undefined if it does not exist. */ - async get(id: string): Promise { - const index = await this.getIndexName(); - return this.esClient - .get({ index, id }) - .then(this.processHit) - .catch((error) => { - if (isNotFoundError(error)) { - return undefined; - } - this.logger.error(`Error getting migration ${id}: ${error}`); - throw error; - }); - } -} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/data/field_maps.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/data/field_maps.ts index 34b9f4e82fed7..43d7c88f450bf 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/data/field_maps.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/data/field_maps.ts @@ -26,6 +26,7 @@ export const dashboardMigrationsDashboardsFieldMap: FieldMap< migration_id: { type: 'keyword', required: true }, created_by: { type: 'keyword', required: true }, status: { type: 'keyword', required: true }, + translation_result: { type: 'keyword', required: true }, updated_at: { type: 'date', required: true }, updated_by: { type: 'keyword', required: true }, original_dashboard: { type: 'object', required: true }, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/lib/parsers/splunk/splunk_xml_dashboard_parser.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/lib/parsers/splunk/splunk_xml_dashboard_parser.test.ts new file mode 100644 index 0000000000000..762e9ed957190 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/lib/parsers/splunk/splunk_xml_dashboard_parser.test.ts @@ -0,0 +1,379 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SplunkXmlDashboardParser } from './splunk_xml_dashboard_parser'; +import type { VizType } from '../types'; + +describe('SplunkXmlDashboardParser', () => { + const createBasicXml = (content: string) => ` + + + + ${content} + + + `; + + const createPanelXml = ( + title: string, + query: string, + chartType?: string, + stackMode?: string, + overlayMode?: string, + hasMetric?: boolean + ) => { + const chartElement = chartType ? `` : ''; + const vizElement = chartType ? `` : ''; + const chartOption = chartType ? `` : ''; + const stackOption = stackMode + ? `` + : ''; + const overlayOption = overlayMode + ? `` + : ''; + const metricElement = hasMetric ? 'some metric content' : ''; + + return ` + + ${title} + + ${query} + + ${chartElement} + ${vizElement} + ${chartOption} + ${stackOption} + ${overlayOption} + ${metricElement} + + `; + }; + + describe('extractPanels', () => { + it('should extract basic panel information', async () => { + const xml = createBasicXml(createPanelXml('Test Panel', 'index=main | stats count by host')); + const parser = new SplunkXmlDashboardParser(xml); + const panels = await parser.extractPanels(); + + expect(panels).toHaveLength(1); + expect(panels[0]).toMatchObject({ + title: 'Test Panel', + query: 'index=main | stats count by host', + viz_type: 'table', // default + position: { x: 0, y: 0, w: 48, h: 16 }, + }); + expect(panels[0].id).toBeDefined(); + }); + + it('should handle multiple panels in a row', async () => { + const xml = createBasicXml( + createPanelXml('Panel 1', 'index=main | stats count') + + createPanelXml('Panel 2', 'index=app | stats sum(bytes)') + ); + const parser = new SplunkXmlDashboardParser(xml); + const panels = await parser.extractPanels(); + + expect(panels).toHaveLength(2); + expect(panels[0].title).toBe('Panel 1'); + expect(panels[1].title).toBe('Panel 2'); + + // Check position calculation + expect(panels[0].position).toEqual({ x: 0, y: 0, w: 24, h: 16 }); + expect(panels[1].position).toEqual({ x: 24, y: 0, w: 24, h: 16 }); + }); + + it('should generate fallback titles for panels without titles', async () => { + const xml = createBasicXml(` + + + index=main | stats count + + + `); + const parser = new SplunkXmlDashboardParser(xml); + const panels = await parser.extractPanels(); + + expect(panels).toHaveLength(1); + expect(panels[0].title).toBe('Untitled Panel 0'); + }); + + it('should skip panels without queries', async () => { + const xml = createBasicXml(` + + Panel without query + + `); + const parser = new SplunkXmlDashboardParser(xml); + const panels = await parser.extractPanels(); + + expect(panels).toHaveLength(0); + }); + + describe('chart type mapping', () => { + const testChartType = async ( + chartType: string, + expectedVizType: VizType, + stackMode?: string, + overlayMode?: string, + hasMetric?: boolean + ) => { + const xml = createBasicXml( + createPanelXml('Test', 'index=main', chartType, stackMode, overlayMode, hasMetric) + ); + const parser = new SplunkXmlDashboardParser(xml); + const panels = await parser.extractPanels(); + return panels[0]?.viz_type; + }; + + it('should map bar chart types correctly', async () => { + expect(await testChartType('bar', 'bar_horizontal')).toBe('bar_horizontal'); + expect(await testChartType('column', 'bar_vertical')).toBe('bar_vertical'); + }); + + it('should map stacked chart types correctly', async () => { + expect(await testChartType('bar', 'bar_horizontal_stacked', 'stacked')).toBe( + 'bar_horizontal_stacked' + ); + expect(await testChartType('column', 'bar_vertical_stacked', 'stacked')).toBe( + 'bar_vertical_stacked' + ); + expect(await testChartType('area', 'area_stacked', 'stacked')).toBe('area_stacked'); + }); + + it('should map special chart types correctly', async () => { + expect(await testChartType('pie', 'pie')).toBe('pie'); + expect(await testChartType('line', 'line')).toBe('line'); + expect(await testChartType('area', 'area')).toBe('area'); + expect(await testChartType('donut', 'donut')).toBe('donut'); + expect(await testChartType('radialGauge', 'gauge')).toBe('gauge'); + expect(await testChartType('treemap', 'treemap')).toBe('treemap'); + }); + + it('should map heatmap overlay mode correctly', async () => { + expect(await testChartType('table', 'heatmap', undefined, 'heatmap')).toBe('heatmap'); + }); + + it('should map metric type correctly', async () => { + expect(await testChartType('table', 'metric', undefined, undefined, true)).toBe('metric'); + }); + + it('should default to table for unknown types', async () => { + expect(await testChartType('unknown', 'table')).toBe('table'); + }); + }); + + it('should handle multiple rows', async () => { + const xml = ` + + + + ${createPanelXml('Row 1 Panel 1', 'index=main')} + + + ${createPanelXml('Row 2 Panel 1', 'index=app')} + ${createPanelXml('Row 2 Panel 2', 'index=web')} + + + `; + const parser = new SplunkXmlDashboardParser(xml); + const panels = await parser.extractPanels(); + + expect(panels).toHaveLength(3); + + // Check row positioning + expect(panels[0].position.y).toBe(0); // Row 1 + expect(panels[1].position.y).toBe(16); // Row 2 + expect(panels[2].position.y).toBe(16); // Row 2 + + // Check column positioning in row 2 + expect(panels[1].position.x).toBe(0); + expect(panels[2].position.x).toBe(24); + }); + + it('should handle deeply nested XML structures', async () => { + const xml = ` + + +

+
+ + ${createPanelXml('Nested Panel', 'index=main | stats count')} + +
+ + + `; + const parser = new SplunkXmlDashboardParser(xml); + const panels = await parser.extractPanels(); + + expect(panels).toHaveLength(1); + expect(panels[0].title).toBe('Nested Panel'); + }); + }); + + describe('extractQueries', () => { + it('should extract queries from panels', async () => { + const xml = createBasicXml( + createPanelXml('Panel 1', 'index=main | stats count by host') + + createPanelXml('Panel 2', 'index=app | stats sum(bytes)') + ); + const parser = new SplunkXmlDashboardParser(xml); + const queries = await parser.extractQueries(); + + expect(queries).toHaveLength(2); + expect(queries).toContain('index=main | stats count by host'); + expect(queries).toContain('index=app | stats sum(bytes)'); + }); + + it('should deduplicate identical queries', async () => { + const xml = createBasicXml( + createPanelXml('Panel 1', 'index=main | stats count') + + createPanelXml('Panel 2', 'index=main | stats count') + ); + const parser = new SplunkXmlDashboardParser(xml); + const queries = await parser.extractQueries(); + + expect(queries).toHaveLength(1); + expect(queries[0]).toBe('index=main | stats count'); + }); + + it('should handle panels without queries', async () => { + const xml = ` + + + + Panel without query + + ${createPanelXml('Panel with query', 'index=main')} + + + `; + const parser = new SplunkXmlDashboardParser(xml); + const queries = await parser.extractQueries(); + + expect(queries).toHaveLength(1); + expect(queries[0]).toBe('index=main'); + }); + + it('should extract queries from deeply nested panels', async () => { + const xml = ` + +
+
+ + + + index=nested | stats count + + + +
+
+ + `; + const parser = new SplunkXmlDashboardParser(xml); + const queries = await parser.extractQueries(); + + expect(queries).toHaveLength(1); + expect(queries[0]).toBe('index=nested | stats count'); + }); + }); + + describe('edge cases', () => { + it('should handle empty XML', async () => { + const parser = new SplunkXmlDashboardParser(''); + const panels = await parser.extractPanels(); + const queries = await parser.extractQueries(); + + expect(panels).toHaveLength(0); + expect(queries).toHaveLength(0); + }); + + it('should handle XML without dashboard element', async () => { + const parser = new SplunkXmlDashboardParser(''); + const panels = await parser.extractPanels(); + const queries = await parser.extractQueries(); + + expect(panels).toHaveLength(0); + expect(queries).toHaveLength(0); + }); + + it('should handle malformed XML gracefully', async () => { + const parser = new SplunkXmlDashboardParser('not valid xml'); + + await expect(parser.extractPanels()).rejects.toThrow(); + await expect(parser.extractQueries()).rejects.toThrow(); + }); + + it('should trim whitespace from queries and titles', async () => { + const xml = createBasicXml(` + + Whitespace Title + + index=main | stats count + + + `); + const parser = new SplunkXmlDashboardParser(xml); + const panels = await parser.extractPanels(); + + expect(panels[0].title).toBe('Whitespace Title'); + expect(panels[0].query).toBe('index=main | stats count'); + }); + + it('should handle empty titles and queries', async () => { + const xml = createBasicXml(` + + + + index=main + + + `); + const parser = new SplunkXmlDashboardParser(xml); + const panels = await parser.extractPanels(); + + expect(panels[0].title).toBe('Untitled Panel 0'); + }); + }); + + describe('chart type detection priority', () => { + it('should prioritize chart option over chart element', async () => { + const xml = createBasicXml(` + + Priority Test + + index=main + + + + + `); + const parser = new SplunkXmlDashboardParser(xml); + const panels = await parser.extractPanels(); + + expect(panels[0].viz_type).toBe('pie'); + }); + + it('should prioritize chart element over viz element', async () => { + const xml = createBasicXml(` + + Priority Test + + index=main + + + + + `); + const parser = new SplunkXmlDashboardParser(xml); + const panels = await parser.extractPanels(); + + expect(panels[0].viz_type).toBe('bar_horizontal'); + }); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/lib/parsers/splunk/splunk_xml_dashboard_parser.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/lib/parsers/splunk/splunk_xml_dashboard_parser.ts new file mode 100644 index 0000000000000..2cbd17431abab --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/lib/parsers/splunk/splunk_xml_dashboard_parser.ts @@ -0,0 +1,308 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { v4 as uuidV4 } from 'uuid'; +import xml2js from 'xml2js'; +import type { ParsedPanel, PanelPosition, VizType } from '../types'; + +interface XmlElement { + $?: { [key: string]: string }; // XML attributes + _?: string; // Text content +} +/** + * Represents a parsed XML object from a Splunk dashboard + */ +interface SplunkXmlElement extends XmlElement { + [key: string]: SplunkXmlElement[] | SplunkXmlElement | string | undefined; +} + +export class SplunkXmlDashboardParser { + constructor(private readonly xml: string) {} + + private async parse(): Promise { + return xml2js.parseStringPromise(this.xml, { + explicitArray: true, + }) as Promise; + } + + public async extractPanels(): Promise { + const root = await this.parse(); + const panels: ParsedPanel[] = []; + + if (!root) return panels; + + const allRows = this.findAllDeep(root, 'row'); + + allRows.forEach((row, rowIndex) => { + const allPanels = this.findAllDeep(row, 'panel'); + const panelCount = allPanels.length; + + allPanels.forEach((panel, panelIndex) => { + if (!panel) return; + + // Use deep search to find query element (equivalent to .//query in Python) + const queryElement = this.findDeep(panel, 'query') as string[] | undefined; + if (!Array.isArray(queryElement) || !queryElement[0]) return; + + const query = queryElement[0].toString().trim(); + + // Extract panel title using deep search (equivalent to .//title in Python) + let title = ''; + const titleElement = this.findDeep(panel, 'title') as string[] | undefined; + if (Array.isArray(titleElement) && titleElement.length > 0) { + title = titleElement[0].toString().trim(); + } + // If still no title, provide a fallback with panel index + if (!title || title === '') { + title = `Untitled Panel ${panelIndex}`; + } + + // Get visualization type using deep search for all chart elements + const vizType = this.getPanelChartType(panel); + const height = 16; // Default height + const position = this.calculatePositions(rowIndex, panelIndex, panelCount, height); + + panels.push({ + id: uuidV4(), + title, + query, + viz_type: vizType, + position, + }); + }); + }); + + return panels; + } + + public async extractQueries(): Promise { + const root = await this.parse(); + const queries: string[] = []; + + const allPanels = this.findAllDeep(root, 'panel'); + + allPanels.forEach((panel) => { + const queryElement = this.findDeep(panel, 'query') as string[] | undefined; + if (Array.isArray(queryElement) && queryElement[0]) { + const query = queryElement[0].toString().trim(); + if (query && !queries.includes(query)) { + queries.push(query); + } + } + }); + + return queries; + } + + // Unified chart type mapping with deep search (equivalent to Python logic) + private getPanelChartType(panel: SplunkXmlElement): VizType { + // Deep search for visualization elements + const metricXml = this.findDeep(panel, 'single'); + const vizXml = this.findDeep(panel, 'viz') as SplunkXmlElement[] | undefined; + const chartXml = this.findDeep(panel, 'chart') as SplunkXmlElement[] | undefined; + + // Deep search for chart options with attribute filtering + const chartOption = this.findDeep(panel, 'option', 'name', 'charting.chart') as + | SplunkXmlElement + | undefined; + const stackMode = this.findDeep(panel, 'option', 'name', 'charting.chart.stackMode') as + | SplunkXmlElement + | undefined; + const overlayMode = this.findDeep(panel, 'option', 'name', 'dataOverlayMode') as + | SplunkXmlElement + | undefined; + + // Extract chart type (combining extractChartType logic) + let chartType = 'table'; + + // Override with viz type if present (matching Python logic) + if (Array.isArray(vizXml) && vizXml[0]?.$ && vizXml[0].$.type) { + chartType = vizXml[0].$.type; + } + + // Override with chart type if present (matching Python logic) + if (Array.isArray(chartXml) && chartXml[0]?.$ && chartXml[0].$.type) { + chartType = chartXml[0].$.type; + } + + // Override with chart option if present (matching Python logic) + if (chartOption && chartOption._) { + chartType = chartOption._; + } + + // Convert to VizType (combining getChartType logic) + return this.mapToVizType(chartType, stackMode, overlayMode, metricXml); + } + + private mapToVizType( + chartType: string, + stackMode: SplunkXmlElement | undefined, + overlayMode: SplunkXmlElement | undefined, + metricXml: unknown + ): VizType { + const isStacked = stackMode?._ && stackMode._.includes('stacked'); + + // Direct mapping to VizType enum values (matching Python logic) + if (chartType === 'bar') { + return isStacked ? 'bar_horizontal_stacked' : 'bar_horizontal'; + } + + if (chartType === 'column') { + return isStacked ? 'bar_vertical_stacked' : 'bar_vertical'; + } + + if (chartType === 'area') { + return isStacked ? 'area_stacked' : 'area'; + } + + // Special case transformations (matching Python logic) + if (chartType === 'table' && overlayMode?._ === 'heatmap') { + return 'heatmap'; + } + + if (chartType === 'radialGauge') { + return 'gauge'; + } + + if (chartType === 'table' && metricXml) { + return 'metric'; + } + + // Direct enum mapping for other types + const typeMap: Record = { + table: 'table', + heatmap: 'heatmap', + gauge: 'gauge', + metric: 'metric', + pie: 'pie', + donut: 'donut', + line: 'line', + treemap: 'treemap', + markdown: 'markdown', + }; + + return typeMap[chartType] || 'table'; + } + + // Unified deep search method (equivalent to Python's .// XPath expressions) + private findDeep( + source: SplunkXmlElement, + elementName: string, + attrName?: string, + attrValue?: string + ): SplunkXmlElement[] | SplunkXmlElement | string | undefined { + if (typeof source !== 'object' || source === null) { + return undefined; + } + // Check if the element exists at this level + if (elementName in source) { + const element = source[elementName]; + + // If no attribute filtering is needed, return the element + if (!attrName || !attrValue) { + return element; + } + + // If attribute filtering is needed, check if it's an array of elements + if (Array.isArray(element)) { + for (const item of element as SplunkXmlElement[]) { + if (item.$ && item.$[attrName] === attrValue) { + return item; + } + } + } + } + + // Search recursively in all properties + for (const key of Object.keys(source)) { + const value = source[key]; + + // Handle array values + if (Array.isArray(value)) { + for (const item of value) { + const result = this.findDeep(item, elementName, attrName, attrValue); + if (result !== undefined) { + return result; + } + } + } + // Handle object values + else if (typeof value === 'object' && value !== null) { + const result = this.findDeep(value, elementName, attrName, attrValue); + if (result !== undefined) { + return result; + } + } + } + + return undefined; + } + + private findAllDeep(source: SplunkXmlElement, elementName: string): SplunkXmlElement[] { + const results: SplunkXmlElement[] = []; + + if (typeof source !== 'object' || source === null) { + return results; + } + + // Check if the element exists at this level + if (elementName in source) { + const element = source[elementName]; + + // If it's an array, add all elements to results + if (Array.isArray(element)) { + results.push(...element); + // Don't search deeper in these found elements, continue with siblings + } else if (element) { + // Single element, add it to results + results.push(element as SplunkXmlElement); + // Don't search deeper in this found element, continue with siblings + } + } + + // Search recursively in all properties (but skip children of found elements) + for (const key of Object.keys(source)) { + if (key === elementName) { + // Skip the element we already processed above + } else { + const value = source[key]; + + // Handle array values + if (Array.isArray(value)) { + for (const item of value) { + const childResults = this.findAllDeep(item, elementName); + results.push(...childResults); + } + } + // Handle object values + else if (typeof value === 'object' && value !== null) { + const childResults = this.findAllDeep(value, elementName); + results.push(...childResults); + } + } + } + + return results; + } + + // Calculate panel positions + private calculatePositions( + rowIndex: number, + panelIndex: number, + totalPanelsInRow: number, + height = 16 + ): PanelPosition { + const panelWidth = Math.floor(48 / totalPanelsInRow); + + return { + x: panelIndex * panelWidth, + y: rowIndex * height, + w: panelWidth, + h: height, + }; + } +} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/lib/parsers/types.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/lib/parsers/types.ts new file mode 100644 index 0000000000000..d81370e4c2d4f --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/lib/parsers/types.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export interface ParsedPanel { + /** The generated panel uuid */ + id: string; + /** The panel title extracted */ + title: string; + /** The extracted query */ + query: string; + /** The visualization type */ + viz_type: VizType; + /** The computed position */ + position: PanelPosition; +} + +export type VizType = + | 'area_stacked' + | 'area' + | 'bar_horizontal_stacked' + | 'bar_horizontal' + | 'bar_vertical_stacked' + | 'bar_vertical' + | 'donut' + | 'gauge' + | 'heatmap' + | 'line' + | 'markdown' + | 'metric' + | 'pie' + | 'table' + | 'treemap'; + +export interface PanelPosition { + x: number; + y: number; + w: number; + h: number; +} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/siem_dashboard_migration_service.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/siem_dashboard_migration_service.ts index 9fb101233a767..18c411b1bd0bb 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/siem_dashboard_migration_service.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/siem_dashboard_migration_service.ts @@ -11,6 +11,8 @@ import type { Subject } from 'rxjs'; import type { DashboardMigrationsDataClient } from './data/dashboard_migrations_data_client'; import { DashboardMigrationsDataService } from './data/dashboard_migrations_data_service'; import type { SiemMigrationsCreateClientParams } from '../common/types'; +import type { DashboardMigrationsTaskClient } from './task/dashboard_migrations_task_client'; +import { DashboardMigrationsTaskService } from './task/dashboard_migrations_task_service'; export interface SiemDashboardsMigrationsSetupParams { esClusterClient: IClusterClient; @@ -20,16 +22,19 @@ export interface SiemDashboardsMigrationsSetupParams { export interface SiemDashboardMigrationsClient { data: DashboardMigrationsDataClient; + task: DashboardMigrationsTaskClient; } export class SiemDashboardMigrationsService { private dataService: DashboardMigrationsDataService; private esClusterClient?: IClusterClient; + private taskService: DashboardMigrationsTaskService; private logger: Logger; constructor(logger: LoggerFactory, kibanaVersion: string, elserInferenceId?: string) { this.logger = logger.get('siemDashboardMigrations'); this.dataService = new DashboardMigrationsDataService(this.logger, kibanaVersion); + this.taskService = new DashboardMigrationsTaskService(this.logger); } setup({ esClusterClient, ...params }: SiemDashboardsMigrationsSetupParams) { @@ -58,8 +63,12 @@ export class SiemDashboardMigrationsService { dependencies, }); - return { data: dataClient }; + const taskClient = this.taskService.createClient({ currentUser, dataClient, dependencies }); + + return { data: dataClient, task: taskClient }; } - stop() {} + stop() { + this.taskService.stopAll(); + } } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/__mocks__/dashboard_migrations_task_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/__mocks__/dashboard_migrations_task_client.ts new file mode 100644 index 0000000000000..869d1ca037a07 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/__mocks__/dashboard_migrations_task_client.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MockDashboardMigrationsTaskClient } from './mocks'; +export const DashboardMigrationsTaskClient = MockDashboardMigrationsTaskClient; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/__mocks__/dashboard_migrations_task_service.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/__mocks__/dashboard_migrations_task_service.ts new file mode 100644 index 0000000000000..9c9e71b048043 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/__mocks__/dashboard_migrations_task_service.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MockDashboardMigrationsTaskService } from './mocks'; +export const DashboardMigrationsTaskService = MockDashboardMigrationsTaskService; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/__mocks__/dashboard_migrations_telemetry_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/__mocks__/dashboard_migrations_telemetry_client.ts new file mode 100644 index 0000000000000..199e630d2f1de --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/__mocks__/dashboard_migrations_telemetry_client.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MockSiemMigrationTelemetryClient } from './mocks'; +export const SiemMigrationTelemetryClient = MockSiemMigrationTelemetryClient; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/__mocks__/mocks.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/__mocks__/mocks.ts new file mode 100644 index 0000000000000..fd59bf8c97354 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/__mocks__/mocks.ts @@ -0,0 +1,129 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { PublicMethodsOf } from '@kbn/utility-types'; +import { FakeLLM } from '@langchain/core/utils/testing'; +import { AsyncLocalStorageProviderSingleton } from '@langchain/core/singletons'; +import type { DashboardMigrationTelemetryClient } from '../dashboard_migrations_telemetry_client'; +import type { BaseLLMParams } from '@langchain/core/language_models/llms'; + +export const createSiemMigrationTelemetryClientMock = () => { + // Mock for the object returned by startSiemMigrationTask + const mockStartDashboardTranslationReturn = { + success: jest.fn(), + failure: jest.fn(), + }; + + // Mock for the function returned by startSiemMigrationTask + const mockStartDashboardTranslation = jest + .fn() + .mockReturnValue(mockStartDashboardTranslationReturn); + + // Mock for startSiemMigrationTask return value + const mockStartSiemMigrationTaskReturn = { + startDashboardTranslation: mockStartDashboardTranslation, + success: jest.fn(), + failure: jest.fn(), + aborted: jest.fn(), + }; + + return { + reportIntegrationsMatch: jest.fn(), + reportPrebuiltDashboardsMatch: jest.fn(), + startSiemMigrationTask: jest.fn().mockReturnValue(mockStartSiemMigrationTaskReturn), + } as jest.Mocked>; +}; + +// Factory function for the mock class +export const MockSiemMigrationTelemetryClient = jest + .fn() + .mockImplementation(() => createSiemMigrationTelemetryClientMock()); + +export const createDashboardMigrationsTaskClientMock = () => ({ + start: jest.fn().mockResolvedValue({ started: true }), + stop: jest.fn().mockResolvedValue({ stopped: true }), + getStats: jest.fn().mockResolvedValue({ + status: 'done', + items: { + total: 1, + finished: 1, + processing: 0, + pending: 0, + failed: 0, + }, + }), + getAllStats: jest.fn().mockResolvedValue([]), +}); + +export const MockDashboardMigrationsTaskClient = jest + .fn() + .mockImplementation(() => createDashboardMigrationsTaskClientMock()); + +// Dashboard migrations task service +export const mockStopAll = jest.fn(); +export const mockCreateClient = jest.fn(() => createDashboardMigrationsTaskClientMock()); + +export const MockDashboardMigrationsTaskService = jest.fn().mockImplementation(() => ({ + createClient: mockCreateClient, + stopAll: mockStopAll, +})); + +export interface NodeResponse { + nodeId: string; + response: string; +} + +interface SiemMigrationFakeLLMParams extends BaseLLMParams { + nodeResponses: NodeResponse[]; +} + +export class SiemMigrationFakeLLM extends FakeLLM { + private nodeResponses: NodeResponse[]; + private defaultResponse: string; + private callCount: Map; + private totalCount: number; + + constructor(fields: SiemMigrationFakeLLMParams) { + super({ + response: 'unexpected node call', + ...fields, + }); + this.nodeResponses = fields.nodeResponses; + this.defaultResponse = 'unexpected node call'; + this.callCount = new Map(); + this.totalCount = 0; + } + + _llmType(): string { + return 'fake'; + } + + async _call(prompt: string, _options: this['ParsedCallOptions']): Promise { + // Get the current runnable config metadata + const item = AsyncLocalStorageProviderSingleton.getRunnableConfig(); + for (const nodeResponse of this.nodeResponses) { + if (item.metadata.langgraph_node === nodeResponse.nodeId) { + const currentCount = this.callCount.get(nodeResponse.nodeId) || 0; + this.callCount.set(nodeResponse.nodeId, currentCount + 1); + this.totalCount += 1; + return nodeResponse.response; + } + } + return this.defaultResponse; + } + + getNodeCallCount(nodeId: string): number { + return this.callCount.get(nodeId) || 0; + } + + getTotalCallCount(): number { + return this.totalCount; + } + + resetCallCounts(): void { + this.callCount.clear(); + } +} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/graph.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/graph.test.ts new file mode 100644 index 0000000000000..9f39c5a54917d --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/graph.test.ts @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import fs from 'fs'; +import type { ActionsClientChatOpenAI } from '@kbn/langchain/server/language_models'; +import { loggerMock } from '@kbn/logging-mocks'; +import type { NodeResponse } from '../__mocks__/mocks'; +import { SiemMigrationFakeLLM, MockSiemMigrationTelemetryClient } from '../__mocks__/mocks'; +import { MockEsqlKnowledgeBase } from '../../../common/task/util/__mocks__/mocks'; +import { MockDashboardMigrationsRetriever } from '../retrievers/__mocks__/mocks'; +import { getDashboardMigrationAgent } from './graph'; +import type { OriginalDashboard } from '../../../../../../common/siem_migrations/model/dashboard_migration.gen'; + +const mockOriginalDashboardData = fs.readFileSync( + `${__dirname}/../../__mocks__/original_dashboard_example.xml` +); + +const mockOriginalDashboard: OriginalDashboard = { + id: 'b12c89bc-9d06-11eb-a592-acde48001122', + vendor: 'splunk' as const, + title: 'Office Document Executing Macro Code', + description: + 'The following analytic identifies office documents executing macro code. It leverages Sysmon EventCode 7 to detect when processes like WINWORD.EXE or EXCEL.EXE load specific DLLs associated with macros (e.g., VBE7.DLL). This activity is significant because macros are a common attack vector for delivering malicious payloads, such as malware. If confirmed malicious, this could lead to unauthorized code execution, data exfiltration, or further compromise of the system. Disabling macros by default is recommended to mitigate this risk.', + data: mockOriginalDashboardData, + format: 'xml', +}; + +const logger = loggerMock.create(); +let fakeLLM: SiemMigrationFakeLLM; +let mockRetriever = new MockDashboardMigrationsRetriever(); +let mockEsqlKnowledgeBase = new MockEsqlKnowledgeBase(); +let mockTelemetryClient = new MockSiemMigrationTelemetryClient(); + +const setupAgent = (responses: NodeResponse[]) => { + fakeLLM = new SiemMigrationFakeLLM({ nodeResponses: responses }); + const model = fakeLLM as unknown as ActionsClientChatOpenAI; + const graph = getDashboardMigrationAgent({ + model, + esqlKnowledgeBase: mockEsqlKnowledgeBase, + dashboardMigrationsRetriever: mockRetriever, + logger, + telemetryClient: mockTelemetryClient, + }); + return graph; +}; + +describe('getDashboardMigrationAgent', () => { + beforeEach(() => { + mockRetriever = new MockDashboardMigrationsRetriever(); + mockTelemetryClient = new MockSiemMigrationTelemetryClient(); + mockEsqlKnowledgeBase = new MockEsqlKnowledgeBase(); + jest.clearAllMocks(); + }); + + it('should compile graph', () => { + setupAgent([{ nodeId: '', response: '' }]); + }); + + it('should run graph', async () => { + const agent = setupAgent([{ nodeId: '', response: '' }]); + const result = await agent.invoke({ + id: 'testId', + original_dashboard: mockOriginalDashboard, + resources: {}, + }); + + expect(result).toEqual(expect.any(Object)); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/graph.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/graph.ts new file mode 100644 index 0000000000000..0387f1a52902c --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/graph.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { END, START, StateGraph } from '@langchain/langgraph'; +import { getParseOriginalDashboardNode } from './nodes/parse_original_dashboard'; +import { migrateDashboardConfigSchema, migrateDashboardState } from './state'; +import type { MigrateDashboardGraphParams } from './types'; +import { getTranslatePanelNode } from './nodes/translate_panel/translate_panel'; +import { getAggregateDashboardNode } from './nodes/aggregate_dashboard'; + +export function getDashboardMigrationAgent(params: MigrateDashboardGraphParams) { + const parseOriginalDashboardNode = getParseOriginalDashboardNode(); + const translatePanel = getTranslatePanelNode(params); + const aggregateDashboardNode = getAggregateDashboardNode(); + + const siemMigrationAgentGraph = new StateGraph( + migrateDashboardState, + migrateDashboardConfigSchema + ) + // Nodes + .addNode('parseOriginalDashboard', parseOriginalDashboardNode) + .addNode('translatePanel', translatePanel.node, { subgraphs: [translatePanel.subgraph] }) + .addNode('aggregateDashboard', aggregateDashboardNode) + // Edges + .addEdge(START, 'parseOriginalDashboard') + .addConditionalEdges('parseOriginalDashboard', translatePanel.conditionalEdge, [ + 'translatePanel', + ]) + .addEdge('translatePanel', 'aggregateDashboard') + .addEdge('aggregateDashboard', END); + + const graph = siemMigrationAgentGraph.compile(); + graph.name = 'Dashboard Migration Graph'; // Customizes the name displayed in LangSmith + return graph; +} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/index.ts new file mode 100644 index 0000000000000..f3c73aa6e453c --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { getDashboardMigrationAgent } from './graph'; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/nodes/aggregate_dashboard/aggregate_dashboard.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/nodes/aggregate_dashboard/aggregate_dashboard.ts new file mode 100644 index 0000000000000..ab92becd6b22b --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/nodes/aggregate_dashboard/aggregate_dashboard.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import path from 'path'; +import fs from 'fs'; +import { MigrationTranslationResult } from '../../../../../../../../common/siem_migrations/constants'; +import type { GraphNode } from '../../types'; + +interface DashboardData { + attributes: { + title: string; + panelsJSON: string; + }; +} + +export const getAggregateDashboardNode = (): GraphNode => { + return async (state) => { + let dashboardData: DashboardData; + try { + const templatePath = path.join(__dirname, `./dashboard.json`); + const template = fs.readFileSync(templatePath, 'utf-8'); + + if (!template) { + throw new Error(`Dashboard template not found`); + } + dashboardData = JSON.parse(template); + } catch (error) { + // TODO: log the error + return { + // TODO: add comment: "panel chart type not supported" + translation_result: MigrationTranslationResult.UNTRANSLATABLE, + }; + } + + const panels = state.translated_panels.sort((a, b) => a.index - b.index); + + dashboardData.attributes.title = state.original_dashboard.title; + dashboardData.attributes.panelsJSON = JSON.stringify(panels.map(({ data }) => data)); + + // TODO: Use individual translation results for each panel: + // panels.map((panel) => panel.translation_result) + // and aggregate the top level translation_result here + let translationResult; + if (state.translated_panels.length > 0) { + if (state.translated_panels.length > 0) { + translationResult = MigrationTranslationResult.PARTIAL; + } else { + translationResult = MigrationTranslationResult.FULL; + } + } else { + translationResult = MigrationTranslationResult.UNTRANSLATABLE; + } + + return { + elastic_dashboard: { + title: state.original_dashboard.title, + data: JSON.stringify(dashboardData), + }, + translation_result: translationResult, + }; + }; +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/nodes/aggregate_dashboard/dashboard.json b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/nodes/aggregate_dashboard/dashboard.json new file mode 100644 index 0000000000000..7cfaf18340ada --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/nodes/aggregate_dashboard/dashboard.json @@ -0,0 +1,23 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" + }, + "optionsJSON": "{\"useMargins\":true,\"syncColors\":false,\"syncCursor\":true,\"syncTooltips\":false,\"hidePanelTitles\":false}", + "panelsJSON": "", + "timeRestore": false, + "title": "", + "version": 2 + }, + "coreMigrationVersion": "8.8.0", + "created_at": "2024-11-22T22:38:58.430Z", + "created_by": "u_3965676980_cloud", + "id": "cb50f38e-2cdd-42cf-bd51-16063303dda1", + "managed": false, + "type": "dashboard", + "typeMigrationVersion": "10.2.0", + "updated_at": "2024-11-23T00:49:12.254Z", + "updated_by": "u_3965676980_cloud", + "version": "WzEzODMsNF0=" +} \ No newline at end of file diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/nodes/aggregate_dashboard/index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/nodes/aggregate_dashboard/index.ts new file mode 100644 index 0000000000000..4859c811ad430 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/nodes/aggregate_dashboard/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +export { getAggregateDashboardNode } from './aggregate_dashboard'; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/nodes/parse_original_dashboard/index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/nodes/parse_original_dashboard/index.ts new file mode 100644 index 0000000000000..2ed917e114c34 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/nodes/parse_original_dashboard/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { getParseOriginalDashboardNode } from './parse_original_dashboard'; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/nodes/parse_original_dashboard/parse_original_dashboard.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/nodes/parse_original_dashboard/parse_original_dashboard.ts new file mode 100644 index 0000000000000..074056f1be320 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/nodes/parse_original_dashboard/parse_original_dashboard.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SplunkXmlDashboardParser } from '../../../../lib/parsers/splunk/splunk_xml_dashboard_parser'; +import type { GraphNode } from '../../types'; + +export const getParseOriginalDashboardNode = (): GraphNode => { + return async (state) => { + if (state.original_dashboard.vendor !== 'splunk') { + throw new Error('Unsupported dashboard vendor'); + } + + const parser = new SplunkXmlDashboardParser(state.original_dashboard.data); + const panels = await parser.extractPanels(); + + return { + parsed_original_dashboard: { + title: state.original_dashboard.title, + panels, + }, + }; + }; +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/nodes/translate_panel/index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/nodes/translate_panel/index.ts new file mode 100644 index 0000000000000..5a2010ebafee1 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/nodes/translate_panel/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +export { getTranslatePanelNode } from './translate_panel'; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/nodes/translate_panel/translate_panel.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/nodes/translate_panel/translate_panel.ts new file mode 100644 index 0000000000000..1f4100444a0e1 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/nodes/translate_panel/translate_panel.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Send } from '@langchain/langgraph'; +import type { MigrateDashboardState, TranslatePanelNodeParams } from '../../types'; +import { getTranslatePanelGraph } from '../../sub_graphs/translate_panel'; +import type { TranslatePanelGraphParams } from '../../sub_graphs/translate_panel/types'; + +export type TranslatePanelNode = (( + params: TranslatePanelNodeParams +) => Promise>) & { + subgraph?: ReturnType; +}; + +export interface TranslatePanel { + node: TranslatePanelNode; + conditionalEdge: (state: MigrateDashboardState) => Send[]; + subgraph: ReturnType; +} +// This is a special node, it's goal is to use map-reduce to translate the dashboard panels in parallel. +// This is the recommended technique at the time of writing this code. LangGraph docs: https://langchain-ai.github.io/langgraphjs/how-tos/map-reduce/. +export const getTranslatePanelNode = (params: TranslatePanelGraphParams): TranslatePanel => { + const translatePanelSubGraph = getTranslatePanelGraph(params); + return { + // Fan-in: the results of the individual panel translations are aggregated back into the overall dashboard state via state reducer. + node: async ({ panel, index }) => { + try { + if (!panel.query) { + throw new Error('Panel query is missing'); + } + const output = await translatePanelSubGraph.invoke({ parsed_panel: panel }); + return { + // Fan-in: translated panels are concatenated by the state reducer, so the results can be aggregated later + translated_panels: [ + { + index, + data: output.elastic_panel ?? {}, + translation_result: output.translation_result, + }, + ], + }; + } catch (err) { + // Fan-in: failed panels are concatenated by the state reducer, so the results can be aggregated later + return { + failed_panel_translations: [ + { + index, + error_message: err.toString(), + details: err, + }, + ], + }; + } + }, + // Fan-out: for each panel, Send translatePanel to be executed in parallel. + // This function needs to be called inside a `conditionalEdge` + conditionalEdge: (state: MigrateDashboardState) => { + const panels = state.parsed_original_dashboard.panels ?? []; + return panels.map((panel, i) => { + const translatePanelParams: TranslatePanelNodeParams = { panel, index: i }; + return new Send('translatePanel', translatePanelParams); + }); + }, + subgraph: translatePanelSubGraph, // Only for the diagram generation + }; +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/state.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/state.ts new file mode 100644 index 0000000000000..6d43a47c3bed7 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/state.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Annotation } from '@langchain/langgraph'; +import { uniq } from 'lodash/fp'; +import type { MigrationTranslationResult } from '../../../../../../common/siem_migrations/constants'; +import type { + ElasticDashboard, + OriginalDashboard, + DashboardMigrationDashboard, +} from '../../../../../../common/siem_migrations/model/dashboard_migration.gen'; +import type { MigrationResources } from '../../../common/task/retrievers/resource_retriever'; +import type { FailedPanelTranslations, ParsedOriginalDashboard, TranslatedPanels } from './types'; + +export const migrateDashboardState = Annotation.Root({ + id: Annotation(), + original_dashboard: Annotation(), + parsed_original_dashboard: Annotation(), + translated_panels: Annotation({ + reducer: (current, value) => current.concat(value), + default: () => [], + }), + failed_panel_translations: Annotation({ + reducer: (current, value) => current.concat(value), + default: () => [], + }), + elastic_dashboard: Annotation({ + reducer: (current, value) => ({ ...current, ...value }), + }), + resources: Annotation(), + translation_result: Annotation(), + comments: Annotation({ + // Translation subgraph causes the original main graph comments to be concatenated again, we need to deduplicate them. + reducer: (current, value) => uniq(value ? (current ?? []).concat(value) : current), + default: () => [], + }), +}); + +export const migrateDashboardConfigSchema = Annotation.Root({ + skipPrebuiltDashboardsMatching: Annotation(), +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/graph.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/graph.ts new file mode 100644 index 0000000000000..a44015c4a6cf9 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/graph.ts @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { END, START, StateGraph } from '@langchain/langgraph'; +import { isEmpty } from 'lodash/fp'; +import { getEcsMappingNode } from './nodes/ecs_mapping'; +import { getFixQueryErrorsNode } from './nodes/fix_query_errors'; +import { getInlineQueryNode } from './nodes/inline_query'; +import { getTranslateQueryNode } from './nodes/translate_query'; +import { getTranslationResultNode } from './nodes/translation_result'; +import { getValidationNode } from './nodes/validation'; +import { translateDashboardPanelState } from './state'; +import type { TranslatePanelGraphParams, TranslateDashboardPanelState } from './types'; +import { migrateDashboardConfigSchema } from '../../state'; +import { getSelectIndexPatternNode } from './nodes/select_index_pattern/select_index_pattern'; + +// How many times we will try to self-heal when validation fails, to prevent infinite graph recursions +const MAX_VALIDATION_ITERATIONS = 3; + +export function getTranslatePanelGraph(params: TranslatePanelGraphParams) { + const translateQueryNode = getTranslateQueryNode(params); + const inlineQueryNode = getInlineQueryNode(params); + const validationNode = getValidationNode(params); + const fixQueryErrorsNode = getFixQueryErrorsNode(params); + const ecsMappingNode = getEcsMappingNode(params); + const selectIndexPatternNode = getSelectIndexPatternNode(params); + const translationResultNode = getTranslationResultNode(); + + const translateDashboardPanelGraph = new StateGraph( + translateDashboardPanelState, + migrateDashboardConfigSchema + ) + // Nodes + .addNode('inlineQuery', inlineQueryNode) + // TODO: .addNode('createDescription', createDescriptionNode) -> ask the LLM to create a description of the panel + .addNode('selectIndexPattern', selectIndexPatternNode) + + // Consider this block by the entire Assistant nlToEsql graph + .addNode('translateQuery', translateQueryNode) + .addNode('validation', validationNode) + .addNode('fixQueryErrors', fixQueryErrorsNode) + .addNode('ecsMapping', ecsMappingNode) // Not sure about this one, maybe we should keep it anyway, tests need to be done + // Consider this block by the entire Assistant nlToEsql graph + + .addNode('translationResult', translationResultNode) + + // Edges + .addEdge(START, 'inlineQuery') + .addConditionalEdges('inlineQuery', translatableRouter, [ + 'selectIndexPattern', + 'translationResult', + ]) + // .addEdge('inlineQuery', 'createDescription') // createDescription would go after inlineQuery + .addEdge('inlineQuery', 'selectIndexPattern') + // .addEdge('createDescription', 'selectIndexPattern') // And before selectIndexPattern, the description is sent to the selectIndexPattern graph + .addEdge('selectIndexPattern', 'translateQuery') + .addEdge('translateQuery', 'validation') + .addEdge('fixQueryErrors', 'validation') + .addEdge('ecsMapping', 'validation') + .addConditionalEdges('validation', validationRouter, [ + 'fixQueryErrors', + 'ecsMapping', + 'translationResult', + ]) + .addEdge('translationResult', END); + + const graph = translateDashboardPanelGraph.compile(); + graph.name = 'Translate Dashboard Panel Graph'; + return graph; +} + +const translatableRouter = (state: TranslateDashboardPanelState) => { + if (!state.inline_query) { + return 'translationResult'; + } + return 'selectIndexPattern'; +}; + +const validationRouter = (state: TranslateDashboardPanelState) => { + if ( + state.validation_errors.iterations <= MAX_VALIDATION_ITERATIONS && + !isEmpty(state.validation_errors?.esql_errors) + ) { + return 'fixQueryErrors'; + } + if (!state.includes_ecs_mapping) { + return 'ecsMapping'; + } + + return 'translationResult'; +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/index.ts new file mode 100644 index 0000000000000..ce761fe4a02b4 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { getTranslatePanelGraph } from './graph'; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/ecs_mapping/ecs_mapping.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/ecs_mapping/ecs_mapping.ts new file mode 100644 index 0000000000000..1ae49801632db --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/ecs_mapping/ecs_mapping.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + getConvertEsqlSchemaCisToEcs, + type GetConvertEsqlSchemaCisToEcsParams, +} from '../../../../../../../common/task/agent/tools/convert_esql_schema_cim_to_ecs'; +import type { GraphNode } from '../../types'; + +export const getEcsMappingNode = (params: GetConvertEsqlSchemaCisToEcsParams): GraphNode => { + const convertEsqlSchemaCimToEcs = getConvertEsqlSchemaCisToEcs(params); + return async (state) => { + const { query, comments } = await convertEsqlSchemaCimToEcs({ + title: state.parsed_panel.title ?? '', + description: state.description ?? '', + query: state.esql_query ?? '', + originalQuery: state.inline_query ?? '', + }); + + // Set includes_ecs_mapping to indicate that this node has been executed to ensure it only runs once + return { + includes_ecs_mapping: true, + comments, + ...(query && { esql_query: query }), + }; + }; +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/ecs_mapping/index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/ecs_mapping/index.ts new file mode 100644 index 0000000000000..339e6d3dd8e7a --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/ecs_mapping/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +export { getEcsMappingNode } from './ecs_mapping'; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/fix_query_errors/fix_query_errors.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/fix_query_errors/fix_query_errors.ts new file mode 100644 index 0000000000000..fae3ff5ccb5cb --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/fix_query_errors/fix_query_errors.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + getFixEsqlQueryErrors, + type GetFixEsqlQueryErrorsParams, +} from '../../../../../../../common/task/agent/tools/fix_esql_query_errors'; +import type { GraphNode } from '../../types'; + +export const getFixQueryErrorsNode = (params: GetFixEsqlQueryErrorsParams): GraphNode => { + const fixEsqlQueryErrors = getFixEsqlQueryErrors(params); + return async (state) => { + const { query } = await fixEsqlQueryErrors({ + invalidQuery: state.esql_query, + validationErrors: state.validation_errors.esql_errors, + }); + if (!query) { + return {}; + } + return { esql_query: query }; + }; +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/fix_query_errors/index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/fix_query_errors/index.ts new file mode 100644 index 0000000000000..a805331675389 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/fix_query_errors/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +export { getFixQueryErrorsNode } from './fix_query_errors'; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/inline_query/index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/inline_query/index.ts new file mode 100644 index 0000000000000..c466306e99074 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/inline_query/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +export { getInlineQueryNode } from './inline_query'; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/inline_query/inline_query.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/inline_query/inline_query.ts new file mode 100644 index 0000000000000..3aea2dfc6d013 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/inline_query/inline_query.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { + getInlineSplQuery, + type GetInlineSplQueryParams, +} from '../../../../../../../common/task/agent/tools/inline_spl_query'; +import type { GraphNode } from '../../types'; + +export const getInlineQueryNode = (params: GetInlineSplQueryParams): GraphNode => { + const inlineSplQuery = getInlineSplQuery(params); + return async (state) => { + // NOTE: "inputlookup" is not currently supported, to make it supported we need to parametrize the unsupported check logic here, and the Splunk lookups identifier. + const { inlineQuery, isUnsupported, comments } = await inlineSplQuery({ + query: state.parsed_panel.query, + resources: state.resources, + }); + if (isUnsupported) { + // Graph conditional edge detects undefined inline_query as unsupported query + return { inline_query: undefined, comments }; + } + return { + inline_query: inlineQuery ?? state.parsed_panel.query, + comments, + }; + }; +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/select_index_pattern/index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/select_index_pattern/index.ts new file mode 100644 index 0000000000000..b8ad36a18805e --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/select_index_pattern/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +export { getSelectIndexPanelNode as getEcsMappingNode } from './select_index_pattern'; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/select_index_pattern/prompts.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/select_index_pattern/prompts.ts new file mode 100644 index 0000000000000..fcdb2f9dad5c3 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/select_index_pattern/prompts.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ChatPromptTemplate } from '@langchain/core/prompts'; + +export const SELECT_INDEX_PATTERN_PROMPT = ChatPromptTemplate.fromTemplate( + `You are a cybersecurity expert familiar with both Splunk and Elasticsearch. + +Your task is: + +- Analyze the provided Splunk query and its context. + +- Determine the most specific Elastic index pattern that would support this query in the current Elastic cluster. + +Instructions: + +- Respond only with the index pattern string. + +- Do not add explanations, comments, or extra formatting. + +- Prioritize specificity: choose the narrowest index pattern that captures the relevant data. + + + {title} + {description} + {query} + +` +); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/select_index_pattern/select_index_pattern.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/select_index_pattern/select_index_pattern.ts new file mode 100644 index 0000000000000..cfcd962aa1241 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/select_index_pattern/select_index_pattern.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { IScopedClusterClient, Logger } from '@kbn/core/server'; +import { getSelectIndexPatternGraph } from '../../../../../../../../../assistant/tools/esql/graphs/select_index_pattern/select_index_pattern'; +import type { ChatModel } from '../../../../../../../common/task/util/actions_client_chat'; +import type { DashboardMigrationTelemetryClient } from '../../../../../dashboard_migrations_telemetry_client'; +import type { GraphNode } from '../../types'; +import { SELECT_INDEX_PATTERN_PROMPT } from './prompts'; + +interface GetSelectIndexPatternParams { + model: ChatModel; + esScopedClient: IScopedClusterClient; + telemetryClient: DashboardMigrationTelemetryClient; + logger: Logger; +} + +export const getSelectIndexPatternNode = (params: GetSelectIndexPatternParams): GraphNode => { + const selectIndexPatternGraphPromise = getSelectIndexPatternGraph({ + // Using the `asInternalUser` so we can access all indices to find the best index pattern + // we can change it to `asCurrentUser`, but we would be restricted to the indices the user (who started the migration task) has access to. + esClient: params.esScopedClient.asInternalUser, + createLlmInstance: async () => params.model, + }); + + return async (state, config) => { + const selectIndexPatternGraph = await selectIndexPatternGraphPromise; // This will only be awaited the first time the node is executed + + if (!state.inline_query) { + return { index_pattern: '[indexPattern]' }; + } + const question = await SELECT_INDEX_PATTERN_PROMPT.format({ + query: state.inline_query, + title: state.parsed_panel.title, + description: state.description ?? '', + }); + const { selectedIndexPattern } = await selectIndexPatternGraph.invoke( + { input: { question } }, + config + ); + + if (!selectedIndexPattern) { + return { index_pattern: '[indexPattern]' }; + } + return { + index_pattern: selectedIndexPattern, + }; + }; +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translate_query/index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translate_query/index.ts new file mode 100644 index 0000000000000..7d247f755e9da --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translate_query/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +export { getTranslateQueryNode } from './translate_query'; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translate_query/translate_query.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translate_query/translate_query.ts new file mode 100644 index 0000000000000..4a3b72a637345 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translate_query/translate_query.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + getTranslateSplToEsql, + TASK_DESCRIPTION, + type GetTranslateSplToEsqlParams, +} from '../../../../../../../common/task/agent/tools/translate_spl_to_esql'; +import type { GraphNode } from '../../types'; + +export const getTranslateQueryNode = (params: GetTranslateSplToEsqlParams): GraphNode => { + const translateSplToEsql = getTranslateSplToEsql(params); + return async (state) => { + if (!state.inline_query) { + return {}; + } + const { esqlQuery, comments } = await translateSplToEsql({ + title: state.parsed_panel.title, + description: state.description ?? '', + taskDescription: TASK_DESCRIPTION.migrate_dashboard, + inlineQuery: state.inline_query, + indexPattern: state.index_pattern || '[indexPattern]', + }); + + if (!esqlQuery) { + return { comments }; + } + + return { + esql_query: esqlQuery, + comments, + }; + }; +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/index.ts new file mode 100644 index 0000000000000..6779a9c99ebc8 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +export { getTranslationResultNode } from './translation_result'; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/process_panel.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/process_panel.ts new file mode 100644 index 0000000000000..0e87d17a03b2e --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/process_panel.ts @@ -0,0 +1,273 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getQueryColumnsFromESQLQuery } from '@kbn/esql-utils'; +import type { ParsedPanel } from '../../../../../../lib/parsers/types'; + +interface ColumnInfo { + columnId: string; + fieldName: string; + meta: { + type: string; + }; + inMetricDimension?: boolean; +} + +interface PanelJSON { + title?: string; + gridData?: { + x: number; + y: number; + w: number; + h: number; + i: string; + }; + panelIndex?: string; + embeddableConfig?: { + attributes?: { + state?: { + visualization?: Record; // eslint-disable-line @typescript-eslint/no-explicit-any + datasourceStates?: { + textBased?: { + layers?: { + [key: string]: { + query?: { esql: string }; + columns?: ColumnInfo[]; + }; + }; + }; + }; + query?: { esql: string }; + }; + }; + }; + [key: string]: any; // eslint-disable-line @typescript-eslint/no-explicit-any +} + +// Process the panel and return the modified panelJSON +export const processPanel = (panel: object, query: string, parsedPanel: ParsedPanel): object => { + // Apply changes to panel - equivalent to Python convertToKibanaDashboard logic for single panel + const panelJSON = { ...panel } as PanelJSON; + + const { columnList, columns } = parseColumns(query); + const vizType = parsedPanel.viz_type; + + // Set panel basic properties + panelJSON.title = parsedPanel.title; + + // Set position from parsed_panel.position + if (parsedPanel.position) { + panelJSON.gridData = { + x: parsedPanel.position.x, + y: parsedPanel.position.y, + w: parsedPanel.position.w, + h: parsedPanel.position.h, + i: parsedPanel.id, + }; + panelJSON.panelIndex = parsedPanel.id; + } + + // Configure visualization-specific properties + configureChartSpecificProperties(panelJSON, vizType, columns); + + if (vizType === 'treemap') { + configureTreemapProperties(panelJSON, columns); + } + + configureStackedProperties(panelJSON, vizType, columns); + configureDatasourceProperties(panelJSON, query, columnList); + + // Handle metric visualization + if (vizType === 'metric') { + if (panelJSON.embeddableConfig?.attributes?.state?.visualization) { + panelJSON.embeddableConfig.attributes.state.visualization.metricAccessor = columns[0]; + } + } + + // Handle gauge visualization + if (vizType === 'gauge') { + configureGaugeProperties(panelJSON, query, columns); + } + + return panelJSON; +}; + +// Parse columns from ESQL query and build column array for panel JSON +function parseColumns(query: string): { columnList: ColumnInfo[]; columns: string[] } { + const columnNames = getQueryColumnsFromESQLQuery(query); + const columnList: ColumnInfo[] = []; + const columns: string[] = []; + + let metricIndex = 0; + + columnNames.forEach((columnName) => { + // For now, assume numeric columns are metrics (first columns) and string columns are dimensions + // This is a simplification - in real implementation you'd need type information + const isNumeric = metricIndex === 0; // First column is typically the metric + + if (isNumeric) { + columnList.splice(metricIndex, 0, { + columnId: columnName, + fieldName: columnName, + meta: { type: 'number' }, + inMetricDimension: true, + }); + columns.splice(metricIndex, 0, columnName); + metricIndex++; + } else { + columnList.push({ + columnId: columnName, + fieldName: columnName, + meta: { type: 'string' }, + }); + columns.push(columnName); + } + }); + + // Ensure at least one column has inMetricDimension if no numeric columns + if (metricIndex === 0 && columnList.length > 0) { + columnList[0].inMetricDimension = true; + } + + return { columnList, columns }; +} + +// Configure chart-specific properties +function configureChartSpecificProperties( + panelJSON: PanelJSON, + vizType: string, + columns: string[] +): void { + const chartTypes = [ + 'bar', + 'bar_vertical', + 'bar_horizontal', + 'bar_vertical_stacked', + 'bar_horizontal_stacked', + 'area', + 'area_stacked', + 'line', + 'heatmap', + ]; + + if (chartTypes.includes(vizType)) { + if (panelJSON.embeddableConfig?.attributes?.state?.visualization?.layers?.[0]) { + panelJSON.embeddableConfig.attributes.state.visualization.layers[0].xAccessor = + columns[columns.length - 1]; + panelJSON.embeddableConfig.attributes.state.visualization.layers[0].accessors = [columns[0]]; + } + } + + if (vizType === 'pie') { + if (panelJSON.embeddableConfig?.attributes?.state?.visualization?.layers?.[0]) { + panelJSON.embeddableConfig.attributes.state.visualization.layers[0].primaryGroups = [ + columns[columns.length - 1], + ]; + panelJSON.embeddableConfig.attributes.state.visualization.layers[0].metrics = [[columns[0]]]; + } + } + + if (vizType === 'table') { + if (panelJSON.embeddableConfig?.attributes?.state?.visualization) { + panelJSON.embeddableConfig.attributes.state.visualization.columns = columns.map((column) => ({ + columnId: column, + })); + } + } + + if (vizType === 'heatmap') { + if (panelJSON.embeddableConfig?.attributes?.state?.visualization) { + panelJSON.embeddableConfig.attributes.state.visualization.valueAccessor = columns[0]; + panelJSON.embeddableConfig.attributes.state.visualization.xAccessor = + columns[columns.length - 1]; + if (columns.length > 1) { + panelJSON.embeddableConfig.attributes.state.visualization.yAccessor = + columns[columns.length - 2]; + } + } + } +} + +// Configure treemap specific properties +function configureTreemapProperties(panelJSON: PanelJSON, columns: string[]): void { + if (panelJSON.embeddableConfig?.attributes?.state?.visualization?.layers?.[0]) { + if (panelJSON.embeddableConfig.attributes.state.visualization.layers[0].metrics) { + panelJSON.embeddableConfig.attributes.state.visualization.layers[0].metrics.push(columns[0]); + } + if (panelJSON.embeddableConfig.attributes.state.visualization.layers[0].primaryGroups) { + for (let i = 1; i < columns.length - 1; i++) { + panelJSON.embeddableConfig.attributes.state.visualization.layers[0].primaryGroups.push( + columns[i] + ); + } + } + } +} + +// Configure stacked chart properties +function configureStackedProperties( + panelJSON: PanelJSON, + vizType: string, + columns: string[] +): void { + if ((vizType.includes('stacked') || vizType.includes('line')) && columns.length > 2) { + if (panelJSON.embeddableConfig?.attributes?.state?.visualization?.layers?.[0]) { + panelJSON.embeddableConfig.attributes.state.visualization.layers[0].splitAccessor = + columns[columns.length - 2]; + } + } + + if (vizType.includes('stacked') && columns.length === 2) { + if (panelJSON.embeddableConfig?.attributes?.state?.visualization?.layers?.[0]) { + panelJSON.embeddableConfig.attributes.state.visualization.layers[0].splitAccessor = + columns[columns.length - 1]; + } + } +} + +// Configure datasource properties +function configureDatasourceProperties( + panelJSON: PanelJSON, + query: string, + columnList: ColumnInfo[] +): void { + if (panelJSON.embeddableConfig?.attributes?.state?.datasourceStates?.textBased?.layers) { + const layerId = '3a5310ab-2832-41db-bdbe-1b6939dd5651'; + if (panelJSON.embeddableConfig.attributes.state.datasourceStates.textBased.layers[layerId]) { + panelJSON.embeddableConfig.attributes.state.datasourceStates.textBased.layers[layerId].query = + { esql: query }; + panelJSON.embeddableConfig.attributes.state.datasourceStates.textBased.layers[ + layerId + ].columns = columnList; + } + } + + if (panelJSON.embeddableConfig?.attributes?.state?.query) { + panelJSON.embeddableConfig.attributes.state.query.esql = query; + } +} + +// Configure gauge visualization +function configureGaugeProperties(panelJSON: PanelJSON, query: string, columns: string[]): void { + const gaugeLayerId = '3b1b0102-bb45-40f5-9ef2-419d2eaaa56c'; + if ( + panelJSON.embeddableConfig?.attributes?.state?.datasourceStates?.textBased?.layers?.[ + gaugeLayerId + ] + ) { + const gaugeLayer = + panelJSON.embeddableConfig.attributes.state.datasourceStates.textBased.layers[gaugeLayerId]; + if (gaugeLayer.columns?.[0]) { + gaugeLayer.columns[0].fieldName = columns[0]; + gaugeLayer.columns[0].columnId = columns[0]; + } + gaugeLayer.query = { esql: query }; + } + if (panelJSON.embeddableConfig?.attributes?.state?.visualization) { + panelJSON.embeddableConfig.attributes.state.visualization.metricAccessor = columns[0]; + } +} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/area.viz.json b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/area.viz.json new file mode 100644 index 0000000000000..b390775b84cfd --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/area.viz.json @@ -0,0 +1,129 @@ +{ + "type": "lens", + "gridData": { + "x": 0, + "y": 0, + "w": 12, + "h": 6, + "i": "dea780b5-eacc-4166-9158-994d596956ab" + }, + "panelIndex": "dea780b5-eacc-4166-9158-994d596956ab", + "embeddableConfig": { + "attributes": { + "title": "", + "references": [ + { + "type": "index-pattern", + "id": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "name": "textBasedLanguages-datasource-layer-3d7f0133-8948-4059-885f-1ecc1feb4a50" + } + ], + "state": { + "datasourceStates": { + "textBased": { + "layers": { + "3a5310ab-2832-41db-bdbe-1b6939dd5651": { + "index": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "query": { + "esql": "" + }, + "columns": [], + "timeField": "@timestamp" + } + }, + "indexPatternRefs": [ + { + "id": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "title": "logs-*", + "timeField": "@timestamp" + } + ] + } + }, + "filters": [], + "query": { + "esql": "" + }, + "visualization": { + "legend": { + "isVisible":true, + "position": "right" + }, + "valueLabels": "hide", + "fittingFunction": "None", + "axisTitlesVisibilitySettings": { + "x":true, + "yLeft":true, + "yRight":true + }, + "tickLabelsVisibilitySettings": { + "x":true, + "yLeft":true, + "yRight":true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "gridlinesVisibilitySettings": { + "x":true, + "yLeft":true, + "yRight":true + }, + "preferredSeriesType": "area", + "layers": [ + { + "layerId": "3a5310ab-2832-41db-bdbe-1b6939dd5651", + "seriesType": "area", + "xAccessor": "day", + "accessors": [ + "count()" + ], + "layerType": "data", + "colorMapping": { + "assignments": [], + "specialAssignments": [ + { + "rule": { + "type": "other" + }, + "color": { + "type": "loop" + }, + "touched":false + } + ], + "paletteId": "eui_amsterdam_color_blind", + "colorMode": { + "type": "categorical" + } + } + } + ] + }, + "adHocDataViews": { + "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da": { + "id": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "title": "logs-*", + "timeFieldName": "@timestamp", + "sourceFilters": [], + "type": "esql", + "fieldFormats": {}, + "runtimeFieldMap": {}, + "allowNoIndex": false, + "name": "logs-*", + "allowHidden": false + } + } + }, + "visualizationType": "lnsXY", + "type": "lens" + }, + "disabledActions": [ + "OPEN_FLYOUT_ADD_DRILLDOWN" + ], + "enhancements": {} + }, + "title": "" +} \ No newline at end of file diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/area_stacked.viz.json b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/area_stacked.viz.json new file mode 100644 index 0000000000000..a5866cf0e0a28 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/area_stacked.viz.json @@ -0,0 +1,130 @@ +{ + "type": "lens", + "gridData": { + "x": 0, + "y": 0, + "w": 12, + "h": 6, + "i": "81d5799f-6666-41b1-afcf-9226c99c6829" + }, + "panelIndex": "81d5799f-6666-41b1-afcf-9226c99c6829", + "embeddableConfig": { + "attributes": { + "title": "", + "references": [ + { + "type": "index-pattern", + "id": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "name": "textBasedLanguages-datasource-layer-58de5f95-1ea0-46cc-9593-273de8f4935b" + } + ], + "state": { + "datasourceStates": { + "textBased": { + "layers": { + "3a5310ab-2832-41db-bdbe-1b6939dd5651": { + "index": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "query": { + "esql": "" + }, + "columns": [], + "timeField": "@timestamp" + } + }, + "indexPatternRefs": [ + { + "id": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "title": "logs-*", + "timeField": "@timestamp" + } + ] + } + }, + "filters": [], + "query": { + "esql": "" + }, + "visualization": { + "legend": { + "isVisible":true, + "position": "right" + }, + "valueLabels": "hide", + "fittingFunction": "None", + "axisTitlesVisibilitySettings": { + "x":true, + "yLeft":true, + "yRight":true + }, + "tickLabelsVisibilitySettings": { + "x":true, + "yLeft":true, + "yRight":true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "gridlinesVisibilitySettings": { + "x":true, + "yLeft":true, + "yRight":true + }, + "preferredSeriesType": "area_stacked", + "layers": [ + { + "layerId": "3a5310ab-2832-41db-bdbe-1b6939dd5651", + "seriesType": "area_stacked", + "xAccessor": "day", + "accessors": [ + "count()", + "names" + ], + "layerType": "data", + "colorMapping": { + "assignments": [], + "specialAssignments": [ + { + "rule": { + "type": "other" + }, + "color": { + "type": "loop" + }, + "touched":false + } + ], + "paletteId": "eui_amsterdam_color_blind", + "colorMode": { + "type": "categorical" + } + } + } + ] + }, + "adHocDataViews": { + "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da": { + "id": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "title": "logs-*", + "timeFieldName": "@timestamp", + "sourceFilters": [], + "type": "esql", + "fieldFormats": {}, + "runtimeFieldMap": {}, + "allowNoIndex": false, + "name": "logs-*", + "allowHidden": false + } + } + }, + "visualizationType": "lnsXY", + "type": "lens" + }, + "disabledActions": [ + "OPEN_FLYOUT_ADD_DRILLDOWN" + ], + "enhancements": {} + }, + "title": "" +} \ No newline at end of file diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/bar_horizontal.viz.json b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/bar_horizontal.viz.json new file mode 100644 index 0000000000000..199b647f8c6b1 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/bar_horizontal.viz.json @@ -0,0 +1,123 @@ +{ + "type": "lens", + "gridData": { + "x": 0, + "y": 0, + "w": 12, + "h": 6, + "i": "46688264-e84d-40e9-8dc3-422d451bd364" + }, + "panelIndex": "46688264-e84d-40e9-8dc3-422d451bd364", + "embeddableConfig": { + "attributes": { + "title": "", + "references": [], + "state": { + "datasourceStates": { + "textBased": { + "layers": { + "3a5310ab-2832-41db-bdbe-1b6939dd5651": { + "index": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "query": { + "esql": "" + }, + "columns": [], + "timeField": "@timestamp" + } + }, + "indexPatternRefs": [ + { + "id": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "title": "logs-*", + "timeField": "@timestamp" + } + ] + } + }, + "filters": [], + "query": { + "esql": "" + }, + "visualization": { + "legend": { + "isVisible":true, + "position": "right" + }, + "valueLabels": "hide", + "fittingFunction": "None", + "axisTitlesVisibilitySettings": { + "x":true, + "yLeft":true, + "yRight":true + }, + "tickLabelsVisibilitySettings": { + "x":true, + "yLeft":true, + "yRight":true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "gridlinesVisibilitySettings": { + "x":true, + "yLeft":true, + "yRight":true + }, + "preferredSeriesType": "bar_stacked", + "layers": [ + { + "layerId": "3a5310ab-2832-41db-bdbe-1b6939dd5651", + "seriesType": "bar_horizontal", + "xAccessor": "agent.name", + "accessors": [ + "count()" + ], + "layerType": "data", + "colorMapping": { + "assignments": [], + "specialAssignments": [ + { + "rule": { + "type": "other" + }, + "color": { + "type": "loop" + }, + "touched":false + } + ], + "paletteId": "eui_amsterdam_color_blind", + "colorMode": { + "type": "categorical" + } + } + } + ] + }, + "adHocDataViews": { + "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da": { + "id": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "title": "logs-*", + "timeFieldName": "@timestamp", + "sourceFilters": [], + "type": "esql", + "fieldFormats": {}, + "runtimeFieldMap": {}, + "allowNoIndex": false, + "name": "logs-*", + "allowHidden": false + } + } + }, + "visualizationType": "lnsXY", + "type": "lens" + }, + "disabledActions": [ + "OPEN_FLYOUT_ADD_DRILLDOWN" + ], + "enhancements": {} + }, + "title": "" +} \ No newline at end of file diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/bar_horizontal_stacked.viz.json b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/bar_horizontal_stacked.viz.json new file mode 100644 index 0000000000000..d920e4641ef5c --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/bar_horizontal_stacked.viz.json @@ -0,0 +1,130 @@ +{ + "type": "lens", + "gridData": { + "x": 0, + "y": 0, + "w": 12, + "h": 6, + "i": "e0e7cfe4-bff9-464a-a753-759dad5e865e" + }, + "panelIndex": "e0e7cfe4-bff9-464a-a753-759dad5e865e", + "embeddableConfig": { + "attributes": { + "title": "", + "references": [ + { + "type": "index-pattern", + "id": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "name": "textBasedLanguages-datasource-layer-67d41b7c-dad3-4209-b4a6-c9ece0aa9d69" + } + ], + "state": { + "datasourceStates": { + "textBased": { + "layers": { + "3a5310ab-2832-41db-bdbe-1b6939dd5651": { + "index": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "query": { + "esql": "" + }, + "columns": [], + "timeField": "@timestamp" + } + }, + "indexPatternRefs": [ + { + "id": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "title": "logs-*", + "timeField": "@timestamp" + } + ] + } + }, + "filters": [], + "query": { + "esql": "" + }, + "visualization": { + "legend": { + "isVisible":true, + "position": "right" + }, + "valueLabels": "hide", + "fittingFunction": "None", + "axisTitlesVisibilitySettings": { + "x":true, + "yLeft":true, + "yRight":true + }, + "tickLabelsVisibilitySettings": { + "x":true, + "yLeft":true, + "yRight":true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "gridlinesVisibilitySettings": { + "x":true, + "yLeft":true, + "yRight":true + }, + "preferredSeriesType": "bar_horizontal_stacked", + "layers": [ + { + "layerId": "3a5310ab-2832-41db-bdbe-1b6939dd5651", + "seriesType": "bar_horizontal_stacked", + "xAccessor": "agent.name", + "splitAccessor": "agent.name", + "accessors": [ + "count()" + ], + "layerType": "data", + "colorMapping": { + "assignments": [], + "specialAssignments": [ + { + "rule": { + "type": "other" + }, + "color": { + "type": "loop" + }, + "touched":false + } + ], + "paletteId": "eui_amsterdam_color_blind", + "colorMode": { + "type": "categorical" + } + } + } + ] + }, + "adHocDataViews": { + "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da": { + "id": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "title": "logs-*", + "timeFieldName": "@timestamp", + "sourceFilters": [], + "type": "esql", + "fieldFormats": {}, + "runtimeFieldMap": {}, + "allowNoIndex": false, + "name": "logs-*", + "allowHidden": false + } + } + }, + "visualizationType": "lnsXY", + "type": "lens" + }, + "disabledActions": [ + "OPEN_FLYOUT_ADD_DRILLDOWN" + ], + "enhancements": {} + }, + "title": "" +} \ No newline at end of file diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/bar_vertical.viz.json b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/bar_vertical.viz.json new file mode 100644 index 0000000000000..59a6f0559bbf2 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/bar_vertical.viz.json @@ -0,0 +1,129 @@ +{ + "type": "lens", + "gridData": { + "x": 0, + "y": 0, + "w": 12, + "h": 6, + "i": "da3460eb-dcbe-4da9-b675-320cd9cc17e4" + }, + "panelIndex": "da3460eb-dcbe-4da9-b675-320cd9cc17e4", + "embeddableConfig": { + "attributes": { + "title": "", + "references": [ + { + "type": "index-pattern", + "id": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "name": "textBasedLanguages-datasource-layer-7fb5abc5-850c-4ed2-bf42-815ec57524fe" + } + ], + "state": { + "datasourceStates": { + "textBased": { + "layers": { + "3a5310ab-2832-41db-bdbe-1b6939dd5651": { + "index": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "query": { + "esql": "" + }, + "columns": [], + "timeField": "@timestamp" + } + }, + "indexPatternRefs": [ + { + "id": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "title": "logs-*", + "timeField": "@timestamp" + } + ] + } + }, + "filters": [], + "query": { + "esql": "" + }, + "visualization": { + "legend": { + "isVisible":true, + "position": "right" + }, + "valueLabels": "hide", + "fittingFunction": "None", + "axisTitlesVisibilitySettings": { + "x":true, + "yLeft":true, + "yRight":true + }, + "tickLabelsVisibilitySettings": { + "x":true, + "yLeft":true, + "yRight":true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "gridlinesVisibilitySettings": { + "x":true, + "yLeft":true, + "yRight":true + }, + "preferredSeriesType": "bar", + "layers": [ + { + "layerId": "3a5310ab-2832-41db-bdbe-1b6939dd5651", + "seriesType": "bar", + "xAccessor": "agent.name", + "accessors": [ + "count()" + ], + "layerType": "data", + "colorMapping": { + "assignments": [], + "specialAssignments": [ + { + "rule": { + "type": "other" + }, + "color": { + "type": "loop" + }, + "touched":false + } + ], + "paletteId": "eui_amsterdam_color_blind", + "colorMode": { + "type": "categorical" + } + } + } + ] + }, + "adHocDataViews": { + "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da": { + "id": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "title": "logs-*", + "timeFieldName": "@timestamp", + "sourceFilters": [], + "type": "esql", + "fieldFormats": {}, + "runtimeFieldMap": {}, + "allowNoIndex": false, + "name": "logs-*", + "allowHidden": false + } + } + }, + "visualizationType": "lnsXY", + "type": "lens" + }, + "disabledActions": [ + "OPEN_FLYOUT_ADD_DRILLDOWN" + ], + "enhancements": {} + }, + "title": "" +} \ No newline at end of file diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/bar_vertical_stacked.viz.json b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/bar_vertical_stacked.viz.json new file mode 100644 index 0000000000000..60fdf2f2a43ba --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/bar_vertical_stacked.viz.json @@ -0,0 +1,130 @@ +{ + "type": "lens", + "gridData": { + "x": 0, + "y": 0, + "w": 12, + "h": 6, + "i": "f16cf895-b9fc-4bd5-a065-04bc010a798c" + }, + "panelIndex": "f16cf895-b9fc-4bd5-a065-04bc010a798c", + "embeddableConfig": { + "attributes": { + "title": "", + "references": [ + { + "type": "index-pattern", + "id": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "name": "textBasedLanguages-datasource-layer-52a01fd3-4fd6-47dc-b748-104390a59003" + } + ], + "state": { + "datasourceStates": { + "textBased": { + "layers": { + "3a5310ab-2832-41db-bdbe-1b6939dd5651": { + "index": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "query": { + "esql": "" + }, + "columns": [], + "timeField": "@timestamp" + } + }, + "indexPatternRefs": [ + { + "id": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "title": "logs-*", + "timeField": "@timestamp" + } + ] + } + }, + "filters": [], + "query": { + "esql": "" + }, + "visualization": { + "legend": { + "isVisible":true, + "position": "right" + }, + "valueLabels": "hide", + "fittingFunction": "None", + "axisTitlesVisibilitySettings": { + "x":true, + "yLeft":true, + "yRight":true + }, + "tickLabelsVisibilitySettings": { + "x":true, + "yLeft":true, + "yRight":true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "gridlinesVisibilitySettings": { + "x":true, + "yLeft":true, + "yRight":true + }, + "preferredSeriesType": "bar_stacked", + "layers": [ + { + "layerId": "3a5310ab-2832-41db-bdbe-1b6939dd5651", + "seriesType": "bar_stacked", + "xAccessor": "agent.name", + "splitAccessor": "agent.name", + "accessors": [ + "count()" + ], + "layerType": "data", + "colorMapping": { + "assignments": [], + "specialAssignments": [ + { + "rule": { + "type": "other" + }, + "color": { + "type": "loop" + }, + "touched":false + } + ], + "paletteId": "eui_amsterdam_color_blind", + "colorMode": { + "type": "categorical" + } + } + } + ] + }, + "adHocDataViews": { + "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da": { + "id": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "title": "logs-*", + "timeFieldName": "@timestamp", + "sourceFilters": [], + "type": "esql", + "fieldFormats": {}, + "runtimeFieldMap": {}, + "allowNoIndex": false, + "name": "logs-*", + "allowHidden": false + } + } + }, + "visualizationType": "lnsXY", + "type": "lens" + }, + "disabledActions": [ + "OPEN_FLYOUT_ADD_DRILLDOWN" + ], + "enhancements": {} + }, + "title": "" +} \ No newline at end of file diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/donut.viz.json b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/donut.viz.json new file mode 100644 index 0000000000000..3edd74dbc3ee7 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/donut.viz.json @@ -0,0 +1,108 @@ +{ + "type": "lens", + "gridData": { + "x": 0, + "y": 0, + "w": 12, + "h": 6, + "i": "c927fb91-0322-4e90-9714-65b0e3e6a073" + }, + "panelIndex": "c927fb91-0322-4e90-9714-65b0e3e6a073", + "embeddableConfig": { + "attributes": { + "title": "", + "references": [ + { + "type": "index-pattern", + "id": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "name": "textBasedLanguages-datasource-layer-69a4b241-c0ac-4834-8c2b-618d3faf53f1" + } + ], + "state": { + "datasourceStates": { + "textBased": { + "layers": { + "3a5310ab-2832-41db-bdbe-1b6939dd5651": { + "index": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "query": { + "esql": "" + }, + "columns": [], + "timeField": "@timestamp" + } + }, + "indexPatternRefs": [ + { + "id": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "title": "logs-*", + "timeField": "@timestamp" + } + ] + } + }, + "filters": [], + "query": { + "esql": "" + }, + "visualization": { + "shape": "donut", + "layers": [ + { + "layerId": "3a5310ab-2832-41db-bdbe-1b6939dd5651", + "primaryGroups": [ + "agent.name" + ], + "metrics": [ + "count()" + ], + "numberDisplay": "percent", + "categoryDisplay": "default", + "legendDisplay": "default", + "nestedLegend":false, + "layerType": "data", + "colorMapping": { + "assignments": [], + "specialAssignments": [ + { + "rule": { + "type": "other" + }, + "color": { + "type": "loop" + }, + "touched":false + } + ], + "paletteId": "eui_amsterdam_color_blind", + "colorMode": { + "type": "categorical" + } + } + } + ] + }, + "adHocDataViews": { + "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da": { + "id": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "title": "logs-*", + "timeFieldName": "@timestamp", + "sourceFilters": [], + "type": "esql", + "fieldFormats": {}, + "runtimeFieldMap": {}, + "allowNoIndex": false, + "name": "logs-*", + "allowHidden": false + } + } + }, + "visualizationType": "lnsPie", + "type": "lens" + }, + "disabledActions": [ + "OPEN_FLYOUT_ADD_DRILLDOWN" + ], + "enhancements": {} + }, + "title": "" +} \ No newline at end of file diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/gauge.viz.json b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/gauge.viz.json new file mode 100644 index 0000000000000..83995cc5f39d9 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/gauge.viz.json @@ -0,0 +1,181 @@ +{ + "type": "lens", + "title": "Requests successful", + "embeddableConfig": + { + "attributes": + { + "title": "Table ", + "references": + [], + "state": + { + "datasourceStates": + { + "textBased": + { + "layers": + { + "3a5310ab-2832-41db-bdbe-1b6939dd5651": + { + "index": "047b9ce1c481e9105458e4238be7cbb304abc176b09c3b4d196d84686c42b5d0", + "query": + { + "esql": "FROM logs-nginx\\n| WHERE event.dataset == 'nginx.access' AND event.outcome == 'success'\\n| STATS a = COUNT(*)" + }, + "columns": + [ + { + "columnId": "a", + "fieldName": "a", + "meta": + { + "type": "number" + }, + "inMetricDimension": true + } + ], + "timeField": "@timestamp" + }, + "3b1b0102-bb45-40f5-9ef2-419d2eaaa56c": + { + "index": "047b9ce1c481e9105458e4238be7cbb304abc176b09c3b4d196d84686c42b5d0", + "query": + { + "esql": "FROM logs-nginx\\n| WHERE event.dataset == 'nginx.access' AND event.outcome == 'success'\\n| STATS a = COUNT(*)" + }, + "columns": + [ + { + "columnId": "aa9a832b-d4bf-4652-b2eb-7cbebb4bbb13", + "fieldName": "a", + "meta": + { + "type": "number", + "esType": "long" + } + } + ] + } + }, + "indexPatternRefs": + [ + { + "id": "047b9ce1c481e9105458e4238be7cbb304abc176b09c3b4d196d84686c42b5d0", + "title": "logs*", + "timeField": "@timestamp" + } + ] + } + }, + "filters": + [], + "query": + { + "esql": "FROM logs-nginx\\n| WHERE event.dataset == 'nginx.access' AND event.outcome == 'success'\\n| STATS a = COUNT(*)" + }, + "visualization": + { + "shape": "semiCircle", + "layerId": "3a5310ab-2832-41db-bdbe-1b6939dd5651", + "layerType": "data", + "ticksPosition": "bands", + "labelMajorMode": "auto", + "metricAccessor": "aa9a832b-d4bf-4652-b2eb-7cbebb4bbb13", + "palette": + { + "name": "custom", + "type": "palette", + "params": + { + "steps": 4, + "name": "custom", + "reverse": false, + "rangeType": "percent", + "rangeMin": null, + "rangeMax": null, + "progression": "fixed", + "stops": + [ + { + "color": "#d7060680", + "stop": 25 + }, + { + "color": "#f8e72e80", + "stop": 50 + }, + { + "color": "#baf50a80", + "stop": 75 + }, + { + "color": "#02bc0c80", + "stop": 100 + } + ], + "colorStops": + [ + { + "color": "#d7060680", + "stop": null + }, + { + "color": "#f8e72e80", + "stop": 25 + }, + { + "color": "#baf50a80", + "stop": 50 + }, + { + "color": "#02bc0c80", + "stop": 75 + } + ], + "continuity": "all", + "maxSteps": 5 + } + }, + "colorMode": "palette" + }, + "adHocDataViews": + { + "047b9ce1c481e9105458e4238be7cbb304abc176b09c3b4d196d84686c42b5d0": + { + "id": "047b9ce1c481e9105458e4238be7cbb304abc176b09c3b4d196d84686c42b5d0", + "title": "logs*", + "timeFieldName": "@timestamp", + "sourceFilters": + [], + "type": "esql", + "fieldFormats": + {}, + "runtimeFieldMap": + {}, + "allowNoIndex": false, + "name": "logs*", + "allowHidden": false + } + } + }, + "visualizationType": "lnsGauge", + "type": "lens" + }, + "disabledActions": + [ + "OPEN_FLYOUT_ADD_DRILLDOWN" + ], + "enhancements": + {} + }, + "panelIndex": "2757c30b-a233-4cce-9db0-f56313513e9f", + "gridData": + { + "x": 0, + "y": 48, + "w": 24, + "h": 16, + "i": "2757c30b-a233-4cce-9db0-f56313513e9f" + } + } \ No newline at end of file diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/heatmap.viz.json b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/heatmap.viz.json new file mode 100644 index 0000000000000..59e876a414001 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/heatmap.viz.json @@ -0,0 +1,92 @@ +{ + "type": "lens", + "gridData": { + "x": 0, + "y": 0, + "w": 12, + "h": 6, + "i": "07fdc318-57e3-4e3a-916e-1b949c047e15" + }, + "panelIndex": "07fdc318-57e3-4e3a-916e-1b949c047e15", + "embeddableConfig": { + "attributes": { + "title": "", + "references": [ + { + "type": "index-pattern", + "id": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "name": "textBasedLanguages-datasource-layer-413744a0-350a-42a5-bff3-2b6d0ebc808c" + } + ], + "state": { + "datasourceStates": { + "textBased": { + "layers": { + "3a5310ab-2832-41db-bdbe-1b6939dd5651": { + "index": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "query": { + "esql": "" + }, + "columns": [], + "timeField": "@timestamp" + } + }, + "indexPatternRefs": [ + { + "id": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "title": "logs-*", + "timeField": "@timestamp" + } + ] + } + }, + "filters": [], + "query": { + "esql": "" + }, + "visualization": { + "shape": "heatmap", + "layerId": "3a5310ab-2832-41db-bdbe-1b6939dd5651", + "layerType": "data", + "legend": { + "isVisible":true, + "position": "right", + "type": "heatmap_legend" + }, + "gridConfig": { + "type": "heatmap_grid", + "isCellLabelVisible":false, + "isYAxisLabelVisible":true, + "isXAxisLabelVisible":true, + "isYAxisTitleVisible":false, + "isXAxisTitleVisible":false + }, + "valueAccessor": "count()", + "xAccessor": "agent.name", + "yAccessor": "agent.id" + }, + "adHocDataViews": { + "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da": { + "id": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "title": "logs-*", + "timeFieldName": "@timestamp", + "sourceFilters": [], + "type": "esql", + "fieldFormats": {}, + "runtimeFieldMap": {}, + "allowNoIndex": false, + "name": "logs-*", + "allowHidden": false + } + } + }, + "visualizationType": "lnsHeatmap", + "type": "lens" + }, + "disabledActions": [ + "OPEN_FLYOUT_ADD_DRILLDOWN" + ], + "enhancements": {} + }, + "title": "" +} \ No newline at end of file diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/line.viz.json b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/line.viz.json new file mode 100644 index 0000000000000..794e2dd4fa344 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/line.viz.json @@ -0,0 +1,129 @@ +{ + "type": "lens", + "gridData": { + "x": 0, + "y": 0, + "w": 12, + "h": 6, + "i": "de95d316-320e-42e6-a71a-24dfdb4d9a10" + }, + "panelIndex": "de95d316-320e-42e6-a71a-24dfdb4d9a10", + "embeddableConfig": { + "attributes": { + "title": "", + "references": [ + { + "type": "index-pattern", + "id": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "name": "textBasedLanguages-datasource-layer-db7c0a39-c039-4f8a-9029-e418c52ce62b" + } + ], + "state": { + "datasourceStates": { + "textBased": { + "layers": { + "3a5310ab-2832-41db-bdbe-1b6939dd5651": { + "index": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "query": { + "esql": "" + }, + "columns": [], + "timeField": "@timestamp" + } + }, + "indexPatternRefs": [ + { + "id": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "title": "logs-*", + "timeField": "@timestamp" + } + ] + } + }, + "filters": [], + "query": { + "esql": "" + }, + "visualization": { + "legend": { + "isVisible":true, + "position": "right" + }, + "valueLabels": "hide", + "fittingFunction": "None", + "axisTitlesVisibilitySettings": { + "x":true, + "yLeft":true, + "yRight":true + }, + "tickLabelsVisibilitySettings": { + "x":true, + "yLeft":true, + "yRight":true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "gridlinesVisibilitySettings": { + "x":true, + "yLeft":true, + "yRight":true + }, + "preferredSeriesType": "line", + "layers": [ + { + "layerId": "3a5310ab-2832-41db-bdbe-1b6939dd5651", + "seriesType": "line", + "xAccessor": "day", + "accessors": [ + "count()" + ], + "layerType": "data", + "colorMapping": { + "assignments": [], + "specialAssignments": [ + { + "rule": { + "type": "other" + }, + "color": { + "type": "loop" + }, + "touched":false + } + ], + "paletteId": "eui_amsterdam_color_blind", + "colorMode": { + "type": "categorical" + } + } + } + ] + }, + "adHocDataViews": { + "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da": { + "id": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "title": "logs-*", + "timeFieldName": "@timestamp", + "sourceFilters": [], + "type": "esql", + "fieldFormats": {}, + "runtimeFieldMap": {}, + "allowNoIndex": false, + "name": "logs-*", + "allowHidden": false + } + } + }, + "visualizationType": "lnsXY", + "type": "lens" + }, + "disabledActions": [ + "OPEN_FLYOUT_ADD_DRILLDOWN" + ], + "enhancements": {} + }, + "title": "" +} \ No newline at end of file diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/markdown.viz.json b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/markdown.viz.json new file mode 100644 index 0000000000000..fd6ba07840328 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/markdown.viz.json @@ -0,0 +1,38 @@ +{ + "type": "visualization", + "gridData": { + "x": 0, + "y": 0, + "w": 12, + "h": 6, + "i": "0735694d-1138-4e11-a315b20" + }, + "panelIndex": "0735694d-1138-4e11-a315-29c0dfdd3b20", + "embeddableConfig": { + "savedVis": { + "id": "", + "title": "", + "description": "", + "type": "markdown", + "params": { + "fontSize": 12, + "openLinksInNewTab": false, + "markdown": "Sample text" + }, + "uiState": {}, + "data": { + "aggs": [], + "searchSource": { + "query": { + "query": "", + "language": "kuery" + }, + "filter": [] + } + } + }, + "description": "", + "enhancements": {} + }, + "title": "Markdown" +} \ No newline at end of file diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/metric.viz.json b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/metric.viz.json new file mode 100644 index 0000000000000..ed0c543743d35 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/metric.viz.json @@ -0,0 +1,69 @@ +{ + "type": "lens", + "gridData": { + "x": 0, + "y": 0, + "w": 12, + "h": 6, + "i": "bfc47c01-6421-428f-85fc-40b7d125901b" + }, + "panelIndex": "bfc47c01-6421-428f-85fc-40b7d125901b", + "embeddableConfig": { + "attributes": { + "title": "", + "references": [], + "state": { + "datasourceStates": { + "textBased": { + "layers": { + "3a5310ab-2832-41db-bdbe-1b6939dd5651": { + "index": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "query": { + "esql": "" + }, + "columns": [], + "timeField": "@timestamp" + } + }, + "indexPatternRefs": [ + { + "id": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "title": "logs-*", + "timeField": "@timestamp" + } + ] + } + }, + "filters": [], + "query": { + "esql": "" + }, + "visualization": { + "layerId": "3a5310ab-2832-41db-bdbe-1b6939dd5651", + "layerType": "data", + "metricAccessor": "count()" + }, + "adHocDataViews": { + "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da": { + "id": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "title": "logs-*", + "timeFieldName": "@timestamp", + "sourceFilters": [], + "type": "esql", + "fieldFormats": {}, + "runtimeFieldMap": {}, + "allowNoIndex": false, + "name": "logs-*", + "allowHidden": false + } + } + }, + "visualizationType": "lnsMetric", + "type": "lens" + }, + "disabledActions": [ + "OPEN_FLYOUT_ADD_DRILLDOWN" + ], + "enhancements": {} + } +} \ No newline at end of file diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/pie.viz.json b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/pie.viz.json new file mode 100644 index 0000000000000..b02f1a0c19a0b --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/pie.viz.json @@ -0,0 +1,104 @@ +{ + "type": "lens", + "gridData": { + "x": 0, + "y": 0, + "w": 12, + "h": 6, + "i": "1f6c3f5b-0869-4a6f-ad27-62b77ccf43f5" + }, + "panelIndex": "1f6c3f5b-0869-4a6f-ad27-62b77ccf43f5", + "embeddableConfig": { + "attributes": { + "title": "", + "references": [ + { + "type": "index-pattern", + "id": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "name": "textBasedLanguages-datasource-layer-69a4b241-c0ac-4834-8c2b-618d3faf53f1" + } + ], + "state": { + "datasourceStates": { + "textBased": { + "layers": { + "3a5310ab-2832-41db-bdbe-1b6939dd5651": { + "index": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "query": { + "esql": "" + }, + "columns": [], + "timeField": "@timestamp" + } + }, + "indexPatternRefs": [ + { + "id": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "title": "logs-*", + "timeField": "@timestamp" + } + ] + } + }, + "filters": [], + "query": { + "esql": "" + }, + "visualization": { + "shape": "pie", + "layers": [ + { + "layerId": "3a5310ab-2832-41db-bdbe-1b6939dd5651", + "primaryGroups": [], + "metrics": [], + "numberDisplay": "percent", + "categoryDisplay": "default", + "legendDisplay": "default", + "nestedLegend":false, + "layerType": "data", + "colorMapping": { + "assignments": [], + "specialAssignments": [ + { + "rule": { + "type": "other" + }, + "color": { + "type": "loop" + }, + "touched":false + } + ], + "paletteId": "eui_amsterdam_color_blind", + "colorMode": { + "type": "categorical" + } + } + } + ] + }, + "adHocDataViews": { + "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da": { + "id": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "title": "logs-*", + "timeFieldName": "@timestamp", + "sourceFilters": [], + "type": "esql", + "fieldFormats": {}, + "runtimeFieldMap": {}, + "allowNoIndex": false, + "name": "logs-*", + "allowHidden": false + } + } + }, + "visualizationType": "lnsPie", + "type": "lens" + }, + "disabledActions": [ + "OPEN_FLYOUT_ADD_DRILLDOWN" + ], + "enhancements": {} + }, + "title": "" +} \ No newline at end of file diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/table.viz.json b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/table.viz.json new file mode 100644 index 0000000000000..685a2afc03ac4 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/table.viz.json @@ -0,0 +1,76 @@ +{ + "type": "lens", + "gridData": { + "x": 0, + "y": 0, + "w": 12, + "h": 6, + "i": "6e0b498e-a8f5-4b5e-9307-9f6cc505e21c" + }, + "panelIndex": "6e0b498e-a8f5-4b5e-9307-9f6cc505e21c", + "embeddableConfig": { + "attributes": { + "title": "", + "references": [ + { + "type": "index-pattern", + "id": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "name": "textBasedLanguages-datasource-layer-3a5310ab-2832-41db-bdbe-1b6939dd5651" + } + ], + "state": { + "datasourceStates": { + "textBased": { + "layers": { + "3a5310ab-2832-41db-bdbe-1b6939dd5651": { + "index": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "query": { + "esql": "" + }, + "columns": [], + "timeField": "@timestamp" + } + }, + "indexPatternRefs": [ + { + "id": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "title": "logs-*", + "timeField": "@timestamp" + } + ] + } + }, + "filters": [], + "query": { + "esql": "" + }, + "visualization": { + "layerId": "3a5310ab-2832-41db-bdbe-1b6939dd5651", + "layerType": "data", + "columns": [] + }, + "adHocDataViews": { + "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da": { + "id": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "title": "logs-*", + "timeFieldName": "@timestamp", + "sourceFilters": [], + "type": "esql", + "fieldFormats": {}, + "runtimeFieldMap": {}, + "allowNoIndex":false, + "name": "logs-*", + "allowHidden":false + } + } + }, + "visualizationType": "lnsDatatable", + "type": "lens" + }, + "disabledActions": [ + "OPEN_FLYOUT_ADD_DRILLDOWN" + ], + "enhancements": {} + }, + "title": "" +} \ No newline at end of file diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/treemap.viz.json b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/treemap.viz.json new file mode 100644 index 0000000000000..26e1eb58e9c7c --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/templates/treemap.viz.json @@ -0,0 +1,108 @@ +{ + "type": "lens", + "gridData": { + "x": 0, + "y": 0, + "w": 12, + "h": 6, + "i": "2bad2a0b-9581-4a29-8cb0-c4e713bab912" + }, + "panelIndex": "2bad2a0b-9581-4a29-8cb0-c4e713bab912", + "embeddableConfig": { + "attributes": { + "title": "", + "references": [ + { + "type": "index-pattern", + "id": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "name": "textBasedLanguages-datasource-layer-69d8f699-0603-479a-b4f6-712008393869" + } + ], + "state": { + "datasourceStates": { + "textBased": { + "layers": { + "3a5310ab-2832-41db-bdbe-1b6939dd5651": { + "index": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "query": { + "esql": "" + }, + "columns": [], + "timeField": "@timestamp" + } + }, + "indexPatternRefs": [ + { + "id": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "title": "logs-*", + "timeField": "@timestamp" + } + ] + } + }, + "filters": [], + "query": { + "esql": "" + }, + "visualization": { + "shape": "treemap", + "layers": [ + { + "layerId": "3a5310ab-2832-41db-bdbe-1b6939dd5651", + "primaryGroups": [ + "agent.name" + ], + "metrics": [ + "count()" + ], + "numberDisplay": "percent", + "categoryDisplay": "default", + "legendDisplay": "default", + "nestedLegend":false, + "layerType": "data", + "colorMapping": { + "assignments": [], + "specialAssignments": [ + { + "rule": { + "type": "other" + }, + "color": { + "type": "loop" + }, + "touched":false + } + ], + "paletteId": "eui_amsterdam_color_blind", + "colorMode": { + "type": "categorical" + } + } + } + ] + }, + "adHocDataViews": { + "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da": { + "id": "d6c911aa50e2dafcebcedfa50190d5c03bbac0070c96d6b16feb18aa1f84b5da", + "title": "logs-*", + "timeFieldName": "@timestamp", + "sourceFilters": [], + "type": "esql", + "fieldFormats": {}, + "runtimeFieldMap": {}, + "allowNoIndex": false, + "name": "logs-*", + "allowHidden": false + } + } + }, + "visualizationType": "lnsPie", + "type": "lens" + }, + "disabledActions": [ + "OPEN_FLYOUT_ADD_DRILLDOWN" + ], + "enhancements": {} + }, + "title": "" +} \ No newline at end of file diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/translation_result.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/translation_result.ts new file mode 100644 index 0000000000000..35884e7a009cc --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/translation_result/translation_result.ts @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import fs from 'fs'; +import path from 'path'; +import { MigrationTranslationResult } from '../../../../../../../../../../common/siem_migrations/constants'; +import type { GraphNode } from '../../types'; +import { processPanel } from './process_panel'; + +export const getTranslationResultNode = (): GraphNode => { + return async (state) => { + const query = state.esql_query; + if (!query) { + return { translation_result: MigrationTranslationResult.UNTRANSLATABLE }; + } + + let translationResult; + // TODO: use placeholder constant + if (query.startsWith('FROM [indexPattern]')) { + translationResult = MigrationTranslationResult.PARTIAL; + } else if (state.validation_errors?.esql_errors) { + translationResult = MigrationTranslationResult.PARTIAL; + } else if (query.match(/\[(macro|lookup):.*?\]/)) { + translationResult = MigrationTranslationResult.PARTIAL; + } else { + translationResult = MigrationTranslationResult.FULL; + } + + const vizType = state.parsed_panel?.viz_type; + let panel: object; + try { + if (!vizType) { + throw new Error('Panel visualization type could not be extracted'); + } + + const templatePath = path.join(__dirname, `./templates/${vizType}.viz.json`); + const template = fs.readFileSync(templatePath, 'utf-8'); + + if (!template) { + throw new Error(`Template not found for visualization type: ${vizType}`); + } + panel = JSON.parse(template); + } catch (error) { + // TODO: log the error + return { + // TODO: add comment: "panel chart type not supported" + translation_result: MigrationTranslationResult.UNTRANSLATABLE, + }; + } + + const panelJSON = processPanel(panel, query, state.parsed_panel); + + return { + elastic_panel: panelJSON, + translation_result: translationResult, + }; + }; +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/validation/index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/validation/index.ts new file mode 100644 index 0000000000000..a8c0f55191e42 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/validation/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +export { getValidationNode } from './validation'; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/validation/validation.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/validation/validation.ts new file mode 100644 index 0000000000000..46332bda73971 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/nodes/validation/validation.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + getValidateEsql, + type GetValidateEsqlParams, +} from '../../../../../../../common/task/agent/tools/validate_esql/validation'; +import type { GraphNode } from '../../types'; + +/** + * This node runs all validation steps, and will redirect to the END of the graph if no errors are found. + * Any new validation steps should be added here. + */ +export const getValidationNode = (params: GetValidateEsqlParams): GraphNode => { + const validateEsql = getValidateEsql(params); + return async (state) => { + const iterations = state.validation_errors.iterations + 1; + if (!state.esql_query) { + params.logger.warn('Missing query in validation node'); + return { iterations }; + } + + const { error } = await validateEsql({ query: state.esql_query }); + + return { validation_errors: { iterations, esql_errors: error } }; + }; +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/state.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/state.ts new file mode 100644 index 0000000000000..7ec982823b507 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/state.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Annotation } from '@langchain/langgraph'; +import { MigrationTranslationResult } from '../../../../../../../../common/siem_migrations/constants'; +import type { DashboardMigrationDashboard } from '../../../../../../../../common/siem_migrations/model/dashboard_migration.gen'; +import type { MigrationResources } from '../../../../../common/task/retrievers/resource_retriever'; +import type { ValidationErrors } from './types'; +import type { ParsedPanel } from '../../../../lib/parsers/types'; + +export const translateDashboardPanelState = Annotation.Root({ + parsed_panel: Annotation(), + elastic_panel: Annotation(), // The visualization panel object + index_pattern: Annotation(), + resources: Annotation(), + includes_ecs_mapping: Annotation({ + reducer: (current, value) => value ?? current, + default: () => false, + }), + inline_query: Annotation(), + description: Annotation(), + esql_query: Annotation(), + validation_errors: Annotation({ + reducer: (current, value) => value ?? current, + default: () => ({ iterations: 0 }), + }), + translation_result: Annotation({ + reducer: (current, value) => value ?? current, + default: () => MigrationTranslationResult.UNTRANSLATABLE, + }), + comments: Annotation({ + reducer: (current, value) => (value ? (current ?? []).concat(value) : current), + default: () => [], + }), +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/types.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/types.ts new file mode 100644 index 0000000000000..0149c5d8ca2c6 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/sub_graphs/translate_panel/types.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger, IScopedClusterClient } from '@kbn/core/server'; +import type { RunnableConfig } from '@langchain/core/runnables'; +import type { ChatModel } from '../../../../../common/task/util/actions_client_chat'; +import type { EsqlKnowledgeBase } from '../../../../../common/task/util/esql_knowledge_base'; +import type { DashboardMigrationsRetriever } from '../../../retrievers'; +import type { DashboardMigrationTelemetryClient } from '../../../dashboard_migrations_telemetry_client'; +import type { translateDashboardPanelState } from './state'; +import type { migrateDashboardConfigSchema } from '../../state'; + +export type TranslateDashboardPanelState = typeof translateDashboardPanelState.State; +export type TranslateDashboardPanelGraphConfig = RunnableConfig< + (typeof migrateDashboardConfigSchema)['State'] +>; +export type GraphNode = ( + state: TranslateDashboardPanelState, + config: TranslateDashboardPanelGraphConfig +) => Promise>; + +export interface TranslatePanelGraphParams { + model: ChatModel; + esScopedClient: IScopedClusterClient; + esqlKnowledgeBase: EsqlKnowledgeBase; + dashboardMigrationsRetriever: DashboardMigrationsRetriever; + telemetryClient: DashboardMigrationTelemetryClient; + logger: Logger; +} + +export interface ValidationErrors { + iterations: number; + esql_errors?: string; +} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/types.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/types.ts new file mode 100644 index 0000000000000..b38a4ecb73e72 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/agent/types.ts @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger, IScopedClusterClient } from '@kbn/core/server'; +import type { RunnableConfig } from '@langchain/core/runnables'; +import type { MigrationTranslationResult } from '../../../../../../common/siem_migrations/constants'; +import type { DashboardMigrationsRetriever } from '../retrievers'; +import type { EsqlKnowledgeBase } from '../../../common/task/util/esql_knowledge_base'; +import type { ChatModel } from '../../../common/task/util/actions_client_chat'; +import type { migrateDashboardConfigSchema, migrateDashboardState } from './state'; +import type { DashboardMigrationTelemetryClient } from '../dashboard_migrations_telemetry_client'; +import type { ParsedPanel } from '../../lib/parsers/types'; + +export type MigrateDashboardState = typeof migrateDashboardState.State; +export type MigrateDashboardConfigSchema = (typeof migrateDashboardConfigSchema)['State']; +export type MigrateDashboardConfig = RunnableConfig; + +export type GraphNode = ( + state: MigrateDashboardState, + config: MigrateDashboardConfig +) => Promise>; + +export interface DashboardMigrationAgentRunOptions { + skipPrebuiltDashboardsMatching: boolean; +} + +export interface MigrateDashboardGraphParams { + model: ChatModel; + esScopedClient: IScopedClusterClient; + esqlKnowledgeBase: EsqlKnowledgeBase; + dashboardMigrationsRetriever: DashboardMigrationsRetriever; + logger: Logger; + telemetryClient: DashboardMigrationTelemetryClient; +} + +export interface ParsedOriginalDashboard { + title: string; + panels: Array; +} + +export type TranslatedPanels = Array<{ + /** + * The index in the panels array, to keep the same order as in the original dashboard. + * this is probably not necessary since we have already calculated the `position` of each panel, but maintained for consistency + */ + index: number; + /* The visualization json */ + data: object; + /* The individual panel translation result */ + translation_result: MigrationTranslationResult; +}>; + +export type FailedPanelTranslations = Array<{ + index: number; + error_message: string; + details: unknown; +}>; + +export interface TranslatePanelNodeParams { + panel: ParsedPanel; + index: number; +} +export type TranslatePanelNode = ( + params: TranslatePanelNodeParams +) => Promise>; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/dashboard_migrations_task_client.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/dashboard_migrations_task_client.test.ts new file mode 100644 index 0000000000000..cd83a985a1aff --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/dashboard_migrations_task_client.test.ts @@ -0,0 +1,515 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { AuthenticatedUser } from '@kbn/core/server'; +import type { DashboardMigrationsRunning } from './dashboard_migrations_task_client'; +import { DashboardMigrationsTaskClient } from './dashboard_migrations_task_client'; +import { + SiemMigrationStatus, + SiemMigrationTaskStatus, +} from '../../../../../common/siem_migrations/constants'; +import { DashboardMigrationTaskRunner } from './dashboard_migrations_task_runner'; +import type { MockedLogger } from '@kbn/logging-mocks'; +import { loggerMock } from '@kbn/logging-mocks'; +import type { StoredDashboardMigration } from '../types'; +import type { DashboardMigrationTaskStartParams } from './types'; +import { createDashboardMigrationsDataClientMock } from '../data/__mocks__/mocks'; +import type { DashboardMigrationFilters } from '../../../../../common/siem_migrations/dashboards/types'; +import type { SiemMigrationsClientDependencies } from '../../common/types'; +import type { SiemMigrationDataStats } from '../../common/data/types'; + +jest.mock('./dashboard_migrations_task_runner', () => { + return { + DashboardMigrationTaskRunner: jest.fn().mockImplementation(() => { + return { + setup: jest.fn().mockResolvedValue(undefined), + run: jest.fn().mockResolvedValue(undefined), + abortController: { abort: jest.fn() }, + }; + }), + }; +}); + +const currentUser = {} as AuthenticatedUser; +const dependencies = {} as SiemMigrationsClientDependencies; +const migrationId = 'migration1'; + +describe('DashboardMigrationsTaskClient', () => { + let migrationsRunning: DashboardMigrationsRunning; + let logger: MockedLogger; + let data: ReturnType; + const params: DashboardMigrationTaskStartParams = { + migrationId, + connectorId: 'connector1', + invocationConfig: {}, + }; + + beforeEach(() => { + migrationsRunning = new Map(); + logger = loggerMock.create(); + + data = createDashboardMigrationsDataClientMock(); + // @ts-expect-error resetting private property for each test. + DashboardMigrationsTaskClient.migrationsLastError = new Map(); + jest.clearAllMocks(); + }); + + describe('start', () => { + it('should not start if migration is already running', async () => { + // Pre-populate with the migration id. + migrationsRunning.set(migrationId, {} as DashboardMigrationTaskRunner); + const client = new DashboardMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + const result = await client.start(params); + expect(result).toEqual({ exists: true, started: false }); + expect(data.items.updateStatus).not.toHaveBeenCalled(); + }); + + it('should not start if there are no dashboards to migrate (total = 0)', async () => { + data.items.getStats.mockResolvedValue({ + items: { total: 0, pending: 0, completed: 0, failed: 0 }, + } as SiemMigrationDataStats); + const client = new DashboardMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + const result = await client.start(params); + expect(data.items.updateStatus).toHaveBeenCalledWith( + migrationId, + { status: SiemMigrationStatus.PROCESSING }, + SiemMigrationStatus.PENDING, + { refresh: true } + ); + expect(result).toEqual({ exists: false, started: false }); + }); + + it('should not start if there are no pending dashboards', async () => { + data.items.getStats.mockResolvedValue({ + items: { total: 10, pending: 0, completed: 10, failed: 0 }, + } as SiemMigrationDataStats); + const client = new DashboardMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + const result = await client.start(params); + expect(result).toEqual({ exists: true, started: false }); + }); + + it('should start migration successfully', async () => { + data.items.getStats.mockResolvedValue({ + items: { total: 10, pending: 5, completed: 0, failed: 0 }, + } as SiemMigrationDataStats); + const mockedRunnerInstance = { + setup: jest.fn().mockResolvedValue(undefined), + run: jest.fn().mockResolvedValue(undefined), + abortController: { abort: jest.fn() }, + }; + // Use our custom mock for this test. + (DashboardMigrationTaskRunner as jest.Mock).mockImplementationOnce( + () => mockedRunnerInstance + ); + + const client = new DashboardMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + const result = await client.start(params); + expect(result).toEqual({ exists: true, started: true }); + expect(logger.get).toHaveBeenCalledWith(migrationId); + expect(mockedRunnerInstance.setup).toHaveBeenCalledWith(params.connectorId); + expect(logger.get(migrationId).info).toHaveBeenCalledWith('Starting migration'); + expect(migrationsRunning.has(migrationId)).toBe(true); + + // Allow the asynchronous run() call to complete its finally callback. + await new Promise(process.nextTick); + expect(migrationsRunning.has(migrationId)).toBe(false); + // @ts-expect-error check private property + expect(DashboardMigrationsTaskClient.migrationsLastError.has(migrationId)).toBe(false); + }); + + it('should throw error if a race condition occurs after setup', async () => { + data.items.getStats.mockResolvedValue({ + items: { total: 10, pending: 5, completed: 0, failed: 0 }, + } as SiemMigrationDataStats); + const mockedRunnerInstance = { + setup: jest.fn().mockImplementationOnce(() => { + // Simulate a race condition by setting the migration as running during setup. + migrationsRunning.set(migrationId, {} as DashboardMigrationTaskRunner); + return Promise.resolve(); + }), + run: jest.fn().mockResolvedValue(undefined), + abortController: { abort: jest.fn() }, + }; + (DashboardMigrationTaskRunner as jest.Mock).mockImplementation(() => mockedRunnerInstance); + + const client = new DashboardMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + await expect(client.start(params)).rejects.toThrow('Task already running for this migration'); + }); + + it('should mark migration as started by calling saveAsStarted', async () => { + data.items.getStats.mockResolvedValue({ + items: { total: 10, pending: 5, completed: 0, failed: 0 }, + } as SiemMigrationDataStats); + + const client = new DashboardMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + + await client.start(params); + expect(data.migrations.saveAsStarted).toHaveBeenCalledWith({ + id: migrationId, + connectorId: params.connectorId, + }); + }); + + it('should mark migration as ended by calling saveAsEnded if run completes successfully', async () => { + migrationsRunning = new Map(); + data.items.getStats.mockResolvedValue({ + items: { total: 10, pending: 5, completed: 0, failed: 0 }, + } as SiemMigrationDataStats); + + const client = new DashboardMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + + await client.start(params); + // Allow the asynchronous run() call to complete its finally callback. + await new Promise(process.nextTick); + expect(data.migrations.saveAsFinished).toHaveBeenCalledWith({ id: migrationId }); + }); + }); + + describe('updateToRetry', () => { + it('should not update if migration is currently running', async () => { + migrationsRunning.set(migrationId, {} as DashboardMigrationTaskRunner); + const client = new DashboardMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + const filter: DashboardMigrationFilters = { fullyTranslated: true }; + const result = await client.updateToRetry(migrationId, filter); + expect(result).toEqual({ updated: false }); + expect(data.items.updateStatus).not.toHaveBeenCalled(); + }); + + it('should update to retry if migration is not running', async () => { + const client = new DashboardMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + const filter: DashboardMigrationFilters = { fullyTranslated: true }; + const result = await client.updateToRetry(migrationId, filter); + expect(filter.installed).toBe(false); + expect(data.items.updateStatus).toHaveBeenCalledWith( + migrationId, + { fullyTranslated: true, installed: false }, + SiemMigrationStatus.PENDING, + { refresh: true } + ); + expect(result).toEqual({ updated: true }); + }); + }); + + describe('getStats', () => { + it('should return RUNNING status if migration is running', async () => { + migrationsRunning.set(migrationId, {} as DashboardMigrationTaskRunner); // migration is running + data.items.getStats.mockResolvedValue({ + items: { total: 10, pending: 5, completed: 3, failed: 2 }, + } as SiemMigrationDataStats); + + data.migrations.get.mockResolvedValue({ + id: migrationId, + } as unknown as StoredDashboardMigration); + + const client = new DashboardMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + const stats = await client.getStats(migrationId); + expect(stats.status).toEqual(SiemMigrationTaskStatus.RUNNING); + }); + + it('should return READY status if pending equals total', async () => { + data.items.getStats.mockResolvedValue({ + items: { total: 10, pending: 10, completed: 0, failed: 0 }, + } as SiemMigrationDataStats); + data.migrations.get.mockResolvedValue({ + id: migrationId, + } as unknown as StoredDashboardMigration); + + const client = new DashboardMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + const stats = await client.getStats(migrationId); + expect(stats.status).toEqual(SiemMigrationTaskStatus.READY); + }); + + it('should return FINISHED status if completed+failed equals total', async () => { + data.items.getStats.mockResolvedValue({ + items: { total: 10, pending: 0, completed: 5, failed: 5 }, + } as SiemMigrationDataStats); + + data.migrations.get.mockResolvedValue({ + id: migrationId, + } as unknown as StoredDashboardMigration); + const client = new DashboardMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + const stats = await client.getStats(migrationId); + expect(stats.status).toEqual(SiemMigrationTaskStatus.FINISHED); + }); + + it('should return STOPPED status for other cases', async () => { + data.items.getStats.mockResolvedValue({ + items: { total: 10, pending: 2, completed: 3, failed: 2 }, + } as SiemMigrationDataStats); + const client = new DashboardMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + const stats = await client.getStats(migrationId); + expect(stats.status).toEqual(SiemMigrationTaskStatus.INTERRUPTED); + }); + + it('should include error if one exists', async () => { + const errorMessage = 'Test error'; + data.items.getStats.mockResolvedValue({ + id: 'migration-1', + items: { total: 10, pending: 2, completed: 3, failed: 2 }, + } as SiemMigrationDataStats); + + data.migrations.get.mockResolvedValue({ + id: 'migration-1', + name: 'Test Migration', + created_at: new Date().toISOString(), + created_by: 'test-user', + last_execution: { + error: errorMessage, + }, + }); + + data.migrations.get.mockResolvedValue({ + id: migrationId, + last_execution: { + error: 'Test error', + }, + } as unknown as StoredDashboardMigration); + + const client = new DashboardMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + const stats = await client.getStats(migrationId); + expect(stats.last_execution?.error).toEqual('Test error'); + }); + }); + + describe('getAllStats', () => { + it('should return combined stats for all migrations', async () => { + const statsArray = [ + { + id: 'm1', + items: { total: 10, pending: 10, completed: 0, failed: 0 }, + } as SiemMigrationDataStats, + { + id: 'm2', + items: { total: 10, pending: 2, completed: 3, failed: 2 }, + } as SiemMigrationDataStats, + ]; + const migrations = [{ id: 'm1' }, { id: 'm2' }] as unknown as StoredDashboardMigration[]; + data.items.getAllStats.mockResolvedValue(statsArray); + data.migrations.getAll.mockResolvedValue(migrations); + // Mark migration m1 as running. + migrationsRunning.set('m1', {} as DashboardMigrationTaskRunner); + const client = new DashboardMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + const allStats = await client.getAllStats(); + const m1Stats = allStats.find((s) => s.id === 'm1'); + const m2Stats = allStats.find((s) => s.id === 'm2'); + expect(m1Stats?.status).toEqual(SiemMigrationTaskStatus.RUNNING); + expect(m2Stats?.status).toEqual(SiemMigrationTaskStatus.INTERRUPTED); + }); + }); + + describe('stop', () => { + it('should stop a running migration', async () => { + const abortMock = jest.fn(); + const migrationRunner = { + abortController: { abort: abortMock }, + } as unknown as DashboardMigrationTaskRunner; + migrationsRunning.set(migrationId, migrationRunner); + const client = new DashboardMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + const result = await client.stop(migrationId); + expect(result).toEqual({ exists: true, stopped: true }); + expect(abortMock).toHaveBeenCalled(); + }); + + it('should return stopped even if migration is already stopped', async () => { + data.items.getStats.mockResolvedValue({ + items: { total: 10, pending: 10, completed: 0, failed: 0 }, + } as SiemMigrationDataStats); + const client = new DashboardMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + const result = await client.stop(migrationId); + expect(result).toEqual({ exists: true, stopped: true }); + }); + + it('should return exists false if migration is not running and total equals 0', async () => { + data.items.getStats.mockResolvedValue({ + items: { total: 0, pending: 0, completed: 0, failed: 0 }, + } as SiemMigrationDataStats); + const client = new DashboardMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + const result = await client.stop(migrationId); + expect(result).toEqual({ exists: false, stopped: true }); + }); + + it('should catch errors and return exists true, stopped false', async () => { + const error = new Error('Stop error'); + data.items.getStats.mockRejectedValue(error); + const client = new DashboardMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + const result = await client.stop(migrationId); + expect(result).toEqual({ exists: true, stopped: false }); + expect(logger.error).toHaveBeenCalledWith( + `Error stopping migration ID:${migrationId}`, + error + ); + }); + + it('should mark migration task as stopped when manually stopping a running migration', async () => { + const abortMock = jest.fn(); + const migrationRunner = { + abortController: { abort: abortMock }, + } as unknown as DashboardMigrationTaskRunner; + migrationsRunning.set(migrationId, migrationRunner); + data.migrations.setIsStopped.mockResolvedValue(undefined); + + const client = new DashboardMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + await client.stop(migrationId); + expect(data.migrations.setIsStopped).toHaveBeenCalledWith({ id: migrationId }); + }); + }); + describe('task error', () => { + it('should call saveAsFailed when there has been an error during the migration', async () => { + data.items.getStats.mockResolvedValue({ + items: { total: 10, pending: 10, completed: 0, failed: 0 }, + } as SiemMigrationDataStats); + const error = new Error('Migration error'); + + const mockedRunnerInstance = { + setup: jest.fn().mockResolvedValue(undefined), + run: jest.fn().mockRejectedValue(error), + } as unknown as DashboardMigrationTaskRunner; + + (DashboardMigrationTaskRunner as jest.Mock).mockImplementation(() => mockedRunnerInstance); + + const client = new DashboardMigrationsTaskClient( + migrationsRunning, + logger, + data, + currentUser, + dependencies + ); + + const response = await client.start(params); + + // Allow the asynchronous run() call to complete its finally callback. + await new Promise(process.nextTick); + + expect(response).toEqual({ exists: true, started: true }); + + expect(data.migrations.saveAsFailed).toHaveBeenCalledWith({ + id: migrationId, + error: error.message, + }); + }); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/dashboard_migrations_task_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/dashboard_migrations_task_client.ts new file mode 100644 index 0000000000000..34e8ddd407a92 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/dashboard_migrations_task_client.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RunnableConfig } from '@langchain/core/runnables'; +import type { + DashboardMigration, + DashboardMigrationDashboard, +} from '../../../../../common/siem_migrations/model/dashboard_migration.gen'; +import type { + DashboardMigrationTaskInput, + DashboardMigrationTaskOutput, +} from './dashboard_migrations_task_runner'; +import { DashboardMigrationTaskRunner } from './dashboard_migrations_task_runner'; +import { SiemMigrationsTaskClient } from '../../common/task/siem_migrations_task_client'; +import type { MigrateDashboardConfigSchema } from './agent/types'; +import { DashboardMigrationTaskEvaluator } from './dashboard_migrations_task_evaluator'; + +export type DashboardMigrationsRunning = Map; +export class DashboardMigrationsTaskClient extends SiemMigrationsTaskClient< + DashboardMigration, + DashboardMigrationDashboard, + DashboardMigrationTaskInput, + MigrateDashboardConfigSchema, + DashboardMigrationTaskOutput +> { + protected readonly TaskRunnerClass = DashboardMigrationTaskRunner; + protected readonly EvaluatorClass = DashboardMigrationTaskEvaluator; + + // Dashboards specific last_execution config + protected getLastExecutionConfig( + invocationConfig: RunnableConfig + ): Record { + return { + skipPrebuiltDashboardsMatching: + invocationConfig.configurable?.skipPrebuiltDashboardsMatching ?? false, + }; + } +} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/dashboard_migrations_task_evaluator.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/dashboard_migrations_task_evaluator.ts new file mode 100644 index 0000000000000..e9f5fac943eb8 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/dashboard_migrations_task_evaluator.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { EvaluationResult } from 'langsmith/evaluation'; +import type { Run, Example } from 'langsmith/schemas'; +import { DashboardMigrationTaskRunner } from './dashboard_migrations_task_runner'; +import { SiemMigrationTaskEvaluable } from '../../common/task/siem_migrations_task_evaluator'; + +type CustomEvaluatorResult = Omit; +export type CustomEvaluator = (args: { run: Run; example: Example }) => CustomEvaluatorResult; + +export class DashboardMigrationTaskEvaluator extends SiemMigrationTaskEvaluable( + DashboardMigrationTaskRunner +) { + protected readonly evaluators: Record = { + // TODO: Implement custom evaluators for dashboard migrations + }; +} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/dashboard_migrations_task_runner.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/dashboard_migrations_task_runner.ts new file mode 100644 index 0000000000000..6a8d6f6997f34 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/dashboard_migrations_task_runner.ts @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { AuthenticatedUser, Logger } from '@kbn/core/server'; +import type { + ElasticDashboard, + DashboardMigration, + DashboardMigrationDashboard, +} from '../../../../../common/siem_migrations/model/dashboard_migration.gen'; +import type { DashboardMigrationsDataClient } from '../data/dashboard_migrations_data_client'; +import type { MigrateDashboardConfigSchema, MigrateDashboardState } from './agent/types'; +import { getDashboardMigrationAgent } from './agent'; +import { DashboardMigrationsRetriever } from './retrievers'; +import type { StoredDashboardMigrationDashboard } from '../types'; +import { EsqlKnowledgeBase } from '../../common/task/util/esql_knowledge_base'; +import type { SiemMigrationsClientDependencies } from '../../common/types'; +import { SiemMigrationTaskRunner } from '../../common/task/siem_migrations_task_runner'; +import { DashboardMigrationTelemetryClient } from './dashboard_migrations_telemetry_client'; +import type { MigrationResources } from '../../common/task/retrievers/resource_retriever'; +import { nullifyMissingProperties } from '../../common/task/util/nullify_missing_properties'; + +export interface DashboardMigrationTaskInput + extends Pick { + resources: MigrationResources; +} +export type DashboardMigrationTaskOutput = MigrateDashboardState; + +export class DashboardMigrationTaskRunner extends SiemMigrationTaskRunner< + DashboardMigration, + DashboardMigrationDashboard, + DashboardMigrationTaskInput, + MigrateDashboardConfigSchema, + DashboardMigrationTaskOutput +> { + private retriever: DashboardMigrationsRetriever; + + constructor( + public readonly migrationId: string, + public readonly startedBy: AuthenticatedUser, + public readonly abortController: AbortController, + protected readonly data: DashboardMigrationsDataClient, + protected readonly logger: Logger, + protected readonly dependencies: SiemMigrationsClientDependencies + ) { + super(migrationId, startedBy, abortController, data, logger, dependencies); + this.retriever = new DashboardMigrationsRetriever(this.migrationId, { + data: this.data, + }); + } + + /** Retrieves the connector and creates the migration agent */ + public async setup(connectorId: string): Promise { + const { inferenceClient } = this.dependencies; + const model = await this.actionsClientChat.createModel({ + connectorId, + migrationId: this.migrationId, + abortController: this.abortController, + }); + const modelName = this.actionsClientChat.getModelName(model); + + // TODO: inference model name + // console.log('modelName', modelName); + + const telemetryClient = new DashboardMigrationTelemetryClient( + this.dependencies.telemetry, + this.logger, + this.migrationId, + modelName + ); + + const esqlKnowledgeBase = new EsqlKnowledgeBase( + connectorId, + this.migrationId, + inferenceClient, + this.logger + ); + + const agent = getDashboardMigrationAgent({ + model, + esScopedClient: this.data.esScopedClient, + esqlKnowledgeBase, + dashboardMigrationsRetriever: this.retriever, + logger: this.logger, + telemetryClient, + }); + + this.telemetry = telemetryClient; + this.task = (input, config) => agent.invoke(input, config); + } + + protected async initialize() { + await this.retriever.initialize(); + } + + protected async prepareTaskInput( + migrationDashboard: StoredDashboardMigrationDashboard + ): Promise { + const resources = await this.retriever.resources.getResources(migrationDashboard); + return { + id: migrationDashboard.id, + original_dashboard: migrationDashboard.original_dashboard, + resources, + }; + } + + protected processTaskOutput( + migrationDashboard: StoredDashboardMigrationDashboard, + migrationOutput: DashboardMigrationTaskOutput + ): StoredDashboardMigrationDashboard { + return { + ...migrationDashboard, + elastic_dashboard: nullifyMissingProperties({ + source: migrationDashboard.elastic_dashboard, + target: migrationOutput.elastic_dashboard as ElasticDashboard, + }), + translation_result: migrationOutput.translation_result, + comments: migrationOutput.comments, + }; + } +} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/dashboard_migrations_task_service.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/dashboard_migrations_task_service.ts new file mode 100644 index 0000000000000..c5b485251c980 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/dashboard_migrations_task_service.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; +import type { DashboardMigrationTaskCreateClientParams } from './types'; +import { + DashboardMigrationsTaskClient, + type DashboardMigrationsRunning, +} from './dashboard_migrations_task_client'; + +export class DashboardMigrationsTaskService { + private migrationsRunning: DashboardMigrationsRunning; + + constructor(private logger: Logger) { + this.migrationsRunning = new Map(); + } + + public createClient({ + currentUser, + dataClient, + dependencies, + }: DashboardMigrationTaskCreateClientParams): DashboardMigrationsTaskClient { + return new DashboardMigrationsTaskClient( + this.migrationsRunning, + this.logger, + dataClient, + currentUser, + dependencies + ); + } + + /** Stops all running migrations */ + stopAll() { + this.migrationsRunning.forEach((migrationRunning) => { + migrationRunning.abortController.abort('Server shutdown'); + }); + this.migrationsRunning.clear(); + } +} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/dashboard_migrations_telemetry_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/dashboard_migrations_telemetry_client.ts new file mode 100644 index 0000000000000..72a9cab9491dd --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/dashboard_migrations_telemetry_client.ts @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { AnalyticsServiceSetup, Logger, EventTypeOpts } from '@kbn/core/server'; +import type { DashboardMigrationDashboard } from '../../../../../common/siem_migrations/model/dashboard_migration.gen'; +import { + SIEM_MIGRATIONS_MIGRATION_ABORTED, + SIEM_MIGRATIONS_MIGRATION_FAILURE, + SIEM_MIGRATIONS_MIGRATION_SUCCESS, + SIEM_MIGRATIONS_RULE_TRANSLATION_FAILURE, + SIEM_MIGRATIONS_RULE_TRANSLATION_SUCCESS, +} from '../../../telemetry/event_based/events'; +import { siemMigrationEventNames } from '../../../telemetry/event_based/event_meta'; +import { SiemMigrationsEventTypes } from '../../../telemetry/event_based/types'; +import type { MigrateDashboardState } from './agent/types'; +import type { SiemMigrationTelemetryClient } from '../../common/task/siem_migrations_telemetry_client'; + +export class DashboardMigrationTelemetryClient + implements SiemMigrationTelemetryClient +{ + constructor( + private readonly telemetry: AnalyticsServiceSetup, + private readonly logger: Logger, + private readonly migrationId: string, + private readonly modelName: string = '' + ) {} + + private reportEvent(eventTypeOpts: EventTypeOpts, data: T): void { + try { + this.telemetry.reportEvent(eventTypeOpts.eventType, data); + } catch (e) { + this.logger.error(`Error reporting event ${eventTypeOpts.eventType}: ${e.message}`); + } + } + + public startSiemMigrationTask() { + const startTime = Date.now(); + const stats = { completed: 0, failed: 0 }; + + return { + startItemTranslation: () => { + const dashboardStartTime = Date.now(); + return { + success: (migrationResult: MigrateDashboardState) => { + stats.completed++; + this.reportEvent(SIEM_MIGRATIONS_RULE_TRANSLATION_SUCCESS, { + migrationId: this.migrationId, + translationResult: migrationResult.translation_result || '', + duration: Date.now() - dashboardStartTime, + model: this.modelName, + prebuiltMatch: migrationResult.elastic_dashboard?.prebuilt_dashboard_id + ? true + : false, + eventName: siemMigrationEventNames[SiemMigrationsEventTypes.TranslationSuccess], + }); + }, + failure: (error: Error) => { + stats.failed++; + this.reportEvent(SIEM_MIGRATIONS_RULE_TRANSLATION_FAILURE, { + migrationId: this.migrationId, + error: error.message, + model: this.modelName, + eventName: siemMigrationEventNames[SiemMigrationsEventTypes.TranslationFailure], + }); + }, + }; + }, + success: () => { + const duration = Date.now() - startTime; + this.reportEvent(SIEM_MIGRATIONS_MIGRATION_SUCCESS, { + migrationId: this.migrationId, + model: this.modelName || '', + completed: stats.completed, + failed: stats.failed, + total: stats.completed + stats.failed, + duration, + eventName: siemMigrationEventNames[SiemMigrationsEventTypes.MigrationSuccess], + }); + }, + failure: (error: Error) => { + const duration = Date.now() - startTime; + this.reportEvent(SIEM_MIGRATIONS_MIGRATION_FAILURE, { + migrationId: this.migrationId, + model: this.modelName || '', + completed: stats.completed, + failed: stats.failed, + total: stats.completed + stats.failed, + duration, + error: error.message, + eventName: siemMigrationEventNames[SiemMigrationsEventTypes.MigrationFailure], + }); + }, + aborted: (error: Error) => { + const duration = Date.now() - startTime; + this.reportEvent(SIEM_MIGRATIONS_MIGRATION_ABORTED, { + migrationId: this.migrationId, + model: this.modelName || '', + completed: stats.completed, + failed: stats.failed, + total: stats.completed + stats.failed, + duration, + reason: error.message, + eventName: siemMigrationEventNames[SiemMigrationsEventTypes.MigrationAborted], + }); + }, + }; + } +} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/retrievers/__mocks__/mocks.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/retrievers/__mocks__/mocks.ts new file mode 100644 index 0000000000000..b4779b99a47ee --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/retrievers/__mocks__/mocks.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { PublicMethodsOf } from '@kbn/utility-types'; +import type { DashboardMigrationsRetriever } from '..'; + +export const createDashboardMigrationsRetrieverMock = () => { + const mockResources = { + initialize: jest.fn().mockResolvedValue(undefined), + getResources: jest.fn().mockResolvedValue({}), + }; + + const mockIntegrations = { + populateIndex: jest.fn().mockResolvedValue(undefined), + search: jest.fn().mockResolvedValue([]), + }; + + const mockPrebuiltDashboards = { + populateIndex: jest.fn().mockResolvedValue(undefined), + search: jest.fn().mockResolvedValue([]), + }; + + const mockRetriever = { + resources: mockResources, + integrations: mockIntegrations, + prebuiltDashboards: mockPrebuiltDashboards, + initialize: jest.fn().mockResolvedValue(undefined), + }; + + return mockRetriever as jest.Mocked>; +}; + +export const MockDashboardMigrationsRetriever = jest + .fn() + .mockImplementation(() => createDashboardMigrationsRetrieverMock()); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/retrievers/__mocks__/rule_migrations_retriever.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/retrievers/__mocks__/rule_migrations_retriever.ts new file mode 100644 index 0000000000000..d7977fd3495e6 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/retrievers/__mocks__/rule_migrations_retriever.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MockRuleMigrationsRetriever } from './mocks'; +export const RuleMigrationsRetriever = MockRuleMigrationsRetriever; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/retrievers/dashboard_migrations_retriever.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/retrievers/dashboard_migrations_retriever.ts new file mode 100644 index 0000000000000..9ca4244e87f59 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/retrievers/dashboard_migrations_retriever.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { DashboardMigrationsDataClient } from '../../data/dashboard_migrations_data_client'; +import { DashboardResourceRetriever } from './dashboard_resource_retriever'; + +export interface DashboardMigrationsRetrieverClients { + data: DashboardMigrationsDataClient; +} + +/** + * DashboardMigrationsRetriever is a class that is responsible for retrieving all the necessary data during the dashboard migration process. + * It is composed of multiple retrievers that are responsible for retrieving specific types of data. + * Such as dashboard integrations, prebuilt dashboards, and dashboard resources. + */ +export class DashboardMigrationsRetriever { + public readonly resources: DashboardResourceRetriever; + + constructor(migrationId: string, clients: DashboardMigrationsRetrieverClients) { + this.resources = new DashboardResourceRetriever(migrationId, clients.data.resources); + } + + public async initialize() { + await this.resources.initialize(); + } +} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/retrievers/dashboard_resource_retriever.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/retrievers/dashboard_resource_retriever.ts new file mode 100644 index 0000000000000..050f16ef22ac9 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/retrievers/dashboard_resource_retriever.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DashboardResourceIdentifier } from '../../../../../../common/siem_migrations/dashboards/resources'; +import type { DashboardMigrationDashboard } from '../../../../../../common/siem_migrations/model/dashboard_migration.gen'; +import { ResourceRetriever } from '../../../common/task/retrievers/resource_retriever'; + +export class DashboardResourceRetriever extends ResourceRetriever { + protected ResourceIdentifierClass = DashboardResourceIdentifier; +} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/retrievers/index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/retrievers/index.ts new file mode 100644 index 0000000000000..34a8293ce40e6 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/retrievers/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { DashboardMigrationsRetriever } from './dashboard_migrations_retriever'; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/types.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/types.ts new file mode 100644 index 0000000000000..b3837eac23495 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/task/types.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { AuthenticatedUser } from '@kbn/core/server'; +import type { LangSmithEvaluationOptions } from '../../../../../common/siem_migrations/model/common.gen'; +import type { DashboardMigrationsDataClient } from '../data/dashboard_migrations_data_client'; +import type { StoredDashboardMigrationDashboard } from '../types'; +import type { getDashboardMigrationAgent } from './agent'; +import type { DashboardMigrationTelemetryClient } from './dashboard_migrations_telemetry_client'; +import type { ChatModel } from '../../common/task/util/actions_client_chat'; +import type { DashboardMigrationsRetriever } from './retrievers'; +import type { MigrateDashboardGraphConfig } from './agent/types'; +import type { SiemMigrationsClientDependencies } from '../../common/types'; +import type { MigrationResources } from '../../common/task/retrievers/resource_retriever'; + +export type MigrationAgent = ReturnType; + +export interface DashboardMigrationInput + extends Pick { + resources: MigrationResources; +} + +export interface DashboardMigrationTaskCreateClientParams { + currentUser: AuthenticatedUser; + dataClient: DashboardMigrationsDataClient; + dependencies: SiemMigrationsClientDependencies; +} + +export interface DashboardMigrationTaskStartParams { + migrationId: string; + connectorId: string; + invocationConfig: MigrateDashboardGraphConfig; +} + +export interface DashboardMigrationTaskRunParams extends DashboardMigrationTaskStartParams { + model: ChatModel; + abortController: AbortController; +} + +export interface DashboardMigrationTaskCreateAgentParams { + connectorId: string; + retriever: DashboardMigrationsRetriever; + telemetryClient: DashboardMigrationTelemetryClient; + model: ChatModel; +} + +export interface DashboardMigrationTaskStartResult { + started: boolean; + exists: boolean; +} + +export interface DashboardMigrationTaskStopResult { + stopped: boolean; + exists: boolean; +} + +export interface DashboardMigrationTaskEvaluateParams { + evaluationId: string; + connectorId: string; + langsmithOptions: LangSmithEvaluationOptions; + invocationConfig: MigrateDashboardGraphConfig; + abortController: AbortController; +} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/types.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/types.ts index 8e2f950a8496f..0fdc074e9e906 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/types.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/dashboards/types.ts @@ -6,10 +6,16 @@ */ import type { IndexPatternAdapter } from '@kbn/index-adapter'; +import type { + DashboardMigration, + DashboardMigrationDashboard, +} from '../../../../common/siem_migrations/model/dashboard_migration.gen'; +import type { Stored } from '../types'; export interface DashboardMigrationAdapters { migrations: IndexPatternAdapter; dashboards: IndexPatternAdapter; + resources: IndexPatternAdapter; } export type DashboardMigrationAdapterId = keyof DashboardMigrationAdapters; @@ -19,3 +25,6 @@ export type DashboardMigrationIndexNameProviders = Record< DashboardMigrationAdapterId, DashboardMigrationIndexNameProvider >; + +export type StoredDashboardMigration = Stored; +export type StoredDashboardMigrationDashboard = Stored; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/routes.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/routes.ts index 8acf9b10c6808..33093a4f23b0a 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/routes.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/routes.ts @@ -18,9 +18,9 @@ export const registerSiemMigrationsRoutes = ( ) => { if (!config.experimentalFeatures.siemMigrationsDisabled) { registerSiemRuleMigrationsRoutes(router, config, logger); - } - if (config.experimentalFeatures.automaticDashboardsMigration) { - registerSiemDashboardMigrationsRoutes(router, logger); + if (config.experimentalFeatures.automaticDashboardsMigration) { + registerSiemDashboardMigrationsRoutes(router, config, logger); + } } }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/create.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/create.ts index 40aa2e2e88938..d4e52c9492345 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/create.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/create.ts @@ -13,9 +13,9 @@ import { CreateRuleMigrationRequestBody, } from '../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; import type { SecuritySolutionPluginRouter } from '../../../../types'; -import { SiemMigrationAuditLogger } from '../../common/utils/audit'; -import { authz } from '../../common/utils/authz'; -import { withLicense } from '../../common/utils/with_license'; +import { SiemMigrationAuditLogger } from '../../common/api/util/audit'; +import { authz } from '../../common/api/util/authz'; +import { withLicense } from '../../common/api/util/with_license'; export const registerSiemRuleMigrationsCreateRoute = ( router: SecuritySolutionPluginRouter, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/delete.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/delete.ts index 39cdcb1e0f967..0b8192b5341b6 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/delete.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/delete.ts @@ -10,9 +10,9 @@ import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; import { SIEM_RULE_MIGRATION_PATH } from '../../../../../common/siem_migrations/constants'; import { GetRuleMigrationRequestParams } from '../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; import type { SecuritySolutionPluginRouter } from '../../../../types'; -import { SiemMigrationAuditLogger } from '../../common/utils/audit'; -import { authz } from '../../common/utils/authz'; -import { withLicense } from '../../common/utils/with_license'; +import { SiemMigrationAuditLogger } from '../../common/api/util/audit'; +import { authz } from '../../common/api/util/authz'; +import { withLicense } from '../../common/api/util/with_license'; import { withExistingMigration } from './util/with_existing_migration_id'; export const registerSiemRuleMigrationsDeleteRoute = ( diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/evaluation/evaluate.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/evaluation/evaluate.ts index f773e9b2300ba..6dfc2e0d81aa5 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/evaluation/evaluate.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/evaluation/evaluate.ts @@ -12,11 +12,11 @@ import { z } from '@kbn/zod'; import { RuleMigrationTaskExecutionSettings } from '../../../../../../common/siem_migrations/model/rule_migration.gen'; import { LangSmithEvaluationOptions } from '../../../../../../common/siem_migrations/model/common.gen'; import { SIEM_RULE_MIGRATION_EVALUATE_PATH } from '../../../../../../common/siem_migrations/constants'; -import { createTracersCallbacks } from '../util/tracing'; +import { createTracersCallbacks } from '../../../common/api/util/tracing'; import type { SecuritySolutionPluginRouter } from '../../../../../types'; -import { authz } from '../../../common/utils/authz'; -import { withLicense } from '../../../common/utils/with_license'; -import type { MigrateRuleGraphConfig } from '../../task/agent/types'; +import { authz } from '../../../common/api/util/authz'; +import { withLicense } from '../../../common/api/util/with_license'; +import type { MigrateRuleConfig } from '../../task/agent/types'; const REQUEST_TIMEOUT = 10 * 60 * 1000; // 10 minutes @@ -62,7 +62,7 @@ export const registerSiemRuleMigrationsEvaluateRoute = ( const securitySolutionContext = await context.securitySolution; const ruleMigrationsClient = securitySolutionContext.siemMigrations.getRulesClient(); - const invocationConfig: MigrateRuleGraphConfig = { + const invocationConfig: MigrateRuleConfig = { callbacks: createTracersCallbacks(langsmithOptions, logger), configurable: { skipPrebuiltRulesMatching, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/get.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/get.ts index 67eeec952b90e..f2d91f9c1057d 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/get.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/get.ts @@ -11,9 +11,9 @@ import { SIEM_RULE_MIGRATION_PATH } from '../../../../../common/siem_migrations/ import type { GetRuleMigrationResponse } from '../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; import { GetRuleMigrationRequestParams } from '../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; import type { SecuritySolutionPluginRouter } from '../../../../types'; -import { SiemMigrationAuditLogger } from '../../common/utils/audit'; -import { authz } from '../../common/utils/authz'; -import { withLicense } from '../../common/utils/with_license'; +import { SiemMigrationAuditLogger } from '../../common/api/util/audit'; +import { authz } from '../../common/api/util/authz'; +import { withLicense } from '../../common/api/util/with_license'; import { MIGRATION_ID_NOT_FOUND } from '../../common/translations'; export const registerSiemRuleMigrationsGetRoute = ( @@ -44,9 +44,7 @@ export const registerSiemRuleMigrationsGetRoute = ( const ruleMigrationsClient = ctx.securitySolution.siemMigrations.getRulesClient(); await siemMigrationAuditLogger.logGetMigration({ migrationId }); - const storedMigration = await ruleMigrationsClient.data.migrations.get({ - id: migrationId, - }); + const storedMigration = await ruleMigrationsClient.data.migrations.get(migrationId); if (!storedMigration) { return res.notFound({ diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/get_integrations.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/get_integrations.ts index ddbdb7d529814..ad476eae401e5 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/get_integrations.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/get_integrations.ts @@ -10,8 +10,8 @@ import type { RelatedIntegration } from '../../../../../common/api/detection_eng import { type GetRuleMigrationIntegrationsResponse } from '../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; import { SIEM_RULE_MIGRATIONS_INTEGRATIONS_PATH } from '../../../../../common/siem_migrations/constants'; import type { SecuritySolutionPluginRouter } from '../../../../types'; -import { authz } from '../../common/utils/authz'; -import { withLicense } from '../../common/utils/with_license'; +import { authz } from '../../common/api/util/authz'; +import { withLicense } from '../../common/api/util/with_license'; export const registerSiemRuleMigrationsIntegrationsRoute = ( router: SecuritySolutionPluginRouter, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/get_prebuilt_rules.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/get_prebuilt_rules.ts index 7ed8c9d4dedc0..7b7ed9b175d49 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/get_prebuilt_rules.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/get_prebuilt_rules.ts @@ -11,8 +11,8 @@ import type { GetRuleMigrationPrebuiltRulesResponse } from '../../../../../commo import { GetRuleMigrationPrebuiltRulesRequestParams } from '../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; import { SIEM_RULE_MIGRATIONS_PREBUILT_RULES_PATH } from '../../../../../common/siem_migrations/constants'; import type { SecuritySolutionPluginRouter } from '../../../../types'; -import { authz } from '../../common/utils/authz'; -import { withLicense } from '../../common/utils/with_license'; +import { authz } from '../../common/api/util/authz'; +import { withLicense } from '../../common/api/util/with_license'; import { getPrebuiltRulesForMigration } from './util/prebuilt_rules'; export const registerSiemRuleMigrationsPrebuiltRulesRoute = ( diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/install.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/install.ts index 9d40d027f451a..b6f81dad7e340 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/install.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/install.ts @@ -14,10 +14,10 @@ import { InstallMigrationRulesRequestParams, } from '../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; import type { SecuritySolutionPluginRouter } from '../../../../types'; -import { SiemMigrationAuditLogger } from '../../common/utils/audit'; +import { SiemMigrationAuditLogger } from '../../common/api/util/audit'; import { installTranslated } from './util/installation'; -import { authz } from '../../common/utils/authz'; -import { withLicense } from '../../common/utils/with_license'; +import { authz } from '../../common/api/util/authz'; +import { withLicense } from '../../common/api/util/with_license'; export const registerSiemRuleMigrationsInstallRoute = ( router: SecuritySolutionPluginRouter, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/integrations_stats.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/integrations_stats.ts index 1c6db8b675925..41eaa865bffa7 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/integrations_stats.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/integrations_stats.ts @@ -9,9 +9,9 @@ import type { IKibanaResponse, Logger } from '@kbn/core/server'; import { type GetRuleMigrationIntegrationsStatsResponse } from '../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; import { SIEM_RULE_MIGRATIONS_INTEGRATIONS_STATS_PATH } from '../../../../../common/siem_migrations/constants'; import type { SecuritySolutionPluginRouter } from '../../../../types'; -import { authz } from '../../common/utils/authz'; -import { withLicense } from '../../common/utils/with_license'; -import { SiemMigrationAuditLogger } from '../../common/utils/audit'; +import { authz } from '../../common/api/util/authz'; +import { withLicense } from '../../common/api/util/with_license'; +import { SiemMigrationAuditLogger } from '../../common/api/util/audit'; export const registerSiemRuleMigrationsIntegrationsStatsRoute = ( router: SecuritySolutionPluginRouter, @@ -41,7 +41,7 @@ export const registerSiemRuleMigrationsIntegrationsStatsRoute = ( await siemMigrationAuditLogger.logGetAllIntegrationsStats(); const allIntegrationsStats = - await ruleMigrationsClient.data.rules.getAllIntegrationsStats(); + await ruleMigrationsClient.data.items.getAllIntegrationsStats(); return res.ok({ body: allIntegrationsStats }); } catch (error) { diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/privileges/get_missing_privileges.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/privileges/get_missing_privileges.ts index 656263e304566..dedf72953ffe6 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/privileges/get_missing_privileges.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/privileges/get_missing_privileges.ts @@ -13,8 +13,8 @@ import { LOOKUPS_INDEX_PREFIX, } from '../../../../../../common/siem_migrations/constants'; import type { SecuritySolutionPluginRouter } from '../../../../../types'; -import { authz } from '../../../common/utils/authz'; -import { withLicense } from '../../../common/utils/with_license'; +import { authz } from '../../../common/api/util/authz'; +import { withLicense } from '../../../common/api/util/with_license'; export const registerSiemRuleMigrationsGetMissingPrivilegesRoute = ( router: SecuritySolutionPluginRouter, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/get.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/get.ts index 5618d2cf6eef3..6c3daba5b5d18 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/get.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/get.ts @@ -14,9 +14,9 @@ import { type GetRuleMigrationResourcesResponse, } from '../../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; import type { SecuritySolutionPluginRouter } from '../../../../../types'; -import { SiemMigrationAuditLogger } from '../../../common/utils/audit'; -import { authz } from '../../../common/utils/authz'; -import { withLicense } from '../../../common/utils/with_license'; +import { SiemMigrationAuditLogger } from '../../../common/api/util/audit'; +import { authz } from '../../../common/api/util/authz'; +import { withLicense } from '../../../common/api/util/with_license'; export const registerSiemRuleMigrationsResourceGetRoute = ( router: SecuritySolutionPluginRouter, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/missing.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/missing.ts index ced119bb54cf5..06f5abf8abf19 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/missing.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/missing.ts @@ -14,8 +14,8 @@ import { } from '../../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; import { SIEM_RULE_MIGRATION_RESOURCES_MISSING_PATH } from '../../../../../../common/siem_migrations/constants'; import type { SecuritySolutionPluginRouter } from '../../../../../types'; -import { authz } from '../../../common/utils/authz'; -import { withLicense } from '../../../common/utils/with_license'; +import { authz } from '../../../common/api/util/authz'; +import { withLicense } from '../../../common/api/util/with_license'; export const registerSiemRuleMigrationsResourceGetMissingRoute = ( router: SecuritySolutionPluginRouter, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/upsert.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/upsert.ts index 2567e9314c95a..b53b5b3bbcca7 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/upsert.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/upsert.ts @@ -16,11 +16,11 @@ import { } from '../../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; import { ResourceIdentifier } from '../../../../../../common/siem_migrations/rules/resources'; import type { SecuritySolutionPluginRouter } from '../../../../../types'; -import type { CreateRuleMigrationResourceInput } from '../../data/rule_migrations_data_resources_client'; -import { SiemMigrationAuditLogger } from '../../../common/utils/audit'; -import { authz } from '../../../common/utils/authz'; +import { SiemMigrationAuditLogger } from '../../../common/api/util/audit'; +import { authz } from '../../../common/api/util/authz'; import { processLookups } from '../util/lookups'; -import { withLicense } from '../../../common/utils/with_license'; +import { withLicense } from '../../../common/api/util/with_license'; +import type { CreateSiemMigrationResourceInput } from '../../../common/data/siem_migrations_data_resources_client'; export const registerSiemRuleMigrationsResourceUpsertRoute = ( router: SecuritySolutionPluginRouter, @@ -59,7 +59,7 @@ export const registerSiemRuleMigrationsResourceUpsertRoute = ( await siemMigrationAuditLogger.logUploadResources({ migrationId }); // Check if the migration exists - const { data } = await ruleMigrationsClient.data.rules.get(migrationId, { size: 1 }); + const { data } = await ruleMigrationsClient.data.items.get(migrationId, { size: 1 }); const [rule] = data; if (!rule) { return res.notFound({ body: { message: 'Migration not found' } }); @@ -79,7 +79,7 @@ export const registerSiemRuleMigrationsResourceUpsertRoute = ( const resourceIdentifier = new ResourceIdentifier(rule.original_rule.vendor); const resourcesToCreate = resourceIdentifier .fromResources(resources) - .map((resource) => ({ + .map((resource) => ({ ...resource, migration_id: migrationId, })); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/rules/create.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/rules/create.ts index b975e7906b7bc..9ad2f81a0e37b 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/rules/create.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/rules/create.ts @@ -15,11 +15,11 @@ import { } from '../../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; import { ResourceIdentifier } from '../../../../../../common/siem_migrations/rules/resources'; import type { SecuritySolutionPluginRouter } from '../../../../../types'; -import type { AddRuleMigrationRulesInput } from '../../data/rule_migrations_data_rules_client'; -import { SiemMigrationAuditLogger } from '../../../common/utils/audit'; -import { authz } from '../../../common/utils/authz'; +import type { CreateRuleMigrationRulesInput } from '../../data/rule_migrations_data_rules_client'; +import { SiemMigrationAuditLogger } from '../../../common/api/util/audit'; +import { authz } from '../../../common/api/util/authz'; import { withExistingMigration } from '../util/with_existing_migration_id'; -import { withLicense } from '../../../common/utils/with_license'; +import { withLicense } from '../../../common/api/util/with_license'; export const registerSiemRuleMigrationsCreateRulesRoute = ( router: SecuritySolutionPluginRouter, @@ -60,14 +60,14 @@ export const registerSiemRuleMigrationsCreateRulesRoute = ( count: rulesCount, }); - const ruleMigrations = originalRules.map( + const ruleMigrations = originalRules.map( (originalRule) => ({ migration_id: migrationId, original_rule: originalRule, }) ); - await ruleMigrationsClient.data.rules.create(ruleMigrations); + await ruleMigrationsClient.data.items.create(ruleMigrations); // Create identified resource documents without content to keep track of them const resourceIdentifier = new ResourceIdentifier(firstOriginalRule.vendor); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/rules/get.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/rules/get.ts index 55386d9bdfa2e..0b8cbf02806d0 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/rules/get.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/rules/get.ts @@ -15,9 +15,9 @@ import { } from '../../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; import type { SecuritySolutionPluginRouter } from '../../../../../types'; import type { RuleMigrationGetRulesOptions } from '../../data/rule_migrations_data_rules_client'; -import { SiemMigrationAuditLogger } from '../../../common/utils/audit'; -import { authz } from '../../../common/utils/authz'; -import { withLicense } from '../../../common/utils/with_license'; +import { SiemMigrationAuditLogger } from '../../../common/api/util/audit'; +import { authz } from '../../../common/api/util/authz'; +import { withLicense } from '../../../common/api/util/with_license'; import { withExistingMigration } from '../util/with_existing_migration_id'; export const registerSiemRuleMigrationsGetRulesRoute = ( @@ -67,7 +67,7 @@ export const registerSiemRuleMigrationsGetRulesRoute = ( from: page && size ? page * size : 0, }; - const result = await ruleMigrationsClient.data.rules.get(migrationId, options); + const result = await ruleMigrationsClient.data.items.get(migrationId, options); await siemMigrationAuditLogger.logGetMigrationRules({ migrationId }); return res.ok({ body: result }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/rules/update.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/rules/update.ts index 6ed7b47d22539..043fbd05a9c43 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/rules/update.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/rules/update.ts @@ -14,10 +14,10 @@ import { UpdateRuleMigrationRulesRequestParams, } from '../../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; import type { SecuritySolutionPluginRouter } from '../../../../../types'; -import { authz } from '../../../common/utils/authz'; -import { SiemMigrationAuditLogger } from '../../../common/utils/audit'; +import { authz } from '../../../common/api/util/authz'; +import { SiemMigrationAuditLogger } from '../../../common/api/util/audit'; import { transformToInternalUpdateRuleMigrationData } from '../util/update_rules'; -import { withLicense } from '../../../common/utils/with_license'; +import { withLicense } from '../../../common/api/util/with_license'; import { withExistingMigration } from '../util/with_existing_migration_id'; export const registerSiemRuleMigrationsUpdateRulesRoute = ( @@ -61,7 +61,7 @@ export const registerSiemRuleMigrationsUpdateRulesRoute = ( const transformedRuleToUpdate = rulesToUpdate.map( transformToInternalUpdateRuleMigrationData ); - await ruleMigrationsClient.data.rules.update(transformedRuleToUpdate); + await ruleMigrationsClient.data.items.update(transformedRuleToUpdate); return res.ok({ body: { updated: true } }); } catch (error) { diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/start.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/start.ts index 2841643f5202f..5264d84423bdc 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/start.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/start.ts @@ -14,11 +14,11 @@ import { type StartRuleMigrationResponse, } from '../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; import type { SecuritySolutionPluginRouter } from '../../../../types'; -import { SiemMigrationAuditLogger } from '../../common/utils/audit'; -import { authz } from '../../common/utils/authz'; -import { getRetryFilter } from './util/retry'; -import { withLicense } from '../../common/utils/with_license'; -import { createTracersCallbacks } from './util/tracing'; +import { SiemMigrationAuditLogger } from '../../common/api/util/audit'; +import { authz } from '../../common/api/util/authz'; +import { getRetryFilter } from '../../common/api/util/retry'; +import { withLicense } from '../../common/api/util/with_license'; +import { createTracersCallbacks } from '../../common/api/util/tracing'; import { withExistingMigration } from './util/with_existing_migration_id'; export const registerSiemRuleMigrationsStartRoute = ( diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/stats.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/stats.ts index 8e8e46ed44f87..f25cf22a3f339 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/stats.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/stats.ts @@ -13,8 +13,8 @@ import { } from '../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; import { SIEM_RULE_MIGRATION_STATS_PATH } from '../../../../../common/siem_migrations/constants'; import type { SecuritySolutionPluginRouter } from '../../../../types'; -import { authz } from '../../common/utils/authz'; -import { withLicense } from '../../common/utils/with_license'; +import { authz } from '../../common/api/util/authz'; +import { withLicense } from '../../common/api/util/with_license'; import { withExistingMigration } from './util/with_existing_migration_id'; export const registerSiemRuleMigrationsStatsRoute = ( @@ -44,7 +44,7 @@ export const registerSiemRuleMigrationsStatsRoute = ( const stats = await ruleMigrationsClient.task.getStats(migrationId); - if (stats.rules.total === 0) { + if (stats.items.total === 0) { return res.noContent(); } return res.ok({ body: stats }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/stats_all.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/stats_all.ts index 5d9151ac6b524..d169caa82b57a 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/stats_all.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/stats_all.ts @@ -9,8 +9,8 @@ import type { IKibanaResponse, Logger } from '@kbn/core/server'; import type { GetAllStatsRuleMigrationResponse } from '../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; import { SIEM_RULE_MIGRATIONS_ALL_STATS_PATH } from '../../../../../common/siem_migrations/constants'; import type { SecuritySolutionPluginRouter } from '../../../../types'; -import { authz } from '../../common/utils/authz'; -import { withLicense } from '../../common/utils/with_license'; +import { authz } from '../../common/api/util/authz'; +import { withLicense } from '../../common/api/util/with_license'; export const registerSiemRuleMigrationsStatsAllRoute = ( router: SecuritySolutionPluginRouter, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/stop.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/stop.ts index a317943162b65..bb1d1b4060fb8 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/stop.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/stop.ts @@ -13,9 +13,9 @@ import { type StopRuleMigrationResponse, } from '../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; import type { SecuritySolutionPluginRouter } from '../../../../types'; -import { SiemMigrationAuditLogger } from '../../common/utils/audit'; -import { authz } from '../../common/utils/authz'; -import { withLicense } from '../../common/utils/with_license'; +import { SiemMigrationAuditLogger } from '../../common/api/util/audit'; +import { authz } from '../../common/api/util/authz'; +import { withLicense } from '../../common/api/util/with_license'; import { withExistingMigration } from './util/with_existing_migration_id'; export const registerSiemRuleMigrationsStopRoute = ( diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/translation_stats.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/translation_stats.ts index 7c67c3c3a4a32..4ab1c04988b11 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/translation_stats.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/translation_stats.ts @@ -11,8 +11,8 @@ import type { GetRuleMigrationTranslationStatsResponse } from '../../../../../co import { GetRuleMigrationTranslationStatsRequestParams } from '../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; import { SIEM_RULE_MIGRATION_TRANSLATION_STATS_PATH } from '../../../../../common/siem_migrations/constants'; import type { SecuritySolutionPluginRouter } from '../../../../types'; -import { authz } from '../../common/utils/authz'; -import { withLicense } from '../../common/utils/with_license'; +import { authz } from '../../common/api/util/authz'; +import { withLicense } from '../../common/api/util/with_license'; import { withExistingMigration } from './util/with_existing_migration_id'; export const registerSiemRuleMigrationsTranslationStatsRoute = ( @@ -46,7 +46,7 @@ export const registerSiemRuleMigrationsTranslationStatsRoute = ( const ctx = await context.resolve(['securitySolution']); const ruleMigrationsClient = ctx.securitySolution.siemMigrations.getRulesClient(); - const stats = await ruleMigrationsClient.data.rules.getTranslationStats(migrationId); + const stats = await ruleMigrationsClient.data.items.getTranslationStats(migrationId); if (stats.rules.total === 0) { return res.noContent(); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/update.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/update.ts index 75b192720c103..4418991e96216 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/update.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/update.ts @@ -13,9 +13,9 @@ import { UpdateRuleMigrationRequestParams, } from '../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; import type { SecuritySolutionPluginRouter } from '../../../../types'; -import { SiemMigrationAuditLogger } from '../../common/utils/audit'; -import { authz } from '../../common/utils/authz'; -import { withLicense } from '../../common/utils/with_license'; +import { SiemMigrationAuditLogger } from '../../common/api/util/audit'; +import { authz } from '../../common/api/util/authz'; +import { withLicense } from '../../common/api/util/with_license'; import { withExistingMigration } from './util/with_existing_migration_id'; export const registerSiemRuleMigrationsUpdateRoute = ( diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/util/installation.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/util/installation.ts index 7968176928a93..f350e909b1341 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/util/installation.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/util/installation.ts @@ -15,7 +15,7 @@ import { performTimelinesInstallation } from '../../../../detection_engine/prebu import { createPrebuiltRules } from '../../../../detection_engine/prebuilt_rules/logic/rule_objects/create_prebuilt_rules'; import type { IDetectionRulesClient } from '../../../../detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface'; import type { RuleResponse } from '../../../../../../common/api/detection_engine'; -import type { StoredRuleMigration } from '../../types'; +import type { StoredRuleMigrationRule } from '../../types'; import { getPrebuiltRules, getUniquePrebuiltRuleIds } from './prebuilt_rules'; import { convertMigrationCustomRuleToSecurityRulePayload, @@ -26,7 +26,7 @@ import { getVendorTag } from './tags'; const MAX_CUSTOM_RULES_TO_CREATE_IN_PARALLEL = 50; const installPrebuiltRules = async ( - rulesToInstall: StoredRuleMigration[], + rulesToInstall: StoredRuleMigrationRule[], enabled: boolean, securitySolutionContext: SecuritySolutionApiRequestHandlerContext, rulesClient: RulesClient, @@ -88,7 +88,7 @@ const installPrebuiltRules = async ( }; export const installCustomRules = async ( - rulesToInstall: StoredRuleMigration[], + rulesToInstall: StoredRuleMigrationRule[], enabled: boolean, detectionRulesClient: IDetectionRulesClient ): Promise<{ @@ -179,7 +179,7 @@ export const installTranslated = async ({ const installationErrors: Error[] = []; // Install rules that matched Elastic prebuilt rules - const prebuiltRuleBatches = ruleMigrationsClient.data.rules.searchBatches(migrationId, { + const prebuiltRuleBatches = ruleMigrationsClient.data.items.searchBatches(migrationId, { filters: { ids, installable: true, prebuilt: true }, }); let prebuiltRulesToInstall = await prebuiltRuleBatches.next(); @@ -194,7 +194,7 @@ export const installTranslated = async ({ ); installedCount += rulesToUpdate.length; installationErrors.push(...errors); - await ruleMigrationsClient.data.rules.update(rulesToUpdate); + await ruleMigrationsClient.data.items.update(rulesToUpdate); prebuiltRulesToInstall = await prebuiltRuleBatches.next(); } @@ -205,7 +205,7 @@ export const installTranslated = async ({ } // Install rules with custom translation - const customRuleBatches = ruleMigrationsClient.data.rules.searchBatches(migrationId, { + const customRuleBatches = ruleMigrationsClient.data.items.searchBatches(migrationId, { filters: { ids, installable: true, prebuilt: false }, }); let customRulesToInstall = await customRuleBatches.next(); @@ -217,7 +217,7 @@ export const installTranslated = async ({ ); installedCount += rulesToUpdate.length; installationErrors.push(...errors); - await ruleMigrationsClient.data.rules.update(rulesToUpdate); + await ruleMigrationsClient.data.items.update(rulesToUpdate); customRulesToInstall = await customRuleBatches.next(); } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/util/prebuilt_rules.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/util/prebuilt_rules.ts index efdbb255f980c..278cb13930a3e 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/util/prebuilt_rules.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/util/prebuilt_rules.ts @@ -99,7 +99,7 @@ export const getPrebuiltRulesForMigration = async ( savedObjectsClient: SavedObjectsClientContract ): Promise> => { const options = { filters: { prebuilt: true } }; - const batches = ruleMigrationsClient.data.rules.searchBatches(migrationId, options); + const batches = ruleMigrationsClient.data.items.searchBatches(migrationId, options); const rulesIds = new Set(); let results = await batches.next(); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/util/update_rules.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/util/update_rules.ts index 77ead9cb4082a..db6cb5ec1f3b7 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/util/update_rules.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/util/update_rules.ts @@ -6,11 +6,9 @@ */ import { parseEsqlQuery } from '@kbn/securitysolution-utils'; -import { - RuleMigrationTranslationResultEnum, - type RuleMigrationTranslationResult, - type UpdateRuleMigrationRule, -} from '../../../../../../common/siem_migrations/model/rule_migration.gen'; +import type { MigrationTranslationResult } from '../../../../../../common/siem_migrations/model/migration.gen'; +import { MigrationTranslationResultEnum } from '../../../../../../common/siem_migrations/model/migration.gen'; +import { type UpdateRuleMigrationRule } from '../../../../../../common/siem_migrations/model/rule_migration.gen'; import type { InternalUpdateRuleMigrationRule } from '../../types'; export const isValidEsqlQuery = (esqlQuery: string) => { @@ -31,13 +29,13 @@ export const isValidEsqlQuery = (esqlQuery: string) => { export const convertEsqlQueryToTranslationResult = ( esqlQuery: string -): RuleMigrationTranslationResult | undefined => { +): MigrationTranslationResult | undefined => { if (esqlQuery === '') { - return RuleMigrationTranslationResultEnum.untranslatable; + return MigrationTranslationResultEnum.untranslatable; } return isValidEsqlQuery(esqlQuery) - ? RuleMigrationTranslationResultEnum.full - : RuleMigrationTranslationResultEnum.partial; + ? MigrationTranslationResultEnum.full + : MigrationTranslationResultEnum.partial; }; export const transformToInternalUpdateRuleMigrationData = ( diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/util/with_existing_migration_id.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/util/with_existing_migration_id.ts index 6da298f5392d6..90012ff419f34 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/util/with_existing_migration_id.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/util/with_existing_migration_id.ts @@ -28,9 +28,7 @@ export const withExistingMigration = < const { migration_id: migrationId } = req.params; const ctx = await context.resolve(['securitySolution']); const ruleMigrationsClient = ctx.securitySolution.siemMigrations.getRulesClient(); - const storedMigration = await ruleMigrationsClient.data.migrations.get({ - id: migrationId, - }); + const storedMigration = await ruleMigrationsClient.data.migrations.get(migrationId); if (!storedMigration) { return res.notFound({ diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/dsl_queries.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/dsl_queries.ts new file mode 100644 index 0000000000000..b271de659368b --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/dsl_queries.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; +import { dsl as genericDsl } from '../../common/data/dsl_queries'; + +export const dsl = { + isInstalled(): QueryDslQueryContainer { + return { exists: { field: 'elastic_rule.id' } }; + }, + isNotInstalled(): QueryDslQueryContainer { + return { bool: { must_not: dsl.isInstalled() } }; + }, + isPrebuilt(): QueryDslQueryContainer { + return { exists: { field: 'elastic_rule.prebuilt_rule_id' } }; + }, + isCustom(): QueryDslQueryContainer { + return { bool: { must_not: dsl.isPrebuilt() } }; + }, + matchTitle(title: string): QueryDslQueryContainer { + return { match: { 'elastic_rule.title': title } }; + }, + isInstallable(): QueryDslQueryContainer[] { + return [genericDsl.isFullyTranslated(), dsl.isNotInstalled()]; + }, + isNotInstallable(): QueryDslQueryContainer[] { + return [genericDsl.isNotFullyTranslated(), dsl.isInstalled()]; + }, +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_field_maps.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/field_maps.ts similarity index 100% rename from x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_field_maps.ts rename to x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/field_maps.ts diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_client.ts index ad389dd21ccf6..da71749b730aa 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_client.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_client.ts @@ -8,20 +8,18 @@ import type { AuthenticatedUser, IScopedClusterClient, Logger } from '@kbn/core/server'; import { RuleMigrationsDataIntegrationsClient } from './rule_migrations_data_integrations_client'; import { RuleMigrationsDataPrebuiltRulesClient } from './rule_migrations_data_prebuilt_rules_client'; -import { RuleMigrationsDataResourcesClient } from './rule_migrations_data_resources_client'; import { RuleMigrationsDataRulesClient } from './rule_migrations_data_rules_client'; import { RuleMigrationsDataLookupsClient } from './rule_migrations_data_lookups_client'; import type { RuleMigrationIndexNameProviders } from '../types'; import { RuleMigrationsDataMigrationClient } from './rule_migrations_data_migration_client'; import type { SiemMigrationsClientDependencies } from '../../common/types'; +import { SiemMigrationsDataClient } from '../../common/data/siem_migrations_data_client'; +import { SiemMigrationsDataResourcesClient } from '../../common/data/siem_migrations_data_resources_client'; -export class RuleMigrationsDataClient { - protected logger: Logger; - protected esClient: IScopedClusterClient['asInternalUser']; - +export class RuleMigrationsDataClient extends SiemMigrationsDataClient { public readonly migrations: RuleMigrationsDataMigrationClient; - public readonly rules: RuleMigrationsDataRulesClient; - public readonly resources: RuleMigrationsDataResourcesClient; + public readonly items: RuleMigrationsDataRulesClient; + public readonly resources: SiemMigrationsDataResourcesClient; public readonly integrations: RuleMigrationsDataIntegrationsClient; public readonly prebuiltRules: RuleMigrationsDataPrebuiltRulesClient; public readonly lookups: RuleMigrationsDataLookupsClient; @@ -34,6 +32,8 @@ export class RuleMigrationsDataClient { spaceId: string, dependencies: SiemMigrationsClientDependencies ) { + super(esScopedClient, logger); + this.migrations = new RuleMigrationsDataMigrationClient( indexNameProviders.migrations, currentUser, @@ -41,14 +41,14 @@ export class RuleMigrationsDataClient { logger, dependencies ); - this.rules = new RuleMigrationsDataRulesClient( + this.items = new RuleMigrationsDataRulesClient( indexNameProviders.rules, currentUser, esScopedClient, logger, dependencies ); - this.resources = new RuleMigrationsDataResourcesClient( + this.resources = new SiemMigrationsDataResourcesClient( indexNameProviders.resources, currentUser, esScopedClient, @@ -75,40 +75,5 @@ export class RuleMigrationsDataClient { logger, spaceId ); - - this.logger = logger; - this.esClient = esScopedClient.asInternalUser; - } - - /** - * - * Deletes a migration and all its associated rules and resources. - * - */ - async deleteMigration(migrationId: string) { - const migrationDeleteOperations = await this.migrations.prepareDelete({ - id: migrationId, - }); - - const rulesByMigrationIdDeleteOperations = await this.rules.prepareDelete(migrationId); - - const resourcesByMigrationIdDeleteOperations = await this.resources.prepareDelete(migrationId); - - return this.esClient - .bulk({ - refresh: 'wait_for', - operations: [ - ...migrationDeleteOperations, - ...rulesByMigrationIdDeleteOperations, - ...resourcesByMigrationIdDeleteOperations, - ], - }) - .then(() => { - this.logger.info(`Deleted migration ${migrationId}`); - }) - .catch((error) => { - this.logger.error(`Error deleting migration ${migrationId}: ${error}`); - throw error; - }); } } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_migration_client.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_migration_client.test.ts index 48e80f59905db..6f054193701b3 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_migration_client.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_migration_client.test.ts @@ -91,7 +91,7 @@ describe('RuleMigrationsDataMigrationClient', () => { esClient.asInternalUser.get as unknown as jest.MockedFn ).mockResolvedValueOnce(response); - const result = await ruleMigrationsDataMigrationClient.get({ id }); + const result = await ruleMigrationsDataMigrationClient.get(id); expect(result).toEqual({ ...response._source, @@ -112,7 +112,7 @@ describe('RuleMigrationsDataMigrationClient', () => { message: JSON.stringify(response), }); - const result = await ruleMigrationsDataMigrationClient.get({ id }); + const result = await ruleMigrationsDataMigrationClient.get(id); expect(result).toBeUndefined(); }); @@ -123,7 +123,7 @@ describe('RuleMigrationsDataMigrationClient', () => { esClient.asInternalUser.get as unknown as jest.MockedFn ).mockRejectedValueOnce(new Error('Test error')); - await expect(ruleMigrationsDataMigrationClient.get({ id })).rejects.toThrow('Test error'); + await expect(ruleMigrationsDataMigrationClient.get(id)).rejects.toThrow('Test error'); expect(esClient.asInternalUser.get).toHaveBeenCalled(); expect(logger.error).toHaveBeenCalledWith(`Error getting migration ${id}: Error: Test error`); @@ -137,9 +137,7 @@ describe('RuleMigrationsDataMigrationClient', () => { const migrationId = 'testId'; const index = '.kibana-siem-rule-migrations'; - const operations = await ruleMigrationsDataMigrationClient.prepareDelete({ - id: migrationId, - }); + const operations = await ruleMigrationsDataMigrationClient.prepareDelete(migrationId); expect(operations).toMatchObject([ { diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_migration_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_migration_client.ts index 3acece32299c8..6d860ecc1bf54 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_migration_client.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_migration_client.ts @@ -5,97 +5,15 @@ * 2.0. */ -import { v4 as uuidV4 } from 'uuid'; -import type { BulkOperationContainer } from '@elastic/elasticsearch/lib/api/types'; -import type { RuleMigrationLastExecution } from '../../../../../common/siem_migrations/model/rule_migration.gen'; -import type { StoredSiemMigration } from '../types'; -import { SiemMigrationsDataBaseClient } from '../../common/data/siem_migrations_data_base_client'; -import { isNotFoundError } from '../../common/utils/is_not_found_error'; -import { MAX_ES_SEARCH_SIZE } from '../constants'; - -export class RuleMigrationsDataMigrationClient extends SiemMigrationsDataBaseClient { - async create(name: string): Promise { - const migrationId = uuidV4(); - const index = await this.getIndexName(); - const profileUid = await this.getProfileUid(); - const createdAt = new Date().toISOString(); - - await this.esClient - .create({ - refresh: 'wait_for', - id: migrationId, - index, - document: { - created_by: profileUid, - created_at: createdAt, - name, - }, - }) - .catch((error) => { - this.logger.error(`Error creating migration ${migrationId}: ${error}`); - throw error; - }); - - return migrationId; - } - - /** - * - * Gets the migration document by id or returns undefined if it does not exist. - * - * */ - async get({ id }: { id: string }): Promise { - const index = await this.getIndexName(); - return this.esClient - .get({ index, id }) - .then(this.processHit) - .catch((error) => { - if (isNotFoundError(error)) { - return undefined; - } - this.logger.error(`Error getting migration ${id}: ${error}`); - throw error; - }); - } - - /** - * Gets all migrations from the index. - */ - async getAll(): Promise { - const index = await this.getIndexName(); - return this.esClient - .search({ - index, - size: MAX_ES_SEARCH_SIZE, // Adjust size as needed - query: { match_all: {} }, - _source: true, - }) - .then((result) => this.processResponseHits(result)) - .catch((error) => { - this.logger.error(`Error getting all migrations:- ${error}`); - throw error; - }); - } +import type { RuleMigration } from '../../../../../common/siem_migrations/model/rule_migration.gen'; +import { SiemMigrationsDataMigrationClient } from '../../common/data/siem_migrations_data_migration_client'; +export class RuleMigrationsDataMigrationClient extends SiemMigrationsDataMigrationClient { /** + * Saves a migration as started. * - * Prepares bulk ES delete operation for a migration document based on its id. - * - */ - async prepareDelete({ id }: { id: string }): Promise { - const index = await this.getIndexName(); - const migrationDeleteOperation = { - delete: { - _index: index, - _id: id, - }, - }; - - return [migrationDeleteOperation]; - } - - /** - * Saves a migration as started, updating the last execution parameters with the current timestamp. + * Overloads the `saveAsStarted` method of the SiemMigrationsDataMigrationClient class + * to receive and store the `skipPrebuiltRulesMatching` value which is specific of rule migrations. */ async saveAsStarted({ id, @@ -115,56 +33,4 @@ export class RuleMigrationsDataMigrationClient extends SiemMigrationsDataBaseCli skip_prebuilt_rules_matching: skipPrebuiltRulesMatching, }); } - - /** - * Saves a migration as ended, updating the last execution parameters with the current timestamp. - */ - async saveAsFinished({ id }: { id: string }): Promise { - await this.updateLastExecution(id, { finished_at: new Date().toISOString() }); - } - - /** - * Saves a migration as failed, updating the last execution parameters with the provided error message. - */ - async saveAsFailed({ id, error }: { id: string; error: string }): Promise { - await this.updateLastExecution(id, { error, finished_at: new Date().toISOString() }); - } - - /** - * Sets `is_stopped` flag for migration document. - * It does not update `finished_at` timestamp, `saveAsFinished` or `saveAsFailed` should be called separately. - */ - async setIsStopped({ id }: { id: string }): Promise { - await this.updateLastExecution(id, { is_stopped: true }); - } - - /** - * Updates the last execution parameters for a migration document. - */ - private async updateLastExecution( - id: string, - lastExecutionParams: RuleMigrationLastExecution - ): Promise { - const index = await this.getIndexName(); - const doc = { last_execution: lastExecutionParams }; - await this.esClient - .update({ index, id, refresh: 'wait_for', doc, retry_on_conflict: 1 }) - .catch((error) => { - this.logger.error(`Error updating last execution for migration ${id}: ${error}`); - throw error; - }); - } - - /** - * Updates the migration document with the provided values. - */ - async update(id: string, doc: Partial): Promise { - const index = await this.getIndexName(); - await this.esClient - .update({ index, id, doc, refresh: 'wait_for', retry_on_conflict: 1 }) - .catch((error) => { - this.logger.error(`Error updating migration: ${error}`); - throw error; - }); - } } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_rules_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_rules_client.ts index f14932a2eaae5..18a53ebfbb4c4 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_rules_client.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_rules_client.ts @@ -8,20 +8,13 @@ import type { AggregationsAggregationContainer, AggregationsFilterAggregate, - AggregationsMaxAggregate, - AggregationsMinAggregate, AggregationsStringTermsAggregate, AggregationsStringTermsBucket, QueryDslQueryContainer, - Duration, - BulkOperationContainer, } from '@elastic/elasticsearch/lib/api/types'; -import type { RuleMigrationFilters } from '../../../../../common/siem_migrations/types'; -import type { InternalUpdateRuleMigrationRule, StoredRuleMigration } from '../types'; -import { - SiemMigrationStatus, - RuleTranslationResult, -} from '../../../../../common/siem_migrations/constants'; +import type { estypes } from '@elastic/elasticsearch'; +import type { RuleMigrationFilters } from '../../../../../common/siem_migrations/rules/types'; +import { SiemMigrationStatus } from '../../../../../common/siem_migrations/constants'; import { type RuleMigrationTaskStats, type RuleMigrationTranslationStats, @@ -29,14 +22,15 @@ import { type RuleMigrationRule, } from '../../../../../common/siem_migrations/model/rule_migration.gen'; import { getSortingOptions, type RuleMigrationSort } from './sort'; -import { conditions as searchConditions } from './search'; -import { SiemMigrationsDataBaseClient } from '../../common/data/siem_migrations_data_base_client'; +import { dsl } from './dsl_queries'; import { MAX_ES_SEARCH_SIZE } from '../constants'; +import type { + CreateMigrationItemInput, + SiemMigrationItemSort, +} from '../../common/data/siem_migrations_data_item_client'; +import { SiemMigrationsDataItemClient } from '../../common/data/siem_migrations_data_item_client'; -export type AddRuleMigrationRulesInput = Omit< - RuleMigrationRule, - '@timestamp' | 'id' | 'status' | 'created_by' ->; +export type CreateRuleMigrationRulesInput = CreateMigrationItemInput; export type RuleMigrationDataStats = Omit; export type RuleMigrationAllDataStats = RuleMigrationDataStats[]; @@ -47,187 +41,11 @@ export interface RuleMigrationGetRulesOptions { size?: number; } -/* BULK_MAX_SIZE defines the number to break down the bulk operations by. - * The 500 number was chosen as a reasonable number to avoid large payloads. It can be adjusted if needed. */ -const BULK_MAX_SIZE = 500 as const; -/* DEFAULT_SEARCH_BATCH_SIZE defines the default number of documents to retrieve per search operation - * when retrieving search results in batches. */ -const DEFAULT_SEARCH_BATCH_SIZE = 500 as const; - -export class RuleMigrationsDataRulesClient extends SiemMigrationsDataBaseClient { - /** Indexes an array of rule migrations to be processed */ - async create(ruleMigrations: AddRuleMigrationRulesInput[]): Promise { - const index = await this.getIndexName(); - const profileId = await this.getProfileUid(); - - let ruleMigrationsSlice: AddRuleMigrationRulesInput[]; - const createdAt = new Date().toISOString(); - while ((ruleMigrationsSlice = ruleMigrations.splice(0, BULK_MAX_SIZE)).length) { - await this.esClient - .bulk({ - refresh: 'wait_for', - operations: ruleMigrationsSlice.flatMap((ruleMigration) => [ - { create: { _index: index } }, - { - ...ruleMigration, - '@timestamp': createdAt, - status: SiemMigrationStatus.PENDING, - created_by: profileId, - updated_by: profileId, - updated_at: createdAt, - }, - ]), - }) - .catch((error) => { - this.logger.error(`Error creating rule migrations: ${error.message}`); - throw error; - }); - } - } - - /** Updates an array of rule migrations to be processed */ - async update(ruleMigrations: InternalUpdateRuleMigrationRule[]): Promise { - const index = await this.getIndexName(); - const profileId = await this.getProfileUid(); - - let ruleMigrationsSlice: InternalUpdateRuleMigrationRule[]; - const updatedAt = new Date().toISOString(); - while ((ruleMigrationsSlice = ruleMigrations.splice(0, BULK_MAX_SIZE)).length) { - await this.esClient - .bulk({ - refresh: 'wait_for', - operations: ruleMigrationsSlice.flatMap((ruleMigration) => { - const { id, ...rest } = ruleMigration; - return [ - { update: { _index: index, _id: id } }, - { - doc: { - ...rest, - updated_by: profileId, - updated_at: updatedAt, - }, - }, - ]; - }), - }) - .catch((error) => { - this.logger.error(`Error updating rule migrations: ${error.message}`); - throw error; - }); - } - } - - /** Retrieves an array of rule documents of a specific migrations */ - async get( - migrationId: string, - { filters = {}, sort: sortParam = {}, from, size }: RuleMigrationGetRulesOptions = {} - ): Promise<{ total: number; data: StoredRuleMigration[] }> { - const index = await this.getIndexName(); - const query = this.getFilterQuery(migrationId, filters); - const sort = sortParam.sortField ? getSortingOptions(sortParam) : undefined; - - const result = await this.esClient - .search({ index, query, sort, from, size }) - .catch((error) => { - this.logger.error(`Error searching rule migrations: ${error.message}`); - throw error; - }); - return { - total: this.getTotalHits(result), - data: this.processResponseHits(result), - }; - } - - /** Returns batching functions to traverse all the migration rules search results */ - searchBatches( - migrationId: string, - options: { scroll?: Duration; size?: number; filters?: RuleMigrationFilters } = {} - ) { - const { size = DEFAULT_SEARCH_BATCH_SIZE, filters = {}, scroll } = options; - const query = this.getFilterQuery(migrationId, filters); - const search = { query, sort: '_doc', scroll, size }; // sort by _doc to ensure consistent order - try { - return this.getSearchBatches(search); - } catch (error) { - this.logger.error(`Error scrolling rule migrations: ${error.message}`); - throw error; - } - } - - /** Updates one rule migration status to `processing` */ - async saveProcessing(id: string): Promise { - const index = await this.getIndexName(); - const profileId = await this.getProfileUid(); - const doc = { - status: SiemMigrationStatus.PROCESSING, - updated_by: profileId, - updated_at: new Date().toISOString(), - }; - await this.esClient.update({ index, id, doc, refresh: 'wait_for' }).catch((error) => { - this.logger.error(`Error updating rule migration status to processing: ${error.message}`); - throw error; - }); - } - - /** Updates one rule migration with the provided data and sets the status to `completed` */ - async saveCompleted({ id, ...ruleMigration }: StoredRuleMigration): Promise { - const index = await this.getIndexName(); - const profileId = await this.getProfileUid(); - const doc = { - ...ruleMigration, - status: SiemMigrationStatus.COMPLETED, - updated_by: profileId, - updated_at: new Date().toISOString(), - }; - await this.esClient.update({ index, id, doc, refresh: 'wait_for' }).catch((error) => { - this.logger.error(`Error updating rule migration status to completed: ${error.message}`); - throw error; - }); - } - - /** Updates one rule migration with the provided data and sets the status to `failed` */ - async saveError({ id, ...ruleMigration }: StoredRuleMigration): Promise { - const index = await this.getIndexName(); - const profileId = await this.getProfileUid(); - const doc = { - ...ruleMigration, - status: SiemMigrationStatus.FAILED, - updated_by: profileId, - updated_at: new Date().toISOString(), - }; - await this.esClient.update({ index, id, doc, refresh: 'wait_for' }).catch((error) => { - this.logger.error(`Error updating rule migration status to failed: ${error.message}`); - throw error; - }); - } - - /** Updates all the rule migration with the provided id with status `processing` back to `pending` */ - async releaseProcessing(migrationId: string): Promise { - return this.updateStatus( - migrationId, - { status: SiemMigrationStatus.PROCESSING }, - SiemMigrationStatus.PENDING - ); - } - - /** Updates all the rule migration with the provided id and with status `statusToQuery` to `statusToUpdate` */ - async updateStatus( - migrationId: string, - filter: RuleMigrationFilters, - statusToUpdate: SiemMigrationStatus, - { refresh = false }: { refresh?: boolean } = {} - ): Promise { - const index = await this.getIndexName(); - const query = this.getFilterQuery(migrationId, filter); - const script = { source: `ctx._source['status'] = '${statusToUpdate}'` }; - await this.esClient.updateByQuery({ index, query, script, refresh }).catch((error) => { - this.logger.error(`Error updating rule migrations status: ${error.message}`); - throw error; - }); - } +export class RuleMigrationsDataRulesClient extends SiemMigrationsDataItemClient { + protected type = 'rule' as const; /** Retrieves the translation stats for the rule migrations with the provided id */ - async getTranslationStats(migrationId: string): Promise { + public async getTranslationStats(migrationId: string): Promise { const index = await this.getIndexName(); const query = this.getFilterQuery(migrationId); @@ -236,8 +54,8 @@ export class RuleMigrationsDataRulesClient extends SiemMigrationsDataBaseClient filter: { term: { status: SiemMigrationStatus.COMPLETED } }, aggs: { result: { terms: { field: 'translation_result' } }, - installable: { filter: { bool: { must: searchConditions.isInstallable() } } }, - prebuilt: { filter: searchConditions.isPrebuilt() }, + installable: { filter: { bool: { must: dsl.isInstallable() } } }, + prebuilt: { filter: dsl.isPrebuilt() }, }, }, failed: { filter: { term: { status: SiemMigrationStatus.FAILED } } }, @@ -269,72 +87,8 @@ export class RuleMigrationsDataRulesClient extends SiemMigrationsDataBaseClient }; } - /** Retrieves the stats for the rule migrations with the provided id */ - async getStats(migrationId: string): Promise { - const index = await this.getIndexName(); - const query = this.getFilterQuery(migrationId); - const aggregations = { - status: { terms: { field: 'status' } }, - createdAt: { min: { field: '@timestamp' } }, - lastUpdatedAt: { max: { field: 'updated_at' } }, - }; - const result = await this.esClient - .search({ index, query, aggregations, _source: false }) - .catch((error) => { - this.logger.error(`Error getting rule migrations stats: ${error.message}`); - throw error; - }); - - const aggs = result.aggregations ?? {}; - - return { - id: migrationId, - rules: { - total: this.getTotalHits(result), - ...this.statusAggCounts(aggs.status as AggregationsStringTermsAggregate), - }, - created_at: (aggs.createdAt as AggregationsMinAggregate)?.value_as_string ?? '', - last_updated_at: (aggs.lastUpdatedAt as AggregationsMaxAggregate)?.value_as_string ?? '', - }; - } - - /** Retrieves the stats for all the rule migrations aggregated by migration id, in creation order */ - async getAllStats(): Promise { - const index = await this.getIndexName(); - const aggregations: { migrationIds: AggregationsAggregationContainer } = { - migrationIds: { - terms: { field: 'migration_id', order: { createdAt: 'asc' }, size: MAX_ES_SEARCH_SIZE }, - aggregations: { - status: { terms: { field: 'status' } }, - createdAt: { min: { field: '@timestamp' } }, - lastUpdatedAt: { max: { field: 'updated_at' } }, - }, - }, - }; - const result = await this.esClient - .search({ index, aggregations, _source: false }) - .catch((error) => { - this.logger.error(`Error getting all rule migrations stats: ${error.message}`); - throw error; - }); - - const migrationsAgg = result.aggregations?.migrationIds as AggregationsStringTermsAggregate; - const buckets = (migrationsAgg?.buckets as AggregationsStringTermsBucket[]) ?? []; - return buckets.map((bucket) => ({ - id: `${bucket.key}`, - rules: { - total: bucket.doc_count, - ...this.statusAggCounts(bucket.status as AggregationsStringTermsAggregate), - }, - created_at: (bucket.createdAt as AggregationsMinAggregate | undefined) - ?.value_as_string as string, - last_updated_at: (bucket.lastUpdatedAt as AggregationsMaxAggregate | undefined) - ?.value_as_string as string, - })); - } - /** Retrieves the stats for the integrations of all the migration rules */ - async getAllIntegrationsStats(): Promise { + public async getAllIntegrationsStats(): Promise { const index = await this.getIndexName(); const aggregations: { integrationIds: AggregationsAggregationContainer } = { integrationIds: { @@ -360,107 +114,30 @@ export class RuleMigrationsDataRulesClient extends SiemMigrationsDataBaseClient })); } - private statusAggCounts( - statusAgg: AggregationsStringTermsAggregate - ): Record { - const buckets = statusAgg.buckets as AggregationsStringTermsBucket[]; - return { - [SiemMigrationStatus.PENDING]: - buckets.find(({ key }) => key === SiemMigrationStatus.PENDING)?.doc_count ?? 0, - [SiemMigrationStatus.PROCESSING]: - buckets.find(({ key }) => key === SiemMigrationStatus.PROCESSING)?.doc_count ?? 0, - [SiemMigrationStatus.COMPLETED]: - buckets.find(({ key }) => key === SiemMigrationStatus.COMPLETED)?.doc_count ?? 0, - [SiemMigrationStatus.FAILED]: - buckets.find(({ key }) => key === SiemMigrationStatus.FAILED)?.doc_count ?? 0, - }; - } - - private translationResultAggCount( - resultAgg: AggregationsStringTermsAggregate - ): Record { - const buckets = resultAgg.buckets as AggregationsStringTermsBucket[]; - return { - [RuleTranslationResult.FULL]: - buckets.find(({ key }) => key === RuleTranslationResult.FULL)?.doc_count ?? 0, - [RuleTranslationResult.PARTIAL]: - buckets.find(({ key }) => key === RuleTranslationResult.PARTIAL)?.doc_count ?? 0, - [RuleTranslationResult.UNTRANSLATABLE]: - buckets.find(({ key }) => key === RuleTranslationResult.UNTRANSLATABLE)?.doc_count ?? 0, - }; - } - - private getFilterQuery( + protected getFilterQuery( migrationId: string, filters: RuleMigrationFilters = {} - ): QueryDslQueryContainer { - const filter: QueryDslQueryContainer[] = [{ term: { migration_id: migrationId } }]; - if (filters.status) { - if (Array.isArray(filters.status)) { - filter.push({ terms: { status: filters.status } }); - } else { - filter.push({ term: { status: filters.status } }); - } - } - if (filters.ids) { - filter.push({ terms: { _id: filters.ids } }); - } + ): { bool: { filter: QueryDslQueryContainer[] } } { + const { filter } = super.getFilterQuery(migrationId, filters).bool; + + // Rules specific filters if (filters.searchTerm?.length) { - filter.push(searchConditions.matchTitle(filters.searchTerm)); - } - if (filters.installed === true) { - filter.push(searchConditions.isInstalled()); - } else if (filters.installed === false) { - filter.push(searchConditions.isNotInstalled()); - } - if (filters.installable === true) { - filter.push(...searchConditions.isInstallable()); - } else if (filters.installable === false) { - filter.push(...searchConditions.isNotInstallable()); - } - if (filters.prebuilt === true) { - filter.push(searchConditions.isPrebuilt()); - } else if (filters.prebuilt === false) { - filter.push(searchConditions.isCustom()); + filter.push(dsl.matchTitle(filters.searchTerm)); } - if (filters.failed === true) { - filter.push(searchConditions.isFailed()); - } else if (filters.failed === false) { - filter.push(searchConditions.isNotFailed()); + if (filters.installed != null) { + filter.push(filters.installed ? dsl.isInstalled() : dsl.isNotInstalled()); } - if (filters.fullyTranslated === true) { - filter.push(searchConditions.isFullyTranslated()); - } else if (filters.fullyTranslated === false) { - filter.push(searchConditions.isNotFullyTranslated()); + if (filters.installable != null) { + filter.push(...(filters.installable ? dsl.isInstallable() : dsl.isNotInstallable())); } - if (filters.partiallyTranslated === true) { - filter.push(searchConditions.isPartiallyTranslated()); - } else if (filters.partiallyTranslated === false) { - filter.push(searchConditions.isNotPartiallyTranslated()); - } - if (filters.untranslatable === true) { - filter.push(searchConditions.isUntranslatable()); - } else if (filters.untranslatable === false) { - filter.push(searchConditions.isNotUntranslatable()); + if (filters.prebuilt != null) { + filter.push(filters.prebuilt ? dsl.isPrebuilt() : dsl.isCustom()); } + return { bool: { filter } }; } - /** - * - * Prepares bulk ES delete operations for the rules based on migrationId. - * - * */ - async prepareDelete(migrationId: string): Promise { - const index = await this.getIndexName(); - const rulesToBeDeleted = await this.get(migrationId, { size: MAX_ES_SEARCH_SIZE }); - const rulesToBeDeletedDocIds = rulesToBeDeleted.data.map((rule) => rule.id); - - return rulesToBeDeletedDocIds.map((docId) => ({ - delete: { - _id: docId, - _index: index, - }, - })); + protected getSortOptions(sort: SiemMigrationItemSort = {}): estypes.Sort { + return getSortingOptions(sort); } } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.test.ts index 6a6115d7b19f3..8ecb1a9a44d9f 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.test.ts @@ -12,7 +12,7 @@ import { IndexPatternAdapter, IndexAdapter } from '@kbn/index-adapter'; import { Subject } from 'rxjs'; import type { RuleMigrationIndexNameProviders } from '../types'; import type { SetupParams } from './rule_migrations_data_service'; -import { INDEX_PATTERN, RuleMigrationsDataService } from './rule_migrations_data_service'; +import { RuleMigrationsDataService } from './rule_migrations_data_service'; import { RuleMigrationIndexMigrator } from '../index_migrators'; import type { SiemMigrationsClientDependencies } from '../../common/types'; @@ -28,6 +28,9 @@ jest.mock('./rule_migrations_data_client', () => ({ }), })); +// @ts-expect-error accessing protected property +const INDEX_PATTERN = new RuleMigrationsDataService().baseIndexName; + const MockedIndexPatternAdapter = IndexPatternAdapter as unknown as jest.MockedClass< typeof IndexPatternAdapter >; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.ts index 2445865fe424e..fef3c0d1485bc 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.ts @@ -24,13 +24,11 @@ import { migrationsFieldMaps, ruleMigrationResourcesFieldMap, ruleMigrationsFieldMap, -} from './rule_migrations_field_maps'; +} from './field_maps'; import { RuleMigrationIndexMigrator } from '../index_migrators'; import { SiemMigrationsBaseDataService } from '../../common/siem_migrations_base_service'; import type { SiemMigrationsClientDependencies } from '../../common/types'; -export const INDEX_PATTERN = '.kibana-siem-rule-migrations'; - interface CreateClientParams { spaceId: string; currentUser: AuthenticatedUser; @@ -47,6 +45,8 @@ export interface SetupParams extends Omit { } export class RuleMigrationsDataService extends SiemMigrationsBaseDataService { + protected readonly baseIndexName = '.kibana-siem-rule-migrations'; + private readonly adapters: RuleMigrationAdapters; constructor(private logger: Logger, protected kibanaVersion: string, elserInferenceId?: string) { @@ -75,10 +75,6 @@ export class RuleMigrationsDataService extends SiemMigrationsBaseDataService { }; } - private getAdapterIndexName(adapterId: RuleMigrationAdapterId) { - return `${INDEX_PATTERN}-${adapterId}`; - } - private createRuleIndexPatternAdapter({ adapterId, fieldMap }: CreateRuleAdapterParams) { const name = this.getAdapterIndexName(adapterId); return this.createIndexPatternAdapter({ name, fieldMap }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/index_migrators/rule_migrations_per_space_index_migrator.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/index_migrators/rule_migrations_per_space_index_migrator.test.ts index 38df357d295c7..308cf85eda3cf 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/index_migrators/rule_migrations_per_space_index_migrator.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/index_migrators/rule_migrations_per_space_index_migrator.test.ts @@ -6,7 +6,7 @@ */ import type { ElasticsearchClient, Logger } from '@kbn/core/server'; -import type { RuleMigrationAdapters, StoredSiemMigration } from '../types'; +import type { RuleMigrationAdapters, StoredRuleMigration } from '../types'; import { RuleMigrationSpaceIndexMigrator } from './rule_migrations_per_space_index_migrator'; import type { SearchResponseBody } from 'elasticsearch-8.x/lib/api/types'; @@ -33,7 +33,7 @@ const mockMigrationsIndexResult = { hits: { hits: [], }, -} as unknown as SearchResponseBody; +} as unknown as SearchResponseBody; const getMockedESSearchFunction = ( rulesIndexAggResult: typeof mockRuleIndexAggregationsResult = mockRuleIndexAggregationsResult, @@ -114,7 +114,7 @@ describe('RuleMigrationSpaceIndexMigrator', () => { }, ], }, - } as unknown as SearchResponseBody; + } as unknown as SearchResponseBody; esClientMock.search.mockImplementation( getMockedESSearchFunction( mockRuleIndexAggregationsResult, @@ -164,7 +164,7 @@ describe('RuleMigrationSpaceIndexMigrator', () => { }, ], }, - } as unknown as SearchResponseBody; + } as unknown as SearchResponseBody; esClientMock.search.mockImplementation( getMockedESSearchFunction( diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/index_migrators/rule_migrations_per_space_index_migrator.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/index_migrators/rule_migrations_per_space_index_migrator.ts index b03def0e2eece..efa372bc288b9 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/index_migrators/rule_migrations_per_space_index_migrator.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/index_migrators/rule_migrations_per_space_index_migrator.ts @@ -12,7 +12,7 @@ import type { AggregationsStringTermsAggregate, AggregationsStringTermsBucket, } from '@elastic/elasticsearch/lib/api/types'; -import type { RuleMigrationAdapters, StoredSiemMigration } from '../types'; +import type { RuleMigrationAdapters, StoredRuleMigration } from '../types'; import { MAX_ES_SEARCH_SIZE } from '../constants'; export class RuleMigrationSpaceIndexMigrator { @@ -101,7 +101,7 @@ export class RuleMigrationSpaceIndexMigrator { /** * Creates migration documents in the migrations index. */ - private async createMigrationDocs(docs: Array>) { + private async createMigrationDocs(docs: Array>) { const _index = this.ruleMigrationIndexAdapters.migrations.getIndexName(this.spaceId); const operations = docs.flatMap(({ id: _id, ...doc }) => [ { create: { _id, _index } }, @@ -113,7 +113,7 @@ export class RuleMigrationSpaceIndexMigrator { /** * Updates migration documents in the migrations index. */ - private async updateMigrationDocs(docs: Array>) { + private async updateMigrationDocs(docs: Array>) { const _index = this.ruleMigrationIndexAdapters.migrations.getIndexName(this.spaceId); const operations = docs.flatMap(({ id: _id, ...doc }) => [ { update: { _id, _index } }, @@ -159,7 +159,7 @@ export class RuleMigrationSpaceIndexMigrator { */ private async getMigrationIdsFromMigrationsIndex(): Promise { const index = this.ruleMigrationIndexAdapters.migrations.getIndexName(this.spaceId); - const result = await this.esClient.search({ + const result = await this.esClient.search({ index, size: MAX_ES_SEARCH_SIZE, query: { match_all: {} }, @@ -178,7 +178,7 @@ export class RuleMigrationSpaceIndexMigrator { private async getMigrationIdsWithoutName(): Promise { const index = this.ruleMigrationIndexAdapters.migrations.getIndexName(this.spaceId); - const result = await this.esClient.search({ + const result = await this.esClient.search({ index, query: { bool: { must_not: { exists: { field: 'name' } } } }, size: MAX_ES_SEARCH_SIZE, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts index beee88dfb42d4..4f0aa1344091d 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts @@ -11,8 +11,8 @@ import type { LoggerFactory, IClusterClient, Logger } from '@kbn/core/server'; import { RuleMigrationsDataService } from './data/rule_migrations_data_service'; import type { RuleMigrationsDataClient } from './data/rule_migrations_data_client'; import type { RuleMigrationsTaskClient } from './task/rule_migrations_task_client'; -import { RuleMigrationsTaskService } from './task/rule_migrations_task_service'; import type { SiemMigrationsCreateClientParams } from '../common/types'; +import { RuleMigrationsTaskService } from './task/rule_migrations_task_service'; export interface SiemRulesMigrationsSetupParams { esClusterClient: IClusterClient; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/__mocks__/mocks.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/__mocks__/mocks.ts index 22f47d820ddec..00a703a1863be 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/__mocks__/mocks.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/__mocks__/mocks.ts @@ -7,7 +7,7 @@ import type { PublicMethodsOf } from '@kbn/utility-types'; import { FakeLLM } from '@langchain/core/utils/testing'; import { AsyncLocalStorageProviderSingleton } from '@langchain/core/singletons'; -import type { SiemMigrationTelemetryClient } from '../rule_migrations_telemetry_client'; +import type { RuleMigrationTelemetryClient } from '../rule_migrations_telemetry_client'; import type { BaseLLMParams } from '@langchain/core/language_models/llms'; export const createSiemMigrationTelemetryClientMock = () => { @@ -32,7 +32,7 @@ export const createSiemMigrationTelemetryClientMock = () => { reportIntegrationsMatch: jest.fn(), reportPrebuiltRulesMatch: jest.fn(), startSiemMigrationTask: jest.fn().mockReturnValue(mockStartSiemMigrationTaskReturn), - } as jest.Mocked>; + } as jest.Mocked>; }; // Factory function for the mock class @@ -45,7 +45,7 @@ export const createRuleMigrationsTaskClientMock = () => ({ stop: jest.fn().mockResolvedValue({ stopped: true }), getStats: jest.fn().mockResolvedValue({ status: 'done', - rules: { + items: { total: 1, finished: 1, processing: 0, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.test.ts index 53f79b5207e87..3b2b002905ef9 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.test.ts @@ -9,7 +9,7 @@ import type { ActionsClientChatOpenAI } from '@kbn/langchain/server/language_mod import { loggerMock } from '@kbn/logging-mocks'; import type { NodeResponse } from '../__mocks__/mocks'; import { SiemMigrationFakeLLM, MockSiemMigrationTelemetryClient } from '../__mocks__/mocks'; -import { MockEsqlKnowledgeBase } from '../util/__mocks__/mocks'; +import { MockEsqlKnowledgeBase } from '../../../common/task/util/__mocks__/mocks'; import { MockRuleMigrationsRetriever } from '../retrievers/__mocks__/mocks'; import { getRuleMigrationAgent } from './graph'; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts index b78aabce394f4..ddd8c49cea1c8 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts @@ -10,7 +10,7 @@ import { getCreateSemanticQueryNode } from './nodes/create_semantic_query'; import { getMatchPrebuiltRuleNode } from './nodes/match_prebuilt_rule'; import { migrateRuleConfigSchema, migrateRuleState } from './state'; import { getTranslateRuleGraph } from './sub_graphs/translate_rule'; -import type { MigrateRuleGraphConfig, MigrateRuleGraphParams, MigrateRuleState } from './types'; +import type { MigrateRuleConfig, MigrateRuleGraphParams, MigrateRuleState } from './types'; export function getRuleMigrationAgent({ model, @@ -57,7 +57,7 @@ export function getRuleMigrationAgent({ return graph; } -const skipPrebuiltRuleConditional = (_state: MigrateRuleState, config: MigrateRuleGraphConfig) => { +const skipPrebuiltRuleConditional = (_state: MigrateRuleState, config: MigrateRuleConfig) => { if (config.configurable?.skipPrebuiltRulesMatching) { return 'translationSubGraph'; } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/create_semantic_query/create_semantic_query.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/create_semantic_query/create_semantic_query.ts index 446b96234711a..0af5f46f79ab4 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/create_semantic_query/create_semantic_query.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/create_semantic_query/create_semantic_query.ts @@ -6,7 +6,7 @@ */ import { JsonOutputParser } from '@langchain/core/output_parsers'; -import type { ChatModel } from '../../../util/actions_client_chat'; +import type { ChatModel } from '../../../../../common/task/util/actions_client_chat'; import type { GraphNode } from '../../types'; import { CREATE_SEMANTIC_QUERY_PROMPT } from './prompts'; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/match_prebuilt_rule.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/match_prebuilt_rule.ts index 8ec7870eaa2b8..c0ee9cdec3e4d 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/match_prebuilt_rule.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/match_prebuilt_rule.ts @@ -7,11 +7,11 @@ import type { Logger } from '@kbn/core/server'; import { JsonOutputParser } from '@langchain/core/output_parsers'; +import type { ChatModel } from '../../../../../common/task/util/actions_client_chat'; import { RuleTranslationResult } from '../../../../../../../../common/siem_migrations/constants'; import type { RuleMigrationsRetriever } from '../../../retrievers'; -import type { SiemMigrationTelemetryClient } from '../../../rule_migrations_telemetry_client'; -import type { ChatModel } from '../../../util/actions_client_chat'; -import { cleanMarkdown, generateAssistantComment } from '../../../util/comments'; +import type { RuleMigrationTelemetryClient } from '../../../rule_migrations_telemetry_client'; +import { cleanMarkdown, generateAssistantComment } from '../../../../../common/task/util/comments'; import type { GraphNode } from '../../types'; import { MATCH_PREBUILT_RULE_PROMPT } from './prompts'; import { @@ -22,7 +22,7 @@ import { interface GetMatchPrebuiltRuleNodeParams { model: ChatModel; logger: Logger; - telemetryClient: SiemMigrationTelemetryClient; + telemetryClient: RuleMigrationTelemetryClient; ruleMigrationsRetriever: RuleMigrationsRetriever; } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/state.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/state.ts index 0216adb44c5dc..f60269f7f9641 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/state.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/state.ts @@ -13,11 +13,12 @@ import type { OriginalRule, RuleMigrationRule, } from '../../../../../../common/siem_migrations/model/rule_migration.gen'; -import type { RuleMigrationResources } from '../retrievers/rule_resource_retriever'; +import type { MigrationResources } from '../../../common/task/retrievers/resource_retriever'; export const migrateRuleState = Annotation.Root({ + id: Annotation(), original_rule: Annotation(), - resources: Annotation(), + resources: Annotation(), elastic_rule: Annotation({ reducer: (state, action) => ({ ...state, ...action }), }), diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/ecs_mapping/cim_ecs_map.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/ecs_mapping/cim_ecs_map.ts deleted file mode 100644 index 3bafaf2fc6518..0000000000000 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/ecs_mapping/cim_ecs_map.ts +++ /dev/null @@ -1,181 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export const SIEM_RULE_MIGRATION_CIM_ECS_MAP = ` -datamodel,object,source_field,ecs_field,data_type -Application_State,All_Application_State,dest,service.node.name,string -Application_State,All_Application_State,process,process.title,string -Application_State,All_Application_State,user,user.name,string -Application_State,Ports,dest_port,destination.port,number -Application_State,Ports,transport,network.transport,string -Application_State,Ports,transport_dest_port,destination.port,string -Application_State,Services,service,service.name,string -Application_State,Services,service_id,service.id,string -Application_State,Services,status,service.state,string -Authentication,Authentication,action,event.action,string -Authentication,Authentication,app,process.name,string -Authentication,Authentication,dest,host.name,string -Authentication,Authentication,duration,event.duration,number -Authentication,Authentication,signature,event.code,string -Authentication,Authentication,signature_id,event.reason,string -Authentication,Authentication,src,source.address,string -Authentication,Authentication,src_nt_domain,source.domain,string -Authentication,Authentication,user,user.name,string -Certificates,All_Certificates,dest_port,destination.port,number -Certificates,All_Certificates,duration,event.duration,number -Certificates,All_Certificates,src,source.address,string -Certificates,All_Certificates,src_port,source.port,number -Certificates,All_Certificates,transport,network.protocol,string -Certificates,SSL,ssl_end_time,tls.server.not_after,time -Certificates,SSL,ssl_hash,tls.server.hash,string -Certificates,SSL,ssl_issuer_common_name,tls.server.issuer,string -Certificates,SSL,ssl_issuer_locality,x509.issuer.locality,string -Certificates,SSL,ssl_issuer_organization,x509.issuer.organization,string -Certificates,SSL,ssl_issuer_state,x509.issuer.state_or_province,string -Certificates,SSL,ssl_issuer_unit,x509.issuer.organizational_unit,string -Certificates,SSL,ssl_publickey_algorithm,x509.public_key_algorithm,string -Certificates,SSL,ssl_serial,x509.serial_number,string -Certificates,SSL,ssl_signature_algorithm,x509.signature_algorithm,string -Certificates,SSL,ssl_start_time,x509.not_before,time -Certificates,SSL,ssl_subject,x509.subject.distinguished_name,string -Certificates,SSL,ssl_subject_common_name,x509.subject.common_name,string -Certificates,SSL,ssl_subject_locality,x509.subject.locality,string -Certificates,SSL,ssl_subject_organization,x509.subject.organization,string -Certificates,SSL,ssl_subject_state,x509.subject.state_or_province,string -Certificates,SSL,ssl_subject_unit,x509.subject.organizational_unit,string -Certificates,SSL,ssl_version,tls.version,string -Change,All_Changes,action,event.action,string -Change,Account_Management,dest_nt_domain,destination.domain,string -Change,Account_Management,src_nt_domain,source.domain,string -Change,Account_Management,src_user,source.user,string -Intrusion_Detection,IDS_Attacks,action,event.action,string -Intrusion_Detection,IDS_Attacks,dest,destination.address,string -Intrusion_Detection,IDS_Attacks,dest_port,destination.port,number -Intrusion_Detection,IDS_Attacks,dvc,observer.hostname,string -Intrusion_Detection,IDS_Attacks,severity,event.severity,string -Intrusion_Detection,IDS_Attacks,src,source.ip,string -Intrusion_Detection,IDS_Attacks,user,source.user,string -JVM,OS,os,host.os.name,string -JVM,OS,os_architecture,host.architecture,string -JVM,OS,os_version,host.os.version,string -Malware,Malware_Attacks,action,event.action,string -Malware,Malware_Attacks,date,event.created,string -Malware,Malware_Attacks,dest,host.hostname,string -Malware,Malware_Attacks,file_hash,file.hash.*,string -Malware,Malware_Attacks,file_name,file.name,string -Malware,Malware_Attacks,file_path,file.path,string -Malware,Malware_Attacks,Sender,source.user.email,string -Malware,Malware_Attacks,src,source.ip,string -Malware,Malware_Attacks,user,related.user,string -Malware,Malware_Attacks,url,rule.reference,string -Network_Resolution,DNS,answer,dns.answers,string -Network_Resolution,DNS,dest,destination.address,string -Network_Resolution,DNS,dest_port,destination.port,number -Network_Resolution,DNS,duration,event.duration,number -Network_Resolution,DNS,message_type,dns.type,string -Network_Resolution,DNS,name,dns.question.name,string -Network_Resolution,DNS,query,dns.question.name,string -Network_Resolution,DNS,query_type,dns.op_code,string -Network_Resolution,DNS,record_type,dns.question.type,string -Network_Resolution,DNS,reply_code,dns.response_code,string -Network_Resolution,DNS,reply_code_id,dns.id,number -Network_Resolution,DNS,response_time,event.duration,number -Network_Resolution,DNS,src,source.address,string -Network_Resolution,DNS,src_port,source.port,number -Network_Resolution,DNS,transaction_id,dns.id,number -Network_Resolution,DNS,transport,network.transport,string -Network_Resolution,DNS,ttl,dns.answers.ttl,number -Network_Sessions,All_Sessions,action,event.action,string -Network_Sessions,All_Sessions,dest_ip,destination.ip,string -Network_Sessions,All_Sessions,dest_mac,destination.mac,string -Network_Sessions,All_Sessions,duration,event.duration,number -Network_Sessions,All_Sessions,src_dns,source.registered_domain,string -Network_Sessions,All_Sessions,src_ip,source.ip,string -Network_Sessions,All_Sessions,src_mac,source.mac,string -Network_Sessions,All_Sessions,user,user.name,string -Network_Traffic,All_Traffic,action,event.action,string -Network_Traffic,All_Traffic,app,network.protocol,string -Network_Traffic,All_Traffic,bytes,network.bytes,number -Network_Traffic,All_Traffic,dest,destination.ip,string -Network_Traffic,All_Traffic,dest_ip,destination.ip,string -Network_Traffic,All_Traffic,dest_mac,destination.mac,string -Network_Traffic,All_Traffic,dest_port,destination.port,number -Network_Traffic,All_Traffic,dest_translated_ip,destination.nat.ip,string -Network_Traffic,All_Traffic,dest_translated_port,destination.nat.port,number -Network_Traffic,All_Traffic,direction,network.direction,string -Network_Traffic,All_Traffic,duration,event.duration,number -Network_Traffic,All_Traffic,dvc,observer.name,string -Network_Traffic,All_Traffic,dvc_ip,observer.ip,string -Network_Traffic,All_Traffic,dvc_mac,observer.mac,string -Network_Traffic,All_Traffic,dvc_zone,observer.egress.zone,string -Network_Traffic,All_Traffic,packets,network.packets,number -Network_Traffic,All_Traffic,packets_in,source.packets,number -Network_Traffic,All_Traffic,packets_out,destination.packets,number -Network_Traffic,All_Traffic,protocol,network.protocol,string -Network_Traffic,All_Traffic,rule,rule.name,string -Network_Traffic,All_Traffic,src,source.address,string -Network_Traffic,All_Traffic,src_ip,source.ip,string -Network_Traffic,All_Traffic,src_mac,source.mac,string -Network_Traffic,All_Traffic,src_port,source.port,number -Network_Traffic,All_Traffic,src_translated_ip,source.nat.ip,string -Network_Traffic,All_Traffic,src_translated_port,source.nat.port,number -Network_Traffic,All_Traffic,transport,network.transport,string -Network_Traffic,All_Traffic,vlan,vlan.name,string -Vulnerabilities,Vulnerabilities,category,vulnerability.category,string -Vulnerabilities,Vulnerabilities,cve,vulnerability.id,string -Vulnerabilities,Vulnerabilities,cvss,vulnerability.score.base,number -Vulnerabilities,Vulnerabilities,dest,host.name,string -Vulnerabilities,Vulnerabilities,dvc,vulnerability.scanner.vendor,string -Vulnerabilities,Vulnerabilities,severity,vulnerability.severity,string -Vulnerabilities,Vulnerabilities,url,vulnerability.reference,string -Vulnerabilities,Vulnerabilities,user,related.user,string -Vulnerabilities,Vulnerabilities,vendor_product,vulnerability.scanner.vendor,string -Endpoint,Ports,creation_time,@timestamp,timestamp -Endpoint,Ports,dest_port,destination.port,number -Endpoint,Ports,process_id,process.pid,string -Endpoint,Ports,transport,network.transport,string -Endpoint,Ports,transport_dest_port,destination.port,string -Endpoint,Processes,action,event.action,string -Endpoint,Processes,os,os.full,string -Endpoint,Processes,parent_process_exec,process.parent.name,string -Endpoint,Processes,parent_process_id,process.ppid,number -Endpoint,Processes,parent_process_guid,process.parent.entity_id,string -Endpoint,Processes,parent_process_path,process.parent.executable,string -Endpoint,Processes,process_current_directory,process.parent.working_directory, -Endpoint,Processes,process_exec,process.name,string -Endpoint,Processes,process_hash,process.hash.*,string -Endpoint,Processes,process_guid,process.entity_id,string -Endpoint,Processes,process_id,process.pid,number -Endpoint,Processes,process_path,process.executable,string -Endpoint,Processes,user_id,related.user,string -Endpoint,Services,description,service.name,string -Endpoint,Services,process_id,service.id,string -Endpoint,Services,service_dll,dll.name,string -Endpoint,Services,service_dll_path,dll.path,string -Endpoint,Services,service_dll_hash,dll.hash.*,string -Endpoint,Services,service_dll_signature_exists,dll.code_signature.exists,boolean -Endpoint,Services,service_dll_signature_verified,dll.code_signature.valid,boolean -Endpoint,Services,service_exec,service.name,string -Endpoint,Services,service_hash,hash.*,string -Endpoint,Filesystem,file_access_time,file.accessed,timestamp -Endpoint,Filesystem,file_create_time,file.created,timestamp -Endpoint,Filesystem,file_modify_time,file.mtime,timestamp -Endpoint,Filesystem,process_id,process.pid,string -Endpoint,Registry,process_id,process.id,string -Web,Web,action,event.action,string -Web,Web,app,observer.product,string -Web,Web,bytes_in,http.request.bytes,number -Web,Web,bytes_out,http.response.bytes,number -Web,Web,dest,destination.ip,string -Web,Web,duration,event.duration,number -Web,Web,http_method,http.request.method,string -Web,Web,http_referrer,http.request.referrer,string -Web,Web,http_user_agent,user_agent.name,string -Web,Web,status,http.response.status_code,string -Web,Web,url,url.full,string -Web,Web,user,url.username,string -Web,Web,vendor_product,observer.product,string`; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/ecs_mapping/ecs_mapping.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/ecs_mapping/ecs_mapping.ts index 91d836e21ce81..0b70bc67ea9f5 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/ecs_mapping/ecs_mapping.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/ecs_mapping/ecs_mapping.ts @@ -5,58 +5,27 @@ * 2.0. */ -import type { Logger } from '@kbn/core/server'; -import type { EsqlKnowledgeBase } from '../../../../../util/esql_knowledge_base'; +import { + getConvertEsqlSchemaCisToEcs, + type GetConvertEsqlSchemaCisToEcsParams, +} from '../../../../../../../common/task/agent/tools/convert_esql_schema_cim_to_ecs'; import type { GraphNode } from '../../types'; -import { SIEM_RULE_MIGRATION_CIM_ECS_MAP } from './cim_ecs_map'; -import { ESQL_TRANSLATE_ECS_MAPPING_PROMPT } from './prompts'; -import { cleanMarkdown, generateAssistantComment } from '../../../../../util/comments'; -interface GetEcsMappingNodeParams { - esqlKnowledgeBase: EsqlKnowledgeBase; - logger: Logger; -} - -export const getEcsMappingNode = ({ - esqlKnowledgeBase, - logger, -}: GetEcsMappingNodeParams): GraphNode => { +export const getEcsMappingNode = (params: GetConvertEsqlSchemaCisToEcsParams): GraphNode => { + const convertEsqlSchemaCimToEcs = getConvertEsqlSchemaCisToEcs(params); return async (state) => { - const elasticRule = { - title: state.elastic_rule.title, - description: state.elastic_rule.description, - query: state.elastic_rule.query, - }; - - const prompt = await ESQL_TRANSLATE_ECS_MAPPING_PROMPT.format({ - field_mapping: SIEM_RULE_MIGRATION_CIM_ECS_MAP, - splunk_query: state.inline_query, - elastic_rule: JSON.stringify(elasticRule, null, 2), + const { query, comments } = await convertEsqlSchemaCimToEcs({ + title: state.elastic_rule.title ?? '', + description: state.elastic_rule.description ?? '', + query: state.elastic_rule.query ?? '', + originalQuery: state.inline_query, }); - const response = await esqlKnowledgeBase.translate(prompt); - - const updatedQuery = response.match(/```esql\n([\s\S]*?)\n```/)?.[1] ?? ''; - if (!updatedQuery) { - logger.warn('Failed to apply ECS mapping to the query'); - const summary = '## Field Mapping Summary\n\nFailed to apply ECS mapping to the query'; - return { - includes_ecs_mapping: true, - comments: [generateAssistantComment(summary)], - }; - } - - const ecsSummary = response.match(/## Field Mapping Summary[\s\S]*$/)?.[0] ?? ''; - - // We set includes_ecs_mapping to true to indicate that the ecs mapping has been applied. - // This is to ensure that the node only runs once + // Set includes_ecs_mapping to indicate that this node has been executed to ensure it only runs once return { - comments: [generateAssistantComment(cleanMarkdown(ecsSummary))], includes_ecs_mapping: true, - elastic_rule: { - ...state.elastic_rule, - query: updatedQuery, - }, + comments, + ...(query && { elastic_rule: { ...state.elastic_rule, query } }), }; }; }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/fix_query_errors/fix_query_errors.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/fix_query_errors/fix_query_errors.ts index dc010840f9f6e..7a79ae48f44cc 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/fix_query_errors/fix_query_errors.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/fix_query_errors/fix_query_errors.ts @@ -5,30 +5,22 @@ * 2.0. */ -import type { Logger } from '@kbn/core/server'; -import type { EsqlKnowledgeBase } from '../../../../../util/esql_knowledge_base'; +import { + getFixEsqlQueryErrors, + type GetFixEsqlQueryErrorsParams, +} from '../../../../../../../common/task/agent/tools/fix_esql_query_errors'; import type { GraphNode } from '../../types'; -import { RESOLVE_ESQL_ERRORS_TEMPLATE } from './prompts'; -interface GetFixQueryErrorsNodeParams { - esqlKnowledgeBase: EsqlKnowledgeBase; - logger: Logger; -} - -export const getFixQueryErrorsNode = ({ - esqlKnowledgeBase, - logger, -}: GetFixQueryErrorsNodeParams): GraphNode => { +export const getFixQueryErrorsNode = (params: GetFixEsqlQueryErrorsParams): GraphNode => { + const fixEsqlQueryErrors = getFixEsqlQueryErrors(params); return async (state) => { - const rule = state.elastic_rule; - const prompt = await RESOLVE_ESQL_ERRORS_TEMPLATE.format({ - esql_errors: state.validation_errors.esql_errors, - esql_query: rule.query, + const { query } = await fixEsqlQueryErrors({ + invalidQuery: state.elastic_rule.query, + validationErrors: state.validation_errors.esql_errors, }); - const response = await esqlKnowledgeBase.translate(prompt); - - const esqlQuery = response.match(/```esql\n([\s\S]*?)\n```/)?.[1] ?? ''; - rule.query = esqlQuery; - return { elastic_rule: rule }; + if (!query) { + return {}; + } + return { elastic_rule: { ...state.elastic_rule, query } }; }; }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/inline_query.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/inline_query.ts index 1232bdd326bc8..83bb9769193de 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/inline_query.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/inline_query/inline_query.ts @@ -4,76 +4,26 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - -import type { Logger } from '@kbn/core/server'; -import { StringOutputParser } from '@langchain/core/output_parsers'; -import { isEmpty } from 'lodash/fp'; -import type { ChatModel } from '../../../../../util/actions_client_chat'; -import type { GraphNode } from '../../../../types'; -import { REPLACE_QUERY_RESOURCE_PROMPT, getResourcesContext } from './prompts'; -import { cleanMarkdown, generateAssistantComment } from '../../../../../util/comments'; - -interface GetInlineQueryNodeParams { - model: ChatModel; - logger: Logger; -} - -export const getInlineQueryNode = ({ model, logger }: GetInlineQueryNodeParams): GraphNode => { +import { + getInlineSplQuery, + type GetInlineSplQueryParams, +} from '../../../../../../../common/task/agent/tools/inline_spl_query'; +import type { GraphNode } from '../../types'; + +export const getInlineQueryNode = (params: GetInlineSplQueryParams): GraphNode => { + const inlineSplQuery = getInlineSplQuery(params); return async (state) => { - const query = state.original_rule.query; - // Check before to avoid unnecessary LLM calls - let unsupportedComment = getUnsupportedComment(query); - if (unsupportedComment) { - return { - inline_query: undefined, // No inline query if unsupported to jump to the end of the graph - comments: [generateAssistantComment(unsupportedComment)], - }; - } - - if (isEmpty(state.resources)) { - // No resources identified in the query, no need to replace - return { inline_query: query }; - } - - const replaceQueryParser = new StringOutputParser(); - const replaceQueryResourcePrompt = - REPLACE_QUERY_RESOURCE_PROMPT.pipe(model).pipe(replaceQueryParser); - const resourceContext = getResourcesContext(state.resources); - const response = await replaceQueryResourcePrompt.invoke({ + const { inlineQuery, isUnsupported, comments } = await inlineSplQuery({ query: state.original_rule.query, - macros: resourceContext.macros, - lookups: resourceContext.lookups, + resources: state.resources, }); - const inlineQuery = response.match(/```spl\n([\s\S]*?)\n```/)?.[1].trim() ?? ''; - if (!inlineQuery) { - logger.warn('Failed to retrieve inline query'); - const summary = '## Inlining Summary\n\nFailed to retrieve inline query'; - return { - inline_query: query, - comments: [generateAssistantComment(summary)], - }; - } - - // Check after replacing in case the replacements made it untranslatable - unsupportedComment = getUnsupportedComment(inlineQuery); - if (unsupportedComment) { - return { - inline_query: undefined, // No inline query if unsupported to jump to the end of the graph - comments: [generateAssistantComment(unsupportedComment)], - }; + if (isUnsupported) { + // Graph conditional edge detects undefined inline_query as unsupported query + return { inline_query: undefined, comments }; } - - const inliningSummary = response.match(/## Inlining Summary[\s\S]*$/)?.[0] ?? ''; return { - inline_query: inlineQuery, - comments: [generateAssistantComment(cleanMarkdown(inliningSummary))], + inline_query: inlineQuery ?? state.original_rule.query, + comments, }; }; }; - -const getUnsupportedComment = (query: string): string | undefined => { - const unsupportedText = '## Translation Summary\nCan not create custom translation.\n'; - if (query.includes(' inputlookup')) { - return `${unsupportedText}Reason: \`inputlookup\` command is not supported.`; - } -}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/retrieve_integrations/retrieve_integrations.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/retrieve_integrations/retrieve_integrations.ts index 68046667f01ae..5f6e579308b95 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/retrieve_integrations/retrieve_integrations.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/retrieve_integrations/retrieve_integrations.ts @@ -7,15 +7,18 @@ import { JsonOutputParser } from '@langchain/core/output_parsers'; import type { RuleMigrationsRetriever } from '../../../../../retrievers'; -import type { SiemMigrationTelemetryClient } from '../../../../../rule_migrations_telemetry_client'; -import type { ChatModel } from '../../../../../util/actions_client_chat'; -import { cleanMarkdown, generateAssistantComment } from '../../../../../util/comments'; +import type { RuleMigrationTelemetryClient } from '../../../../../rule_migrations_telemetry_client'; +import type { ChatModel } from '../../../../../../../common/task/util/actions_client_chat'; +import { + cleanMarkdown, + generateAssistantComment, +} from '../../../../../../../common/task/util/comments'; import type { GraphNode } from '../../types'; import { MATCH_INTEGRATION_PROMPT } from './prompts'; interface GetRetrieveIntegrationsNodeParams { model: ChatModel; - telemetryClient: SiemMigrationTelemetryClient; + telemetryClient: RuleMigrationTelemetryClient; ruleMigrationsRetriever: RuleMigrationsRetriever; } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/translate_rule.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/translate_rule.ts index 6cfbdfa0b957f..4701ad5268683 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/translate_rule.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/translate_rule.ts @@ -5,56 +5,37 @@ * 2.0. */ -import type { Logger } from '@kbn/core/server'; -import { cleanMarkdown, generateAssistantComment } from '../../../../../util/comments'; -import type { EsqlKnowledgeBase } from '../../../../../util/esql_knowledge_base'; +import { + getTranslateSplToEsql, + TASK_DESCRIPTION, + type GetTranslateSplToEsqlParams, +} from '../../../../../../../common/task/agent/tools/translate_spl_to_esql'; import type { GraphNode } from '../../types'; -import { ESQL_SYNTAX_TRANSLATION_PROMPT } from './prompts'; import { getElasticRiskScoreFromOriginalRule, getElasticSeverityFromOriginalRule, } from './severity'; -interface GetTranslateRuleNodeParams { - esqlKnowledgeBase: EsqlKnowledgeBase; - logger: Logger; -} - -export const getTranslateRuleNode = ({ - esqlKnowledgeBase, - logger, -}: GetTranslateRuleNodeParams): GraphNode => { +export const getTranslateRuleNode = (params: GetTranslateSplToEsqlParams): GraphNode => { + const translateSplToEsql = getTranslateSplToEsql(params); return async (state) => { const indexPatterns = state.integration?.data_streams?.map((dataStream) => dataStream.index_pattern).join(',') || 'logs-*'; - const splunkRule = { + const { esqlQuery, comments } = await translateSplToEsql({ title: state.original_rule.title, + taskDescription: TASK_DESCRIPTION.migrate_rule, description: state.original_rule.description, - inline_query: state.inline_query, - }; - - const prompt = await ESQL_SYNTAX_TRANSLATION_PROMPT.format({ - splunk_rule: JSON.stringify(splunkRule, null, 2), - indexPatterns, + inlineQuery: state.inline_query, + indexPattern: indexPatterns, }); - const response = await esqlKnowledgeBase.translate(prompt); - const esqlQuery = response.match(/```esql\n([\s\S]*?)\n```/)?.[1].trim() ?? ''; if (!esqlQuery) { - logger.warn('Failed to extract ESQL query from translation response'); - const comment = - '## Translation Summary\n\nFailed to extract ESQL query from translation response'; - return { - comments: [generateAssistantComment(comment)], - }; + return { comments }; } - const translationSummary = response.match(/## Translation Summary[\s\S]*$/)?.[0] ?? ''; - return { - comments: [generateAssistantComment(cleanMarkdown(translationSummary))], elastic_rule: { query: esqlQuery, query_language: 'esql', @@ -62,6 +43,7 @@ export const getTranslateRuleNode = ({ severity: getElasticSeverityFromOriginalRule(state.original_rule), ...(state.integration?.id && { integration_ids: [state.integration.id] }), }, + comments, }; }; }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translation_result/translation_result.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translation_result/translation_result.ts index e17a17ca8c6df..2ee45c9f9219e 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translation_result/translation_result.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translation_result/translation_result.ts @@ -13,7 +13,7 @@ import { RuleTranslationResult } from '../../../../../../../../../../common/siem import type { GraphNode } from '../../types'; export const getTranslationResultNode = (): GraphNode => { - return async (state, config) => { + return async (state) => { // Set defaults const elasticRule = { title: state.original_rule.title, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/validation/validation.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/validation/validation.ts index d16dcd95ada67..ac413bb48d89d 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/validation/validation.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/validation/validation.ts @@ -5,50 +5,27 @@ * 2.0. */ -import type { Logger } from '@kbn/core/server'; -import { isEmpty } from 'lodash/fp'; -import { parseEsqlQuery } from '@kbn/securitysolution-utils'; +import { + getValidateEsql, + type GetValidateEsqlParams, +} from '../../../../../../../common/task/agent/tools/validate_esql/validation'; import type { GraphNode } from '../../types'; -interface GetValidationNodeParams { - logger: Logger; -} - /** * This node runs all validation steps, and will redirect to the END of the graph if no errors are found. * Any new validation steps should be added here. */ -export const getValidationNode = ({ logger }: GetValidationNodeParams): GraphNode => { +export const getValidationNode = (params: GetValidateEsqlParams): GraphNode => { + const validateEsql = getValidateEsql(params); return async (state) => { - const query = state.elastic_rule.query; - - // We want to prevent infinite loops, so we increment the iterations counter for each validation run. - const currentIteration = state.validation_errors.iterations + 1; - let esqlErrors: string = ''; - try { - const sanitizedQuery = query ? removePlaceHolders(query) : ''; - if (!isEmpty(sanitizedQuery)) { - const { errors, isEsqlQueryAggregating, hasMetadataOperator } = - parseEsqlQuery(sanitizedQuery); - if (!isEmpty(errors)) { - esqlErrors = JSON.stringify(errors); - } else if (!isEsqlQueryAggregating && !hasMetadataOperator) { - esqlErrors = `Queries that do't use the STATS...BY function (non-aggregating queries) must include the "metadata _id, _version, _index" operator after the source command. For example: FROM logs* metadata _id, _version, _index.`; - } - } - if (esqlErrors) { - logger.debug(`ESQL query validation failed: ${esqlErrors}`); - } - } catch (error) { - esqlErrors = error.message; - logger.info(`Error parsing ESQL query: ${error.message}`); + const iterations = state.validation_errors.iterations + 1; + if (!state.elastic_rule.query) { + params.logger.warn('Missing query in validation node'); + return { iterations }; } - return { validation_errors: { iterations: currentIteration, esql_errors: esqlErrors } }; + + const { error } = await validateEsql({ query: state.elastic_rule.query }); + + return { validation_errors: { iterations, esql_errors: error } }; }; }; - -function removePlaceHolders(query: string): string { - return query - .replaceAll(/\[(macro|lookup):.*?\]/g, '') // Removes any macro or lookup placeholders - .replaceAll(/\n(\s*?\|\s*?\n)*/g, '\n'); // Removes any empty lines with | (pipe) alone after removing the placeholders -} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/state.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/state.ts index 456d7bde6cf4e..1916ffddfd801 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/state.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/state.ts @@ -12,13 +12,13 @@ import type { OriginalRule, RuleMigrationRule, } from '../../../../../../../../common/siem_migrations/model/rule_migration.gen'; -import type { RuleMigrationResources } from '../../../retrievers/rule_resource_retriever'; +import type { MigrationResources } from '../../../../../common/task/retrievers/resource_retriever'; import type { RuleMigrationIntegration } from '../../../../types'; import type { TranslateRuleValidationErrors } from './types'; export const translateRuleState = Annotation.Root({ original_rule: Annotation(), - resources: Annotation(), + resources: Annotation(), integration: Annotation({ reducer: (current, value) => value ?? current, default: () => ({} as RuleMigrationIntegration), diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/types.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/types.ts index e9ac2eccf6f1d..6086823be491d 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/types.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/types.ts @@ -7,10 +7,10 @@ import type { Logger } from '@kbn/core/server'; import type { RunnableConfig } from '@langchain/core/runnables'; -import type { EsqlKnowledgeBase } from '../../../util/esql_knowledge_base'; +import type { ChatModel } from '../../../../../common/task/util/actions_client_chat'; +import type { EsqlKnowledgeBase } from '../../../../../common/task/util/esql_knowledge_base'; import type { RuleMigrationsRetriever } from '../../../retrievers'; -import type { SiemMigrationTelemetryClient } from '../../../rule_migrations_telemetry_client'; -import type { ChatModel } from '../../../util/actions_client_chat'; +import type { RuleMigrationTelemetryClient } from '../../../rule_migrations_telemetry_client'; import type { translateRuleState } from './state'; import type { migrateRuleConfigSchema } from '../../state'; @@ -25,7 +25,7 @@ export interface TranslateRuleGraphParams { model: ChatModel; esqlKnowledgeBase: EsqlKnowledgeBase; ruleMigrationsRetriever: RuleMigrationsRetriever; - telemetryClient: SiemMigrationTelemetryClient; + telemetryClient: RuleMigrationTelemetryClient; logger: Logger; } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/types.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/types.ts index 756bf87ab7f42..e1cdc34fe7849 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/types.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/types.ts @@ -8,16 +8,17 @@ import type { Logger } from '@kbn/core/server'; import type { RunnableConfig } from '@langchain/core/runnables'; import type { RuleMigrationsRetriever } from '../retrievers'; -import type { EsqlKnowledgeBase } from '../util/esql_knowledge_base'; -import type { SiemMigrationTelemetryClient } from '../rule_migrations_telemetry_client'; -import type { ChatModel } from '../util/actions_client_chat'; +import type { EsqlKnowledgeBase } from '../../../common/task/util/esql_knowledge_base'; +import type { ChatModel } from '../../../common/task/util/actions_client_chat'; import type { migrateRuleConfigSchema, migrateRuleState } from './state'; +import type { RuleMigrationTelemetryClient } from '../rule_migrations_telemetry_client'; export type MigrateRuleState = typeof migrateRuleState.State; -export type MigrateRuleGraphConfig = RunnableConfig<(typeof migrateRuleConfigSchema)['State']>; +export type MigrateRuleConfigSchema = (typeof migrateRuleConfigSchema)['State']; +export type MigrateRuleConfig = RunnableConfig; export type GraphNode = ( state: MigrateRuleState, - config: MigrateRuleGraphConfig + config: MigrateRuleConfig ) => Promise>; export interface RuleMigrationAgentRunOptions { @@ -29,5 +30,5 @@ export interface MigrateRuleGraphParams { model: ChatModel; ruleMigrationsRetriever: RuleMigrationsRetriever; logger: Logger; - telemetryClient: SiemMigrationTelemetryClient; + telemetryClient: RuleMigrationTelemetryClient; } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_migrations_retriever.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_migrations_retriever.ts index 8608f1961fd52..fcfce28b10d0f 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_migrations_retriever.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_migrations_retriever.ts @@ -33,7 +33,7 @@ export class RuleMigrationsRetriever { public readonly prebuiltRules: PrebuiltRulesRetriever; constructor(migrationId: string, clients: RuleMigrationsRetrieverClients) { - this.resources = new RuleResourceRetriever(migrationId, clients.data); + this.resources = new RuleResourceRetriever(migrationId, clients.data.resources); this.integrations = new IntegrationRetriever(clients); this.prebuiltRules = new PrebuiltRulesRetriever(clients); } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_resource_retriever.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_resource_retriever.ts index 03c978812085b..c2538c55aa8b6 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_resource_retriever.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/rule_resource_retriever.ts @@ -5,114 +5,10 @@ * 2.0. */ -import { ResourceIdentifier } from '../../../../../../common/siem_migrations/rules/resources'; -import type { - RuleMigrationResource, - RuleMigrationResourceType, - RuleMigrationRule, -} from '../../../../../../common/siem_migrations/model/rule_migration.gen'; -import type { RuleMigrationsDataClient } from '../../data/rule_migrations_data_client'; +import { RuleResourceIdentifier } from '../../../../../../common/siem_migrations/rules/resources'; +import type { RuleMigrationRule } from '../../../../../../common/siem_migrations/model/rule_migration.gen'; +import { ResourceRetriever } from '../../../common/task/retrievers/resource_retriever'; -export interface RuleMigrationDefinedResource extends RuleMigrationResource { - content: string; // ensures content exists -} -export type RuleMigrationResourcesData = Pick< - RuleMigrationDefinedResource, - 'name' | 'content' | 'type' ->; -export type RuleMigrationResources = Partial< - Record ->; -interface ExistingResources { - macro: Record; - lookup: Record; -} - -export class RuleResourceRetriever { - private existingResources?: ExistingResources; - - constructor( - private readonly migrationId: string, - private readonly dataClient: RuleMigrationsDataClient - ) {} - - public async initialize(): Promise { - const batches = this.dataClient.resources.searchBatches( - this.migrationId, - { filters: { hasContent: true } } // filters out missing (undefined) content resources, empty strings content will be included - ); - - const existingRuleResources: ExistingResources = { macro: {}, lookup: {} }; - let resources; - do { - resources = await batches.next(); - resources.forEach((resource) => { - existingRuleResources[resource.type][resource.name] = resource; - }); - } while (resources.length > 0); - - this.existingResources = existingRuleResources; - } - - public async getResources(migrationRule: RuleMigrationRule): Promise { - const originalRule = migrationRule.original_rule; - const existingResources = this.existingResources; - if (!existingResources) { - throw new Error('initialize must be called before calling getResources'); - } - - const resourceIdentifier = new ResourceIdentifier(originalRule.vendor); - const resourcesIdentifiedFromRule = resourceIdentifier.fromOriginalRule(originalRule); - - const macrosFound = new Map(); - const lookupsFound = new Map(); - resourcesIdentifiedFromRule.forEach((resource) => { - const existingResource = existingResources[resource.type][resource.name]; - if (existingResource) { - if (resource.type === 'macro') { - macrosFound.set(resource.name, existingResource); - } else if (resource.type === 'lookup') { - lookupsFound.set(resource.name, existingResource); - } - } - }); - - const resourcesFound = [...macrosFound.values(), ...lookupsFound.values()]; - if (!resourcesFound.length) { - return {}; - } - - let nestedResourcesFound = resourcesFound; - do { - const nestedResourcesIdentified = resourceIdentifier.fromResources(nestedResourcesFound); - - nestedResourcesFound = []; - nestedResourcesIdentified.forEach((resource) => { - const existingResource = existingResources[resource.type][resource.name]; - if (existingResource) { - nestedResourcesFound.push(existingResource); - if (resource.type === 'macro') { - macrosFound.set(resource.name, existingResource); - } else if (resource.type === 'lookup') { - lookupsFound.set(resource.name, existingResource); - } - } - }); - } while (nestedResourcesFound.length > 0); - - return { - ...(macrosFound.size > 0 ? { macro: this.formatOutput(macrosFound) } : {}), - ...(lookupsFound.size > 0 ? { lookup: this.formatOutput(lookupsFound) } : {}), - }; - } - - private formatOutput( - resources: Map - ): RuleMigrationResourcesData[] { - return Array.from(resources.values()).map(({ name, content, type }) => ({ - name, - content, - type, - })); - } +export class RuleResourceRetriever extends ResourceRetriever { + protected ResourceIdentifierClass = RuleResourceIdentifier; } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_client.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_client.test.ts index 32ff2354bbc1a..89fae9a5bb5cc 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_client.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_client.test.ts @@ -15,11 +15,11 @@ import { import { RuleMigrationTaskRunner } from './rule_migrations_task_runner'; import type { MockedLogger } from '@kbn/logging-mocks'; import { loggerMock } from '@kbn/logging-mocks'; -import type { StoredSiemMigration } from '../types'; +import type { StoredRuleMigration } from '../types'; import type { RuleMigrationTaskStartParams } from './types'; import { createRuleMigrationsDataClientMock } from '../data/__mocks__/mocks'; import type { RuleMigrationDataStats } from '../data/rule_migrations_data_rules_client'; -import type { RuleMigrationFilters } from '../../../../../common/siem_migrations/types'; +import type { RuleMigrationFilters } from '../../../../../common/siem_migrations/rules/types'; import type { SiemMigrationsClientDependencies } from '../../common/types'; jest.mock('./rule_migrations_task_runner', () => { @@ -71,11 +71,11 @@ describe('RuleMigrationsTaskClient', () => { ); const result = await client.start(params); expect(result).toEqual({ exists: true, started: false }); - expect(data.rules.updateStatus).not.toHaveBeenCalled(); + expect(data.items.updateStatus).not.toHaveBeenCalled(); }); it('should not start if there are no rules to migrate (total = 0)', async () => { - data.rules.getStats.mockResolvedValue({ + data.items.getStats.mockResolvedValue({ rules: { total: 0, pending: 0, completed: 0, failed: 0 }, } as RuleMigrationDataStats); const client = new RuleMigrationsTaskClient( @@ -86,7 +86,7 @@ describe('RuleMigrationsTaskClient', () => { dependencies ); const result = await client.start(params); - expect(data.rules.updateStatus).toHaveBeenCalledWith( + expect(data.items.updateStatus).toHaveBeenCalledWith( migrationId, { status: SiemMigrationStatus.PROCESSING }, SiemMigrationStatus.PENDING, @@ -96,7 +96,7 @@ describe('RuleMigrationsTaskClient', () => { }); it('should not start if there are no pending rules', async () => { - data.rules.getStats.mockResolvedValue({ + data.items.getStats.mockResolvedValue({ rules: { total: 10, pending: 0, completed: 10, failed: 0 }, } as RuleMigrationDataStats); const client = new RuleMigrationsTaskClient( @@ -111,7 +111,7 @@ describe('RuleMigrationsTaskClient', () => { }); it('should start migration successfully', async () => { - data.rules.getStats.mockResolvedValue({ + data.items.getStats.mockResolvedValue({ rules: { total: 10, pending: 5, completed: 0, failed: 0 }, } as RuleMigrationDataStats); const mockedRunnerInstance = { @@ -144,7 +144,7 @@ describe('RuleMigrationsTaskClient', () => { }); it('should throw error if a race condition occurs after setup', async () => { - data.rules.getStats.mockResolvedValue({ + data.items.getStats.mockResolvedValue({ rules: { total: 10, pending: 5, completed: 0, failed: 0 }, } as RuleMigrationDataStats); const mockedRunnerInstance = { @@ -169,7 +169,7 @@ describe('RuleMigrationsTaskClient', () => { }); it('should mark migration as started by calling saveAsStarted', async () => { - data.rules.getStats.mockResolvedValue({ + data.items.getStats.mockResolvedValue({ rules: { total: 10, pending: 5, completed: 0, failed: 0 }, } as RuleMigrationDataStats); @@ -190,7 +190,7 @@ describe('RuleMigrationsTaskClient', () => { it('should mark migration as ended by calling saveAsEnded if run completes successfully', async () => { migrationsRunning = new Map(); - data.rules.getStats.mockResolvedValue({ + data.items.getStats.mockResolvedValue({ rules: { total: 10, pending: 5, completed: 0, failed: 0 }, } as RuleMigrationDataStats); @@ -222,7 +222,7 @@ describe('RuleMigrationsTaskClient', () => { const filter: RuleMigrationFilters = { fullyTranslated: true }; const result = await client.updateToRetry(migrationId, filter); expect(result).toEqual({ updated: false }); - expect(data.rules.updateStatus).not.toHaveBeenCalled(); + expect(data.items.updateStatus).not.toHaveBeenCalled(); }); it('should update to retry if migration is not running', async () => { @@ -236,7 +236,7 @@ describe('RuleMigrationsTaskClient', () => { const filter: RuleMigrationFilters = { fullyTranslated: true }; const result = await client.updateToRetry(migrationId, filter); expect(filter.installed).toBe(false); - expect(data.rules.updateStatus).toHaveBeenCalledWith( + expect(data.items.updateStatus).toHaveBeenCalledWith( migrationId, { fullyTranslated: true, installed: false }, SiemMigrationStatus.PENDING, @@ -249,13 +249,13 @@ describe('RuleMigrationsTaskClient', () => { describe('getStats', () => { it('should return RUNNING status if migration is running', async () => { migrationsRunning.set(migrationId, {} as RuleMigrationTaskRunner); // migration is running - data.rules.getStats.mockResolvedValue({ + data.items.getStats.mockResolvedValue({ rules: { total: 10, pending: 5, completed: 3, failed: 2 }, } as RuleMigrationDataStats); data.migrations.get.mockResolvedValue({ id: migrationId, - } as unknown as StoredSiemMigration); + } as unknown as StoredRuleMigration); const client = new RuleMigrationsTaskClient( migrationsRunning, @@ -269,12 +269,12 @@ describe('RuleMigrationsTaskClient', () => { }); it('should return READY status if pending equals total', async () => { - data.rules.getStats.mockResolvedValue({ + data.items.getStats.mockResolvedValue({ rules: { total: 10, pending: 10, completed: 0, failed: 0 }, } as RuleMigrationDataStats); data.migrations.get.mockResolvedValue({ id: migrationId, - } as unknown as StoredSiemMigration); + } as unknown as StoredRuleMigration); const client = new RuleMigrationsTaskClient( migrationsRunning, @@ -288,13 +288,13 @@ describe('RuleMigrationsTaskClient', () => { }); it('should return FINISHED status if completed+failed equals total', async () => { - data.rules.getStats.mockResolvedValue({ + data.items.getStats.mockResolvedValue({ rules: { total: 10, pending: 0, completed: 5, failed: 5 }, } as RuleMigrationDataStats); data.migrations.get.mockResolvedValue({ id: migrationId, - } as unknown as StoredSiemMigration); + } as unknown as StoredRuleMigration); const client = new RuleMigrationsTaskClient( migrationsRunning, logger, @@ -307,7 +307,7 @@ describe('RuleMigrationsTaskClient', () => { }); it('should return STOPPED status for other cases', async () => { - data.rules.getStats.mockResolvedValue({ + data.items.getStats.mockResolvedValue({ rules: { total: 10, pending: 2, completed: 3, failed: 2 }, } as RuleMigrationDataStats); const client = new RuleMigrationsTaskClient( @@ -323,7 +323,7 @@ describe('RuleMigrationsTaskClient', () => { it('should include error if one exists', async () => { const errorMessage = 'Test error'; - data.rules.getStats.mockResolvedValue({ + data.items.getStats.mockResolvedValue({ id: 'migration-1', rules: { total: 10, pending: 2, completed: 3, failed: 2 }, } as RuleMigrationDataStats); @@ -343,7 +343,7 @@ describe('RuleMigrationsTaskClient', () => { last_execution: { error: 'Test error', }, - } as unknown as StoredSiemMigration); + } as unknown as StoredRuleMigration); const client = new RuleMigrationsTaskClient( migrationsRunning, @@ -369,8 +369,8 @@ describe('RuleMigrationsTaskClient', () => { rules: { total: 10, pending: 2, completed: 3, failed: 2 }, } as RuleMigrationDataStats, ]; - const migrations = [{ id: 'm1' }, { id: 'm2' }] as unknown as StoredSiemMigration[]; - data.rules.getAllStats.mockResolvedValue(statsArray); + const migrations = [{ id: 'm1' }, { id: 'm2' }] as unknown as StoredRuleMigration[]; + data.items.getAllStats.mockResolvedValue(statsArray); data.migrations.getAll.mockResolvedValue(migrations); // Mark migration m1 as running. migrationsRunning.set('m1', {} as RuleMigrationTaskRunner); @@ -409,7 +409,7 @@ describe('RuleMigrationsTaskClient', () => { }); it('should return stopped even if migration is already stopped', async () => { - data.rules.getStats.mockResolvedValue({ + data.items.getStats.mockResolvedValue({ rules: { total: 10, pending: 10, completed: 0, failed: 0 }, } as RuleMigrationDataStats); const client = new RuleMigrationsTaskClient( @@ -424,7 +424,7 @@ describe('RuleMigrationsTaskClient', () => { }); it('should return exists false if migration is not running and total equals 0', async () => { - data.rules.getStats.mockResolvedValue({ + data.items.getStats.mockResolvedValue({ rules: { total: 0, pending: 0, completed: 0, failed: 0 }, } as RuleMigrationDataStats); const client = new RuleMigrationsTaskClient( @@ -440,7 +440,7 @@ describe('RuleMigrationsTaskClient', () => { it('should catch errors and return exists true, stopped false', async () => { const error = new Error('Stop error'); - data.rules.getStats.mockRejectedValue(error); + data.items.getStats.mockRejectedValue(error); const client = new RuleMigrationsTaskClient( migrationsRunning, logger, @@ -477,7 +477,7 @@ describe('RuleMigrationsTaskClient', () => { }); describe('task error', () => { it('should call saveAsFailed when there has been an error during the migration', async () => { - data.rules.getStats.mockResolvedValue({ + data.items.getStats.mockResolvedValue({ rules: { total: 10, pending: 10, completed: 0, failed: 0 }, } as RuleMigrationDataStats); const error = new Error('Migration error'); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_client.ts index ad00c9c4081f4..7d99708df8791 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_client.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_client.ts @@ -5,236 +5,37 @@ * 2.0. */ -import type { AuthenticatedUser, Logger } from '@kbn/core/server'; -import { - SiemMigrationStatus, - SiemMigrationTaskStatus, -} from '../../../../../common/siem_migrations/constants'; -import type { RuleMigrationTaskStats } from '../../../../../common/siem_migrations/model/rule_migration.gen'; -import type { RuleMigrationFilters } from '../../../../../common/siem_migrations/types'; -import type { RuleMigrationsDataClient } from '../data/rule_migrations_data_client'; -import type { RuleMigrationDataStats } from '../data/rule_migrations_data_rules_client'; -import type { StoredSiemMigration } from '../types'; +import type { RunnableConfig } from '@langchain/core/runnables'; import type { - RuleMigrationTaskEvaluateParams, - RuleMigrationTaskStartParams, - RuleMigrationTaskStartResult, - RuleMigrationTaskStopResult, -} from './types'; + RuleMigration, + RuleMigrationRule, +} from '../../../../../common/siem_migrations/model/rule_migration.gen'; +import type { + RuleMigrationTaskInput, + RuleMigrationTaskOutput, +} from './rule_migrations_task_runner'; import { RuleMigrationTaskRunner } from './rule_migrations_task_runner'; +import { SiemMigrationsTaskClient } from '../../common/task/siem_migrations_task_client'; +import type { MigrateRuleConfigSchema } from './agent/types'; import { RuleMigrationTaskEvaluator } from './rule_migrations_task_evaluator'; -import type { SiemMigrationsClientDependencies } from '../../common/types'; - -export type MigrationsRunning = Map; - -export class RuleMigrationsTaskClient { - constructor( - private migrationsRunning: MigrationsRunning, - private logger: Logger, - private data: RuleMigrationsDataClient, - private currentUser: AuthenticatedUser, - private dependencies: SiemMigrationsClientDependencies - ) {} - - /** Starts a rule migration task */ - async start(params: RuleMigrationTaskStartParams): Promise { - const { migrationId, connectorId, invocationConfig } = params; - if (this.migrationsRunning.has(migrationId)) { - return { exists: true, started: false }; - } - // Just in case some previous execution was interrupted without cleaning up - await this.data.rules.updateStatus( - migrationId, - { status: SiemMigrationStatus.PROCESSING }, - SiemMigrationStatus.PENDING, - { refresh: true } - ); - - const { rules } = await this.data.rules.getStats(migrationId); - if (rules.total === 0) { - return { exists: false, started: false }; - } - if (rules.pending === 0) { - return { exists: true, started: false }; - } - - const migrationLogger = this.logger.get(migrationId); - const abortController = new AbortController(); - const migrationTaskRunner = new RuleMigrationTaskRunner( - migrationId, - this.currentUser, - abortController, - this.data, - migrationLogger, - this.dependencies - ); - - await migrationTaskRunner.setup(connectorId); - - if (this.migrationsRunning.has(migrationId)) { - // Just to prevent a race condition in the setup - throw new Error('Task already running for this migration'); - } - - migrationLogger.info('Starting migration'); - - this.migrationsRunning.set(migrationId, migrationTaskRunner); - - await this.data.migrations.saveAsStarted({ - id: migrationId, - connectorId, - skipPrebuiltRulesMatching: invocationConfig.configurable?.skipPrebuiltRulesMatching, - }); - - // run the migration in the background without awaiting and resolve the `start` promise - migrationTaskRunner - .run(invocationConfig) - .then(() => { - // The task runner has finished normally. Abort errors are also handled here, it's an expected finish scenario, nothing special should be done. - migrationLogger.debug('Migration execution task finished'); - this.data.migrations.saveAsFinished({ id: migrationId }).catch((error) => { - migrationLogger.error(`Error saving migration as finished: ${error}`); - }); - }) - .catch((error) => { - // Unexpected errors, no use in throwing them since the `start` promise is long gone. Just log and store the error message - migrationLogger.error(`Error executing migration task: ${error}`); - this.data.migrations - .saveAsFailed({ id: migrationId, error: error.message }) - .catch((saveError) => { - migrationLogger.error(`Error saving migration as failed: ${saveError}`); - }); - }) - .finally(() => { - this.migrationsRunning.delete(migrationId); - }); - - return { exists: true, started: true }; - } - - /** Updates all the rules in a migration to be re-executed */ - public async updateToRetry( - migrationId: string, - filter: RuleMigrationFilters - ): Promise<{ updated: boolean }> { - if (this.migrationsRunning.has(migrationId)) { - // not update migrations that are currently running - return { updated: false }; - } - filter.installed = false; // only retry rules that are not installed - await this.data.rules.updateStatus(migrationId, filter, SiemMigrationStatus.PENDING, { - refresh: true, - }); - return { updated: true }; - } - - /** Returns the stats of a migration */ - public async getStats(migrationId: string): Promise { - const migration = await this.data.migrations.get({ id: migrationId }); - if (!migration) { - throw new Error(`Migration with ID ${migrationId} not found`); - } - const dataStats = await this.data.rules.getStats(migrationId); - const taskStats = this.getTaskStats(migration, dataStats.rules); - return { ...taskStats, ...dataStats, name: migration.name }; - } - - /** Returns the stats of all migrations */ - async getAllStats(): Promise { - const allDataStats = await this.data.rules.getAllStats(); - const allMigrations = await this.data.migrations.getAll(); - const allMigrationsMap = new Map( - allMigrations.map((migration) => [migration.id, migration]) - ); - - const allStats: RuleMigrationTaskStats[] = []; - - for (const dataStats of allDataStats) { - const migration = allMigrationsMap.get(dataStats.id); - if (migration) { - const tasksStats = this.getTaskStats(migration, dataStats.rules); - allStats.push({ ...tasksStats, ...dataStats, name: migration.name }); - } - } - return allStats; - } - private getTaskStats( - migration: StoredSiemMigration, - dataStats: RuleMigrationDataStats['rules'] - ): Pick { +export type RuleMigrationsRunning = Map; +export class RuleMigrationsTaskClient extends SiemMigrationsTaskClient< + RuleMigration, + RuleMigrationRule, + RuleMigrationTaskInput, + MigrateRuleConfigSchema, + RuleMigrationTaskOutput +> { + protected readonly TaskRunnerClass = RuleMigrationTaskRunner; + protected readonly EvaluatorClass = RuleMigrationTaskEvaluator; + + // Rules specific last_execution config + protected getLastExecutionConfig( + invocationConfig: RunnableConfig + ): Record { return { - status: this.getTaskStatus(migration, dataStats), - last_execution: migration.last_execution, + skipPrebuiltRulesMatching: invocationConfig.configurable?.skipPrebuiltRulesMatching ?? false, }; } - - private getTaskStatus( - migration: StoredSiemMigration, - dataStats: RuleMigrationDataStats['rules'] - ): SiemMigrationTaskStatus { - const { id: migrationId, last_execution: lastExecution } = migration; - if (this.migrationsRunning.has(migrationId)) { - return SiemMigrationTaskStatus.RUNNING; - } - if (dataStats.completed + dataStats.failed === dataStats.total) { - return SiemMigrationTaskStatus.FINISHED; - } - if (lastExecution?.is_stopped) { - return SiemMigrationTaskStatus.STOPPED; - } - if (dataStats.pending === dataStats.total) { - return SiemMigrationTaskStatus.READY; - } - return SiemMigrationTaskStatus.INTERRUPTED; - } - - /** Stops one running migration */ - async stop(migrationId: string): Promise { - try { - const migrationRunning = this.migrationsRunning.get(migrationId); - if (migrationRunning) { - migrationRunning.abortController.abort('Stopped by user'); - await this.data.migrations.setIsStopped({ id: migrationId }); - return { exists: true, stopped: true }; - } - - const { rules } = await this.data.rules.getStats(migrationId); - if (rules.total > 0) { - return { exists: true, stopped: true }; - } - return { exists: false, stopped: true }; - } catch (err) { - this.logger.error(`Error stopping migration ID:${migrationId}`, err); - return { exists: true, stopped: false }; - } - } - - /** Creates a new evaluator for the rule migration task */ - async evaluate(params: RuleMigrationTaskEvaluateParams): Promise { - const { evaluationId, langsmithOptions, connectorId, invocationConfig, abortController } = - params; - - const migrationLogger = this.logger.get('evaluate'); - - const migrationTaskEvaluator = new RuleMigrationTaskEvaluator( - evaluationId, - this.currentUser, - abortController, - this.data, - migrationLogger, - this.dependencies - ); - - await migrationTaskEvaluator.evaluate({ - connectorId, - langsmithOptions, - invocationConfig, - }); - } - - /** Returns if a migration is running or not */ - isMigrationRunning(migrationId: string): boolean { - return this.migrationsRunning.has(migrationId); - } } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_evaluator.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_evaluator.test.ts index 010804bc92d26..faa50fb1c39c2 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_evaluator.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_evaluator.test.ts @@ -5,13 +5,13 @@ * 2.0. */ -import type { CustomEvaluator } from './rule_migrations_task_evaluator'; import { RuleMigrationTaskEvaluator } from './rule_migrations_task_evaluator'; import type { Run, Example } from 'langsmith/schemas'; import { createRuleMigrationsDataClientMock } from '../data/__mocks__/mocks'; import { loggerMock } from '@kbn/logging-mocks'; import type { AuthenticatedUser } from '@kbn/core/server'; import type { SiemMigrationsClientDependencies } from '../../common/types'; +import type { CustomEvaluator } from '../../common/task/siem_migrations_task_evaluator'; // Mock dependencies jest.mock('langsmith/evaluation', () => ({ @@ -66,65 +66,13 @@ describe('RuleMigrationTaskEvaluator', () => { describe('evaluators', () => { let evaluator: CustomEvaluator; + // Helper to access private evaluator methods const setEvaluator = (name: string) => { // @ts-expect-error (accessing private property) evaluator = taskEvaluator.evaluators[name]; }; - describe('translation_result evaluator', () => { - beforeAll(() => { - setEvaluator('translation_result'); - }); - - it('should return true score when translation results match', () => { - const mockRun = { outputs: { translation_result: 'full' } } as unknown as Run; - const mockExample = { outputs: { translation_result: 'full' } } as unknown as Example; - - const result = evaluator({ run: mockRun, example: mockExample }); - - expect(result).toEqual({ - score: true, - comment: 'Correct', - }); - }); - - it('should return false score when translation results do not match', () => { - const mockRun = { outputs: { translation_result: 'full' } } as unknown as Run; - const mockExample = { outputs: { translation_result: 'partial' } } as unknown as Example; - - const result = evaluator({ run: mockRun, example: mockExample }); - - expect(result).toEqual({ - score: false, - comment: 'Incorrect, expected "partial" but got "full"', - }); - }); - - it('should ignore score when expected result is missing', () => { - const mockRun = { outputs: { translation_result: 'full' } } as unknown as Run; - const mockExample = { outputs: {} } as unknown as Example; - - const result = evaluator({ run: mockRun, example: mockExample }); - - expect(result).toEqual({ - comment: 'No translation result expected', - }); - }); - - it('should return false score when run result is missing', () => { - const mockRun = { outputs: {} } as unknown as Run; - const mockExample = { outputs: { translation_result: 'full' } } as unknown as Example; - - const result = evaluator({ run: mockRun, example: mockExample }); - - expect(result).toEqual({ - score: false, - comment: 'No translation result received', - }); - }); - }); - describe('custom_query_accuracy evaluator', () => { beforeAll(() => { setEvaluator('custom_query_accuracy'); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_evaluator.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_evaluator.ts index c7fbadd5139bf..62f5bfd47bc1d 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_evaluator.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_evaluator.ts @@ -5,110 +5,16 @@ * 2.0. */ -import type { EvaluationResult } from 'langsmith/evaluation'; -import type { Run, Example } from 'langsmith/schemas'; -import { evaluate } from 'langsmith/evaluation'; -import { isLangSmithEnabled } from '@kbn/langchain/server/tracers/langsmith'; -import { Client } from 'langsmith'; import { distance } from 'fastest-levenshtein'; -import type { LangSmithEvaluationOptions } from '../../../../../common/siem_migrations/model/common.gen'; import { RuleMigrationTaskRunner } from './rule_migrations_task_runner'; -import type { MigrateRuleGraphConfig, MigrateRuleState } from './agent/types'; - -export interface EvaluateParams { - connectorId: string; - langsmithOptions: LangSmithEvaluationOptions; - invocationConfig?: MigrateRuleGraphConfig; -} - -export type Evaluator = (args: { run: Run; example: Example }) => EvaluationResult; -type CustomEvaluatorResult = Omit; -export type CustomEvaluator = (args: { run: Run; example: Example }) => CustomEvaluatorResult; - -export class RuleMigrationTaskEvaluator extends RuleMigrationTaskRunner { - public async evaluate({ connectorId, langsmithOptions, invocationConfig }: EvaluateParams) { - if (!isLangSmithEnabled()) { - throw Error('LangSmith is not enabled'); - } - - const client = new Client({ apiKey: langsmithOptions.api_key }); - - // Make sure the dataset exists - const dataset: Example[] = []; - for await (const example of client.listExamples({ datasetName: langsmithOptions.dataset })) { - dataset.push(example); - } - if (dataset.length === 0) { - throw Error(`LangSmith dataset not found: ${langsmithOptions.dataset}`); - } - - // Initialize the the task runner first, this may take some time - await this.initialize(); - - // Check if the connector exists and user has privileges to read it - const connector = await this.dependencies.actionsClient.get({ id: connectorId }); - if (!connector) { - throw Error(`Connector with id ${connectorId} not found`); - } - - // for each connector, setup the evaluator - await this.setup(connectorId); - - // create the migration task after setup - const migrateRuleTask = this.createMigrateRuleTask(invocationConfig); - const evaluators = this.getEvaluators(); - - evaluate(migrateRuleTask, { - data: langsmithOptions.dataset, - experimentPrefix: connector.name, - evaluators, - client, - maxConcurrency: 3, - }) - .then(() => { - this.logger.info('Evaluation finished'); - }) - .catch((err) => { - this.logger.error(`Evaluation error:\n ${JSON.stringify(err, null, 2)}`); - }); - } - - private getEvaluators(): Evaluator[] { - return Object.entries(this.evaluators).map(([key, evaluator]) => { - return (args) => { - const result = evaluator(args); - return { key, ...result }; - }; - }); - } - - /** - * This is a map of custom evaluators that are used to evaluate rule migration tasks - * The object keys are used for the `key` property of the evaluation result, and the value is a function that takes a the `run` and `example` - * and returns a `score` and a `comment` (and any other data needed for the evaluation) - **/ - private readonly evaluators: Record = { - translation_result: ({ run, example }) => { - const runResult = (run?.outputs as MigrateRuleState)?.translation_result; - const expectedResult = (example?.outputs as MigrateRuleState)?.translation_result; - - if (!expectedResult) { - return { comment: 'No translation result expected' }; - } - if (!runResult) { - return { score: false, comment: 'No translation result received' }; - } - - if (runResult === expectedResult) { - return { score: true, comment: 'Correct' }; - } - - return { - score: false, - comment: `Incorrect, expected "${expectedResult}" but got "${runResult}"`, - }; - }, - +import type { MigrateRuleState } from './agent/types'; +import type { CustomEvaluator } from '../../common/task/siem_migrations_task_evaluator'; +import { SiemMigrationTaskEvaluable } from '../../common/task/siem_migrations_task_evaluator'; + +export class RuleMigrationTaskEvaluator extends SiemMigrationTaskEvaluable( + RuleMigrationTaskRunner +) { + protected readonly evaluators: Record = { custom_query_accuracy: ({ run, example }) => { const runQuery = (run?.outputs as MigrateRuleState)?.elastic_rule?.query; const expectedQuery = (example?.outputs as MigrateRuleState)?.elastic_rule?.query; @@ -138,7 +44,8 @@ export class RuleMigrationTaskEvaluator extends RuleMigrationTaskRunner { }, prebuilt_rule_match: ({ run, example }) => { - const runPrebuiltRuleId = (run?.outputs as MigrateRuleState)?.elastic_rule?.prebuilt_rule_id; + const runPrebuiltRuleId = (run?.outputs as MigrateRuleState)?.elastic_rule + ?.prebuilt_rule_id; const expectedPrebuiltRuleId = (example?.outputs as MigrateRuleState)?.elastic_rule ?.prebuilt_rule_id; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.test.ts index d950c856c699d..ee499c58ac68a 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.test.ts @@ -8,7 +8,7 @@ import { RuleMigrationTaskRunner } from './rule_migrations_task_runner'; import { SiemMigrationStatus } from '../../../../../common/siem_migrations/constants'; import type { AuthenticatedUser } from '@kbn/core/server'; -import type { StoredRuleMigration } from '../types'; +import type { StoredRuleMigrationRule } from '../types'; import { createRuleMigrationsDataClientMock } from '../data/__mocks__/mocks'; import { loggerMock } from '@kbn/logging-mocks'; import type { SiemMigrationsClientDependencies } from '../../common/types'; @@ -110,22 +110,22 @@ describe('RuleMigrationTaskRunner', () => { }); it('should handle the migration successfully', async () => { - mockRuleMigrationsDataClient.rules.get.mockResolvedValue({ total: 0, data: [] }); - mockRuleMigrationsDataClient.rules.get.mockResolvedValueOnce({ + mockRuleMigrationsDataClient.items.get.mockResolvedValue({ total: 0, data: [] }); + mockRuleMigrationsDataClient.items.get.mockResolvedValueOnce({ total: 1, - data: [{ id: ruleId, status: SiemMigrationStatus.PENDING }] as StoredRuleMigration[], + data: [{ id: ruleId, status: SiemMigrationStatus.PENDING }] as StoredRuleMigrationRule[], }); await taskRunner.setup('test-connector-id'); await expect(taskRunner.run({})).resolves.toBeUndefined(); - expect(mockRuleMigrationsDataClient.rules.saveProcessing).toHaveBeenCalled(); + expect(mockRuleMigrationsDataClient.items.saveProcessing).toHaveBeenCalled(); expect(mockTimeout).toHaveBeenCalledTimes(1); // random execution sleep expect(mockTimeout).toHaveBeenNthCalledWith(1, expect.any(Function), expect.any(Number)); expect(mockInvoke).toHaveBeenCalledTimes(1); - expect(mockRuleMigrationsDataClient.rules.saveCompleted).toHaveBeenCalled(); - expect(mockRuleMigrationsDataClient.rules.get).toHaveBeenCalledTimes(2); // One with data, one without + expect(mockRuleMigrationsDataClient.items.saveCompleted).toHaveBeenCalled(); + expect(mockRuleMigrationsDataClient.items.get).toHaveBeenCalledTimes(2); // One with data, one without expect(mockLogger.info).toHaveBeenCalledWith('Migration completed successfully'); }); @@ -156,12 +156,12 @@ describe('RuleMigrationTaskRunner', () => { describe('during migration', () => { beforeEach(() => { - mockRuleMigrationsDataClient.rules.get.mockRestore(); - mockRuleMigrationsDataClient.rules.get + mockRuleMigrationsDataClient.items.get.mockRestore(); + mockRuleMigrationsDataClient.items.get .mockResolvedValue({ total: 0, data: [] }) .mockResolvedValueOnce({ total: 1, - data: [{ id: ruleId, status: SiemMigrationStatus.PENDING }] as StoredRuleMigration[], + data: [{ id: ruleId, status: SiemMigrationStatus.PENDING }] as StoredRuleMigrationRule[], }); }); @@ -175,7 +175,7 @@ describe('RuleMigrationTaskRunner', () => { await expect(runPromise).resolves.toBeUndefined(); // Ensure the function handles abort gracefully expect(mockLogger.info).toHaveBeenCalledWith('Abort signal received, stopping migration'); - expect(mockRuleMigrationsDataClient.rules.releaseProcessing).toHaveBeenCalled(); + expect(mockRuleMigrationsDataClient.items.releaseProcessing).toHaveBeenCalled(); }); it('should handle other errors correctly', async () => { @@ -187,7 +187,7 @@ describe('RuleMigrationTaskRunner', () => { expect(mockLogger.error).toHaveBeenCalledWith( `Error translating rule \"${ruleId}\" with error: ${errorMessage}` ); - expect(mockRuleMigrationsDataClient.rules.saveError).toHaveBeenCalled(); + expect(mockRuleMigrationsDataClient.items.saveError).toHaveBeenCalled(); }); describe('during rate limit errors', () => { @@ -195,15 +195,15 @@ describe('RuleMigrationTaskRunner', () => { const error = new Error('429. You did way too many requests to this random LLM API bud'); beforeEach(async () => { - mockRuleMigrationsDataClient.rules.get.mockRestore(); - mockRuleMigrationsDataClient.rules.get + mockRuleMigrationsDataClient.items.get.mockRestore(); + mockRuleMigrationsDataClient.items.get .mockResolvedValue({ total: 0, data: [] }) .mockResolvedValueOnce({ total: 2, data: [ { id: ruleId, status: SiemMigrationStatus.PENDING }, { id: rule2Id, status: SiemMigrationStatus.PENDING }, - ] as StoredRuleMigration[], + ] as StoredRuleMigrationRule[], }); }); @@ -253,7 +253,7 @@ describe('RuleMigrationTaskRunner', () => { `Awaiting backoff task for rule "${rule2Id}"` ); expect(mockInvoke).toHaveBeenCalledTimes(6); // 3 retries + 3 executions - expect(mockRuleMigrationsDataClient.rules.saveCompleted).toHaveBeenCalledTimes(2); // 2 rules + expect(mockRuleMigrationsDataClient.items.saveCompleted).toHaveBeenCalledTimes(2); // 2 rules }); it('should fail when reached maxRetries', async () => { @@ -265,14 +265,14 @@ describe('RuleMigrationTaskRunner', () => { expect(mockInvoke).toHaveBeenCalledTimes(10); // 8 retries + 2 executions expect(mockTimeout).toHaveBeenCalledTimes(10); // 2 execution sleeps + 8 backoff sleeps - expect(mockRuleMigrationsDataClient.rules.saveError).toHaveBeenCalledTimes(2); // 2 rules + expect(mockRuleMigrationsDataClient.items.saveError).toHaveBeenCalledTimes(2); // 2 rules }); it('should fail when reached max recovery attempts', async () => { const rule3Id = 'test-rule-id-3'; const rule4Id = 'test-rule-id-4'; - mockRuleMigrationsDataClient.rules.get.mockRestore(); - mockRuleMigrationsDataClient.rules.get + mockRuleMigrationsDataClient.items.get.mockRestore(); + mockRuleMigrationsDataClient.items.get .mockResolvedValue({ total: 0, data: [] }) .mockResolvedValueOnce({ total: 4, @@ -281,7 +281,7 @@ describe('RuleMigrationTaskRunner', () => { { id: rule2Id, status: SiemMigrationStatus.PENDING }, { id: rule3Id, status: SiemMigrationStatus.PENDING }, { id: rule4Id, status: SiemMigrationStatus.PENDING }, - ] as StoredRuleMigration[], + ] as StoredRuleMigrationRule[], }); // max recovery attempts = 3 @@ -303,17 +303,17 @@ describe('RuleMigrationTaskRunner', () => { await expect(taskRunner.run({})).resolves.toBeUndefined(); // success - expect(mockRuleMigrationsDataClient.rules.saveCompleted).toHaveBeenCalledTimes(3); // rules 1, 2 and 3 - expect(mockRuleMigrationsDataClient.rules.saveError).toHaveBeenCalledTimes(1); // rule 4 + expect(mockRuleMigrationsDataClient.items.saveCompleted).toHaveBeenCalledTimes(3); // rules 1, 2 and 3 + expect(mockRuleMigrationsDataClient.items.saveError).toHaveBeenCalledTimes(1); // rule 4 }); it('should increase the executor sleep time when rate limited', async () => { const getResponse = { total: 1, - data: [{ id: ruleId, status: SiemMigrationStatus.PENDING }] as StoredRuleMigration[], + data: [{ id: ruleId, status: SiemMigrationStatus.PENDING }] as StoredRuleMigrationRule[], }; - mockRuleMigrationsDataClient.rules.get.mockRestore(); - mockRuleMigrationsDataClient.rules.get + mockRuleMigrationsDataClient.items.get.mockRestore(); + mockRuleMigrationsDataClient.items.get .mockResolvedValue({ total: 0, data: [] }) .mockResolvedValueOnce(getResponse) .mockResolvedValueOnce({ total: 0, data: [] }) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts index 00469ddff709e..2e40db266cc40 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts @@ -5,86 +5,53 @@ * 2.0. */ -import assert from 'assert'; import type { AuthenticatedUser, Logger } from '@kbn/core/server'; -import { abortSignalToPromise, AbortError } from '@kbn/kibana-utils-plugin/server'; -import { type ElasticRule } from '../../../../../common/siem_migrations/model/rule_migration.gen'; -import { SiemMigrationStatus } from '../../../../../common/siem_migrations/constants'; -import { initPromisePool } from '../../../../utils/promise_pool'; +import type { + ElasticRule, + RuleMigration, + RuleMigrationRule, +} from '../../../../../common/siem_migrations/model/rule_migration.gen'; import type { RuleMigrationsDataClient } from '../data/rule_migrations_data_client'; -import type { MigrateRuleGraphConfig, MigrateRuleState } from './agent/types'; +import type { MigrateRuleConfigSchema, MigrateRuleState } from './agent/types'; import { getRuleMigrationAgent } from './agent'; import { RuleMigrationsRetriever } from './retrievers'; -import { SiemMigrationTelemetryClient } from './rule_migrations_telemetry_client'; -import type { MigrationAgent, RuleMigrationInput } from './types'; -import { generateAssistantComment } from './util/comments'; -import type { StoredRuleMigration } from '../types'; -import { ActionsClientChat } from './util/actions_client_chat'; -import { EsqlKnowledgeBase } from './util/esql_knowledge_base'; -import { nullifyElasticRule } from './util/nullify_missing_properties'; +import type { StoredRuleMigrationRule } from '../types'; +import { EsqlKnowledgeBase } from '../../common/task/util/esql_knowledge_base'; +import { nullifyMissingProperties } from '../../common/task/util/nullify_missing_properties'; import type { SiemMigrationsClientDependencies } from '../../common/types'; - -/** Number of concurrent rule translations in the pool */ -const TASK_CONCURRENCY = 10 as const; -/** Number of rules loaded in memory to be translated in the pool */ -const TASK_BATCH_SIZE = 100 as const; -/** The timeout of each individual agent invocation in minutes */ -const AGENT_INVOKE_TIMEOUT_MIN = 3 as const; - -/** Exponential backoff configuration to handle rate limit errors */ -const RETRY_CONFIG = { - initialRetryDelaySeconds: 1, - backoffMultiplier: 2, - maxRetries: 8, - // max waiting time 4m15s (1*2^8 = 256s) -} as const; - -/** Executor sleep configuration - * A sleep time applied at the beginning of each single rule translation in the execution pool, - * The objective of this sleep is to spread the load of concurrent translations, and prevent hitting the rate limit repeatedly. - * The sleep time applied is a random number between [0-value]. Every time we hit rate limit the value is increased by the multiplier, up to the limit. - */ -const EXECUTOR_SLEEP = { - initialValueSeconds: 3, - multiplier: 2, - limitSeconds: 96, // 1m36s (5 increases) -} as const; - -/** This limit should never be reached, it's a safety net to prevent infinite loops. - * It represents the max number of consecutive rate limit recovery & failure attempts. - * This can only happen when the API can not process TASK_CONCURRENCY translations at a time, - * even after the executor sleep is increased on every attempt. - **/ -const EXECUTOR_RECOVER_MAX_ATTEMPTS = 3 as const; - -export class RuleMigrationTaskRunner { - private telemetry?: SiemMigrationTelemetryClient; - protected agent?: MigrationAgent; +import { SiemMigrationTaskRunner } from '../../common/task/siem_migrations_task_runner'; +import { RuleMigrationTelemetryClient } from './rule_migrations_telemetry_client'; + +export type RuleMigrationTaskInput = Pick; +export type RuleMigrationTaskOutput = MigrateRuleState; + +export class RuleMigrationTaskRunner extends SiemMigrationTaskRunner< + RuleMigration, + RuleMigrationRule, + RuleMigrationTaskInput, + MigrateRuleConfigSchema, + RuleMigrationTaskOutput +> { private retriever: RuleMigrationsRetriever; - private actionsClientChat: ActionsClientChat; - private abort: ReturnType; - private executorSleepMultiplier: number = EXECUTOR_SLEEP.initialValueSeconds; - public isWaiting: boolean = false; constructor( public readonly migrationId: string, public readonly startedBy: AuthenticatedUser, public readonly abortController: AbortController, - private readonly data: RuleMigrationsDataClient, + protected readonly data: RuleMigrationsDataClient, protected readonly logger: Logger, protected readonly dependencies: SiemMigrationsClientDependencies ) { - this.actionsClientChat = new ActionsClientChat(this.dependencies.actionsClient, this.logger); + super(migrationId, startedBy, abortController, data, logger, dependencies); this.retriever = new RuleMigrationsRetriever(this.migrationId, { data: this.data, rules: this.dependencies.rulesClient, savedObjects: this.dependencies.savedObjectsClient, }); - this.abort = abortSignalToPromise(this.abortController.signal); } /** Retrieves the connector and creates the migration agent */ - public async setup(connectorId: string) { + public async setup(connectorId: string): Promise { const { inferenceClient } = this.dependencies; const model = await this.actionsClientChat.createModel({ @@ -92,6 +59,14 @@ export class RuleMigrationTaskRunner { migrationId: this.migrationId, abortController: this.abortController, }); + const modelName = this.actionsClientChat.getModelName(model); + + const telemetryClient = new RuleMigrationTelemetryClient( + this.dependencies.telemetry, + this.logger, + this.migrationId, + modelName + ); const esqlKnowledgeBase = new EsqlKnowledgeBase( connectorId, @@ -100,20 +75,16 @@ export class RuleMigrationTaskRunner { this.logger ); - this.telemetry = new SiemMigrationTelemetryClient( - this.dependencies.telemetry, - this.logger, - this.migrationId, - model.model - ); - - this.agent = getRuleMigrationAgent({ - model, + const agent = getRuleMigrationAgent({ esqlKnowledgeBase, + model, ruleMigrationsRetriever: this.retriever, - telemetryClient: this.telemetry, logger: this.logger, + telemetryClient, }); + + this.telemetry = telemetryClient; + this.task = (input, config) => agent.invoke(input, config); } /** Initializes the retriever populating ELSER indices. It may take a few minutes */ @@ -121,240 +92,25 @@ export class RuleMigrationTaskRunner { await this.retriever.initialize(); } - public async run(invocationConfig: MigrateRuleGraphConfig): Promise { - assert(this.telemetry, 'telemetry is missing please call setup() first'); - const { telemetry, migrationId } = this; - - const migrationTaskTelemetry = telemetry.startSiemMigrationTask(); - - try { - // TODO: track the duration of the initialization alone in the telemetry - this.logger.debug('Initializing migration'); - await this.withAbort(this.initialize()); // long running operation - } catch (error) { - migrationTaskTelemetry.failure(error); - if (error instanceof AbortError) { - this.logger.info('Abort signal received, stopping initialization'); - return; - } else { - throw new Error(`Migration initialization failed. ${error}`); - } - } - - const migrateRuleTask = this.createMigrateRuleTask(invocationConfig); - this.logger.debug(`Started rule translations. Concurrency is: ${TASK_CONCURRENCY}`); - - try { - do { - const { data: ruleMigrations } = await this.data.rules.get(migrationId, { - filters: { status: SiemMigrationStatus.PENDING }, - size: TASK_BATCH_SIZE, // keep these rules in memory and process them in the promise pool with concurrency limit - }); - if (ruleMigrations.length === 0) { - break; - } - - this.logger.debug(`Start processing batch of ${ruleMigrations.length} rules`); - - const { errors } = await initPromisePool({ - concurrency: TASK_CONCURRENCY, - abortSignal: this.abortController.signal, - items: ruleMigrations, - executor: async (ruleMigration) => { - const ruleTranslationTelemetry = migrationTaskTelemetry.startRuleTranslation(); - try { - await this.saveRuleProcessing(ruleMigration); - - const resources = await this.retriever.resources.getResources(ruleMigration); - - const migrationResult = await migrateRuleTask({ - id: ruleMigration.id, - original_rule: ruleMigration.original_rule, - resources, - }); - - await this.saveRuleCompleted(ruleMigration, migrationResult); - ruleTranslationTelemetry.success(migrationResult); - } catch (error) { - if (this.abortController.signal.aborted) { - throw new AbortError(); - } - ruleTranslationTelemetry.failure(error); - await this.saveRuleFailed(ruleMigration, error); - } - }, - }); - - if (errors.length > 0) { - throw errors[0].error; // Only AbortError is thrown from the pool. The task was aborted - } - - this.logger.debug('Batch processed successfully'); - } while (true); - - migrationTaskTelemetry.success(); - this.logger.info('Migration completed successfully'); - } catch (error) { - await this.data.rules.releaseProcessing(migrationId); - - if (error instanceof AbortError) { - migrationTaskTelemetry.aborted(error); - this.logger.info('Abort signal received, stopping migration'); - } else { - migrationTaskTelemetry.failure(error); - throw new Error(`Error processing migration: ${error}`); - } - } finally { - this.abort.cleanup(); - } - } - - protected createMigrateRuleTask(invocationConfig?: MigrateRuleGraphConfig) { - assert(this.agent, 'agent is missing please call setup() first'); - const { agent } = this; - const config: MigrateRuleGraphConfig = { - timeout: AGENT_INVOKE_TIMEOUT_MIN * 60 * 1000, // milliseconds timeout - ...invocationConfig, - signal: this.abortController.signal, - }; - - // Prepare the invocation with specific config - const invoke = async (input: RuleMigrationInput): Promise => - agent.invoke(input, config); - - // Invokes the rule translation with exponential backoff, should be called only when the rate limit has been hit - const invokeWithBackoff = async ( - ruleMigration: RuleMigrationInput - ): Promise => { - this.logger.debug(`Rate limit backoff started for rule "${ruleMigration.id}"`); - let retriesLeft: number = RETRY_CONFIG.maxRetries; - while (true) { - try { - await this.sleepRetry(retriesLeft); - retriesLeft--; - const result = await invoke(ruleMigration); - this.logger.info( - `Rate limit backoff completed successfully for rule "${ruleMigration.id}" after ${ - RETRY_CONFIG.maxRetries - retriesLeft - } retries` - ); - return result; - } catch (error) { - if (!this.isRateLimitError(error) || retriesLeft === 0) { - this.logger.debug( - `Rate limit backoff completed unsuccessfully for rule "${ruleMigration.id}"` - ); - const logMessage = - retriesLeft === 0 - ? `Rate limit backoff completed unsuccessfully for rule "${ruleMigration.id}"` - : `Rate limit backoff interrupted for rule "${ruleMigration.id}". ${error} `; - this.logger.debug(logMessage); - throw error; - } - this.logger.debug( - `Rate limit backoff not completed for rule "${ruleMigration.id}", retries left: ${retriesLeft}` - ); - } - } - }; - - let backoffPromise: Promise | undefined; - // Migrates one rule, this function will be called concurrently by the promise pool. - // Handles rate limit errors and ensures only one task is executing the backoff retries at a time, the rest of translation will await. - const migrateRule = async (ruleMigration: RuleMigrationInput): Promise => { - let recoverAttemptsLeft: number = EXECUTOR_RECOVER_MAX_ATTEMPTS; - while (true) { - try { - await this.executorSleep(); // Random sleep, increased every time we hit the rate limit. - return await invoke(ruleMigration); - } catch (error) { - if (!this.isRateLimitError(error) || recoverAttemptsLeft === 0) { - throw error; - } - if (!backoffPromise) { - // only one translation handles the rate limit backoff retries, the rest will await it and try again when it's resolved - backoffPromise = invokeWithBackoff(ruleMigration); - this.isWaiting = true; - return backoffPromise.finally(() => { - backoffPromise = undefined; - this.increaseExecutorSleep(); - this.isWaiting = false; - }); - } - this.logger.debug(`Awaiting backoff task for rule "${ruleMigration.id}"`); - await backoffPromise.catch(() => { - throw error; // throw the original error - }); - recoverAttemptsLeft--; - } - } - }; - - return migrateRule; - } - - private isRateLimitError(error: Error) { - return error.message.match(/\b429\b/); // "429" (whole word in the error message): Too Many Requests. + protected async prepareTaskInput( + migrationRule: StoredRuleMigrationRule + ): Promise { + const resources = await this.retriever.resources.getResources(migrationRule); + return { id: migrationRule.id, original_rule: migrationRule.original_rule, resources }; } - private async withAbort(promise: Promise): Promise { - return Promise.race([promise, this.abort.promise]); - } - - private async sleep(seconds: number) { - await this.withAbort(new Promise((resolve) => setTimeout(resolve, seconds * 1000))); - } - - // Exponential backoff implementation - private async sleepRetry(retriesLeft: number) { - const seconds = - RETRY_CONFIG.initialRetryDelaySeconds * - Math.pow(RETRY_CONFIG.backoffMultiplier, RETRY_CONFIG.maxRetries - retriesLeft); - this.logger.debug(`Retry sleep: ${seconds}s`); - await this.sleep(seconds); - } - - private executorSleep = async () => { - const seconds = Math.random() * this.executorSleepMultiplier; - this.logger.debug(`Executor sleep: ${seconds.toFixed(3)}s`); - await this.sleep(seconds); - }; - - private increaseExecutorSleep = () => { - const increasedMultiplier = this.executorSleepMultiplier * EXECUTOR_SLEEP.multiplier; - if (increasedMultiplier > EXECUTOR_SLEEP.limitSeconds) { - this.logger.warn('Executor sleep reached the maximum value'); - return; - } - this.executorSleepMultiplier = increasedMultiplier; - }; - - private async saveRuleProcessing(ruleMigration: StoredRuleMigration) { - this.logger.debug(`Starting translation of rule "${ruleMigration.id}"`); - return this.data.rules.saveProcessing(ruleMigration.id); - } - - private async saveRuleCompleted( - ruleMigration: StoredRuleMigration, - migrationResult: MigrateRuleState - ) { - this.logger.debug(`Translation of rule "${ruleMigration.id}" succeeded`); - const nullifiedElasticRule = nullifyElasticRule( - migrationResult.elastic_rule as ElasticRule, - this.logger.error - ); - const ruleMigrationTranslated = { - ...ruleMigration, - elastic_rule: nullifiedElasticRule as ElasticRule, - translation_result: migrationResult.translation_result, - comments: migrationResult.comments, + protected processTaskOutput( + migrationRule: StoredRuleMigrationRule, + migrationOutput: RuleMigrationTaskOutput + ): StoredRuleMigrationRule { + return { + ...migrationRule, + elastic_rule: nullifyMissingProperties({ + source: migrationRule.elastic_rule, + target: migrationOutput.elastic_rule as ElasticRule, + }), + translation_result: migrationOutput.translation_result, + comments: migrationOutput.comments, }; - return this.data.rules.saveCompleted(ruleMigrationTranslated); - } - - private async saveRuleFailed(ruleMigration: StoredRuleMigration, error: Error) { - this.logger.error(`Error translating rule "${ruleMigration.id}" with error: ${error.message}`); - const comments = [generateAssistantComment(`Error migrating rule: ${error.message}`)]; - return this.data.rules.saveError({ ...ruleMigration, comments }); } } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_service.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_service.ts index 709e63ff49828..8b678056f1404 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_service.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_service.ts @@ -7,10 +7,13 @@ import type { Logger } from '@kbn/core/server'; import type { RuleMigrationTaskCreateClientParams } from './types'; -import { RuleMigrationsTaskClient, type MigrationsRunning } from './rule_migrations_task_client'; +import { + RuleMigrationsTaskClient, + type RuleMigrationsRunning, +} from './rule_migrations_task_client'; export class RuleMigrationsTaskService { - private migrationsRunning: MigrationsRunning; + private migrationsRunning: RuleMigrationsRunning; constructor(private logger: Logger) { this.migrationsRunning = new Map(); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_telemetry_client.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_telemetry_client.test.ts deleted file mode 100644 index adeca580860a5..0000000000000 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_telemetry_client.test.ts +++ /dev/null @@ -1,257 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { coreMock } from '@kbn/core/server/mocks'; -import { loggerMock } from '@kbn/logging-mocks'; -import { SiemMigrationTelemetryClient } from './rule_migrations_telemetry_client'; -import type { RuleMigrationIntegration, RuleMigrationPrebuiltRule } from '../types'; -import type { MigrateRuleState } from './agent/types'; - -const translationResultWithMatchMock = { - translation_result: 'full', - elastic_rule: { prebuilt_rule_id: 'testprebuiltid' }, -} as MigrateRuleState; -const translationResultMock = { - translation_result: 'partial', -} as MigrateRuleState; -const preFilterRulesMock: RuleMigrationPrebuiltRule[] = [ - { - rule_id: 'rule1id', - name: 'rule1', - description: 'rule1description', - elser_embedding: 'rule1embedding', - mitre_attack_ids: ['MitreID1'], - }, - { - rule_id: 'rule2id', - name: 'rule2', - description: 'rule2description', - elser_embedding: 'rule1embedding', - }, -]; - -const postFilterRuleMock: RuleMigrationPrebuiltRule = { - rule_id: 'rule1id', - name: 'rule1', - description: 'rule1description', - elser_embedding: 'rule1embedding', - mitre_attack_ids: ['MitreID1'], -}; - -const postFilterIntegrationMocks: RuleMigrationIntegration = { - id: 'testIntegration1', - title: 'testIntegration1', - description: 'testDescription1', - data_streams: [{ dataset: 'testds1', title: 'testds1', index_pattern: 'testds1-pattern' }], - elser_embedding: 'testEmbedding', -}; - -const preFilterIntegrationMocks: RuleMigrationIntegration[] = [ - { - id: 'testIntegration1', - title: 'testIntegration1', - description: 'testDescription1', - data_streams: [{ dataset: 'testds1', title: 'testds1', index_pattern: 'testds1-pattern' }], - elser_embedding: 'testEmbedding', - }, - { - id: 'testIntegration2', - title: 'testIntegration2', - description: 'testDescription2', - data_streams: [{ dataset: 'testds2', title: 'testds2', index_pattern: 'testds2-pattern' }], - elser_embedding: 'testEmbedding', - }, -]; - -const mockTelemetry = coreMock.createSetup().analytics; -const mockLogger = loggerMock.create(); -const siemTelemetryClient = new SiemMigrationTelemetryClient( - mockTelemetry, - mockLogger, - 'testmigration', - 'testModel' -); - -describe('siemMigrationTelemetry', () => { - beforeEach(() => { - jest.clearAllMocks(); - }); - - afterEach(() => { - jest.clearAllMocks(); - }); - - beforeAll(() => { - jest.useFakeTimers(); - const date = '2024-01-28T04:20:02.394Z'; - jest.setSystemTime(new Date(date)); - }); - - afterAll(() => { - jest.useRealTimers(); - }); - it('start/end migration with error', async () => { - const error = 'test error message'; - const siemMigrationTaskTelemetry = siemTelemetryClient.startSiemMigrationTask(); - const ruleTranslationTelemetry = siemMigrationTaskTelemetry.startRuleTranslation(); - - // 2 success and 2 failures - ruleTranslationTelemetry.success(translationResultMock); - ruleTranslationTelemetry.success(translationResultMock); - ruleTranslationTelemetry.failure(new Error('test')); - ruleTranslationTelemetry.failure(new Error('test')); - - siemMigrationTaskTelemetry.failure(new Error(error)); - - expect(mockTelemetry.reportEvent).toHaveBeenNthCalledWith( - 5, - 'siem_migrations_migration_failure', - { - completed: 2, - duration: 0, - error, - failed: 2, - migrationId: 'testmigration', - model: 'testModel', - total: 4, - eventName: 'Migration failure', - } - ); - }); - it('start/end migration success', async () => { - const siemMigrationTaskTelemetry = siemTelemetryClient.startSiemMigrationTask(); - const ruleTranslationTelemetry = siemMigrationTaskTelemetry.startRuleTranslation(); - - // 2 success and 2 failures - ruleTranslationTelemetry.success(translationResultMock); - ruleTranslationTelemetry.success(translationResultMock); - ruleTranslationTelemetry.failure(new Error('test')); - ruleTranslationTelemetry.failure(new Error('test')); - - siemMigrationTaskTelemetry.success(); - - expect(mockTelemetry.reportEvent).toHaveBeenNthCalledWith( - 5, - 'siem_migrations_migration_success', - { - completed: 2, - duration: 0, - failed: 2, - migrationId: 'testmigration', - model: 'testModel', - total: 4, - eventName: 'Migration success', - } - ); - }); - it('start/end rule translation with error', async () => { - const error = 'test error message'; - const siemMigrationTaskTelemetry = siemTelemetryClient.startSiemMigrationTask(); - const ruleTranslationTelemetry = siemMigrationTaskTelemetry.startRuleTranslation(); - - ruleTranslationTelemetry.failure(new Error(error)); - - expect(mockTelemetry.reportEvent).toHaveBeenCalledWith( - 'siem_migrations_rule_translation_failure', - { error, migrationId: 'testmigration', model: 'testModel', eventName: 'Translation failure' } - ); - }); - it('start/end rule translation success with prebuilt', async () => { - const siemMigrationTaskTelemetry = siemTelemetryClient.startSiemMigrationTask(); - const ruleTranslationTelemetry = siemMigrationTaskTelemetry.startRuleTranslation(); - - ruleTranslationTelemetry.success(translationResultWithMatchMock); - - expect(mockTelemetry.reportEvent).toHaveBeenCalledWith( - 'siem_migrations_rule_translation_success', - { - migrationId: 'testmigration', - model: 'testModel', - duration: 0, - prebuiltMatch: true, - translationResult: 'full', - eventName: 'Translation success', - } - ); - }); - it('start/end rule translation success without prebuilt', async () => { - const siemMigrationTaskTelemetry = siemTelemetryClient.startSiemMigrationTask(); - const ruleTranslationTelemetry = siemMigrationTaskTelemetry.startRuleTranslation(); - - ruleTranslationTelemetry.success(translationResultMock); - - expect(mockTelemetry.reportEvent).toHaveBeenCalledWith( - 'siem_migrations_rule_translation_success', - { - migrationId: 'testmigration', - model: 'testModel', - prebuiltMatch: false, - translationResult: 'partial', - duration: 0, - eventName: 'Translation success', - } - ); - }); - it('reportIntegrationMatch with a match', async () => { - siemTelemetryClient.reportIntegrationsMatch({ - preFilterIntegrations: preFilterIntegrationMocks, - postFilterIntegration: postFilterIntegrationMocks, - }); - expect(mockTelemetry.reportEvent).toHaveBeenCalledWith('siem_migrations_integration_match', { - migrationId: 'testmigration', - model: 'testModel', - postFilterIntegrationCount: 1, - postFilterIntegrationName: 'testIntegration1', - preFilterIntegrationCount: 2, - preFilterIntegrationNames: ['testIntegration1', 'testIntegration2'], - eventName: 'Integrations match', - }); - }); - it('reportIntegrationMatch without postFilter matches', async () => { - siemTelemetryClient.reportIntegrationsMatch({ - preFilterIntegrations: preFilterIntegrationMocks, - }); - expect(mockTelemetry.reportEvent).toHaveBeenCalledWith('siem_migrations_integration_match', { - migrationId: 'testmigration', - model: 'testModel', - postFilterIntegrationCount: 0, - postFilterIntegrationName: '', - preFilterIntegrationCount: 2, - preFilterIntegrationNames: ['testIntegration1', 'testIntegration2'], - eventName: 'Integrations match', - }); - }); - it('reportPrebuiltRulesMatch with a match', async () => { - siemTelemetryClient.reportPrebuiltRulesMatch({ - preFilterRules: preFilterRulesMock, - postFilterRule: postFilterRuleMock, - }); - expect(mockTelemetry.reportEvent).toHaveBeenCalledWith('siem_migrations_prebuilt_rules_match', { - migrationId: 'testmigration', - model: 'testModel', - postFilterRuleCount: 1, - postFilterRuleName: 'rule1id', - preFilterRuleCount: 2, - preFilterRuleNames: ['rule1id', 'rule2id'], - eventName: 'Prebuilt rules match', - }); - }); - it('reportPrebuiltRulesMatch without postFilter matches', async () => { - siemTelemetryClient.reportPrebuiltRulesMatch({ - preFilterRules: preFilterRulesMock, - }); - expect(mockTelemetry.reportEvent).toHaveBeenCalledWith('siem_migrations_prebuilt_rules_match', { - migrationId: 'testmigration', - model: 'testModel', - postFilterRuleCount: 0, - postFilterRuleName: '', - preFilterRuleCount: 2, - preFilterRuleNames: ['rule1id', 'rule2id'], - eventName: 'Prebuilt rules match', - }); - }); -}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_telemetry_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_telemetry_client.ts index 74ffb958bbfd8..d76e8aac06f9d 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_telemetry_client.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_telemetry_client.ts @@ -6,6 +6,7 @@ */ import type { AnalyticsServiceSetup, Logger, EventTypeOpts } from '@kbn/core/server'; +import type { RuleMigrationRule } from '../../../../../common/siem_migrations/model/rule_migration.gen'; import { SIEM_MIGRATIONS_INTEGRATIONS_MATCH, SIEM_MIGRATIONS_MIGRATION_ABORTED, @@ -15,10 +16,11 @@ import { SIEM_MIGRATIONS_RULE_TRANSLATION_FAILURE, SIEM_MIGRATIONS_RULE_TRANSLATION_SUCCESS, } from '../../../telemetry/event_based/events'; -import type { RuleMigrationIntegration, RuleSemanticSearchResult } from '../types'; -import type { MigrateRuleState } from './agent/types'; import { siemMigrationEventNames } from '../../../telemetry/event_based/event_meta'; import { SiemMigrationsEventTypes } from '../../../telemetry/event_based/types'; +import type { RuleMigrationIntegration, RuleSemanticSearchResult } from '../types'; +import type { MigrateRuleState } from './agent/types'; +import type { SiemMigrationTelemetryClient } from '../../common/task/siem_migrations_telemetry_client'; interface IntegrationMatchEvent { preFilterIntegrations: RuleMigrationIntegration[]; @@ -30,7 +32,9 @@ interface PrebuiltRuleMatchEvent { postFilterRule?: RuleSemanticSearchResult; } -export class SiemMigrationTelemetryClient { +export class RuleMigrationTelemetryClient + implements SiemMigrationTelemetryClient +{ constructor( private readonly telemetry: AnalyticsServiceSetup, private readonly logger: Logger, @@ -81,7 +85,7 @@ export class SiemMigrationTelemetryClient { const stats = { completed: 0, failed: 0 }; return { - startRuleTranslation: () => { + startItemTranslation: () => { const ruleStartTime = Date.now(); return { success: (migrationResult: MigrateRuleState) => { diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/types.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/types.ts index d3f82d989b11d..c000cbee577a9 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/types.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/types.ts @@ -8,19 +8,19 @@ import type { AuthenticatedUser } from '@kbn/core/server'; import type { LangSmithEvaluationOptions } from '../../../../../common/siem_migrations/model/common.gen'; import type { RuleMigrationsDataClient } from '../data/rule_migrations_data_client'; -import type { StoredRuleMigration } from '../types'; +import type { StoredRuleMigrationRule } from '../types'; import type { getRuleMigrationAgent } from './agent'; -import type { SiemMigrationTelemetryClient } from './rule_migrations_telemetry_client'; -import type { ChatModel } from './util/actions_client_chat'; -import type { RuleMigrationResources } from './retrievers/rule_resource_retriever'; +import type { RuleMigrationTelemetryClient } from './rule_migrations_telemetry_client'; +import type { ChatModel } from '../../common/task/util/actions_client_chat'; +import type { MigrationResources } from '../../common/task/retrievers/resource_retriever'; import type { RuleMigrationsRetriever } from './retrievers'; -import type { MigrateRuleGraphConfig } from './agent/types'; +import type { MigrateRuleConfig } from './agent/types'; import type { SiemMigrationsClientDependencies } from '../../common/types'; export type MigrationAgent = ReturnType; -export interface RuleMigrationInput extends Pick { - resources: RuleMigrationResources; +export interface RuleMigrationInput extends Pick { + resources: MigrationResources; } export interface RuleMigrationTaskCreateClientParams { @@ -32,7 +32,7 @@ export interface RuleMigrationTaskCreateClientParams { export interface RuleMigrationTaskStartParams { migrationId: string; connectorId: string; - invocationConfig: MigrateRuleGraphConfig; + invocationConfig: MigrateRuleConfig; } export interface RuleMigrationTaskRunParams extends RuleMigrationTaskStartParams { @@ -43,7 +43,7 @@ export interface RuleMigrationTaskRunParams extends RuleMigrationTaskStartParams export interface RuleMigrationTaskCreateAgentParams { connectorId: string; retriever: RuleMigrationsRetriever; - telemetryClient: SiemMigrationTelemetryClient; + telemetryClient: RuleMigrationTelemetryClient; model: ChatModel; } @@ -61,6 +61,6 @@ export interface RuleMigrationTaskEvaluateParams { evaluationId: string; connectorId: string; langsmithOptions: LangSmithEvaluationOptions; - invocationConfig: MigrateRuleGraphConfig; + invocationConfig: MigrateRuleConfig; abortController: AbortController; } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/util/nullify_missing_properties.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/util/nullify_missing_properties.test.ts deleted file mode 100644 index e73195fe9751f..0000000000000 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/util/nullify_missing_properties.test.ts +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { z } from '@kbn/zod'; -import type { ElasticRule } from '../../../../../../common/siem_migrations/model/rule_migration.gen'; -import { nullifyElasticRule, nullifyMissingPropertiesInObject } from './nullify_missing_properties'; - -describe('nullify missing values in object', () => { - describe('nullifyMissingPropertiesInObject', () => { - const someZodObject = z.object({ - foo: z.string(), - bar: z.number().optional(), - baz: z.object({ - qux: z.boolean().optional(), - }), - }); - - const val: z.infer = { - foo: 'test', - baz: { - qux: true, - }, - }; - it('should correctly nullify missing values in zod object at first level', () => { - const result = nullifyMissingPropertiesInObject(someZodObject, val); - expect(result).toMatchObject({ - foo: 'test', - bar: null, - baz: { - qux: true, - }, - }); - }); - - it('should throw if object does not conform to the schema', () => { - const invalidVal = { - foo: 'test', - // Missing 'baz' property - }; - - expect(() => - nullifyMissingPropertiesInObject(someZodObject, invalidVal as z.infer) - ).toThrow(); - }); - }); - - describe('nullifyElasticRule', () => { - it('should return an object with nullified empty values', () => { - const elasticRule: ElasticRule = { - title: 'Some Title', - }; - - const result = nullifyElasticRule(elasticRule); - - expect(result).toMatchObject({ - title: 'Some Title', - description: null, - severity: null, - risk_score: null, - query: null, - query_language: null, - prebuilt_rule_id: null, - integration_ids: null, - id: null, - }); - }); - - it('should return original object and call error callback in case of error', () => { - const elasticRule = { - hero: 'Some Title', - } as unknown as ElasticRule; - - const errorMock = jest.fn(); - - const result = nullifyElasticRule(elasticRule, errorMock); - - expect(result).toMatchObject(elasticRule); - expect(errorMock).toHaveBeenCalled(); - }); - }); -}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/util/nullify_missing_properties.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/util/nullify_missing_properties.ts deleted file mode 100644 index 0942fc67983ad..0000000000000 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/util/nullify_missing_properties.ts +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { z } from '@kbn/zod'; -import type { ElasticRule as ElasticRuleType } from '../../../../../../common/siem_migrations/model/rule_migration.gen'; -import { ElasticRule } from '../../../../../../common/siem_migrations/model/rule_migration.gen'; - -type Nullable = { [K in keyof T]: T[K] | null }; - -/** - * This function takes a Zod schema and an object, and returns a new object - * where any missing values of `only first-level keys` in the object are set to null, according to the schema. - * - * Raises an error if the object does not conform to the schema. - * - * This is specially beneficial for `unsetting` fields in Elasticsearch documents. - */ -export const nullifyMissingPropertiesInObject = ( - zodType: T, - obj: z.infer -): Nullable> => { - const schemaWithNullValues = zodType.transform((value: z.infer) => { - const result: Nullable> = { ...value }; - Object.keys(zodType.shape).forEach((key) => { - if (!(key in value)) { - result[key as keyof z.infer] = null; - } - }); - return result; - }); - - return schemaWithNullValues.parse(obj); -}; - -/** - * This function takes an ElasticRule object and returns a new object - * where any missing values are set to null, according to the ElasticRule schema. - * - * If an error occurs during the transformation, it calls the onError callback - * with the error and returns the original object. - */ -export const nullifyElasticRule = (obj: ElasticRuleType, onError?: (error: Error) => void) => { - try { - return nullifyMissingPropertiesInObject(ElasticRule, obj); - } catch (error) { - onError?.(error); - return obj; - } -}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/types.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/types.ts index 738fa8f740baf..fd19b3a3d1101 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/types.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/types.ts @@ -6,10 +6,10 @@ */ import type { IndexAdapter, IndexPatternAdapter } from '@kbn/index-adapter'; +import type { MigrationTranslationResult } from '../../../../common/siem_migrations/model/migration.gen'; import type { RuleMigration, RuleMigrationRule, - RuleMigrationTranslationResult, UpdateRuleMigrationRule, RuleMigrationResource, } from '../../../../common/siem_migrations/model/rule_migration.gen'; @@ -17,9 +17,8 @@ import type { RuleVersions } from './data/rule_migrations_data_prebuilt_rules_cl import type { Stored } from '../types'; import type { SiemMigrationsIndexNameProvider } from '../common/types'; -export type StoredSiemMigration = Stored; - -export type StoredRuleMigration = Stored; +export type StoredRuleMigration = Stored; +export type StoredRuleMigrationRule = Stored; export type StoredRuleMigrationResource = Stored; export interface RuleMigrationIntegration { @@ -41,7 +40,7 @@ export interface RuleMigrationPrebuiltRule { export type RuleSemanticSearchResult = RuleMigrationPrebuiltRule & RuleVersions; export type InternalUpdateRuleMigrationRule = UpdateRuleMigrationRule & { - translation_result?: RuleMigrationTranslationResult; + translation_result?: MigrationTranslationResult; }; /** diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/siem_migrations_service.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/siem_migrations_service.ts index ff079b57bfa3d..87c02b27c80e2 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/siem_migrations_service.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/siem_migrations_service.ts @@ -31,7 +31,6 @@ export class SiemMigrationsService { kibanaVersion, config.siemRuleMigrations?.elserInferenceId ); - this.dashboardsService = new SiemDashboardMigrationsService( logger, kibanaVersion, @@ -42,10 +41,10 @@ export class SiemMigrationsService { setup(params: SiemMigrationsSetupParams) { if (!this.config.experimentalFeatures.siemMigrationsDisabled) { this.rulesService.setup({ ...params, pluginStop$: this.pluginStop$ }); - } - if (this.config.experimentalFeatures.automaticDashboardsMigration) { - this.dashboardsService.setup({ ...params, pluginStop$: this.pluginStop$ }); + if (this.config.experimentalFeatures.automaticDashboardsMigration) { + this.dashboardsService.setup({ ...params, pluginStop$: this.pluginStop$ }); + } } }