[Automatic Migrations] Dashboard migration agent initial Implementation

x-pack/solutions/security/plugins/security_solution/common/siem_migrations/dashboards/constants.ts

@@ -9,14 +9,26 @@ import { SIEM_MIGRATIONS_PATH } from '../constants';
 
 export const SIEM_DASHBOARD_MIGRATIONS_PATH = `${SIEM_MIGRATIONS_PATH}/dashboards` as const;
 
+export const SIEM_DASHBOARD_MIGRATION_EVALUATE_PATH =
+  `${SIEM_DASHBOARD_MIGRATIONS_PATH}/evaluate` as const;
+
+// Migration ID specific routes
+
 export const SIEM_DASHBOARD_MIGRATION_PATH =
   `${SIEM_DASHBOARD_MIGRATIONS_PATH}/{migration_id}` as const;
 
+export const SIEM_DASHBOARD_MIGRATION_DASHBOARDS_PATH =
+  `${SIEM_DASHBOARD_MIGRATION_PATH}/dashboards` as const;
+
+export const SIEM_DASHBOARD_MIGRATION_START_PATH =
+  `${SIEM_DASHBOARD_MIGRATION_PATH}/start` as const;
+export const SIEM_DASHBOARD_MIGRATION_STOP_PATH = `${SIEM_DASHBOARD_MIGRATION_PATH}/stop` as const;
+
 export const SIEM_DASHBOARD_MIGRATION_STATS_PATH =
   `${SIEM_DASHBOARD_MIGRATION_PATH}/stats` as const;
 
-export const SIEM_DASHBOARD_MIGRATION_DASHBOARDS_PATH =
-  `${SIEM_DASHBOARD_MIGRATION_PATH}/dashboards` as const;
+export const SIEM_DASHBOARD_MIGRATION_TRANSLATION_STATS_PATH =
+  `${SIEM_DASHBOARD_MIGRATION_PATH}/translation_stats` as const;
 
 export const SIEM_DASHBOARD_MIGRATION_RESOURCES_MISSING_PATH =
   `${SIEM_DASHBOARD_MIGRATION_PATH}/resources/missing` as const;

x-pack/solutions/security/plugins/security_solution/common/siem_migrations/parsers/splunk/dashboard_xml.test.ts

@@ -6,7 +6,7 @@
  */
 
 import { SplunkXmlDashboardParser } from './dashboard_xml';
-import type { VizType } from './types';
+import type { VizType } from '../types';
 
 describe('SplunkXmlDashboardParser', () => {
   const createBasicXml = (

x-pack/solutions/security/plugins/security_solution/common/siem_migrations/parsers/splunk/dashboard_xml.ts

@@ -15,7 +15,7 @@
 
 import { v4 as uuidV4 } from 'uuid';
 import xml2js from 'xml2js';
-import type { ParsedPanel, PanelPosition, VizType } from './types';
+import type { ParsedPanel, PanelPosition, VizType } from '../types';
 
 interface XmlElement {
   $?: { [key: string]: string }; // XML attributes
@@ -53,7 +53,6 @@ interface SplunkXmlElement extends XmlElement {
       - [] Report reference
       - [] Base search
       - [] Post-process search
-
   - [] Add interactivity
     - fieldset (form)
     - input (form)

x-pack/solutions/security/plugins/security_solution/common/siem_migrations/resources/resource_identifier_base.ts

@@ -20,14 +20,18 @@ export type ResourceIdentifierConstructor<I extends ItemDocument = ItemDocument>
 ) => ResourceIdentifier<I>;
 
 export abstract class ResourceIdentifier<I> {
+  public abstract fromOriginal(item?: OriginalItem<I>): Promise<SiemMigrationResourceBase[]>;
+
   protected identifier: VendorResourceIdentifier;
 
   constructor(protected readonly vendor: SiemMigrationVendor) {
     // The constructor may need query_language as an argument for other vendors
     this.identifier = identifiers[this.vendor];
   }
 
-  public abstract fromOriginal(item?: OriginalItem<I>): Promise<SiemMigrationResourceBase[]>;
+  public fromQuery(query: string): SiemMigrationResourceBase[] {
+    return this.identifier(query);
+  }
 
   public async fromOriginals(
     originalItem: OriginalItem<I>[]

x-pack/solutions/security/plugins/security_solution/scripts/langgraph/draw_graphs_script.ts

@@ -5,17 +5,17 @@
  * 2.0.
  */
 
-import type {
-  ActionsClientChatOpenAI,
-  ActionsClientSimpleChatModel,
-} from '@kbn/langchain/server/language_models';
 import type { Logger } from '@kbn/logging';
 import { ToolingLog } from '@kbn/tooling-log';
 import { FakeLLM } from '@langchain/core/utils/testing';
 import fs from 'fs/promises';
 import path from 'path';
-import type { ElasticsearchClient, KibanaRequest } from '@kbn/core/server';
+import type { ElasticsearchClient, IScopedClusterClient, KibanaRequest } from '@kbn/core/server';
 import type { InferenceServerStart } from '@kbn/inference-plugin/server';
+import type { InferenceChatModel } from '@kbn/inference-langchain';
+import type { DashboardMigrationsRetriever } from '../../server/lib/siem_migrations/dashboards/task/retrievers';
+import { getDashboardMigrationAgent } from '../../server/lib/siem_migrations/dashboards/task/agent';
+import type { DashboardMigrationTelemetryClient } from '../../server/lib/siem_migrations/dashboards/task/dashboard_migrations_telemetry_client';
 import type { ChatModel } from '../../server/lib/siem_migrations/common/task/util/actions_client_chat';
 import { getGenerateEsqlGraph as getGenerateEsqlAgent } from '../../server/assistant/tools/esql/graphs/generate_esql/generate_esql';
 import { getRuleMigrationAgent } from '../../server/lib/siem_migrations/rules/task/agent';
@@ -30,10 +30,11 @@ interface Drawable {
 
 const mockLlm = new FakeLLM({
   response: JSON.stringify({}, null, 2),
-}) as unknown as ActionsClientChatOpenAI | ActionsClientSimpleChatModel;
+}) as unknown as InferenceChatModel;
 
 const esqlKnowledgeBase = {} as EsqlKnowledgeBase;
 const ruleMigrationsRetriever = {} as RuleMigrationsRetriever;
+const dashboardMigrationsRetriever = {} as DashboardMigrationsRetriever;
 
 const createLlmInstance = () => {
   return mockLlm;
@@ -52,6 +53,21 @@ async function getSiemRuleMigrationGraph(logger: Logger): Promise<Drawable> {
   return graph.getGraphAsync({ xray: true });
 }
 
+async function getSiemDashboardMigrationGraph(logger: Logger): Promise<Drawable> {
+  const model = { bindTools: () => null } as unknown as ChatModel;
+  const telemetryClient = {} as DashboardMigrationTelemetryClient;
+  const esScopedClient = {} as IScopedClusterClient;
+  const graph = getDashboardMigrationAgent({
+    model,
+    esScopedClient,
+    esqlKnowledgeBase,
+    dashboardMigrationsRetriever,
+    logger,
+    telemetryClient,
+  });
+  return graph.getGraphAsync({ xray: true });
+}
+
 async function getGenerateEsqlGraph(logger: Logger): Promise<Drawable> {
   const graph = await getGenerateEsqlAgent({
     esClient: {} as unknown as ElasticsearchClient,
@@ -94,4 +110,8 @@ export const draw = async () => {
     getGraphAsync: getSiemRuleMigrationGraph,
     outputFilename: '../../docs/siem_migration/img/rule_migration_agent_graph.png',
   });
+  await drawGraph({
+    getGraphAsync: getSiemDashboardMigrationGraph,
+    outputFilename: '../../docs/siem_migration/img/dashboard_migration_agent_graph.png',
+  });
 };

x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/constants.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/** The index pattern placeholder used when translated queries can not determine the correct index pattern */
+export const MISSING_INDEX_PATTERN_PLACEHOLDER = '[indexPattern]';

x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/data/siem_migrations_data_base_client.ts

@@ -23,17 +23,15 @@ import type { Stored } from '../../types';
 
 const DEFAULT_PIT_KEEP_ALIVE: Duration = '30s' as const;
 
-export class SiemMigrationsDataBaseClient<
-  D extends SiemMigrationsClientDependencies = SiemMigrationsClientDependencies
-> {
+export class SiemMigrationsDataBaseClient {
   protected esClient: ElasticsearchClient;
 
   constructor(
     protected getIndexName: SiemMigrationsIndexNameProvider,
     protected currentUser: AuthenticatedUser,
     protected esScopedClient: IScopedClusterClient,
     protected logger: Logger,
-    protected dependencies: D
+    protected dependencies: SiemMigrationsClientDependencies
   ) {
     this.esClient = esScopedClient.asInternalUser;
   }

x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/convert_esql_schema_cim_to_ecs/convert_esql_schema_cim_to_ecs.ts → x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/helpers/convert_esql_schema_cim_to_ecs/convert_esql_schema_cim_to_ecs.ts

@@ -11,7 +11,7 @@ import type { EsqlKnowledgeBase } from '../../../util/esql_knowledge_base';
 import { CIM_TO_ECS_MAP } from './cim_ecs_map';
 import { ESQL_CONVERT_CIM_TO_ECS_PROMPT } from './prompts';
 import { cleanMarkdown, generateAssistantComment } from '../../../util/comments';
-import type { NodeToolCreator } from '../types';
+import type { NodeHelperCreator } from '../types';
 
 export interface GetConvertEsqlSchemaCisToEcsParams {
   esqlKnowledgeBase: EsqlKnowledgeBase;
@@ -28,7 +28,7 @@ export interface ConvertEsqlSchemaCisToEcsOutput {
   comments: MigrationComments;
 }
 
-export const getConvertEsqlSchemaCisToEcs: NodeToolCreator<
+export const getConvertEsqlSchemaCisToEcs: NodeHelperCreator<
   GetConvertEsqlSchemaCisToEcsParams,
   ConvertEsqlSchemaCisToEcsInput,
   ConvertEsqlSchemaCisToEcsOutput

x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/convert_esql_schema_cim_to_ecs/prompts.ts → x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/helpers/convert_esql_schema_cim_to_ecs/prompts.ts

@@ -8,8 +8,8 @@
 import { ChatPromptTemplate } from '@langchain/core/prompts';
 
 export const ESQL_CONVERT_CIM_TO_ECS_PROMPT =
-  ChatPromptTemplate.fromTemplate(`You are a helpful cybersecurity (SIEM) expert agent. Your task is to migrate "detection rules" from Splunk SPL to Elasticsearch ESQL.
-Your task is to look at the new ESQL query already generated from its initial Splunk SPL query and translate the Splunk CIM field names to the Elastic Common Schema (ECS) fields.
+  ChatPromptTemplate.fromTemplate(`You are a helpful cybersecurity (SIEM) expert agent. Your task is to migrate SPL queries from Splunk to Elasticsearch ES|QL.
+Your task is to look at the new ES|QL query already generated from its initial Splunk SPL query and translate the Splunk CIM field names to the Elastic Common Schema (ECS) fields.
 Below is the relevant context used when deciding which Elastic Common Schema field to use when translating from Splunk CIM fields:
 
 <context>
@@ -30,6 +30,7 @@ Go through the current esql query above and translate the current field names th
 - Go through each part of the ESQL query, if a part is determined to be related to a splunk CIM field or datamodel use the cim to ecs map above to determine the equivalent ECS field.
 - If a field is not in the cim to ecs map, or no datamodel is used, try to use your existing knowledge about Elastic Common Schema to determine if and which ECS field to use.
 - Do not reuse the same ECS field name for multiple Splunk CIM fields, always try to find the most appropriate ECS field.
+- When mapping to ECS fields, try to use the more specific ECS fields, considering the usage in the query. For example: \`host.name == "some_host"\` is better than \`host == "some_host"\`.
 - If you are uncertain about a field mapping, leave it as is and mention it in the summary.
 
 <guidelines>
@@ -39,8 +40,9 @@ Go through the current esql query above and translate the current field names th
 - Only use and modify the current ESQL query, do not create a new one or modify any other part of the rule.
 </guidelines>
 
-<expected_output>
+<output_format>
 - First, the updated ES|QL query inside an \`\`\`esql code block.
-- At the end, the summary of the the field mapping process followed in markdown, starting with "## Field Mapping Summary". This would include the reason, original field names, the target ECS field name and any fields that were left as is.
-</expected_output>
+- After, the summary of the the field mapping process followed in markdown, starting with "## Field Mapping Summary". This would include the reason, original field names, the target ECS field name and any fields that were left as is.
+- Don't add any other information or explanation before or after these two outputs.
+</output_format>
 `);

x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/fix_esql_query_errors/fix_esql_query_errors.ts → x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/helpers/fix_esql_query_errors/fix_esql_query_errors.ts

@@ -6,7 +6,7 @@
  */
 
 import type { Logger } from '@kbn/core/server';
-import type { NodeToolCreator } from '../types';
+import type { NodeHelperCreator } from '../types';
 import type { EsqlKnowledgeBase } from '../../../util/esql_knowledge_base';
 import { RESOLVE_ESQL_ERRORS_TEMPLATE } from './prompts';
 
@@ -22,7 +22,7 @@ export interface FixEsqlQueryErrorsOutput {
   query?: string;
 }
 
-export const getFixEsqlQueryErrors: NodeToolCreator<
+export const getFixEsqlQueryErrors: NodeHelperCreator<
   GetFixEsqlQueryErrorsParams,
   FixEsqlQueryErrorsInput,
   FixEsqlQueryErrorsOutput

x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/helpers/fix_esql_query_errors/prompts.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ChatPromptTemplate } from '@langchain/core/prompts';
+
+export const RESOLVE_ESQL_ERRORS_TEMPLATE =
+  ChatPromptTemplate.fromTemplate(`You are a helpful ES|QL (Elasticsearch Query Language) expert agent. 
+Your task is to fix the errors in the ES|QL query provided.
+
+<guidelines>
+- You will be provided with a ES|QL query and its related errors.
+- Try to fix the errors in the ES|QL query as best as you can to make it work.
+- You must respond only with the modified query inside a \`\`\`esql code block, nothing else similar to the example response below.
+</guidelines>
+
+<context>
+
+<esql_errors>
+{esql_errors}
+</esql_errors>
+
+<esql_query>
+{esql_query}
+</esql_query>
+
+</context>
+`);

x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/inline_spl_query/inline_spl_query.ts → x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/helpers/inline_spl_query/inline_spl_query.ts

@@ -7,12 +7,11 @@
 
 import type { Logger } from '@kbn/core/server';
 import { StringOutputParser } from '@langchain/core/output_parsers';
-import { isEmpty } from 'lodash/fp';
 import type { MigrationComments } from '../../../../../../../../common/siem_migrations/model/common.gen';
 import type { ChatModel } from '../../../util/actions_client_chat';
 import { cleanMarkdown, generateAssistantComment } from '../../../util/comments';
 import type { MigrationResources } from '../../../retrievers/resource_retriever';
-import type { NodeToolCreator } from '../types';
+import type { NodeHelperCreator } from '../types';
 import { REPLACE_QUERY_RESOURCE_PROMPT, getResourcesContext } from './prompts';
 
 export interface GetInlineSplQueryParams {
@@ -29,7 +28,7 @@ export interface InlineSplQueryOutput {
   comments: MigrationComments;
 }
 
-export const getInlineSplQuery: NodeToolCreator<
+export const getInlineSplQuery: NodeHelperCreator<
   GetInlineSplQueryParams,
   InlineSplQueryInput,
   InlineSplQueryOutput
@@ -44,12 +43,6 @@ export const getInlineSplQuery: NodeToolCreator<
       };
     }
 
-    if (isEmpty(resources)) {
-      // No resources identified in the query, no need to replace anything
-      const summary = '## Inlining Summary\n\nNo macro or lookup found in the query.';
-      return { inlineQuery: query, comments: [generateAssistantComment(summary)] };
-    }
-
     const replaceQueryParser = new StringOutputParser();
     const replaceQueryResourcePrompt =
       REPLACE_QUERY_RESOURCE_PROMPT.pipe(model).pipe(replaceQueryParser);

x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/tools/inline_spl_query/prompts.ts → x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/common/task/agent/helpers/inline_spl_query/prompts.ts

@@ -14,7 +14,10 @@ interface ResourceContext {
 }
 
 export const getResourcesContext = (resources: MigrationResources): ResourceContext => {
-  const context: ResourceContext = { macros: '', lookups: '' };
+  const context: ResourceContext = {
+    macros: 'No macros provided',
+    lookups: 'No lookups provided',
+  };
 
   // Process macros
   if (resources.macro?.length) {
@@ -46,32 +49,40 @@ Here are some context for you to reference for your task, read it carefully as y
 You have to replace the macros syntax in the SPL query and use their value inline, if provided.
 
 Always follow the below guidelines when replacing macros:
-- Macros names have the number of arguments in parentheses, e.g., \`macroName(2)\`. You must replace the correct macro accounting for the number of arguments. 
-- Divide the query up into separate sections and go through each section one at a time to identify the macros used that need to be replaced, using one of two scenarios:
-  - The macro is provided in the list of available macros: Replace it using its actual content.
-  - The macro is not in the list of available macros: add a placeholder ("missing placeholder" from now on) in the query with the format [macro:<macro_name>(argumentCount)] including the [] keys,
-    Example: \`get_duration(firstDate,secondDate)\` -> [macro:get_duration(2)]
+- The macros are always identified by backticks (\`).
+- Macros names can be in any case: 
+  - camelCase eg. \`someMacroName\`
+  - snake_case eg. \`some_macro_name\`
+  - kebab-case eg. \`some-macro-name\`
+  - or any other as long as they are between backticks
+- Macros names have the number of arguments in parentheses, e.g., \`macroName(2)\`. You must replace the correct macro accounting for the number of arguments.
+- When you find a macro there are two scenarios:
+- The macro is provided in the list of available macros -> Replace it using its actual content.
+- The macro is not in the list of available macros -> Replace it by the "missing placeholder" in the query with the format [macro:<macro_name>(argumentCount)] including the [] keys,
+  Example: \`get_duration(firstDate,secondDate)\` -> [macro:get_duration(2)]
+- There must not be any macro call with backticks in the final query.
 
 Having the following macros:
-  \`someSource\`: sourcetype="somesource"
+  \`some_source\`: sourcetype="somesource"
   \`searchTitle(1)\`: search title="$value$"
   \`searchTitle\`: search title=*
   \`searchType\`: search type=*
 
 And the following SPL query:
   \`\`\`spl
-  \`someSource\` \`someFilter\`
+  \`some_source\` \`some_filter\`
   | \`searchTitle("sometitle")\`
+  | \`searchTitle\`
   | \`searchType("sometype")\`
   | \`anotherMacro("someParam","someOtherParam", 10)\`
   | table *
   \`\`\`
 
 The correct replacement would be:
   \`\`\`spl
-  sourcetype="somesource" \`someFilter\`
+  sourcetype="somesource" [macro:some_filter]
   | search title="sometitle"
-  | \`searchType("sometype")\`
+  | [macro:searchType(1)]
   | [macro:anotherMacro(3)]
   | table *
   \`\`\`
@@ -134,7 +145,7 @@ The correct replacement would be:
 \`\`\`
 </spl_query>
 
-<expected_output>
+<output_format>
 - First, the modified SPL query inside an \`\`\`spl code block.
 - At the end, a step by step explanation of the replacements made in markdown format:
   - Start with "## Inlining Summary" title
@@ -143,7 +154,8 @@ The correct replacement would be:
   - Finally, the modified SPL query
   - Inside SPL language code blocks, Please add a line break before each pipe (|) character in the query.
   - Make sure the Markdown is formatted correctly and the values properly escaped.
-</expected_output>
+- Don't add any other information or explanation before or after these two outputs.
+</output_format>
 `,
   ],
   ['ai', 'Please find the modified SPL query below:'],