[8.19] [AI4SOC] Added AI4SOC promotion rules bootstrapping (#217517)

x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/prebuilt_rules/bootstrap_prebuilt_rules/bootstrap_prebuilt_rules.gen.ts

@@ -32,10 +32,61 @@ export const PackageInstallStatus = z.object({
   status: z.enum(['installed', 'already_installed']),
 });
 
+export type RuleBootstrapError = z.infer<typeof RuleBootstrapError>;
+export const RuleBootstrapError = z.object({
+  /**
+   * The list of rules that failed to bootstrap
+   */
+  rules: z.array(
+    z.object({
+      /**
+       * The ID of the rule that failed to bootstrap
+       */
+      rule_id: z.string(),
+    })
+  ),
+  /**
+   * The error message
+   */
+  message: z.string(),
+});
+
+export type RuleBootstrapResults = z.infer<typeof RuleBootstrapResults>;
+export const RuleBootstrapResults = z.object({
+  /**
+   * The total number of rules to be processed. This is a dynamic value and depends on the number of integrations installed that have bootstrappable rules
+   */
+  total: z.number().optional(),
+  /**
+   * The number of rules that were installed
+   */
+  installed: z.number(),
+  /**
+   * The number of rules that were updated
+   */
+  updated: z.number(),
+  /**
+   * The number of rules that were deleted
+   */
+  deleted: z.number(),
+  /**
+   * The number of rules that were skipped (already installed rules with no updates)
+   */
+  skipped: z.number().optional(),
+  /**
+   * The list of bootstrap errors
+   */
+  errors: z.array(RuleBootstrapError),
+});
+
 export type BootstrapPrebuiltRulesResponse = z.infer<typeof BootstrapPrebuiltRulesResponse>;
 export const BootstrapPrebuiltRulesResponse = z.object({
   /**
    * The list of packages that were installed or upgraded
    */
   packages: z.array(PackageInstallStatus),
+  /**
+   * The list of rules that were installed or upgraded
+   */
+  rules: RuleBootstrapResults.optional(),
 });

x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/prebuilt_rules/bootstrap_prebuilt_rules/bootstrap_prebuilt_rules.schema.yaml

@@ -25,6 +25,9 @@ paths:
                     description: The list of packages that were installed or upgraded
                     items:
                       $ref: '#/components/schemas/PackageInstallStatus'
+                  rules:
+                    description: The list of rules that were installed or upgraded
+                    $ref: '#/components/schemas/RuleBootstrapResults'
                 required:
                   - packages
 
@@ -49,3 +52,53 @@ components:
         - name
         - version
         - status
+
+    RuleBootstrapResults:
+      type: object
+      properties:
+        total:
+          type: number
+          description: The total number of rules to be processed. This is a dynamic value and depends on the number of integrations installed that have bootstrappable rules
+        installed:
+          type: number
+          description: The number of rules that were installed
+        updated:
+          type: number
+          description: The number of rules that were updated
+        deleted:
+          type: number
+          description: The number of rules that were deleted
+        skipped:
+          type: number
+          description: The number of rules that were skipped (already installed rules with no updates)
+        errors:
+          type: array
+          description: The list of bootstrap errors
+          items:
+            $ref: '#/components/schemas/RuleBootstrapError'
+      required:
+        - installed
+        - updated
+        - deleted
+        - errors
+
+    RuleBootstrapError:
+      type: object
+      properties:
+        rules:
+          type: array
+          description: The list of rules that failed to bootstrap
+          items:
+            type: object
+            properties:
+              rule_id:
+                type: string
+                description: The ID of the rule that failed to bootstrap
+            required:
+              - rule_id
+        message:
+          type: string
+          description: The error message
+      required:
+        - rules
+        - message

x-pack/solutions/security/plugins/security_solution/common/constants.ts

@@ -538,3 +538,8 @@ export const AI_FOR_SOC_INTEGRATIONS = [
   'sentinel_one',
   'crowdstrike',
 ];
+
+/*
+ * The tag to mark promotion rules that are related to the AI for SOC integrations
+ */
+export const PROMOTION_RULE_TAG = 'Promotion';

x-pack/solutions/security/plugins/security_solution/public/configurations/tabs/integrations.tsx

@@ -10,8 +10,7 @@ import { EuiSkeletonLoading } from '@elastic/eui';
 import type { AvailablePackagesHookType } from '@kbn/fleet-plugin/public';
 import { Routes, Route } from '@kbn/shared-ux-router';
 import { Redirect } from 'react-router-dom';
-import { CONFIGURATIONS_PATH } from '../../../common/constants';
-import { SEARCH_AI_LAKE_ALLOWED_INTEGRATIONS } from '../../common/lib/search_ai_lake/integrations';
+import { AI_FOR_SOC_INTEGRATIONS, CONFIGURATIONS_PATH } from '../../../common/constants';
 import { useEnhancedIntegrationCards } from '../../common/lib/search_ai_lake/hooks';
 import { ConfigurationTabs, IntegrationsFacets } from '../constants';
 import { IntegrationsPage, IntegrationsSkeleton } from './integrations/components';
@@ -28,7 +27,7 @@ export const ConfigurationsIntegrationsHome = React.memo<IntegrationsPageProps>(
     });
 
     const allowedIntegrations = filteredCards.filter((card) =>
-      SEARCH_AI_LAKE_ALLOWED_INTEGRATIONS.includes(card.name)
+      AI_FOR_SOC_INTEGRATIONS.includes(card.name)
     );
 
     const { available, installed } = useEnhancedIntegrationCards(allowedIntegrations);

x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management/api/hooks/prebuilt_rules/use_bootstrap_prebuilt_rules.ts

@@ -13,6 +13,7 @@ import { bootstrapPrebuiltRules } from '../../api';
 import { useInvalidateFetchPrebuiltRulesInstallReviewQuery } from './use_fetch_prebuilt_rules_install_review_query';
 import { useInvalidateFetchPrebuiltRulesStatusQuery } from './use_fetch_prebuilt_rules_status_query';
 import { useInvalidateFetchPrebuiltRulesUpgradeReviewQuery } from './use_fetch_prebuilt_rules_upgrade_review_query';
+import { useInvalidateFindRulesQuery } from '../use_find_rules_query';
 
 export const BOOTSTRAP_PREBUILT_RULES_KEY = ['POST', BOOTSTRAP_PREBUILT_RULES_URL];
 
@@ -22,16 +23,19 @@ export const useBootstrapPrebuiltRulesMutation = (
   const invalidatePrePackagedRulesStatus = useInvalidateFetchPrebuiltRulesStatusQuery();
   const invalidatePrebuiltRulesInstallReview = useInvalidateFetchPrebuiltRulesInstallReviewQuery();
   const invalidatePrebuiltRulesUpdateReview = useInvalidateFetchPrebuiltRulesUpgradeReviewQuery();
+  const invalidateFindRulesQuery = useInvalidateFindRulesQuery();
 
   return useMutation(() => bootstrapPrebuiltRules(), {
     ...options,
     mutationKey: BOOTSTRAP_PREBUILT_RULES_KEY,
     onSuccess: (...args) => {
       const response = args[0];
-      if (
-        response?.packages.find((pkg) => pkg.name === PREBUILT_RULES_PACKAGE_NAME)?.status ===
-        'installed'
-      ) {
+
+      // 'installed' means that the package was installed or updated to a newer version
+      const hasInstalledNewPackageVersion =
+        response.packages.find((pkg) => pkg.name === PREBUILT_RULES_PACKAGE_NAME)?.status ===
+        'installed';
+      if (hasInstalledNewPackageVersion) {
         // Invalidate other pre-packaged rules related queries. We need to do
         // that only in case the prebuilt rules package was installed indicating
         // that there might be new rules to install.
@@ -40,6 +44,12 @@ export const useBootstrapPrebuiltRulesMutation = (
         invalidatePrebuiltRulesUpdateReview();
       }
 
+      const hasRuleUpdates =
+        response.rules?.deleted || response.rules?.installed || response.rules?.updated;
+      if (hasRuleUpdates) {
+        invalidateFindRulesQuery();
+      }
+
       if (options?.onSuccess) {
         options.onSuccess(...args);
       }

x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management/logic/translations.ts

@@ -55,3 +55,10 @@ export const TIMELINE_PREPACKAGED_SUCCESS = i18n.translate(
     defaultMessage: 'Installed pre-packaged timeline templates from elastic',
   }
 );
+
+export const BOOTSTRAP_PREBUILT_RULES_FAILURE = i18n.translate(
+  'xpack.securitySolution.containers.detectionEngine.bootstrapPrebuiltRulesFailure',
+  {
+    defaultMessage: 'Failed to bootstrap prebuilt rules',
+  }
+);

x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management/logic/use_upgrade_security_packages.ts

@@ -11,12 +11,27 @@ import {
   BOOTSTRAP_PREBUILT_RULES_KEY,
   useBootstrapPrebuiltRulesMutation,
 } from '../api/hooks/prebuilt_rules/use_bootstrap_prebuilt_rules';
+import { useAppToasts } from '../../../common/hooks/use_app_toasts';
+import * as i18n from './translations';
 
 /**
  * Install or upgrade the security packages (endpoint and prebuilt rules)
  */
 export const useUpgradeSecurityPackages = () => {
-  const { mutate: bootstrapPrebuiltRules } = useBootstrapPrebuiltRulesMutation();
+  const { addError } = useAppToasts();
+
+  const { mutate: bootstrapPrebuiltRules } = useBootstrapPrebuiltRulesMutation({
+    onError: (error) => {
+      addError(error, { title: i18n.BOOTSTRAP_PREBUILT_RULES_FAILURE });
+    },
+    onSuccess: ({ rules }) => {
+      if (rules?.errors.length) {
+        addError(new Error(rules.errors.map((error) => error.message).join('; ')), {
+          title: i18n.BOOTSTRAP_PREBUILT_RULES_FAILURE,
+        });
+      }
+    },
+  });
 
   useEffect(() => {
     bootstrapPrebuiltRules();

x-pack/solutions/security/plugins/security_solution/public/detections/hooks/alert_summary/use_fetch_integrations.ts

@@ -8,15 +8,7 @@
 import { useMemo } from 'react';
 import type { PackageListItem } from '@kbn/fleet-plugin/common';
 import { installationStatuses, useGetPackagesQuery } from '@kbn/fleet-plugin/public';
-
-// We hardcode these here for now as we currently do not have any other way to filter out all the unwanted integrations.
-const AI_FOR_SOC_INTEGRATIONS = [
-  'splunk', // doesnt yet exist
-  'google_secops',
-  'microsoft_sentinel',
-  'sentinel_one',
-  'crowdstrike',
-];
+import { AI_FOR_SOC_INTEGRATIONS } from '../../../../common/constants';
 
 export interface UseFetchIntegrationsResult {
   /**

x-pack/solutions/security/plugins/security_solution/server/fleet_integration/fleet_integration.ts

@@ -47,7 +47,7 @@ import type { NewPolicyData, PolicyConfig } from '../../common/endpoint/types';
 import type { LicenseService } from '../../common/license';
 import type { ManifestManager } from '../endpoint/services';
 import type { IRequestContextFactory } from '../request_context_factory';
-import { installEndpointSecurityPrebuiltRule } from '../lib/detection_engine/prebuilt_rules/logic/rules_package/install_endpoint_security_prebuilt_rule';
+import { installEndpointSecurityPrebuiltRule } from '../lib/detection_engine/prebuilt_rules/logic/integrations/install_endpoint_security_prebuilt_rule';
 import { createPolicyArtifactManifest } from './handlers/create_policy_artifact_manifest';
 import { createDefaultPolicy } from './handlers/create_default_policy';
 import { validatePolicyAgainstLicense } from './handlers/validate_policy_against_license';

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/bootstrap_prebuilt_rules/bootstrap_prebuilt_rules.test.ts

@@ -10,6 +10,7 @@ import { bootstrapPrebuiltRulesRoute } from './bootstrap_prebuilt_rules';
 import type { Installation, RegistryPackage } from '@kbn/fleet-plugin/common';
 import { requestContextMock, serverMock } from '../../../routes/__mocks__';
 import { getBootstrapRulesRequest } from '../../../routes/__mocks__/request_responses';
+import { createProductFeaturesServiceMock } from '../../../../product_features_service/mocks';
 
 const packageMock: RegistryPackage = {
   name: 'detection_engine',
@@ -43,6 +44,7 @@ describe('bootstrap_prebuilt_rules_route', () => {
     jest.clearAllMocks();
     server = serverMock.create();
     ({ clients, context } = requestContextMock.createTools());
+    clients.productFeaturesService = createProductFeaturesServiceMock([]);
 
     bootstrapPrebuiltRulesRoute(server.router);
   });

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/bootstrap_prebuilt_rules/bootstrap_prebuilt_rules_handler.ts

@@ -6,15 +6,21 @@
  */
 
 import type { IKibanaResponse, KibanaRequest, KibanaResponseFactory } from '@kbn/core/server';
+import { ProductFeatureSecurityKey } from '@kbn/security-solution-features/keys';
 import { transformError } from '@kbn/securitysolution-es-utils';
-import type { BootstrapPrebuiltRulesResponse } from '../../../../../../common/api/detection_engine/prebuilt_rules/bootstrap_prebuilt_rules/bootstrap_prebuilt_rules.gen';
 import { installSecurityAiPromptsPackage } from '../../logic/integrations/install_ai_prompts';
+import type {
+  BootstrapPrebuiltRulesResponse,
+  PackageInstallStatus,
+  RuleBootstrapResults,
+} from '../../../../../../common/api/detection_engine/prebuilt_rules/bootstrap_prebuilt_rules/bootstrap_prebuilt_rules.gen';
 import type { SecuritySolutionRequestHandlerContext } from '../../../../../types';
 import { buildSiemResponse } from '../../../routes/utils';
-import {
-  installEndpointPackage,
-  installPrebuiltRulesPackage,
-} from '../install_prebuilt_rules_and_timelines/install_prebuilt_rules_package';
+import { installEndpointPackage } from '../../logic/integrations/install_endpoint_package';
+import { installPrebuiltRulesPackage } from '../../logic/integrations/install_prebuilt_rules_package';
+import { installPromotionRules } from '../../logic/integrations/install_promotion_rules';
+import { createPrebuiltRuleAssetsClient } from '../../logic/rule_assets/prebuilt_rule_assets_client';
+import { createPrebuiltRuleObjectsClient } from '../../logic/rule_objects/prebuilt_rule_objects_client';
 
 export const bootstrapPrebuiltRulesHandler = async (
   context: SecuritySolutionRequestHandlerContext,
@@ -24,27 +30,53 @@ export const bootstrapPrebuiltRulesHandler = async (
   const siemResponse = buildSiemResponse(response);
 
   try {
-    const ctx = await context.resolve(['securitySolution']);
+    const ctx = await context.resolve(['securitySolution', 'alerting', 'core']);
     const securityContext = ctx.securitySolution;
     const config = securityContext.getConfig();
     const securityAIPromptsEnabled = config.experimentalFeatures.securityAIPromptsEnabled;
 
+    const savedObjectsClient = ctx.core.savedObjects.client;
+    const detectionRulesClient = securityContext.getDetectionRulesClient();
+    const ruleAssetsClient = createPrebuiltRuleAssetsClient(savedObjectsClient);
+    const rulesClient = await ctx.alerting.getRulesClient();
+    const ruleObjectsClient = createPrebuiltRuleObjectsClient(rulesClient);
+
+    const productFeatureService = securityContext.getProductFeatureService();
+    const isExternalDetectionsEnabled = productFeatureService.isEnabled(
+      ProductFeatureSecurityKey.externalDetections
+    );
+
+    const packageResults: PackageInstallStatus[] = [];
+
+    // Install packages sequentially to avoid high memory usage
     const prebuiltRulesResult = await installPrebuiltRulesPackage(config, securityContext);
-    const endpointResult = await installEndpointPackage(config, securityContext);
+    packageResults.push({
+      name: prebuiltRulesResult.package.name,
+      version: prebuiltRulesResult.package.version,
+      status: prebuiltRulesResult.status,
+    });
+
+    let ruleResults: RuleBootstrapResults | undefined;
+    if (isExternalDetectionsEnabled) {
+      ruleResults = await installPromotionRules({
+        rulesClient,
+        detectionRulesClient,
+        ruleAssetsClient,
+        ruleObjectsClient,
+        fleetServices: securityContext.getInternalFleetServices(),
+      });
+    } else {
+      const endpointResult = await installEndpointPackage(securityContext);
+      packageResults.push({
+        name: endpointResult.package.name,
+        version: endpointResult.package.version,
+        status: endpointResult.status,
+      });
+    }
 
     const responseBody: BootstrapPrebuiltRulesResponse = {
-      packages: [
-        {
-          name: prebuiltRulesResult.package.name,
-          version: prebuiltRulesResult.package.version,
-          status: prebuiltRulesResult.status,
-        },
-        {
-          name: endpointResult.package.name,
-          version: endpointResult.package.version,
-          status: endpointResult.status,
-        },
-      ],
+      packages: packageResults,
+      rules: ruleResults,
     };
 
     const securityAiPromptsResult = securityAIPromptsEnabled