[8.18] [Security Solution] Add debug logging to endpoints that call Fleet APIs (#230503)

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/fleet_integrations/api/get_all_integrations/route.ts

@@ -71,6 +71,7 @@ export const getAllIntegrationsRoute = (router: SecuritySolutionPluginRouter, lo
 
           return response.ok({ body });
         } catch (err) {
+          logger.error(`getAllIntegrationsRoute: Caught error:`, err);
           const error = transformError(err);
           return siemResponse.error({
             body: error.message,

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/fleet_integrations/api/get_installed_integrations/route.ts

@@ -66,6 +66,7 @@ export const getInstalledIntegrationsRoute = (
 
           return response.ok({ body });
         } catch (err) {
+          logger.error(`getInstalledIntegrationsRoute: Caught error:`, err);
           const error = transformError(err);
           return siemResponse.error({
             body: error.message,

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/bootstrap_prebuilt_rules/bootstrap_prebuilt_rules_handler.ts

@@ -52,6 +52,7 @@ export const bootstrapPrebuiltRulesHandler = async (
       body: responseBody,
     });
   } catch (err) {
+    logger.error(`bootstrapPrebuiltRulesHandler: Caught error:`, err);
     const error = transformError(err);
     return siemResponse.error({
       body: error.message,

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/get_prebuilt_rules_and_timelines_status/get_prebuilt_rules_and_timelines_status_route.test.ts

@@ -81,7 +81,7 @@ describe('get_prepackaged_rule_status_route', () => {
       prepackagedTimelines: [],
     });
 
-    getPrebuiltRulesAndTimelinesStatusRoute(server.router);
+    getPrebuiltRulesAndTimelinesStatusRoute(server.router, clients.logger);
   });
 
   describe('status codes', () => {

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/get_prebuilt_rules_and_timelines_status/get_prebuilt_rules_and_timelines_status_route.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import type { Logger } from '@kbn/core/server';
 import { transformError } from '@kbn/securitysolution-es-utils';
 import { InstallPrepackedTimelinesRequestBody } from '../../../../../../common/api/timeline';
 import { buildSiemResponse } from '../../../routes/utils';
@@ -25,7 +26,10 @@ import { rulesToMap } from '../../logic/utils';
 import { buildFrameworkRequest } from '../../../../timeline/utils/common';
 import { checkTimelinesStatus } from '../../../../timeline/utils/check_timelines_status';
 
-export const getPrebuiltRulesAndTimelinesStatusRoute = (router: SecuritySolutionPluginRouter) => {
+export const getPrebuiltRulesAndTimelinesStatusRoute = (
+  router: SecuritySolutionPluginRouter,
+  logger: Logger
+) => {
   router.versioned
     .get({
       access: 'public',
@@ -62,7 +66,7 @@ export const getPrebuiltRulesAndTimelinesStatusRoute = (router: SecuritySolution
           });
 
           const installedPrebuiltRules = rulesToMap(
-            await getExistingPrepackagedRules({ rulesClient })
+            await getExistingPrepackagedRules({ rulesClient, logger })
           );
 
           const rulesToInstall = getRulesToInstall(latestPrebuiltRules, installedPrebuiltRules);

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/install_prebuilt_rules_and_timelines/install_prebuilt_rules_and_timelines_route.ts

@@ -53,6 +53,7 @@ export const installPrebuiltRulesAndTimelinesRoute = (
           );
           return response.ok({ body: validated ?? {} });
         } catch (err) {
+          logger.error(`installPrebuiltRulesAndTimelinesRoute: Caught error:`, err);
           const error = transformError(err);
           return siemResponse.error({
             body: error.message,

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/install_prebuilt_rules_and_timelines/legacy_create_prepackaged_rules.ts

@@ -60,18 +60,25 @@ export const legacyCreatePrepackagedRules = async (
     logger
   );
 
-  const installedPrebuiltRules = rulesToMap(await getExistingPrepackagedRules({ rulesClient }));
+  const installedPrebuiltRules = rulesToMap(
+    await getExistingPrepackagedRules({ rulesClient, logger })
+  );
   const rulesToInstall = getRulesToInstall(latestPrebuiltRules, installedPrebuiltRules);
   const rulesToUpdate = getRulesToUpdate(latestPrebuiltRules, installedPrebuiltRules);
 
-  const result = await createPrebuiltRules(detectionRulesClient, rulesToInstall);
-  if (result.errors.length > 0) {
-    throw new AggregateError(result.errors, 'Error installing new prebuilt rules');
+  const ruleCreationResult = await createPrebuiltRules(
+    detectionRulesClient,
+    rulesToInstall,
+    logger
+  );
+
+  if (ruleCreationResult.errors.length > 0) {
+    throw new AggregateError(ruleCreationResult.errors, 'Error installing new prebuilt rules');
   }
 
   const { result: timelinesResult } = await performTimelinesInstallation(context);
 
-  await upgradePrebuiltRules(detectionRulesClient, rulesToUpdate);
+  await upgradePrebuiltRules(detectionRulesClient, rulesToUpdate, logger);
 
   return {
     rules_installed: rulesToInstall.length,

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/perform_rule_installation/perform_rule_installation_handler.ts

@@ -104,7 +104,11 @@ export const performRuleInstallationHandler = async (
       const rulesToInstall = ruleInstallQueue.splice(0, BATCH_SIZE);
       const ruleAssets = await ruleAssetsClient.fetchAssetsByVersion(rulesToInstall);
 
-      const { results, errors } = await createPrebuiltRules(detectionRulesClient, ruleAssets);
+      const { results, errors } = await createPrebuiltRules(
+        detectionRulesClient,
+        ruleAssets,
+        logger
+      );
       installedRules.push(...results);
       ruleErrors.push(...errors);
     }
@@ -137,6 +141,7 @@ export const performRuleInstallationHandler = async (
 
     return response.ok({ body });
   } catch (err) {
+    logger.error(`performRuleInstallationHandler: Caught error:`, err);
     const error = transformError(err);
     return siemResponse.error({
       body: error.message,

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/perform_rule_upgrade/perform_rule_upgrade_handler.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import type { KibanaRequest, KibanaResponseFactory } from '@kbn/core/server';
+import type { Logger, KibanaRequest, KibanaResponseFactory } from '@kbn/core/server';
 import { transformError } from '@kbn/securitysolution-es-utils';
 import type {
   PerformRuleUpgradeRequestBody,
@@ -45,8 +45,9 @@ import { getPossibleUpgrades } from '../../logic/utils';
 
 export const performRuleUpgradeHandler = async (
   context: SecuritySolutionRequestHandlerContext,
-  request: KibanaRequest<undefined, undefined, PerformRuleUpgradeRequestBody>,
-  response: KibanaResponseFactory
+  request: KibanaRequest<unknown, unknown, PerformRuleUpgradeRequestBody>,
+  response: KibanaResponseFactory,
+  logger: Logger
 ) => {
   const siemResponse = buildSiemResponse(response);
 
@@ -198,7 +199,8 @@ export const performRuleUpgradeHandler = async (
       } else {
         const { results: upgradeResults, errors: installationErrors } = await upgradePrebuiltRules(
           detectionRulesClient,
-          modifiedPrebuiltRuleAssets
+          modifiedPrebuiltRuleAssets,
+          logger
         );
         ruleErrors.push(...installationErrors);
         updatedRules.push(...upgradeResults.map(({ result }) => result));

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/perform_rule_upgrade/perform_rule_upgrade_route.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import type { Logger } from '@kbn/core/server';
 import { buildRouteValidationWithZod } from '@kbn/zod-helpers';
 import {
   PERFORM_RULE_UPGRADE_URL,
@@ -18,7 +19,7 @@ import {
 } from '../../constants';
 import { performRuleUpgradeHandler } from './perform_rule_upgrade_handler';
 
-export const performRuleUpgradeRoute = (router: SecuritySolutionPluginRouter) => {
+export const performRuleUpgradeRoute = (router: SecuritySolutionPluginRouter, logger: Logger) => {
   router.versioned
     .post({
       access: 'internal',
@@ -44,6 +45,6 @@ export const performRuleUpgradeRoute = (router: SecuritySolutionPluginRouter) =>
           },
         },
       },
-      performRuleUpgradeHandler
+      (context, request, response) => performRuleUpgradeHandler(context, request, response, logger)
     );
 };

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/register_routes.ts

@@ -21,13 +21,13 @@ export const registerPrebuiltRulesRoutes = (
   logger: Logger
 ) => {
   // Legacy endpoints that we're going to deprecate
-  getPrebuiltRulesAndTimelinesStatusRoute(router);
+  getPrebuiltRulesAndTimelinesStatusRoute(router, logger);
   installPrebuiltRulesAndTimelinesRoute(router, logger);
 
   // New endpoints for the rule upgrade and installation workflows
   getPrebuiltRulesStatusRoute(router);
   performRuleInstallationRoute(router, logger);
-  performRuleUpgradeRoute(router);
+  performRuleUpgradeRoute(router, logger);
   reviewRuleInstallationRoute(router);
   reviewRuleUpgradeRoute(router);
   bootstrapPrebuiltRulesRoute(router, logger);

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/ensure_latest_rules_package_installed.ts

@@ -15,13 +15,24 @@ export async function ensureLatestRulesPackageInstalled(
   securityContext: SecuritySolutionApiRequestHandlerContext,
   logger: Logger
 ) {
+  logger.debug(
+    'ensureLatestRulesPackageInstalled: Fetching latest versions of prebuilt rule assets'
+  );
   let latestPrebuiltRules = await ruleAssetsClient.fetchLatestAssets();
+  logger.debug(
+    `ensureLatestRulesPackageInstalled: Fetching latest versions of prebuilt rule assets - done. Fetched assets: ${latestPrebuiltRules.length}.`
+  );
   if (latestPrebuiltRules.length === 0) {
     // Seems no packages with prepackaged rules were installed, try to install the default rules package
     await installPrebuiltRulesPackage(securityContext, logger);
 
-    // Try to get the prepackaged rules again
+    logger.debug(
+      'ensureLatestRulesPackageInstalled: Re-fetching latest versions of prebuilt rule assets after package installation'
+    );
     latestPrebuiltRules = await ruleAssetsClient.fetchLatestAssets();
+    logger.debug(
+      `ensureLatestRulesPackageInstalled: Re-fetched latest versions of prebuilt rule assets after package installation - done. Fetched assets: ${latestPrebuiltRules.length}.`
+    );
   }
   return latestPrebuiltRules;
 }

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/integrations/install_ai_prompts.ts

@@ -29,8 +29,11 @@ export async function installSecurityAiPromptsPackage(
       pkgVersion,
       logger
     );
-  } catch (e) {
-    // fail silently
+  } catch (error) {
+    logger.error(
+      'installSecurityAiPromptsPackage: Security AI prompts package failed to install',
+      error
+    );
     return null;
   }
 }

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_objects/create_prebuilt_rules.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import type { Logger } from '@kbn/core/server';
 import { MAX_RULES_TO_UPDATE_IN_PARALLEL } from '../../../../../../common/constants';
 import { initPromisePool } from '../../../../../utils/promise_pool';
 import { withSecuritySpan } from '../../../../../utils/with_security_span';
@@ -13,9 +14,13 @@ import type { IDetectionRulesClient } from '../../../rule_management/logic/detec
 
 export const createPrebuiltRules = (
   detectionRulesClient: IDetectionRulesClient,
-  rules: PrebuiltRuleAsset[]
+  rules: PrebuiltRuleAsset[],
+  logger?: Logger
 ) => {
   return withSecuritySpan('createPrebuiltRules', async () => {
+    logger?.debug(
+      `createPrebuiltRules: Creating prebuilt rules - started. Rules to create: ${rules.length}`
+    );
     const result = await initPromisePool({
       concurrency: MAX_RULES_TO_UPDATE_IN_PARALLEL,
       items: rules,
@@ -25,6 +30,9 @@ export const createPrebuiltRules = (
         });
       },
     });
+    logger?.debug(
+      `createPrebuiltRules: Creating prebuilt rules - done. Rules created: ${result.results}. Rules failed to create: ${result.errors.length}.`
+    );
 
     return result;
   });

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_objects/upgrade_prebuilt_rules.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import type { Logger } from '@kbn/core/server';
 import { MAX_RULES_TO_UPDATE_IN_PARALLEL } from '../../../../../../common/constants';
 import { initPromisePool } from '../../../../../utils/promise_pool';
 import { withSecuritySpan } from '../../../../../utils/with_security_span';
@@ -17,19 +18,27 @@ import type { IDetectionRulesClient } from '../../../rule_management/logic/detec
  * avoid being a "noisy neighbor".
  * @param detectionRulesClient IDetectionRulesClient
  * @param rules The rules to apply the update for
+ * @param logger Logger to log debug messages
  */
 export const upgradePrebuiltRules = async (
   detectionRulesClient: IDetectionRulesClient,
-  rules: PrebuiltRuleAsset[]
+  rules: PrebuiltRuleAsset[],
+  logger: Logger
 ) =>
   withSecuritySpan('upgradePrebuiltRules', async () => {
+    logger.debug(
+      `upgradePrebuiltRules: Upgrading prebuilt rules - started. Rules to upgrade: ${rules.length}`
+    );
     const result = await initPromisePool({
       concurrency: MAX_RULES_TO_UPDATE_IN_PARALLEL,
       items: rules,
       executor: async (rule) => {
         return detectionRulesClient.upgradePrebuiltRule({ ruleAsset: rule });
       },
     });
+    logger.debug(
+      `upgradePrebuiltRules: Upgrading prebuilt rules - done. Upgraded: ${result.results.length}. Failed to upgrade: ${result.errors.length}.`
+    );
 
     return result;
   });

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rules_package/install_endpoint_security_prebuilt_rule.ts

@@ -81,7 +81,7 @@ export const installEndpointSecurityPrebuiltRule = async ({
       return;
     }
     const ruleAssetsToInstall = await ruleAssetsClient.fetchAssetsByVersion(latestRuleVersion);
-    await createPrebuiltRules(detectionRulesClient, ruleAssetsToInstall);
+    await createPrebuiltRules(detectionRulesClient, ruleAssetsToInstall, logger);
   } catch (err) {
     logger.error(
       `Unable to create Endpoint Security rule automatically (${err.statusCode}): ${err.message}`

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts

@@ -219,6 +219,7 @@ export const importRulesRoute = (
 
           return response.ok({ body: ImportRulesResponse.parse(importRulesResponse) });
         } catch (err) {
+          logger.error(`importRulesRoute: Caught error:`, err);
           const error = transformError(err);
           return siemResponse.error({
             body: error.message,

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/search/get_existing_prepackaged_rules.test.ts

@@ -18,17 +18,20 @@ import {
   getRules,
   getRulesCount,
 } from './get_existing_prepackaged_rules';
+import { requestContextMock } from '../../../routes/__mocks__';
 
 describe('get_existing_prepackaged_rules', () => {
   afterEach(() => {
     jest.resetAllMocks();
   });
 
   describe('getExistingPrepackagedRules', () => {
+    const { clients } = requestContextMock.createTools();
+
     test('should return a single item in a single page', async () => {
       const rulesClient = rulesClientMock.create();
       rulesClient.find.mockResolvedValue(getFindResultWithSingleHit());
-      const rules = await getExistingPrepackagedRules({ rulesClient });
+      const rules = await getExistingPrepackagedRules({ rulesClient, logger: clients.logger });
       expect(rules).toEqual([getRuleMock(getQueryRuleParams())]);
     });
 
@@ -57,7 +60,7 @@ describe('get_existing_prepackaged_rules', () => {
         })
       );
 
-      const rules = await getExistingPrepackagedRules({ rulesClient });
+      const rules = await getExistingPrepackagedRules({ rulesClient, logger: clients.logger });
       expect(rules).toEqual([result1, result2, result3]);
     });
   });

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/search/get_existing_prepackaged_rules.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import type { Logger } from '@kbn/core/server';
 import type { RulesClient } from '@kbn/alerting-plugin/server';
 
 import {
@@ -78,15 +79,23 @@ export const getExistingPrepackagedRules = async ({
   rulesClient,
   page,
   perPage,
+  logger,
 }: {
   rulesClient: RulesClient;
   page?: number;
   perPage?: number;
+  logger: Logger;
 }): Promise<RuleAlertType[]> => {
-  return getRules({
+  logger.debug('getExistingPrepackagedRules: Fetching installed prebuilt rules');
+  const existingPrepackagedRules = await getRules({
     rulesClient,
     page,
     perPage,
     filter: KQL_FILTER_IMMUTABLE_RULES,
   });
+  logger.debug(
+    `getExistingPrepackagedRules: Fetching installed prebuilt rules - done. Fetched: ${existingPrepackagedRules.length} rules.`
+  );
+
+  return existingPrepackagedRules;
 };