[8.16] [8.x] [Telemetry][Security Solution] Index metadata collector (#194004) (#204311)

.buildkite/ftr_security_serverless_configs.yml

@@ -72,6 +72,7 @@ disabled:
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_read/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_read/basic_license_essentials_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/telemetry/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/user_roles/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/genai/nlp_cleanup_task/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/genai/nlp_cleanup_task/basic_license_essentials_tier/configs/serverless.config.ts

.buildkite/ftr_security_stateful_configs.yml

@@ -60,6 +60,7 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_read/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_read/basic_license_essentials_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/telemetry/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/user_roles/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/basic_license_essentials_tier/configs/ess.config.ts

x-pack/plugins/security_solution/server/integration_tests/configuration.test.ts

@@ -0,0 +1,152 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import Path from 'path';
+import axios from 'axios';
+
+import { cloneDeep } from 'lodash';
+
+import { telemetryConfiguration } from '../lib/telemetry/configuration';
+import {
+  TaskManagerPlugin,
+  type TaskManagerStartContract,
+} from '@kbn/task-manager-plugin/server/plugin';
+
+import {
+  setupTestServers,
+  removeFile,
+  mockAxiosPost,
+  DEFAULT_GET_ROUTES,
+  mockAxiosGet,
+  getRandomInt,
+} from './lib/helpers';
+
+import {
+  type TestElasticsearchUtils,
+  type TestKibanaUtils,
+} from '@kbn/core-test-helpers-kbn-server';
+import { Plugin as SecuritySolutionPlugin } from '../plugin';
+import { getTelemetryTasks, runSoonConfigTask } from './lib/telemetry_helpers';
+import type { SecurityTelemetryTask } from '../lib/telemetry/task';
+
+jest.mock('axios');
+
+const logFilePath = Path.join(__dirname, 'config.logs.log');
+const taskManagerStartSpy = jest.spyOn(TaskManagerPlugin.prototype, 'start');
+const securitySolutionStartSpy = jest.spyOn(SecuritySolutionPlugin.prototype, 'start');
+
+const mockedAxiosGet = jest.spyOn(axios, 'get');
+const mockedAxiosPost = jest.spyOn(axios, 'post');
+
+const securitySolutionPlugin = jest.spyOn(SecuritySolutionPlugin.prototype, 'start');
+
+describe('configuration', () => {
+  let esServer: TestElasticsearchUtils;
+  let kibanaServer: TestKibanaUtils;
+  let taskManagerPlugin: TaskManagerStartContract;
+  let tasks: SecurityTelemetryTask[];
+
+  beforeAll(async () => {
+    await removeFile(logFilePath);
+
+    const servers = await setupTestServers(logFilePath);
+
+    esServer = servers.esServer;
+    kibanaServer = servers.kibanaServer;
+
+    expect(taskManagerStartSpy).toHaveBeenCalledTimes(1);
+    taskManagerPlugin = taskManagerStartSpy.mock.results[0].value;
+
+    expect(securitySolutionStartSpy).toHaveBeenCalledTimes(1);
+
+    tasks = getTelemetryTasks(securitySolutionStartSpy);
+
+    expect(securitySolutionPlugin).toHaveBeenCalledTimes(1);
+  });
+
+  afterAll(async () => {
+    if (kibanaServer) {
+      await kibanaServer.stop();
+    }
+    if (esServer) {
+      await esServer.stop();
+    }
+  });
+
+  beforeEach(async () => {
+    jest.clearAllMocks();
+    mockAxiosPost(mockedAxiosPost);
+  });
+
+  afterEach(async () => {});
+
+  describe('configuration task', () => {
+    it('should keep default values when no new config was provided', async () => {
+      const before = cloneDeep(telemetryConfiguration);
+
+      await runSoonConfigTask(tasks, taskManagerPlugin);
+
+      expect(telemetryConfiguration).toEqual(before);
+    });
+
+    it('should update values with new manifest', async () => {
+      const expected = {
+        telemetry_max_buffer_size: getRandomInt(1, 100),
+        max_security_list_telemetry_batch: getRandomInt(1, 100),
+        max_endpoint_telemetry_batch: getRandomInt(1, 100),
+        max_detection_rule_telemetry_batch: getRandomInt(1, 100),
+        max_detection_alerts_batch: getRandomInt(1, 100),
+        use_async_sender: true,
+        pagination_config: {
+          max_page_size_bytes: getRandomInt(1, 100),
+          num_docs_to_sample: getRandomInt(1, 100),
+        },
+        sender_channels: {
+          default: {
+            buffer_time_span_millis: getRandomInt(1, 100),
+            inflight_events_threshold: getRandomInt(1, 100),
+            max_payload_size_bytes: getRandomInt(1, 100),
+          },
+        },
+        indices_metadata_config: {
+          indices_threshold: getRandomInt(1, 100),
+          datastreams_threshold: getRandomInt(1, 100),
+          max_prefixes: getRandomInt(1, 100),
+          max_group_size: getRandomInt(1, 100),
+        },
+      };
+
+      mockAxiosGet(mockedAxiosGet, [
+        ...DEFAULT_GET_ROUTES,
+        [/.*telemetry-buffer-and-batch-sizes-v1.*/, { status: 200, data: cloneDeep(expected) }],
+      ]);
+
+      await runSoonConfigTask(tasks, taskManagerPlugin);
+
+      expect(telemetryConfiguration.telemetry_max_buffer_size).toEqual(
+        expected.telemetry_max_buffer_size
+      );
+      expect(telemetryConfiguration.max_security_list_telemetry_batch).toEqual(
+        expected.max_security_list_telemetry_batch
+      );
+      expect(telemetryConfiguration.max_endpoint_telemetry_batch).toEqual(
+        expected.max_endpoint_telemetry_batch
+      );
+      expect(telemetryConfiguration.max_detection_rule_telemetry_batch).toEqual(
+        expected.max_detection_rule_telemetry_batch
+      );
+      expect(telemetryConfiguration.max_detection_alerts_batch).toEqual(
+        expected.max_detection_alerts_batch
+      );
+      expect(telemetryConfiguration.use_async_sender).toEqual(expected.use_async_sender);
+      expect(telemetryConfiguration.sender_channels).toEqual(expected.sender_channels);
+      expect(telemetryConfiguration.pagination_config).toEqual(expected.pagination_config);
+      expect(telemetryConfiguration.indices_metadata_config).toEqual(
+        expected.indices_metadata_config
+      );
+    });
+  });
+});

x-pack/plugins/security_solution/server/integration_tests/lib/helpers.ts

@@ -10,6 +10,20 @@ import Util from 'util';
 import type { ElasticsearchClient } from '@kbn/core/server';
 import deepmerge from 'deepmerge';
 import { createTestServers, createRootWithCorePlugins } from '@kbn/core-test-helpers-kbn-server';
+
+export const DEFAULT_GET_ROUTES: Array<[RegExp, unknown]> = [
+  [new RegExp('.*/ping$'), { status: 200 }],
+  [
+    /.*kibana\/manifest\/artifacts.*/,
+    {
+      status: 200,
+      data: 'x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/kibana-artifacts.zip',
+    },
+  ],
+];
+
+export const DEFAULT_POST_ROUTES: Array<[RegExp, unknown]> = [[/.*/, { status: 200 }]];
+
 const asyncUnlink = Util.promisify(Fs.unlink);
 
 /**
@@ -127,3 +141,35 @@ export function updateTimestamps(data: object[]): object[] {
     return { ...d, '@timestamp': new Date(currentTimeMillis + (i + 1) * 100) };
   });
 }
+
+export function mockAxiosPost(
+  postSpy: jest.SpyInstance,
+  routes: Array<[RegExp, unknown]> = DEFAULT_POST_ROUTES
+) {
+  postSpy.mockImplementation(async (url: string) => {
+    for (const [route, returnValue] of routes) {
+      if (route.test(url)) {
+        return returnValue;
+      }
+    }
+    return { status: 404 };
+  });
+}
+
+export function mockAxiosGet(
+  getSpy: jest.SpyInstance,
+  routes: Array<[RegExp, unknown]> = DEFAULT_GET_ROUTES
+) {
+  getSpy.mockImplementation(async (url: string) => {
+    for (const [route, returnValue] of routes) {
+      if (route.test(url)) {
+        return returnValue;
+      }
+    }
+    return { status: 404 };
+  });
+}
+
+export function getRandomInt(min: number, max: number): number {
+  return Math.floor(Math.random() * (max - min + 1)) + min;
+}

x-pack/plugins/security_solution/server/integration_tests/lib/telemetry_helpers.ts

@@ -24,12 +24,13 @@ import {
   deleteExceptionListItem,
 } from '@kbn/lists-plugin/server/services/exception_lists';
 import { LEGACY_AGENT_POLICY_SAVED_OBJECT_TYPE } from '@kbn/fleet-plugin/common/constants';
+import type { TaskManagerStartContract } from '@kbn/task-manager-plugin/server';
 
 import { packagePolicyService } from '@kbn/fleet-plugin/server/services';
 
 import { ENDPOINT_ARTIFACT_LISTS } from '@kbn/securitysolution-list-constants';
 import { DETECTION_TYPE, NAMESPACE_TYPE } from '@kbn/lists-plugin/common/constants.mock';
-import { bulkInsert, updateTimestamps } from './helpers';
+import { bulkInsert, eventually, updateTimestamps } from './helpers';
 import { TelemetryEventsSender } from '../../lib/telemetry/sender';
 import type {
   SecuritySolutionPluginStart,
@@ -397,3 +398,24 @@ export function getTelemetryTaskType(task: SecurityTelemetryTask): string {
     return '';
   }
 }
+
+export async function runSoonConfigTask(
+  tasks: SecurityTelemetryTask[],
+  taskManagerPlugin: TaskManagerStartContract
+) {
+  const configTaskType = 'security:telemetry-configuration';
+  const configTask = getTelemetryTask(tasks, configTaskType);
+  const runAfter = new Date();
+  await eventually(async () => {
+    await taskManagerPlugin.runSoon(configTask.getTaskId());
+  });
+
+  // wait until the task finishes
+  await eventually(async () => {
+    const hasRun = await taskManagerPlugin
+      .get(configTask.getTaskId())
+      .then((t) => new Date(t.state.lastExecutionTimestamp) > runAfter)
+      .catch(() => false);
+    expect(hasRun).toBe(true);
+  });
+}