From ddee4f290c53ac9be490376e8ae4df3ca99ecff6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Mon, 19 Feb 2024 13:53:48 +0100 Subject: [PATCH 01/19] Update receiver api --- .../server/lib/telemetry/receiver.ts | 22 +++++++------------ .../server/lib/telemetry/tasks/endpoint.ts | 3 ++- .../server/lib/telemetry/types.ts | 9 ++++++++ 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts index 6571822e57461..fa62e8d47a173 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -36,7 +36,7 @@ import type { SortResults, } from '@elastic/elasticsearch/lib/api/types'; import type { TransportResult } from '@elastic/elasticsearch'; -import type { Agent, AgentPolicy, Installation } from '@kbn/fleet-plugin/common'; +import type { AgentPolicy, Installation } from '@kbn/fleet-plugin/common'; import type { AgentClient, AgentPolicyServiceInterface, @@ -68,6 +68,8 @@ import type { ValueListItemsResponseAggregation, ValueListExceptionListResponseAggregation, ValueListIndicatorMatchResponseAggregation, + Nullable, + FleetAgentResponse, } from './types'; import { telemetryConfiguration } from './configuration'; import { ENDPOINT_METRICS_INDEX } from '../../../common/constants'; @@ -84,27 +86,19 @@ export interface ITelemetryReceiver { packageService?: PackageService ): Promise; - getClusterInfo(): ESClusterInfo | undefined; + getClusterInfo(): Nullable; fetchClusterInfo(): Promise; - fetchLicenseInfo(): Promise; + fetchLicenseInfo(): Promise>; openPointInTime(indexPattern: string): Promise; closePointInTime(pitId: string): Promise; - fetchDetectionRulesPackageVersion(): Promise; + fetchDetectionRulesPackageVersion(): Promise>; - fetchFleetAgents(): Promise< - | { - agents: Agent[]; - total: number; - page: number; - perPage: number; - } - | undefined - >; + fetchFleetAgents(): Promise>; fetchEndpointPolicyResponses( executeFrom: string, @@ -260,7 +254,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { return this.packageService?.asInternalUser.getInstallation(PREBUILT_RULES_PACKAGE_NAME); } - public async fetchFleetAgents() { + public async fetchFleetAgents(): Promise> { if (this.esClient === undefined || this.esClient === null) { throw Error('elasticsearch client is unavailable: cannot retrieve fleet agents'); } diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts index 4455ae1833e4b..951931d6a04fe 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts @@ -18,6 +18,7 @@ import type { ESClusterInfo, ESLicense, Nullable, + FleetAgentResponse, } from '../types'; import type { ITelemetryReceiver } from '../receiver'; import type { TaskExecutionPeriod } from '../task'; @@ -37,7 +38,7 @@ import { TELEMETRY_CHANNEL_ENDPOINT_META } from '../constants'; // Endpoint agent uses this Policy ID while it's installing. const DefaultEndpointPolicyIdToIgnore = '00000000-0000-0000-0000-000000000000'; -const EmptyFleetAgentResponse = { +const EmptyFleetAgentResponse: FleetAgentResponse = { agents: [], total: 0, page: 0, diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/types.ts b/x-pack/plugins/security_solution/server/lib/telemetry/types.ts index 46a8bc21faaef..b29777a121f7a 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/types.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/types.ts @@ -5,6 +5,8 @@ * 2.0. */ +import type { Agent } from '@kbn/fleet-plugin/common'; + import type { AlertEvent, ResolverNode, SafeResolverEvent } from '../../../common/endpoint/types'; import type { AllowlistFields } from './filterlists/types'; @@ -501,3 +503,10 @@ export interface TimelineResult { events: number; timeline: TimelineTelemetryTemplate | undefined; } + +export interface FleetAgentResponse { + agents: Agent[]; + total: number; + page: number; + perPage: number; +} From e91911e545714860378586ad5417310278501f9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Mon, 19 Feb 2024 15:23:29 +0100 Subject: [PATCH 02/19] Update receiver api --- .../server/lib/telemetry/helpers.ts | 4 + .../server/lib/telemetry/receiver.ts | 96 ++++++++----------- .../telemetry/tasks/prebuilt_rule_alerts.ts | 43 ++++----- 3 files changed, 63 insertions(+), 80 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts b/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts index de49dad3b1c26..df8d117e595f4 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts @@ -83,6 +83,10 @@ export const getPreviousDailyTaskTimestamp = ( return lastExecutionTimestamp; }; +export function safeValue(promise: PromiseSettledResult, defaultValue: unknown = {}): T { + return promise.status === 'fulfilled' ? promise.value : (defaultValue as T); +} + /** * Chunks an Array into an Array> * This is to prevent overloading the telemetry channel + user resources diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts index fa62e8d47a173..87893c4f81564 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -92,8 +92,6 @@ export interface ITelemetryReceiver { fetchLicenseInfo(): Promise>; - openPointInTime(indexPattern: string): Promise; - closePointInTime(pitId: string): Promise; fetchDetectionRulesPackageVersion(): Promise>; @@ -155,14 +153,9 @@ export interface ITelemetryReceiver { }>; fetchPrebuiltRuleAlertsBatch( - pitId: string, - searchAfterValue: SortResults | undefined - ): Promise<{ - moreToFetch: boolean; - newPitId: string; - searchAfter: SortResults | undefined; - alerts: TelemetryEvent[]; - }>; + executeFrom: string, + executeTo: string + ): AsyncGenerator; copyLicenseFields(lic: ESLicense): { issuer?: string | undefined; @@ -212,7 +205,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { private readonly maxRecords = 10_000 as const; constructor(logger: Logger) { - this.logger = logger.get('telemetry_events'); + this.logger = logger.get('telemetry_events.receiver'); } public async start( @@ -634,17 +627,17 @@ export class TelemetryReceiver implements ITelemetryReceiver { }; } - public async fetchPrebuiltRuleAlertsBatch( - pitId: string, - searchAfterValue: SortResults | undefined - ) { + public async *fetchPrebuiltRuleAlertsBatch(executeFrom: string, executeTo: string) { if (this.esClient === undefined || this.esClient === null) { throw Error('es client is unavailable: cannot retrieve pre-built rule alert batches'); } - let newPitId = pitId; + tlog(this.logger, `Searching prebuilt rule alerts from ${executeFrom} to ${executeTo}`); + + let pitId = await this.openPointInTime(DEFAULT_DIAGNOSTIC_INDEX); let fetchMore = true; - let searchAfter: SortResults | undefined = searchAfterValue; + let searchAfter: SortResults | undefined; + const query: ESSearchRequest = { query: { bool: { @@ -729,8 +722,8 @@ export class TelemetryReceiver implements ITelemetryReceiver { { range: { '@timestamp': { - gte: 'now-1h', - lte: 'now', + gte: executeFrom, + lte: executeTo, }, }, }, @@ -748,49 +741,44 @@ export class TelemetryReceiver implements ITelemetryReceiver { }; let response = null; - try { - response = await this.esClient.search(query); - const numOfHits = response?.hits.hits.length; + while (fetchMore) { + try { + response = await this.esClient.search(query); + const numOfHits = response?.hits.hits.length; - if (numOfHits > 0) { - const lastHit = response?.hits.hits[numOfHits - 1]; - searchAfter = lastHit?.sort; - } + if (numOfHits > 0) { + const lastHit = response?.hits.hits[numOfHits - 1]; + query.search_after = lastHit?.sort; + } else { + fetchMore = false; + } - fetchMore = numOfHits > 0 && numOfHits < 1_000; - } catch (e) { - tlog(this.logger, e); - fetchMore = false; - } + fetchMore = numOfHits > 0 && numOfHits < 1_000; + } catch (e) { + tlog(this.logger, e); + fetchMore = false; + } - if (response == null) { - return { - moreToFetch: false, - newPitId: pitId, - searchAfter, - alerts: [] as TelemetryEvent[], - }; - } + if (response == null) { + await this.closePointInTime(pitId); + return; + } - const alerts: TelemetryEvent[] = response.hits.hits.flatMap((h) => - h._source != null ? ([h._source] as TelemetryEvent[]) : [] - ); + const alerts: TelemetryEvent[] = response.hits.hits.flatMap((h) => + h._source != null ? ([h._source] as TelemetryEvent[]) : [] + ); - if (response?.pit_id != null) { - newPitId = response?.pit_id; - } + if (response?.pit_id != null) { + pitId = response?.pit_id; + } - tlog(this.logger, `Prebuilt rule alerts to return: ${alerts.length}`); + tlog(this.logger, `Prebuilt rule alerts to return: ${alerts.length}`); - return { - moreToFetch: fetchMore, - newPitId, - searchAfter, - alerts, - }; + yield alerts; + } } - public async openPointInTime(indexPattern: string) { + private async openPointInTime(indexPattern: string) { if (this.esClient === undefined || this.esClient === null) { throw Error('es client is unavailable: cannot retrieve pre-built rule alert batches'); } @@ -1004,7 +992,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { return this.esClient.search(query); } - public async fetchValueListMetaData(interval: number) { + public async fetchValueListMetaData(_interval: number) { if (this.esClient === undefined || this.esClient === null) { throw Error('elasticsearch client is unavailable: cannot retrieve diagnostic alerts'); } diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/prebuilt_rule_alerts.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/prebuilt_rule_alerts.ts index 9d0324b8144c2..ee6b2aa859eb7 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/prebuilt_rule_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/prebuilt_rule_alerts.ts @@ -6,14 +6,19 @@ */ import type { Logger } from '@kbn/core/server'; -import type { SortResults } from '@elastic/elasticsearch/lib/api/types'; import type { ITelemetryEventsSender } from '../sender'; import type { ITelemetryReceiver } from '../receiver'; import type { ITaskMetricsService } from '../task_metrics.types'; -import type { ESClusterInfo, ESLicense, TelemetryEvent } from '../types'; +import type { TelemetryEvent } from '../types'; import type { TaskExecutionPeriod } from '../task'; import { TELEMETRY_CHANNEL_DETECTION_ALERTS } from '../constants'; -import { batchTelemetryRecords, processK8sUsernames, newTelemetryLogger } from '../helpers'; +import { + batchTelemetryRecords, + processK8sUsernames, + newTelemetryLogger, + getPreviousDailyTaskTimestamp, + safeValue, +} from '../helpers'; import { copyAllowlistedFields, filterList } from '../filterlists'; export function createTelemetryPrebuiltRuleAlertsTaskConfig(maxTelemetryBatch: number) { @@ -26,6 +31,7 @@ export function createTelemetryPrebuiltRuleAlertsTaskConfig(maxTelemetryBatch: n interval: '1h', timeout: '15m', version: taskVersion, + getLastExecutionTime: getPreviousDailyTaskTimestamp, runTask: async ( taskId: string, logger: Logger, @@ -48,16 +54,10 @@ export function createTelemetryPrebuiltRuleAlertsTaskConfig(maxTelemetryBatch: n receiver.fetchDetectionRulesPackageVersion(), ]); - const clusterInfo = - clusterInfoPromise.status === 'fulfilled' - ? clusterInfoPromise.value - : ({} as ESClusterInfo); - const licenseInfo = - licenseInfoPromise.status === 'fulfilled' - ? licenseInfoPromise.value - : ({} as ESLicense | undefined); - const packageInfo = - packageVersion.status === 'fulfilled' ? packageVersion.value : undefined; + const clusterInfo = safeValue(clusterInfoPromise); + const licenseInfo = safeValue(licenseInfoPromise); + const packageInfo = safeValue(packageVersion, undefined); + const index = receiver.getAlertsIndex(); if (index === undefined) { @@ -66,23 +66,15 @@ export function createTelemetryPrebuiltRuleAlertsTaskConfig(maxTelemetryBatch: n return 0; } - let fetchMore = true; - let searchAfterValue: SortResults | undefined; - let pitId = await receiver.openPointInTime(index); - - while (fetchMore) { - const { moreToFetch, newPitId, searchAfter, alerts } = - await receiver.fetchPrebuiltRuleAlertsBatch(pitId, searchAfterValue); - + for await (const alerts of receiver.fetchPrebuiltRuleAlertsBatch( + taskExecutionPeriod.last ?? 'now-1h', + taskExecutionPeriod.current + )) { if (alerts.length === 0) { taskMetricsService.end(trace); return 0; } - fetchMore = moreToFetch; - searchAfterValue = searchAfter; - pitId = newPitId; - const processedAlerts = alerts.map( (event: TelemetryEvent): TelemetryEvent => copyAllowlistedFields(filterList.prebuiltRulesAlerts, event) @@ -115,7 +107,6 @@ export function createTelemetryPrebuiltRuleAlertsTaskConfig(maxTelemetryBatch: n } taskMetricsService.end(trace); - await receiver.closePointInTime(pitId); return 0; } catch (err) { logger.error('could not complete prebuilt alerts telemetry task'); From 24f29fbd5156a746330989a3319e4c4124f14593 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Mon, 19 Feb 2024 18:28:12 +0100 Subject: [PATCH 03/19] Paginate telemetry receiver ES query --- .../lib/telemetry_helpers.ts | 25 +++ .../server/integration_tests/receiver.test.ts | 170 +++++++++++++++ .../server/lib/telemetry/receiver.ts | 200 +++++++++++------- 3 files changed, 320 insertions(+), 75 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/integration_tests/receiver.test.ts diff --git a/x-pack/plugins/security_solution/server/integration_tests/lib/telemetry_helpers.ts b/x-pack/plugins/security_solution/server/integration_tests/lib/telemetry_helpers.ts index 3eb8d7aca8883..abf6d1a5ef9d0 100644 --- a/x-pack/plugins/security_solution/server/integration_tests/lib/telemetry_helpers.ts +++ b/x-pack/plugins/security_solution/server/integration_tests/lib/telemetry_helpers.ts @@ -33,6 +33,7 @@ import type { import type { SecurityTelemetryTask } from '../../lib/telemetry/task'; import { Plugin as SecuritySolutionPlugin } from '../../plugin'; import { AsyncTelemetryEventsSender } from '../../lib/telemetry/async_sender'; +import { type ITelemetryReceiver, TelemetryReceiver } from '../../lib/telemetry/receiver'; import { DEFAULT_DIAGNOSTIC_INDEX } from '../../lib/telemetry/constants'; import mockEndpointAlert from '../__mocks__/endpoint-alert.json'; import mockedRule from '../__mocks__/rule.json'; @@ -86,6 +87,30 @@ export function getAsyncTelemetryEventSender( } } +export function getTelemetryReceiver( + spy: jest.SpyInstance< + SecuritySolutionPluginStart, + [core: CoreStart, plugins: SecuritySolutionPluginStartDependencies] + > +): ITelemetryReceiver { + const pluginInstances = spy.mock.instances; + if (pluginInstances.length === 0) { + throw new Error('security_solution plugin not started'); + } + const plugin = pluginInstances[0]; + if (plugin instanceof SecuritySolutionPlugin) { + /* eslint dot-notation: "off" */ + const receiver = plugin['telemetryReceiver']; + if (receiver instanceof TelemetryReceiver) { + return receiver; + } else { + throw new Error('security_solution plugin not started'); + } + } else { + throw new Error('security_solution plugin not started'); + } +} + export function getTelemetryTask( tasks: SecurityTelemetryTask[], id: string diff --git a/x-pack/plugins/security_solution/server/integration_tests/receiver.test.ts b/x-pack/plugins/security_solution/server/integration_tests/receiver.test.ts new file mode 100644 index 0000000000000..ad366e1847654 --- /dev/null +++ b/x-pack/plugins/security_solution/server/integration_tests/receiver.test.ts @@ -0,0 +1,170 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import Path from 'path'; +import type { ElasticsearchClient } from '@kbn/core/server'; + +import { setupTestServers, removeFile } from './lib/helpers'; +import { getTelemetryReceiver } from './lib/telemetry_helpers'; + +import type { ITelemetryReceiver } from '../lib/telemetry/receiver'; + +import { + type TestElasticsearchUtils, + type TestKibanaUtils, +} from '@kbn/core-test-helpers-kbn-server'; +import { Plugin as SecuritySolutionPlugin } from '../plugin'; +import type { SearchRequest } from '@elastic/elasticsearch/lib/api/types'; +import type { Nullable } from '../lib/telemetry/types'; + +// not needed, but it avoids some error messages like "Error: Cross origin http://localhost forbidden" +jest.mock('axios'); + +const logFilePath = Path.join(__dirname, 'logs.log'); + +const securitySolutionPlugin = jest.spyOn(SecuritySolutionPlugin.prototype, 'start'); + +describe('ITelemetryReceiver', () => { + let esServer: TestElasticsearchUtils; + let kibanaServer: TestKibanaUtils; + let telemetryReceiver: ITelemetryReceiver; + let esClient: ElasticsearchClient; + const TEST_INDEX = 'test'; + + beforeAll(async () => { + await removeFile(logFilePath); + + const servers = await setupTestServers(logFilePath); + esServer = servers.esServer; + kibanaServer = servers.kibanaServer; + + expect(securitySolutionPlugin).toHaveBeenCalledTimes(1); + + telemetryReceiver = getTelemetryReceiver(securitySolutionPlugin); + + esClient = kibanaServer.coreStart.elasticsearch.client.asInternalUser; + }); + + afterAll(async () => { + if (kibanaServer) { + await kibanaServer.stop(); + } + if (esServer) { + await esServer.stop(); + } + }); + + beforeEach(async () => { + jest.clearAllMocks(); + await esClient.indices.create({ index: TEST_INDEX }).catch(() => {}); + }); + + afterEach(async () => { + await esClient.indices.delete({ index: TEST_INDEX }).catch(() => {}); + }); + + describe('paginate', () => { + const numOfDocs = 400; + const maxPageSizeBytes = 1_000; + + beforeEach(async () => { + telemetryReceiver.setMaxPageSizeBytes(maxPageSizeBytes); + }); + + afterEach(async () => { + await esClient.deleteByQuery({ index: TEST_INDEX }).catch(() => {}); + }); + + it('should paginate queries', async () => { + const docs = mockedDocs(numOfDocs); + await bulkInsert(docs); + + const results = telemetryReceiver.paginate(TEST_INDEX, testQuery()); + + const pages = await getPages(results); + + const pageSize = Math.floor(maxPageSizeBytes / JSON.stringify(docs[0]).length); + expect(pages.length).toEqual(Math.ceil(numOfDocs / pageSize)); + expect(pages.flat()).toEqual(docs); + }); + + it('should return only expected data', async () => { + const from = new Date().toISOString(); + const batchOne = mockedDocs(numOfDocs); + const to = new Date().toISOString(); + + // wait for 500 ms to ensure that the timestamp of the next batch has different @timestamps + await new Promise((resolve) => setTimeout(resolve, 500)); + const batchTwo = mockedDocs(numOfDocs, 'batchTwo'); + + await bulkInsert(batchOne); + await bulkInsert(batchTwo); + + const results = telemetryReceiver.paginate(TEST_INDEX, testQuery(from, to)); + + const pages = await getPages(results); + + const pageSize = Math.floor(maxPageSizeBytes / JSON.stringify(batchOne[0]).length); + expect(pages.length).toEqual(Math.ceil(numOfDocs / pageSize)); + expect(pages.flat()).toEqual(batchOne); + }); + + it('should manage empty response', async () => { + await bulkInsert(mockedDocs(numOfDocs)); + + const results = telemetryReceiver.paginate(TEST_INDEX, testQuery('now-2d', 'now-1d')); + + const pages = await getPages(results); + expect(pages.length).toEqual(0); + }); + }); + + function mockedDocs(length: number, valuePrefix: string = 'value'): object[] { + const docs = Array.from({ length }, (_, i) => ({ + '@timestamp': new Date().toISOString(), + data: { + key: `${valuePrefix}_${i}`, + }, + })); + return docs; + } + + function testQuery( + from: Nullable = undefined, + to: Nullable = undefined + ): SearchRequest { + return { + query: { + range: { + '@timestamp': { + lte: to ?? new Date().toISOString(), + gte: from ?? 'now-1d', + }, + }, + }, + sort: [ + { + '@timestamp': { + order: 'asc' as const, + }, + }, + ], + }; + } + + async function bulkInsert(data: unknown[]): Promise { + const bulk = data.flatMap((d) => [{ index: { _index: TEST_INDEX } }, d]); + await esClient.bulk({ body: bulk, refresh: 'wait_for' }).catch((_) => {}); + } + + async function getPages(it: AsyncGenerator): Promise { + const pages = []; + for await (const doc of it) { + pages.push(doc); + } + return pages; + } +}); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts index 87893c4f81564..1cb0c80b12452 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -5,6 +5,9 @@ * 2.0. */ +import os from 'os'; +import { cloneDeep } from 'lodash'; + import type { Logger, CoreStart, @@ -124,6 +127,21 @@ export interface ITelemetryReceiver { executeTo: string ): AsyncGenerator; + /** + * Using a PIT executes the given query and returns the results in pages. + * The page size is calculated using the mean of a sample of N documents + * executing the same query. The query must have a sort attribute. + * + * @param index The index to search + * @param query The query to use + * @returns An async generator of pages of results + * + * @see {@link https://www.elastic.co/guide/en/elasticsearch/reference/current/paginate-search-results.html} + * @see {ITelemetryReceiver#setMaxPageSizeBytes} + * @see {ITelemetryReceiver#setNumDocsToSample} + */ + paginate(index: string, query: ESSearchRequest): AsyncGenerator; + fetchPolicyConfigs(id: string): Promise; fetchTrustedApplications(): Promise<{ @@ -187,13 +205,16 @@ export interface ITelemetryReceiver { getAlertsIndex(): string | undefined; getExperimentalFeatures(): ExperimentalFeatures | undefined; + + setMaxPageSizeBytes(bytes: number): void; + setNumDocsToSample(n: number): void; } export class TelemetryReceiver implements ITelemetryReceiver { private readonly logger: Logger; private agentClient?: AgentClient; private agentPolicyService?: AgentPolicyServiceInterface; - private esClient?: ElasticsearchClient; + private _esClient?: ElasticsearchClient; private exceptionListClient?: ExceptionListClient; private soClient?: SavedObjectsClientContract; private getIndexForType?: (type: string) => string; @@ -204,6 +225,11 @@ export class TelemetryReceiver implements ITelemetryReceiver { private experimentalFeatures: ExperimentalFeatures | undefined; private readonly maxRecords = 10_000 as const; + // default to 2% of host's total memory or 80MiB, whichever is smaller + private maxPageSizeBytes: number = Math.min(os.totalmem() * 0.02, 80 * 1024 * 1024); + // number of docs to query to estimate the size of a single doc + private numDocsToSample: number = 10; + constructor(logger: Logger) { this.logger = logger.get('telemetry_events.receiver'); } @@ -220,7 +246,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { this.alertsIndex = alertsIndex; this.agentClient = endpointContextService?.getInternalFleetServices().agent; this.agentPolicyService = endpointContextService?.getInternalFleetServices().agentPolicy; - this.esClient = core?.elasticsearch.client.asInternalUser; + this._esClient = core?.elasticsearch.client.asInternalUser; this.exceptionListClient = exceptionListClient; this.packageService = packageService; this.soClient = @@ -261,12 +287,6 @@ export class TelemetryReceiver implements ITelemetryReceiver { } public async fetchEndpointPolicyResponses(executeFrom: string, executeTo: string) { - if (this.esClient === undefined || this.esClient === null) { - throw Error( - 'elasticsearch client is unavailable: cannot retrieve elastic endpoint policy responses' - ); - } - const query: SearchRequest = { expand_wildcards: ['open' as const, 'hidden' as const], index: `.ds-metrics-endpoint.policy*`, @@ -306,14 +326,10 @@ export class TelemetryReceiver implements ITelemetryReceiver { }, }; - return this.esClient.search(query, { meta: true }); + return this.esClient().search(query, { meta: true }); } public async fetchEndpointMetrics(executeFrom: string, executeTo: string) { - if (this.esClient === undefined || this.esClient === null) { - throw Error('elasticsearch client is unavailable: cannot retrieve elastic endpoint metrics'); - } - const query: SearchRequest = { expand_wildcards: ['open' as const, 'hidden' as const], index: ENDPOINT_METRICS_INDEX, @@ -358,14 +374,10 @@ export class TelemetryReceiver implements ITelemetryReceiver { }, }; - return this.esClient.search(query, { meta: true }); + return this.esClient().search(query, { meta: true }); } public async fetchEndpointMetadata(executeFrom: string, executeTo: string) { - if (this.esClient === undefined || this.esClient === null) { - throw Error('elasticsearch client is unavailable: cannot retrieve elastic endpoint metrics'); - } - const query: SearchRequest = { expand_wildcards: ['open' as const, 'hidden' as const], index: `.ds-metrics-endpoint.metadata-*`, @@ -405,13 +417,11 @@ export class TelemetryReceiver implements ITelemetryReceiver { }, }; - return this.esClient.search(query, { meta: true }); + return this.esClient().search(query, { meta: true }); } public async *fetchDiagnosticAlertsBatch(executeFrom: string, executeTo: string) { - if (this.esClient === undefined || this.esClient === null) { - throw Error('elasticsearch client is unavailable: cannot retrieve diagnostic alerts'); - } + tlog(this.logger, `Searching diagnostic alerts from ${executeFrom} to ${executeTo}`); let pitId = await this.openPointInTime(DEFAULT_DIAGNOSTIC_INDEX); let fetchMore = true; @@ -442,7 +452,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { let response = null; while (fetchMore) { try { - response = await this.esClient.search(query); + response = await this.esClient().search(query); const numOfHits = response?.hits.hits.length; if (numOfHits > 0) { @@ -547,10 +557,6 @@ export class TelemetryReceiver implements ITelemetryReceiver { * @returns The elastic rules */ public async fetchDetectionRules() { - if (this.esClient === undefined || this.esClient === null) { - throw Error('elasticsearch client is unavailable: cannot retrieve detection rules'); - } - const query: SearchRequest = { expand_wildcards: ['open' as const, 'hidden' as const], index: this.getIndexForType?.('alert'), @@ -593,7 +599,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { }, }, }; - return this.esClient.search(query, { meta: true }); + return this.esClient().search(query, { meta: true }); } public async fetchDetectionExceptionList(listId: string, ruleVersion: number) { @@ -628,10 +634,6 @@ export class TelemetryReceiver implements ITelemetryReceiver { } public async *fetchPrebuiltRuleAlertsBatch(executeFrom: string, executeTo: string) { - if (this.esClient === undefined || this.esClient === null) { - throw Error('es client is unavailable: cannot retrieve pre-built rule alert batches'); - } - tlog(this.logger, `Searching prebuilt rule alerts from ${executeFrom} to ${executeTo}`); let pitId = await this.openPointInTime(DEFAULT_DIAGNOSTIC_INDEX); @@ -743,7 +745,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { let response = null; while (fetchMore) { try { - response = await this.esClient.search(query); + response = await this.esClient().search(query); const numOfHits = response?.hits.hits.length; if (numOfHits > 0) { @@ -779,13 +781,9 @@ export class TelemetryReceiver implements ITelemetryReceiver { } private async openPointInTime(indexPattern: string) { - if (this.esClient === undefined || this.esClient === null) { - throw Error('es client is unavailable: cannot retrieve pre-built rule alert batches'); - } - const keepAlive = '5m'; const pitId: OpenPointInTimeResponse['id'] = ( - await this.esClient.openPointInTime({ + await this.esClient().openPointInTime({ index: `${indexPattern}*`, keep_alive: keepAlive, expand_wildcards: ['open' as const, 'hidden' as const], @@ -796,28 +794,20 @@ export class TelemetryReceiver implements ITelemetryReceiver { } public async closePointInTime(pitId: string) { - if (this.esClient === undefined || this.esClient === null) { - throw Error('es client is unavailable: cannot retrieve pre-built rule alert batches'); - } - try { - await this.esClient.closePointInTime({ id: pitId }); + await this.esClient().closePointInTime({ id: pitId }); } catch (error) { tlog(this.logger, `Error trying to close point in time: "${pitId}". Error is: "${error}"`); } } async fetchTimelineAlerts(index: string, rangeFrom: string, rangeTo: string) { - if (this.esClient === undefined || this.esClient === null) { - throw Error('elasticsearch client is unavailable: cannot retrieve cluster infomation'); - } - // default is from looking at Kibana saved objects and online documentation const keepAlive = '5m'; // create and assign an initial point in time let pitId: OpenPointInTimeResponse['id'] = ( - await this.esClient.openPointInTime({ + await this.esClient().openPointInTime({ index: `${index}*`, keep_alive: keepAlive, }) @@ -881,11 +871,9 @@ export class TelemetryReceiver implements ITelemetryReceiver { size: 1000, }; - tlog(this.logger, `Getting alerts with point in time (PIT) query: ${JSON.stringify(query)}`); - let response = null; try { - response = await this.esClient.search(query); + response = await this.esClient().search(query); const numOfHits = response?.hits.hits.length; if (numOfHits > 0) { @@ -908,7 +896,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { } try { - await this.esClient.closePointInTime({ id: pitId }); + await this.esClient().closePointInTime({ id: pitId }); } catch (error) { tlog( this.logger, @@ -949,10 +937,6 @@ export class TelemetryReceiver implements ITelemetryReceiver { } public async fetchTimelineEvents(nodeIds: string[]) { - if (this.esClient === undefined || this.esClient === null) { - throw Error('elasticsearch client is unavailable: cannot retrieve timeline endpoint events'); - } - const query: SearchRequest = { expand_wildcards: ['open' as const, 'hidden' as const], index: [`${this.alertsIndex}*`, 'logs-*'], @@ -989,14 +973,10 @@ export class TelemetryReceiver implements ITelemetryReceiver { }, }; - return this.esClient.search(query); + return this.esClient().search(query); } public async fetchValueListMetaData(_interval: number) { - if (this.esClient === undefined || this.esClient === null) { - throw Error('elasticsearch client is unavailable: cannot retrieve diagnostic alerts'); - } - const listQuery: SearchRequest = { expand_wildcards: ['open' as const, 'hidden' as const], index: '.lists-*', @@ -1076,10 +1056,10 @@ export class TelemetryReceiver implements ITelemetryReceiver { }; const [listMetrics, itemMetrics, exceptionListMetrics, indicatorMatchMetrics] = await Promise.all([ - this.esClient.search(listQuery), - this.esClient.search(itemQuery), - this.esClient.search(exceptionListQuery), - this.esClient.search(indicatorMatchRuleQuery), + this.esClient().search(listQuery), + this.esClient().search(itemQuery), + this.esClient().search(exceptionListQuery), + this.esClient().search(indicatorMatchRuleQuery), ]); const listMetricsResponse = listMetrics as unknown as ValueListResponseAggregation; const itemMetricsResponse = itemMetrics as unknown as ValueListItemsResponseAggregation; @@ -1096,21 +1076,13 @@ export class TelemetryReceiver implements ITelemetryReceiver { } public async fetchClusterInfo(): Promise { - if (this.esClient === undefined || this.esClient === null) { - throw Error('elasticsearch client is unavailable: cannot retrieve cluster infomation'); - } - // @ts-expect-error version.build_date is of type estypes.DateTime - return this.esClient.info(); + return this.esClient().info(); } public async fetchLicenseInfo(): Promise { - if (this.esClient === undefined || this.esClient === null) { - throw Error('elasticsearch client is unavailable: cannot retrieve license information'); - } - try { - const ret = (await this.esClient.transport.request({ + const ret = (await this.esClient().transport.request({ method: 'GET', path: '/_license', querystring: { @@ -1134,4 +1106,82 @@ export class TelemetryReceiver implements ITelemetryReceiver { ...(lic.issuer ? { issuer: lic.issuer } : {}), }; } + + // calculates the number of documents that can be returned per page + // or "-1" if the query returns no documents + private async docsPerPage(index: string, query: ESSearchRequest): Promise { + const sampleQuery: ESSearchRequest = { + query: cloneDeep(query.query), + size: this.numDocsToSample, + index, + }; + const sampleSizeBytes = await this.esClient() + .search(sampleQuery) + .then((r) => r.hits.hits.reduce((sum, hit) => JSON.stringify(hit._source).length + sum, 0)); + const docSizeBytes = sampleSizeBytes / this.numDocsToSample; + + if (docSizeBytes === 0) { + return -1; + } + + return Math.max(Math.floor(this.maxPageSizeBytes / docSizeBytes), 1); + } + + public async *paginate(index: string, query: ESSearchRequest) { + if (query.sort == null) { + throw Error('Not possible to paginate a query without a sort attribute'); + } + + const size = await this.docsPerPage(index, query); + if (size === -1) { + return; + } + + const pit = { + id: await this.openPointInTime(index), + }; + const esQuery = { + ...cloneDeep(query), + pit, + size, + }; + try { + do { + const response = await this.esClient().search(esQuery); + const hits = response?.hits.hits.length ?? 0; + + if (hits === 0) { + return; + } + + esQuery.search_after = response?.hits.hits[hits - 1]?.sort; + + const data = response?.hits.hits.flatMap((h) => + h._source != null ? ([h._source] as T[]) : [] + ); + + yield data; + } while (esQuery.search_after !== undefined); + } catch (e) { + tlog(this.logger, `Error running paginated query: ${e}`); + throw e; + } finally { + await this.closePointInTime(pit.id); + } + } + + public setMaxPageSizeBytes(bytes: number) { + this.maxPageSizeBytes = bytes; + } + + public setNumDocsToSample(n: number) { + this.numDocsToSample = n; + } + + private esClient(): ElasticsearchClient { + if (this._esClient === undefined || this._esClient === null) { + throw Error('elasticsearch client is unavailable'); + } + return this._esClient; + } } From 444882f3e643a749d341edbf93eb268f4de4a280 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Tue, 20 Feb 2024 12:27:48 +0100 Subject: [PATCH 04/19] Fix type validations --- .../plugins/security_solution/server/lib/telemetry/helpers.ts | 2 +- .../security_solution/server/lib/telemetry/sender_helpers.ts | 4 ++-- .../server/lib/telemetry/tasks/detection_rule.ts | 2 +- .../server/lib/telemetry/tasks/security_lists.ts | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts b/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts index df8d117e595f4..2897586bb1561 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts @@ -451,7 +451,7 @@ export class TelemetryTimelineFetcher { const clusterInfo: ESClusterInfo = clusterInfoPromise.status === 'fulfilled' ? clusterInfoPromise.value : ({} as ESClusterInfo); - const licenseInfo: ESLicense | undefined = + const licenseInfo: Nullable = licenseInfoPromise.status === 'fulfilled' ? licenseInfoPromise.value : ({} as ESLicense); return { clusterInfo, licenseInfo }; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/sender_helpers.ts b/x-pack/plugins/security_solution/server/lib/telemetry/sender_helpers.ts index c7bb9d74ce1c5..4bd39865d55e0 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/sender_helpers.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/sender_helpers.ts @@ -15,8 +15,8 @@ import { createUsageCounterLabel } from './helpers'; export interface SenderMetadata { telemetryUrl: string; - licenseInfo: ESLicense | undefined; - clusterInfo: ESClusterInfo | undefined; + licenseInfo: Nullable; + clusterInfo: Nullable; telemetryRequestHeaders: () => RawAxiosRequestHeaders; isTelemetryOptedIn(): Promise; isTelemetryServicesReachable(): Promise; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts index e8acc4e222958..5640fc20902da 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts @@ -59,7 +59,7 @@ export function createTelemetryDetectionRuleListsTaskConfig(maxTelemetryBatch: n const licenseInfo = licenseInfoPromise.status === 'fulfilled' ? licenseInfoPromise.value - : ({} as ESLicense | undefined); + : ({} as Nullable); // Lists Telemetry: Detection Rules diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/security_lists.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/security_lists.ts index 45fe385e76329..fb3191888b0ee 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/security_lists.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/security_lists.ts @@ -69,7 +69,7 @@ export function createTelemetrySecurityListTaskConfig(maxTelemetryBatch: number) const licenseInfo = licenseInfoPromise.status === 'fulfilled' ? licenseInfoPromise.value - : ({} as ESLicense | undefined); + : ({} as Nullable); const FETCH_VALUE_LIST_META_DATA_INTERVAL_IN_HOURS = 24; // Lists Telemetry: Trusted Applications From a26ba17e82d82443f34a93baebe1da367d45a0dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Tue, 20 Feb 2024 13:22:22 +0100 Subject: [PATCH 05/19] Fix type validations --- .../security_solution/server/lib/telemetry/helpers.ts | 7 ++++--- .../server/lib/telemetry/sender_helpers.ts | 8 +++++++- .../server/lib/telemetry/tasks/detection_rule.ts | 8 +++++++- .../server/lib/telemetry/tasks/endpoint.ts | 2 +- .../server/lib/telemetry/tasks/security_lists.ts | 2 +- .../security_solution/server/lib/telemetry/types.ts | 2 +- 6 files changed, 21 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts b/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts index 2897586bb1561..51caf34dbb073 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts @@ -22,6 +22,7 @@ import type { ExceptionListItem, ExtraInfo, ListTemplate, + Nullable, TelemetryEvent, TimeFrame, TimelineResult, @@ -188,7 +189,7 @@ export const ruleExceptionListItemToTelemetryEvent = ( export const templateExceptionList = ( listData: ExceptionListItem[], clusterInfo: ESClusterInfo, - licenseInfo: ESLicense | undefined, + licenseInfo: Nullable, listType: string ) => { return listData.map((item) => { @@ -232,7 +233,7 @@ export const templateExceptionList = ( /** * Convert counter label list to kebab case * - * @param label_list the list of labels to create standardized UsageCounter from + * @param labelList the list of labels to create standardized UsageCounter from * @returns a string label for usage in the UsageCounter */ export const createUsageCounterLabel = (labelList: string[]): string => labelList.join('-'); @@ -254,7 +255,7 @@ export const addDefaultAdvancedPolicyConfigSettings = (policyConfig: PolicyConfi export const formatValueListMetaData = ( valueListResponse: ValueListResponse, clusterInfo: ESClusterInfo, - licenseInfo: ESLicense | undefined + licenseInfo: Nullable ) => ({ '@timestamp': moment().toISOString(), cluster_uuid: clusterInfo.cluster_uuid, diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/sender_helpers.ts b/x-pack/plugins/security_solution/server/lib/telemetry/sender_helpers.ts index 4bd39865d55e0..2471b3463eb8f 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/sender_helpers.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/sender_helpers.ts @@ -10,7 +10,13 @@ import type { TelemetryPluginStart, TelemetryPluginSetup } from '@kbn/telemetry- import type { RawAxiosRequestHeaders } from 'axios'; import { type IUsageCounter } from '@kbn/usage-collection-plugin/server/usage_counters/usage_counter'; import type { ITelemetryReceiver } from './receiver'; -import type { ESClusterInfo, ESLicense, TelemetryChannel, TelemetryCounter } from './types'; +import type { + ESClusterInfo, + ESLicense, + Nullable, + TelemetryChannel, + TelemetryCounter, +} from './types'; import { createUsageCounterLabel } from './helpers'; export interface SenderMetadata { diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts index 5640fc20902da..8c101baa213ba 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts @@ -16,7 +16,13 @@ import { } from '../helpers'; import type { ITelemetryEventsSender } from '../sender'; import type { ITelemetryReceiver } from '../receiver'; -import type { ExceptionListItem, ESClusterInfo, ESLicense, RuleSearchResult } from '../types'; +import type { + ExceptionListItem, + ESClusterInfo, + ESLicense, + RuleSearchResult, + Nullable, +} from '../types'; import type { TaskExecutionPeriod } from '../task'; import type { ITaskMetricsService } from '../task_metrics.types'; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts index 951931d6a04fe..348146361e8f7 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts @@ -147,7 +147,7 @@ export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) { */ const agentsResponse = endpointData.fleetAgentsResponse; - if (agentsResponse === undefined) { + if (agentsResponse === undefined || agentsResponse === null) { log('no fleet agent information available'); taskMetricsService.end(trace); return 0; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/security_lists.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/security_lists.ts index fb3191888b0ee..bc7207c73b06a 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/security_lists.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/security_lists.ts @@ -13,7 +13,7 @@ import { LIST_TRUSTED_APPLICATION, TELEMETRY_CHANNEL_LISTS, } from '../constants'; -import type { ESClusterInfo, ESLicense } from '../types'; +import type { ESClusterInfo, ESLicense, Nullable } from '../types'; import { batchTelemetryRecords, templateExceptionList, diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/types.ts b/x-pack/plugins/security_solution/server/lib/telemetry/types.ts index b29777a121f7a..59306a0a6537c 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/types.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/types.ts @@ -490,7 +490,7 @@ export type Nullable = T | null | undefined; export interface ExtraInfo { clusterInfo: ESClusterInfo; - licenseInfo: ESLicense | undefined; + licenseInfo: Nullable; } export interface TimeFrame { From 821211f2f3f128917e908c07df576e889f536fa4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Wed, 21 Feb 2024 12:37:03 +0100 Subject: [PATCH 06/19] Code style --- .../server/integration_tests/lib/helpers.ts | 3 --- .../integration_tests/telemetry.test.ts | 8 +++---- .../server/lib/telemetry/async_sender.ts | 23 ++++++++++++++++++- .../server/lib/telemetry/receiver.ts | 2 +- 4 files changed, 27 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/security_solution/server/integration_tests/lib/helpers.ts b/x-pack/plugins/security_solution/server/integration_tests/lib/helpers.ts index 20d98303a62a7..db9ab8c656c3c 100644 --- a/x-pack/plugins/security_solution/server/integration_tests/lib/helpers.ts +++ b/x-pack/plugins/security_solution/server/integration_tests/lib/helpers.ts @@ -54,9 +54,6 @@ export async function setupTestServers(logFilePath: string, settings = {}) { const root = createRootWithCorePlugins( deepmerge( { - server: { - port: 9991, - }, logging: { appenders: { file: { diff --git a/x-pack/plugins/security_solution/server/integration_tests/telemetry.test.ts b/x-pack/plugins/security_solution/server/integration_tests/telemetry.test.ts index e2357d6d614ed..67034b45e08c9 100644 --- a/x-pack/plugins/security_solution/server/integration_tests/telemetry.test.ts +++ b/x-pack/plugins/security_solution/server/integration_tests/telemetry.test.ts @@ -46,7 +46,7 @@ jest.mock('axios'); const logFilePath = Path.join(__dirname, 'logs.log'); const taskManagerStartSpy = jest.spyOn(TaskManagerPlugin.prototype, 'start'); -const telemetrySenderStartSpy = jest.spyOn(SecuritySolutionPlugin.prototype, 'start'); +const securitySolutionStartSpy = jest.spyOn(SecuritySolutionPlugin.prototype, 'start'); const mockedAxiosGet = jest.spyOn(axios, 'get'); const mockedAxiosPost = jest.spyOn(axios, 'post'); @@ -69,10 +69,10 @@ describe('telemetry tasks', () => { expect(taskManagerStartSpy).toHaveBeenCalledTimes(1); taskManagerPlugin = taskManagerStartSpy.mock.results[0].value; - expect(telemetrySenderStartSpy).toHaveBeenCalledTimes(1); + expect(securitySolutionStartSpy).toHaveBeenCalledTimes(1); - tasks = getTelemetryTasks(telemetrySenderStartSpy); - asyncTelemetryEventSender = getAsyncTelemetryEventSender(telemetrySenderStartSpy); + tasks = getTelemetryTasks(securitySolutionStartSpy); + asyncTelemetryEventSender = getAsyncTelemetryEventSender(securitySolutionStartSpy); // update queue config to not wait for a long bufferTimeSpanMillis asyncTelemetryEventSender.updateQueueConfig(TelemetryChannel.TASK_METRICS, { diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/async_sender.ts b/x-pack/plugins/security_solution/server/lib/telemetry/async_sender.ts index 48afbddc4e1d7..bc6c6ffd7d83e 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/async_sender.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/async_sender.ts @@ -233,8 +233,11 @@ export class AsyncTelemetryEventsSender implements IAsyncTelemetryEventsSender { // exclude empty buffers rx.filter((n: Event[]) => n.length > 0), + // enrich the events + rx.map((events) => events.map((e) => this.enrich(e))), + // serialize the payloads - rx.map((events) => events.map((e) => JSON.stringify(e.payload))), + rx.map((events) => events.map((e) => this.serialize(e))), // chunk by size rx.map((values) => @@ -262,6 +265,24 @@ export class AsyncTelemetryEventsSender implements IAsyncTelemetryEventsSender { ) as rx.Observable; } + private enrich(event: Event): Event { + const clusterInfo = this.telemetryReceiver?.getClusterInfo(); + + if (typeof event.payload === 'object') { + event.payload = { + ...event.payload, + // TODO(szaffarano): include license info if available + cluster_uuid: clusterInfo?.cluster_uuid, + cluster_name: clusterInfo?.cluster_name, + }; + } + return event; + } + + private serialize(event: Event): string { + return JSON.stringify(event.payload); + } + private async sendEvents(channel: TelemetryChannel, events: string[]): Promise { this.logger.l(`Sending ${events.length} telemetry events to channel "${channel}"`); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts index 1cb0c80b12452..f7caf38ee73cb 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -1080,7 +1080,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { return this.esClient().info(); } - public async fetchLicenseInfo(): Promise { + public async fetchLicenseInfo(): Promise> { try { const ret = (await this.esClient().transport.request({ method: 'GET', From 41b18d39ff3830e7562fcacdefb8e8d4d57684d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Wed, 21 Feb 2024 12:44:36 +0100 Subject: [PATCH 07/19] Code style --- .../server/lib/telemetry/helpers.ts | 10 ++++++++++ .../server/lib/telemetry/receiver.ts | 18 ------------------ .../server/lib/telemetry/sender.ts | 4 ++-- .../lib/telemetry/tasks/detection_rule.ts | 19 ++++--------------- 4 files changed, 16 insertions(+), 35 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts b/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts index 51caf34dbb073..056bb0d4038f9 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts @@ -367,6 +367,16 @@ export const ranges = ( return { rangeFrom, rangeTo }; }; +export const copyLicenseFields = (lic: ESLicense) => { + return { + uid: lic.uid, + status: lic.status, + type: lic.type, + ...(lic.issued_to ? { issued_to: lic.issued_to } : {}), + ...(lic.issuer ? { issuer: lic.issuer } : {}), + }; +}; + export class TelemetryTimelineFetcher { private receiver: ITelemetryReceiver; private extraInfo: Promise; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts index f7caf38ee73cb..07a3b5914816b 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -175,14 +175,6 @@ export interface ITelemetryReceiver { executeTo: string ): AsyncGenerator; - copyLicenseFields(lic: ESLicense): { - issuer?: string | undefined; - issued_to?: string | undefined; - uid: string; - status: string; - type: string; - }; - fetchTimelineAlerts( index: string, rangeFrom: string, @@ -1097,16 +1089,6 @@ export class TelemetryReceiver implements ITelemetryReceiver { } } - public copyLicenseFields(lic: ESLicense) { - return { - uid: lic.uid, - status: lic.status, - type: lic.type, - ...(lic.issued_to ? { issued_to: lic.issued_to } : {}), - ...(lic.issuer ? { issuer: lic.issuer } : {}), - }; - } - // calculates the number of documents that can be returned per page // or "-1" if the query returns no documents private async docsPerPage(index: string, query: ESSearchRequest): Promise { diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts index 7addae79f148d..a788a59b63b71 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts @@ -21,7 +21,7 @@ import type { import type { ITelemetryReceiver } from './receiver'; import { copyAllowlistedFields, filterList } from './filterlists'; import { createTelemetryTaskConfigs } from './tasks'; -import { createUsageCounterLabel, tlog } from './helpers'; +import { copyLicenseFields, createUsageCounterLabel, tlog } from './helpers'; import type { TelemetryChannel, TelemetryEvent } from './types'; import type { SecurityTelemetryTaskConfig } from './task'; import { SecurityTelemetryTask } from './task'; @@ -317,7 +317,7 @@ export class TelemetryEventsSender implements ITelemetryEventsSender { const toSend: TelemetryEvent[] = cloneDeep(this.queue).map((event) => ({ ...event, - ...(licenseInfo ? { license: this.receiver?.copyLicenseFields(licenseInfo) } : {}), + ...(licenseInfo ? { license: copyLicenseFields(licenseInfo) } : {}), cluster_uuid: clusterInfo?.cluster_uuid, cluster_name: clusterInfo?.cluster_name, })); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts index 8c101baa213ba..80db4fdac562e 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts @@ -13,16 +13,11 @@ import { templateExceptionList, newTelemetryLogger, createUsageCounterLabel, + safeValue, } from '../helpers'; import type { ITelemetryEventsSender } from '../sender'; import type { ITelemetryReceiver } from '../receiver'; -import type { - ExceptionListItem, - ESClusterInfo, - ESLicense, - RuleSearchResult, - Nullable, -} from '../types'; +import type { ExceptionListItem, RuleSearchResult } from '../types'; import type { TaskExecutionPeriod } from '../task'; import type { ITaskMetricsService } from '../task_metrics.types'; @@ -58,14 +53,8 @@ export function createTelemetryDetectionRuleListsTaskConfig(maxTelemetryBatch: n receiver.fetchLicenseInfo(), ]); - const clusterInfo = - clusterInfoPromise.status === 'fulfilled' - ? clusterInfoPromise.value - : ({} as ESClusterInfo); - const licenseInfo = - licenseInfoPromise.status === 'fulfilled' - ? licenseInfoPromise.value - : ({} as Nullable); + const clusterInfo = safeValue(clusterInfoPromise); + const licenseInfo = safeValue(licenseInfoPromise); // Lists Telemetry: Detection Rules From 56d652a8f61747be57dd4610e9a86c251efc1f4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Wed, 21 Feb 2024 17:10:46 +0100 Subject: [PATCH 08/19] wip --- .../__mocks__/endpoint-metadata.json | 251 ++ .../__mocks__/endpoint-metrics.json | 2677 +++++++++++++++++ .../__mocks__/endpoint-policy.json | 1205 ++++++++ .../__mocks__/fleet-agents.json | 146 + .../server/integration_tests/lib/helpers.ts | 14 + .../lib/telemetry_helpers.ts | 66 + .../server/integration_tests/receiver.test.ts | 15 +- .../integration_tests/telemetry.test.ts | 50 + .../server/lib/telemetry/tasks/endpoint.ts | 4 +- 9 files changed, 4417 insertions(+), 11 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/integration_tests/__mocks__/endpoint-metadata.json create mode 100644 x-pack/plugins/security_solution/server/integration_tests/__mocks__/endpoint-metrics.json create mode 100644 x-pack/plugins/security_solution/server/integration_tests/__mocks__/endpoint-policy.json create mode 100644 x-pack/plugins/security_solution/server/integration_tests/__mocks__/fleet-agents.json diff --git a/x-pack/plugins/security_solution/server/integration_tests/__mocks__/endpoint-metadata.json b/x-pack/plugins/security_solution/server/integration_tests/__mocks__/endpoint-metadata.json new file mode 100644 index 0000000000000..c46e574421c7d --- /dev/null +++ b/x-pack/plugins/security_solution/server/integration_tests/__mocks__/endpoint-metadata.json @@ -0,0 +1,251 @@ +[ + { + "agent": { + "build": { + "original": "version: 8.6.0, compiled: Mon Jan 2 23:00:00 2023, branch: 8.6, commit: e2d09ff1b8e49bfb5f8940d317eb4ac96672d956" + }, + "id": "123", + "type": "endpoint", + "version": "8.6.0" + }, + "@timestamp": "2024-01-29T18:35:51.819450273Z", + "Endpoint": { + "capabilities": [ + "kill_process", + "suspend_process", + "running_processes", + "isolation", + "ebpf(auto)" + ], + "configuration": { + "isolation": false + }, + "state": { + "isolation": false + }, + "policy": { + "applied": { + "name": "123", + "id": "123", + "endpoint_policy_version": "1", + "version": "55", + "status": "success" + } + }, + "status": "enrolled" + }, + "ecs": { + "version": "1.11.0" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "endpoint.metadata" + }, + "elastic": { + "agent": { + "id": "123" + } + }, + "host": { + "hostname": "123", + "os": { + "Ext": { + "variant": "Amazon Linux" + }, + "kernel": "5.10.102-99.473.amzn2.x86_64 #1 SMP Wed Mar 2 19:14:12 UTC 2022", + "name": "Linux", + "family": "amazon linux", + "type": "linux", + "version": "2", + "platform": "amazon linux", + "full": "Amazon Linux 2" + }, + "ip": ["127.0.0.1", "::1"], + "name": "123", + "id": "123", + "mac": ["aa:aa:aa:aa:aa:aa"], + "architecture": "x86_64" + }, + "event": { + "agent_id_status": "verified", + "sequence": 34893855, + "ingested": "2024-01-29T18:35:53Z", + "created": "2024-01-29T18:35:51.819450273Z", + "kind": "metric", + "module": "endpoint", + "action": "endpoint_metadata", + "id": "123", + "category": ["host"], + "type": ["info"], + "dataset": "endpoint.metadata" + }, + "message": "Endpoint metadata" + }, + { + "agent": { + "build": { + "original": "version: 8.6.0, compiled: Mon Jan 2 23:00:00 2023, branch: 8.6, commit: e2d09ff1b8e49bfb5f8940d317eb4ac96672d956" + }, + "id": "123", + "type": "endpoint", + "version": "8.6.0" + }, + "@timestamp": "2024-01-29T16:38:37.863555622Z", + "Endpoint": { + "capabilities": [ + "kill_process", + "suspend_process", + "running_processes", + "isolation", + "ebpf(auto)" + ], + "configuration": { + "isolation": false + }, + "state": { + "isolation": false + }, + "policy": { + "applied": { + "name": "123", + "id": "123", + "endpoint_policy_version": "1", + "version": "26", + "status": "success" + } + }, + "status": "enrolled" + }, + "ecs": { + "version": "1.11.0" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "endpoint.metadata" + }, + "elastic": { + "agent": { + "id": "123" + } + }, + "host": { + "hostname": "123", + "os": { + "Ext": { + "variant": "Amazon Linux" + }, + "kernel": "5.10.102-99.473.amzn2.x86_64 #1 SMP Wed Mar 2 19:14:12 UTC 2022", + "name": "Linux", + "family": "amazon linux", + "type": "linux", + "version": "2", + "platform": "amazon linux", + "full": "Amazon Linux 2" + }, + "ip": ["127.0.0.1", "::1"], + "name": "123", + "id": "123", + "mac": ["aa:aa:aa:aa:aa:aa"], + "architecture": "x86_64" + }, + "event": { + "agent_id_status": "verified", + "sequence": 33347679, + "ingested": "2024-01-29T16:38:39Z", + "created": "2024-01-29T16:38:37.863555622Z", + "kind": "metric", + "module": "endpoint", + "action": "endpoint_metadata", + "id": "123", + "category": ["host"], + "type": ["info"], + "dataset": "endpoint.metadata" + }, + "message": "Endpoint metadata" + }, + { + "agent": { + "build": { + "original": "version: 8.6.0, compiled: Mon Jan 2 23:00:00 2023, branch: 8.6, commit: e2d09ff1b8e49bfb5f8940d317eb4ac96672d956" + }, + "id": "123", + "type": "endpoint", + "version": "8.6.0" + }, + "@timestamp": "2024-01-29T16:35:51.210383141Z", + "Endpoint": { + "capabilities": [ + "kill_process", + "suspend_process", + "running_processes", + "isolation", + "ebpf(auto)" + ], + "configuration": { + "isolation": false + }, + "state": { + "isolation": false + }, + "policy": { + "applied": { + "name": "123", + "id": "123", + "endpoint_policy_version": "1", + "version": "55", + "status": "success" + } + }, + "status": "enrolled" + }, + "ecs": { + "version": "1.11.0" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "endpoint.metadata" + }, + "elastic": { + "agent": { + "id": "123" + } + }, + "host": { + "hostname": "123", + "os": { + "Ext": { + "variant": "Amazon Linux" + }, + "kernel": "5.10.102-99.473.amzn2.x86_64 #1 SMP Wed Mar 2 19:14:12 UTC 2022", + "name": "Linux", + "family": "amazon linux", + "type": "linux", + "version": "2", + "platform": "amazon linux", + "full": "Amazon Linux 2" + }, + "ip": ["127.0.0.1", "::1"], + "name": "123", + "id": "123", + "mac": ["aa:aa:aa:aa:aa:aa"], + "architecture": "x86_64" + }, + "event": { + "agent_id_status": "verified", + "sequence": 34885625, + "ingested": "2024-01-29T16:35:53Z", + "created": "2024-01-29T16:35:51.210383141Z", + "kind": "metric", + "module": "endpoint", + "action": "endpoint_metadata", + "id": "123", + "category": ["host"], + "type": ["info"], + "dataset": "endpoint.metadata" + }, + "message": "Endpoint metadata" + } +] diff --git a/x-pack/plugins/security_solution/server/integration_tests/__mocks__/endpoint-metrics.json b/x-pack/plugins/security_solution/server/integration_tests/__mocks__/endpoint-metrics.json new file mode 100644 index 0000000000000..a5af045d2f4c3 --- /dev/null +++ b/x-pack/plugins/security_solution/server/integration_tests/__mocks__/endpoint-metrics.json @@ -0,0 +1,2677 @@ +[ + { + "agent": { + "build": { + "original": "version: 8.6.2, compiled: Fri Feb 10 14:00:00 2023, branch: 8.6, commit: eaee25e73cc58a52387e50d5bce542ec8965f638" + }, + "id": "123", + "type": "endpoint", + "version": "8.6.2" + }, + "@timestamp": "2024-01-26T15:06:50.254822749Z", + "Endpoint": { + "metrics": { + "system_impact": [ + { + "process": { + "executable": "/usr/sbin/sshd" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 304 + }, + "process_events": { + "week_idle_ms": 31791, + "week_ms": 110821 + }, + "network_events": { + "week_idle_ms": 2030, + "week_ms": 53 + }, + "overall": { + "week_idle_ms": 33821, + "week_ms": 111178 + } + }, + { + "process": { + "executable": "unknown" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 4552 + }, + "overall": { + "week_idle_ms": 0, + "week_ms": 4552 + } + }, + { + "file_events": { + "week_idle_ms": 130, + "week_ms": 7 + }, + "process": { + "executable": "/lib/systemd/systemd" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 1920 + }, + "process_events": { + "week_idle_ms": 1273, + "week_ms": 1750 + }, + "overall": { + "week_idle_ms": 1403, + "week_ms": 3677 + } + }, + { + "file_events": { + "week_idle_ms": 7, + "week_ms": 0 + }, + "process": { + "executable": "/bin/systemctl" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 3055 + }, + "process_events": { + "week_idle_ms": 0, + "week_ms": 9 + }, + "overall": { + "week_idle_ms": 7, + "week_ms": 3064 + } + }, + { + "file_events": { + "week_idle_ms": 3104, + "week_ms": 1 + }, + "process": { + "executable": "/usr/bin/apt-key" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 61 + }, + "process_events": { + "week_idle_ms": 178857, + "week_ms": 2062 + }, + "overall": { + "week_idle_ms": 181961, + "week_ms": 2124 + } + }, + { + "process": { + "executable": "/usr/bin/gce_workload_cert_refresh" + }, + "process_events": { + "week_idle_ms": 230, + "week_ms": 1405 + }, + "network_events": { + "week_idle_ms": 135, + "week_ms": 21 + }, + "overall": { + "week_idle_ms": 365, + "week_ms": 1426 + } + }, + { + "process": { + "executable": "/usr/bin/apt-config" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 65 + }, + "process_events": { + "week_idle_ms": 18708, + "week_ms": 840 + }, + "overall": { + "week_idle_ms": 18708, + "week_ms": 905 + } + }, + { + "process": { + "executable": "/usr/lib/apt/apt.systemd.daily" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 273 + }, + "process_events": { + "week_idle_ms": 1064, + "week_ms": 352 + }, + "overall": { + "week_idle_ms": 1064, + "week_ms": 625 + } + }, + { + "process": { + "executable": "/usr/bin/dpkg" + }, + "process_events": { + "week_idle_ms": 11479, + "week_ms": 558 + }, + "overall": { + "week_idle_ms": 11479, + "week_ms": 558 + } + }, + { + "process": { + "executable": "/usr/sbin/cron" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 80 + }, + "process_events": { + "week_idle_ms": 270, + "week_ms": 451 + }, + "overall": { + "week_idle_ms": 270, + "week_ms": 531 + } + }, + { + "process": { + "executable": "/usr/bin/cmp" + }, + "process_events": { + "week_idle_ms": 39452, + "week_ms": 513 + }, + "overall": { + "week_idle_ms": 39452, + "week_ms": 513 + } + }, + { + "file_events": { + "week_idle_ms": 144, + "week_ms": 0 + }, + "process": { + "executable": "/sbin/dhclient-script" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 251 + }, + "process_events": { + "week_idle_ms": 2412, + "week_ms": 186 + }, + "overall": { + "week_idle_ms": 2556, + "week_ms": 437 + } + }, + { + "process": { + "executable": "/usr/bin/env" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 413 + }, + "process_events": { + "week_idle_ms": 45, + "week_ms": 4 + }, + "overall": { + "week_idle_ms": 45, + "week_ms": 417 + } + }, + { + "process": { + "executable": "/usr/bin/cat" + }, + "process_events": { + "week_idle_ms": 39679, + "week_ms": 359 + }, + "overall": { + "week_idle_ms": 39679, + "week_ms": 359 + } + }, + { + "process": { + "executable": "/bin/sh" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 35 + }, + "process_events": { + "week_idle_ms": 444, + "week_ms": 197 + }, + "overall": { + "week_idle_ms": 444, + "week_ms": 232 + } + }, + { + "process": { + "executable": "/usr/lib/apt/apt-helper" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 100 + }, + "process_events": { + "week_idle_ms": 8, + "week_ms": 62 + }, + "overall": { + "week_idle_ms": 8, + "week_ms": 162 + } + }, + { + "process": { + "executable": "/usr/bin/date" + }, + "process_events": { + "week_idle_ms": 1024, + "week_ms": 143 + }, + "overall": { + "week_idle_ms": 1024, + "week_ms": 143 + } + }, + { + "file_events": { + "week_idle_ms": 5315, + "week_ms": 2 + }, + "process": { + "executable": "/usr/bin/apt-get" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 53 + }, + "process_events": { + "week_idle_ms": 576, + "week_ms": 76 + }, + "overall": { + "week_idle_ms": 5891, + "week_ms": 131 + } + }, + { + "process": { + "executable": "/usr/bin/unattended-upgrade" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 27 + }, + "process_events": { + "week_idle_ms": 139, + "week_ms": 95 + }, + "overall": { + "week_idle_ms": 139, + "week_ms": 122 + } + }, + { + "process": { + "executable": "/usr/bin/gpgconf" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 26 + }, + "process_events": { + "week_idle_ms": 12075, + "week_ms": 93 + }, + "overall": { + "week_idle_ms": 12075, + "week_ms": 119 + } + } + ], + "memory": { + "endpoint": { + "private": { + "mean": 373780821, + "latest": 411860992 + } + } + }, + "disks": [ + { + "total": 0, + "free": 0, + "device": "sysfs", + "mount": "/sys", + "fstype": "sysfs" + }, + { + "total": 0, + "free": 0, + "device": "proc", + "mount": "/proc", + "fstype": "proc" + }, + { + "total": 2049777664, + "free": 2049777664, + "device": "udev", + "mount": "/dev", + "fstype": "devtmpfs" + }, + { + "total": 0, + "free": 0, + "device": "devpts", + "mount": "/dev/pts", + "fstype": "devpts" + }, + { + "total": 412172288, + "free": 411795456, + "device": "tmpfs", + "mount": "/run", + "fstype": "tmpfs" + }, + { + "endpoint_drive": true, + "total": 10331889664, + "free": 4091109376, + "device": "/dev/sda1", + "mount": "/", + "fstype": "ext4" + }, + { + "total": 0, + "free": 0, + "device": "securityfs", + "mount": "/sys/kernel/security", + "fstype": "securityfs" + }, + { + "total": 2060849152, + "free": 2060849152, + "device": "tmpfs", + "mount": "/dev/shm", + "fstype": "tmpfs" + }, + { + "total": 5242880, + "free": 5242880, + "device": "tmpfs", + "mount": "/run/lock", + "fstype": "tmpfs" + }, + { + "total": 0, + "free": 0, + "device": "cgroup2", + "mount": "/sys/fs/cgroup", + "fstype": "cgroup2" + }, + { + "total": 0, + "free": 0, + "device": "pstore", + "mount": "/sys/fs/pstore", + "fstype": "pstore" + }, + { + "total": 0, + "free": 0, + "device": "efivarfs", + "mount": "/sys/firmware/efi/efivars", + "fstype": "efivarfs" + }, + { + "total": 0, + "free": 0, + "device": "none", + "mount": "/sys/fs/bpf", + "fstype": "bpf" + }, + { + "total": 0, + "free": 0, + "device": "systemd-1", + "mount": "/proc/sys/fs/binfmt_misc", + "fstype": "autofs" + }, + { + "total": 0, + "free": 0, + "device": "hugetlbfs", + "mount": "/dev/hugepages", + "fstype": "hugetlbfs" + }, + { + "total": 0, + "free": 0, + "device": "mqueue", + "mount": "/dev/mqueue", + "fstype": "mqueue" + }, + { + "total": 0, + "free": 0, + "device": "debugfs", + "mount": "/sys/kernel/debug", + "fstype": "debugfs" + }, + { + "total": 0, + "free": 0, + "device": "tracefs", + "mount": "/sys/kernel/tracing", + "fstype": "tracefs" + }, + { + "total": 0, + "free": 0, + "device": "fusectl", + "mount": "/sys/fs/fuse/connections", + "fstype": "fusectl" + }, + { + "total": 0, + "free": 0, + "device": "configfs", + "mount": "/sys/kernel/config", + "fstype": "configfs" + }, + { + "total": 129751040, + "free": 118589440, + "device": "/dev/sda15", + "mount": "/boot/efi", + "fstype": "vfat" + }, + { + "total": 0, + "free": 0, + "device": "binfmt_misc", + "mount": "/proc/sys/fs/binfmt_misc", + "fstype": "binfmt_misc" + }, + { + "total": 0, + "free": 0, + "device": "tracefs", + "mount": "/sys/kernel/debug/tracing", + "fstype": "tracefs" + } + ], + "documents_volume": { + "file_events": { + "suppressed_count": 1651293, + "suppressed_bytes": 0, + "sent_count": 0, + "sent_bytes": 0 + }, + "process_events": { + "suppressed_count": 0, + "suppressed_bytes": 0, + "sent_count": 11026887, + "sent_bytes": 23412176619 + }, + "network_events": { + "suppressed_count": 3270009, + "suppressed_bytes": 0, + "sent_count": 0, + "sent_bytes": 0 + }, + "overall": { + "suppressed_count": 4921302, + "suppressed_bytes": 0, + "sent_count": 11026887, + "sent_bytes": 23412176619 + } + }, + "malicious_behavior_rules": [ + { + "id": "123", + "endpoint_uptime_percent": 0 + } ], + "cpu": { + "endpoint": { + "histogram": { + "counts": [ + 3934326, + 520, + 140, + 110, + 212, + 78, + 41, + 18, + 12, + 25, + 9, + 5, + 5, + 2, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "values": [ + 5, + 10, + 15, + 20, + 25, + 30, + 35, + 40, + 45, + 50, + 55, + 60, + 65, + 70, + 75, + 80, + 85, + 90, + 95, + 100 + ] + }, + "mean": 0.2697424700222798, + "latest": 0.1 + } + }, + "threads": [ + { + "name": "Cron", + "cpu": { + "mean": 0.005330585583239717 + } + }, + { + "name": "FileLogThread", + "cpu": { + "mean": 0.01387886588350829 + } + }, + { + "name": "LoggingLimitThread", + "cpu": { + "mean": 0.0003385089239880071 + } + }, + { + "name": "DocumentLoggingThread", + "cpu": { + "mean": 0.002019893909071295 + } + }, + { + "name": "DocumentLoggingMaintenance", + "cpu": { + "mean": 0.0001711144011367948 + } + }, + { + "name": "BulkConsumerThread", + "cpu": { + "mean": 0.01437360969549076 + } + }, + { + "name": "DocumentLoggingConsumerThread", + "cpu": { + "mean": 0.1566068758230231 + } + }, + { + "name": "DocumentLoggingLimitThread", + "cpu": { + "mean": 0.000383147463414997 + } + }, + { + "name": "ArtifactManifestDownload", + "cpu": { + "mean": 0.00202733366564246 + } + }, + { + "name": "PolicyReloadThread", + "cpu": { + "mean": 0.00001487951314232998 + } + }, + { + "name": "PerformanceMonitorWorkerThread", + "cpu": { + "mean": 0.005423582540379278 + } + }, + { + "name": "MetadataThread", + "cpu": { + "mean": 0.00001115963485674749 + } + }, + { + "name": "EventsQueueThread", + "cpu": { + "mean": 0.05409447002894065 + } + }, + { + "name": "DelayedAlertEnrichment", + "cpu": { + "mean": 0 + } + }, + { + "name": "MaintainProcessMap", + "cpu": { + "mean": 0.002905224941039929 + } + }, + { + "name": "FileScoreThread", + "cpu": { + "mean": 0.003567363275873613 + } + }, + { + "name": "DiagnosticMalwareThread", + "cpu": { + "mean": 0 + } + }, + { + "name": "QuarantineManagerWorkerThread", + "cpu": { + "mean": 0.002083131839926198 + } + }, + { + "name": "EventProcessingThread", + "cpu": { + "mean": 0.01754294664738331 + } + }, + { + "name": "HostIsolationMonitorThread", + "cpu": { + "mean": 0.0008481322806622976 + } + }, + { + "name": "MountMonitor", + "cpu": { + "mean": 0.003463206812704382 + } + }, + { + "name": "responseActionsUploadThread", + "cpu": { + "mean": 0.00270807149264102 + } + }, + { + "name": "responseActionsProcessThread", + "cpu": { + "mean": 0.002689472100521233 + } + }, + { + "name": "serviceCommsThread", + "cpu": { + "mean": 0 + } + }, + { + "name": "grpcConnectionManagerThread", + "cpu": { + "mean": 0.0110926774602411 + } + }, + { + "name": "checkinAPIThread", + "cpu": { + "mean": 0.0003124700433295513 + } + }, + { + "name": "actionsAPIThread", + "cpu": { + "mean": 0.0004910243538035807 + } + }, + { + "name": "stateReportThread", + "cpu": { + "mean": 0.00539010824743476 + } + }, + { + "name": "EventsLoopThread", + "cpu": { + "mean": 0.003266057265515501 + } + }, + { + "name": "FanotifyWatchdog", + "cpu": { + "mean": 0.0002231929794201937 + } + }, + { + "name": "FanotifySyncConsumer", + "cpu": { + "mean": 0.001885980676100637 + } + }, + { + "name": "FanotifyAsyncConsumer", + "cpu": { + "mean": 0.0002492321603525497 + } + }, + { + "name": "FanotifyConsumer", + "cpu": { + "mean": 0.03793164685246193 + } + }, + { + "name": "RulesEngineThread", + "cpu": { + "mean": 0.02721466395730229 + } + } + ], + "event_filter": { + "active_global_count": 0, + "active_user_count": 0 + }, + "uptime": { + "endpoint": 26882599, + "system": 26883343 + } + } + }, + "ecs": { + "version": "1.11.0" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "endpoint.metrics" + }, + "elastic": { + "agent": { + "id": "123" + } + }, + "host": { + "hostname": "123", + "os": { + "Ext": { + "variant": "Debian" + }, + "kernel": "5.10.0-21-cloud-amd64 #1 SMP Debian 5.10.162-1 (2023-01-21)", + "name": "Linux", + "family": "debian", + "type": "linux", + "version": "11.8", + "platform": "debian", + "full": "Debian 11.8" + }, + "ip": [ + "127.0.0.1", + "::1" + ], + "name": "123", + "id": "123", + "mac": [ + "aa:aa:aa:aa:aa:aa" + ], + "architecture": "x86_64" + }, + "event": { + "agent_id_status": "verified", + "sequence": 16194584, + "ingested": "2024-01-26T15:06:52Z", + "created": "2024-01-26T15:06:50.254822749Z", + "kind": "metric", + "module": "endpoint", + "action": "endpoint_metrics", + "id": "123", + "category": [ + "host" + ], + "type": [ + "info" + ], + "dataset": "endpoint.metrics" + }, + "message": "Endpoint metrics" + }, + { + "agent": { + "build": { + "original": "version: 8.6.0, compiled: Mon Jan 2 23:00:00 2023, branch: 8.6, commit: e2d09ff1b8e49bfb5f8940d317eb4ac96672d956" + }, + "id": "123", + "type": "endpoint", + "version": "8.6.0" + }, + "@timestamp": "2024-01-26T14:38:30.628421712Z", + "Endpoint": { + "metrics": { + "system_impact": [ + { + "process": { + "executable": "/usr/bin/amazon-ssm-agent" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 36 + }, + "process_events": { + "week_idle_ms": 5, + "week_ms": 12487 + }, + "overall": { + "week_idle_ms": 5, + "week_ms": 12523 + } + }, + { + "process": { + "executable": "/usr/bin/ps" + }, + "process_events": { + "week_idle_ms": 8692, + "week_ms": 10288 + }, + "overall": { + "week_idle_ms": 8692, + "week_ms": 10288 + } + }, + { + "process": { + "executable": "/usr/sbin/sshd" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 116 + }, + "process_events": { + "week_idle_ms": 3621, + "week_ms": 7204 + }, + "network_events": { + "week_idle_ms": 97, + "week_ms": 0 + }, + "overall": { + "week_idle_ms": 3718, + "week_ms": 7320 + } + }, + { + "process": { + "executable": "/usr/sbin/dhclient-script" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 430 + }, + "process_events": { + "week_idle_ms": 152054, + "week_ms": 6229 + }, + "overall": { + "week_idle_ms": 152054, + "week_ms": 6659 + } + }, + { + "process": { + "executable": "/usr/sbin/crond" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 73 + }, + "process_events": { + "week_idle_ms": 9491, + "week_ms": 2060 + }, + "overall": { + "week_idle_ms": 9491, + "week_ms": 2133 + } + }, + { + "file_events": { + "week_idle_ms": 80958, + "week_ms": 1963 + }, + "process": { + "executable": "/usr/lib/systemd/systemd" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 23 + }, + "process_events": { + "week_idle_ms": 6, + "week_ms": 7 + }, + "overall": { + "week_idle_ms": 80964, + "week_ms": 1993 + } + }, + { + "process": { + "executable": "/bin/logger" + }, + "process_events": { + "week_idle_ms": 25156, + "week_ms": 1097 + }, + "overall": { + "week_idle_ms": 25156, + "week_ms": 1097 + } + }, + { + "file_events": { + "week_idle_ms": 12, + "week_ms": 0 + }, + "process": { + "executable": "/usr/lib64/sa/sadc" + }, + "process_events": { + "week_idle_ms": 4182, + "week_ms": 1005 + }, + "overall": { + "week_idle_ms": 4194, + "week_ms": 1005 + } + }, + { + "process": { + "executable": "unknown" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 981 + }, + "overall": { + "week_idle_ms": 0, + "week_ms": 981 + } + }, + { + "process": { + "executable": "/usr/bin/date" + }, + "process_events": { + "week_idle_ms": 8682, + "week_ms": 856 + }, + "overall": { + "week_idle_ms": 8682, + "week_ms": 856 + } + }, + { + "file_events": { + "week_idle_ms": 89949, + "week_ms": 835 + }, + "process": { + "executable": "/usr/lib/systemd/systemd-logind (deleted)" + }, + "overall": { + "week_idle_ms": 89949, + "week_ms": 835 + } + }, + { + "process": { + "executable": "/bin/cat" + }, + "process_events": { + "week_idle_ms": 28801, + "week_ms": 754 + }, + "overall": { + "week_idle_ms": 28801, + "week_ms": 754 + } + }, + { + "process": { + "executable": "/usr/lib64/sa/sa1" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 69 + }, + "process_events": { + "week_idle_ms": 7188, + "week_ms": 637 + }, + "overall": { + "week_idle_ms": 7188, + "week_ms": 706 + } + }, + { + "process": { + "executable": "/bin/cut" + }, + "process_events": { + "week_idle_ms": 7258, + "week_ms": 660 + }, + "overall": { + "week_idle_ms": 7258, + "week_ms": 660 + } + }, + { + "process": { + "executable": "/bin/ipcalc" + }, + "process_events": { + "week_idle_ms": 6603, + "week_ms": 477 + }, + "overall": { + "week_idle_ms": 6603, + "week_ms": 477 + } + }, + { + "process": { + "executable": "/sbin/ip" + }, + "process_events": { + "week_idle_ms": 14654, + "week_ms": 470 + }, + "overall": { + "week_idle_ms": 14654, + "week_ms": 470 + } + }, + { + "process": { + "executable": "/bin/curl" + }, + "process_events": { + "week_idle_ms": 4696, + "week_ms": 432 + }, + "network_events": { + "week_idle_ms": 3120, + "week_ms": 20 + }, + "overall": { + "week_idle_ms": 7816, + "week_ms": 452 + } + }, + { + "process": { + "executable": "/bin/run-parts" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 66 + }, + "process_events": { + "week_idle_ms": 4219, + "week_ms": 354 + }, + "overall": { + "week_idle_ms": 4219, + "week_ms": 420 + } + }, + { + "process": { + "executable": "/usr/sbin/dhclient" + }, + "process_events": { + "week_idle_ms": 3, + "week_ms": 326 + }, + "overall": { + "week_idle_ms": 3, + "week_ms": 326 + } + }, + { + "process": { + "executable": "/bin/sh" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 51 + }, + "process_events": { + "week_idle_ms": 2569, + "week_ms": 239 + }, + "overall": { + "week_idle_ms": 2569, + "week_ms": 290 + } + } + ], + "memory": { + "endpoint": { + "private": { + "mean": 119318921, + "latest": 136073216 + } + } + }, + "disks": [ + { + "total": 0, + "free": 0, + "device": "sysfs", + "mount": "/sys", + "fstype": "sysfs" + }, + { + "total": 0, + "free": 0, + "device": "proc", + "mount": "/proc", + "fstype": "proc" + }, + { + "total": 4157120512, + "free": 4157120512, + "device": "devtmpfs", + "mount": "/dev", + "fstype": "devtmpfs" + }, + { + "total": 0, + "free": 0, + "device": "securityfs", + "mount": "/sys/kernel/security", + "fstype": "securityfs" + }, + { + "total": 4166328320, + "free": 4166328320, + "device": "tmpfs", + "mount": "/dev/shm", + "fstype": "tmpfs" + }, + { + "total": 0, + "free": 0, + "device": "devpts", + "mount": "/dev/pts", + "fstype": "devpts" + }, + { + "total": 4166328320, + "free": 4165955584, + "device": "tmpfs", + "mount": "/run", + "fstype": "tmpfs" + }, + { + "total": 4166328320, + "free": 4166328320, + "device": "tmpfs", + "mount": "/sys/fs/cgroup", + "fstype": "tmpfs" + }, + { + "total": 0, + "free": 0, + "device": "cgroup", + "mount": "/sys/fs/cgroup/systemd", + "fstype": "cgroup" + }, + { + "total": 0, + "free": 0, + "device": "pstore", + "mount": "/sys/fs/pstore", + "fstype": "pstore" + }, + { + "total": 0, + "free": 0, + "device": "cgroup", + "mount": "/sys/fs/cgroup/devices", + "fstype": "cgroup" + }, + { + "total": 0, + "free": 0, + "device": "cgroup", + "mount": "/sys/fs/cgroup/cpu,cpuacct", + "fstype": "cgroup" + }, + { + "total": 0, + "free": 0, + "device": "cgroup", + "mount": "/sys/fs/cgroup/perf_event", + "fstype": "cgroup" + }, + { + "total": 0, + "free": 0, + "device": "cgroup", + "mount": "/sys/fs/cgroup/pids", + "fstype": "cgroup" + }, + { + "total": 0, + "free": 0, + "device": "cgroup", + "mount": "/sys/fs/cgroup/blkio", + "fstype": "cgroup" + }, + { + "total": 0, + "free": 0, + "device": "cgroup", + "mount": "/sys/fs/cgroup/net_cls,net_prio", + "fstype": "cgroup" + }, + { + "total": 0, + "free": 0, + "device": "cgroup", + "mount": "/sys/fs/cgroup/freezer", + "fstype": "cgroup" + }, + { + "total": 0, + "free": 0, + "device": "cgroup", + "mount": "/sys/fs/cgroup/memory", + "fstype": "cgroup" + }, + { + "total": 0, + "free": 0, + "device": "cgroup", + "mount": "/sys/fs/cgroup/cpuset", + "fstype": "cgroup" + }, + { + "total": 0, + "free": 0, + "device": "cgroup", + "mount": "/sys/fs/cgroup/hugetlb", + "fstype": "cgroup" + }, + { + "endpoint_drive": true, + "total": 8577331200, + "free": 1841799168, + "device": "/dev/xvda1", + "mount": "/", + "fstype": "xfs" + }, + { + "total": 0, + "free": 0, + "device": "mqueue", + "mount": "/dev/mqueue", + "fstype": "mqueue" + }, + { + "total": 0, + "free": 0, + "device": "hugetlbfs", + "mount": "/dev/hugepages", + "fstype": "hugetlbfs" + }, + { + "total": 0, + "free": 0, + "device": "debugfs", + "mount": "/sys/kernel/debug", + "fstype": "debugfs" + }, + { + "total": 0, + "free": 0, + "device": "sunrpc", + "mount": "/var/lib/nfs/rpc_pipefs", + "fstype": "rpc_pipefs" + }, + { + "total": 0, + "free": 0, + "device": "systemd-1", + "mount": "/proc/sys/fs/binfmt_misc", + "fstype": "autofs" + }, + { + "total": 0, + "free": 0, + "device": "binfmt_misc", + "mount": "/proc/sys/fs/binfmt_misc", + "fstype": "binfmt_misc" + }, + { + "total": 0, + "free": 0, + "device": "tracefs", + "mount": "/sys/kernel/debug/tracing", + "fstype": "tracefs" + }, + { + "total": 0, + "free": 0, + "device": "none", + "mount": "/sys/fs/bpf", + "fstype": "bpf" + } + ], + "documents_volume": { + "file_events": { + "suppressed_count": 3928398, + "suppressed_bytes": 0, + "sent_count": 0, + "sent_bytes": 0 + }, + "process_events": { + "suppressed_count": 0, + "suppressed_bytes": 0, + "sent_count": 5689500, + "sent_bytes": 12905842838 + }, + "network_events": { + "suppressed_count": 23529207, + "suppressed_bytes": 0, + "sent_count": 0, + "sent_bytes": 0 + }, + "overall": { + "suppressed_count": 27457605, + "suppressed_bytes": 0, + "sent_count": 5689500, + "sent_bytes": 12905842838 + } + }, + "malicious_behavior_rules": [ + { + "id": "123", + "endpoint_uptime_percent": 0 + } ], + "cpu": { + "endpoint": { + "histogram": { + "counts": [ + 3439059, + 129, + 15, + 0, + 5, + 6, + 4, + 0, + 3, + 4, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "values": [ + 5, + 10, + 15, + 20, + 25, + 30, + 35, + 40, + 45, + 50, + 55, + 60, + 65, + 70, + 75, + 80, + 85, + 90, + 95, + 100 + ] + }, + "mean": 0.2374646952923115, + "latest": 0.1 + } + }, + "threads": [ + { + "name": "Cron", + "cpu": { + "mean": 0.00252963994365742 + } + }, + { + "name": "FileLogThread", + "cpu": { + "mean": 0.0006221101490102125 + } + }, + { + "name": "LoggingLimitThread", + "cpu": { + "mean": 0.0003625542590258192 + } + }, + { + "name": "DocumentLoggingThread", + "cpu": { + "mean": 0.001359578471346822 + } + }, + { + "name": "DocumentLoggingMaintenance", + "cpu": { + "mean": 0.0001029983690414259 + } + }, + { + "name": "BulkConsumerThread", + "cpu": { + "mean": 0.009533529038474382 + } + }, + { + "name": "DocumentLoggingConsumerThread", + "cpu": { + "mean": 0.1315907162873257 + } + }, + { + "name": "DocumentLoggingLimitThread", + "cpu": { + "mean": 0.0002471960856994222 + } + }, + { + "name": "ArtifactManifestDownload", + "cpu": { + "mean": 0.001561455274668017 + } + }, + { + "name": "PolicyReloadThread", + "cpu": { + "mean": 0.00001235980428497111 + } + }, + { + "name": "PerformanceMonitorWorkerThread", + "cpu": { + "mean": 0.004639046541625823 + } + }, + { + "name": "MetadataThread", + "cpu": { + "mean": 0 + } + }, + { + "name": "EventsQueueThread", + "cpu": { + "mean": 0.05644722616946305 + } + }, + { + "name": "DelayedAlertEnrichment", + "cpu": { + "mean": 0 + } + }, + { + "name": "MaintainProcessMap", + "cpu": { + "mean": 0.002167085684631601 + } + }, + { + "name": "FileScoreThread", + "cpu": { + "mean": 0.002241244510341427 + } + }, + { + "name": "DiagnosticMalwareThread", + "cpu": { + "mean": 0 + } + }, + { + "name": "QuarantineManagerWorkerThread", + "cpu": { + "mean": 0.002014648098450291 + } + }, + { + "name": "EventProcessingThread", + "cpu": { + "mean": 0.00583794755726802 + } + }, + { + "name": "HostIsolationMonitorThread", + "cpu": { + "mean": 0.0008281068870930642 + } + }, + { + "name": "MountMonitor", + "cpu": { + "mean": 0.003407186047890369 + } + }, + { + "name": "responseActionsUploadThread", + "cpu": { + "mean": 0.00208468698939846 + } + }, + { + "name": "responseActionsProcessThread", + "cpu": { + "mean": 0.002105286663206745 + } + }, + { + "name": "serviceCommsThread", + "cpu": { + "mean": 0 + } + }, + { + "name": "grpcConnectionManagerThread", + "cpu": { + "mean": 0.002344242879382854 + } + }, + { + "name": "checkinAPIThread", + "cpu": { + "mean": 0.0002513162482505196 + } + }, + { + "name": "actionsAPIThread", + "cpu": { + "mean": 0.0004985125580051291 + } + }, + { + "name": "stateReportThread", + "cpu": { + "mean": 0.004400094313632049 + } + }, + { + "name": "EventsLoopThread", + "cpu": { + "mean": 0.008866112391817567 + } + }, + { + "name": "FanotifyWatchdog", + "cpu": { + "mean": 0.0002142369165309078 + } + }, + { + "name": "FanotifySyncConsumer", + "cpu": { + "mean": 0.001042344997736917 + } + }, + { + "name": "FanotifyAsyncConsumer", + "cpu": { + "mean": 0 + } + }, + { + "name": "FanotifyConsumer", + "cpu": { + "mean": 0.03632551717409642 + } + }, + { + "name": "RulesEngineThread", + "cpu": { + "mean": 0.03752853985923151 + } + } + ], + "event_filter": { + "active_global_count": 0, + "active_user_count": 0 + }, + "uptime": { + "endpoint": 24272228, + "system": 29719735 + } + } + }, + "ecs": { + "version": "1.11.0" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "endpoint.metrics" + }, + "elastic": { + "agent": { + "id": "123" + } + }, + "host": { + "hostname": "123", + "os": { + "Ext": { + "variant": "Amazon Linux" + }, + "kernel": "5.10.102-99.473.amzn2.x86_64 #1 SMP Wed Mar 2 19:14:12 UTC 2022", + "name": "Linux", + "family": "amazon linux", + "type": "linux", + "version": "2", + "platform": "amazon linux", + "full": "Amazon Linux 2" + }, + "ip": [ + "127.0.0.1", + "::1" + ], + "name": "123", + "id": "123", + "mac": [ + "aa:aa:aa:aa:aa:aa" + ], + "architecture": "x86_64" + }, + "event": { + "agent_id_status": "verified", + "sequence": 33235745, + "ingested": "2024-01-26T14:38:32Z", + "created": "2024-01-26T14:38:30.628421712Z", + "kind": "metric", + "module": "endpoint", + "action": "endpoint_metrics", + "id": "123", + "category": [ + "host" + ], + "type": [ + "info" + ], + "dataset": "endpoint.metrics" + }, + "message": "Endpoint metrics" + }, + { + "agent": { + "build": { + "original": "version: 8.6.0, compiled: Mon Jan 2 23:00:00 2023, branch: 8.6, commit: e2d09ff1b8e49bfb5f8940d317eb4ac96672d956" + }, + "id": "123", + "type": "endpoint", + "version": "8.6.0" + }, + "@timestamp": "2024-01-26T14:35:43.564911752Z", + "Endpoint": { + "metrics": { + "system_impact": [ + { + "process": { + "executable": "/usr/bin/amazon-ssm-agent" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 51 + }, + "process_events": { + "week_idle_ms": 2, + "week_ms": 8914 + }, + "overall": { + "week_idle_ms": 2, + "week_ms": 8965 + } + }, + { + "process": { + "executable": "/usr/bin/ps" + }, + "process_events": { + "week_idle_ms": 7543, + "week_ms": 6667 + }, + "overall": { + "week_idle_ms": 7543, + "week_ms": 6667 + } + }, + { + "process": { + "executable": "/usr/sbin/sshd" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 51 + }, + "process_events": { + "week_idle_ms": 3596, + "week_ms": 4596 + }, + "network_events": { + "week_idle_ms": 85, + "week_ms": 5 + }, + "overall": { + "week_idle_ms": 3681, + "week_ms": 4652 + } + }, + { + "process": { + "executable": "/usr/sbin/dhclient-script" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 380 + }, + "process_events": { + "week_idle_ms": 105790, + "week_ms": 3877 + }, + "overall": { + "week_idle_ms": 105790, + "week_ms": 4257 + } + }, + { + "process": { + "executable": "/usr/sbin/crond" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 129 + }, + "process_events": { + "week_idle_ms": 8062, + "week_ms": 1903 + }, + "overall": { + "week_idle_ms": 8062, + "week_ms": 2032 + } + }, + { + "file_events": { + "week_idle_ms": 69051, + "week_ms": 1544 + }, + "process": { + "executable": "/usr/lib/systemd/systemd" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 15 + }, + "process_events": { + "week_idle_ms": 31, + "week_ms": 9 + }, + "overall": { + "week_idle_ms": 69082, + "week_ms": 1568 + } + }, + { + "process": { + "executable": "unknown" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 979 + }, + "overall": { + "week_idle_ms": 0, + "week_ms": 979 + } + }, + { + "file_events": { + "week_idle_ms": 29, + "week_ms": 0 + }, + "process": { + "executable": "/usr/lib64/sa/sadc" + }, + "process_events": { + "week_idle_ms": 5107, + "week_ms": 917 + }, + "overall": { + "week_idle_ms": 5136, + "week_ms": 917 + } + }, + { + "process": { + "executable": "/bin/logger" + }, + "process_events": { + "week_idle_ms": 20585, + "week_ms": 880 + }, + "overall": { + "week_idle_ms": 20585, + "week_ms": 880 + } + }, + { + "file_events": { + "week_idle_ms": 72520, + "week_ms": 852 + }, + "process": { + "executable": "/usr/lib/systemd/systemd-logind (deleted)" + }, + "overall": { + "week_idle_ms": 72520, + "week_ms": 852 + } + }, + { + "process": { + "executable": "/usr/bin/date" + }, + "process_events": { + "week_idle_ms": 9862, + "week_ms": 808 + }, + "overall": { + "week_idle_ms": 9862, + "week_ms": 808 + } + }, + { + "process": { + "executable": "/bin/cat" + }, + "process_events": { + "week_idle_ms": 17468, + "week_ms": 701 + }, + "overall": { + "week_idle_ms": 17468, + "week_ms": 701 + } + }, + { + "process": { + "executable": "/usr/lib64/sa/sa1" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 67 + }, + "process_events": { + "week_idle_ms": 7451, + "week_ms": 618 + }, + "overall": { + "week_idle_ms": 7451, + "week_ms": 685 + } + }, + { + "process": { + "executable": "/bin/curl" + }, + "process_events": { + "week_idle_ms": 5615, + "week_ms": 403 + }, + "network_events": { + "week_idle_ms": 4128, + "week_ms": 6 + }, + "overall": { + "week_idle_ms": 9743, + "week_ms": 409 + } + }, + { + "process": { + "executable": "/bin/run-parts" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 116 + }, + "process_events": { + "week_idle_ms": 3770, + "week_ms": 260 + }, + "overall": { + "week_idle_ms": 3770, + "week_ms": 376 + } + }, + { + "process": { + "executable": "/usr/sbin/dhclient" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 36 + }, + "process_events": { + "week_idle_ms": 1, + "week_ms": 278 + }, + "overall": { + "week_idle_ms": 1, + "week_ms": 314 + } + }, + { + "process": { + "executable": "/bin/cut" + }, + "process_events": { + "week_idle_ms": 5067, + "week_ms": 313 + }, + "overall": { + "week_idle_ms": 5067, + "week_ms": 313 + } + }, + { + "process": { + "executable": "/sbin/ip" + }, + "process_events": { + "week_idle_ms": 10369, + "week_ms": 286 + }, + "overall": { + "week_idle_ms": 10369, + "week_ms": 286 + } + }, + { + "process": { + "executable": "/bin/sh" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 20 + }, + "process_events": { + "week_idle_ms": 2397, + "week_ms": 208 + }, + "overall": { + "week_idle_ms": 2397, + "week_ms": 228 + } + }, + { + "process": { + "executable": "/bin/ipcalc" + }, + "process_events": { + "week_idle_ms": 4827, + "week_ms": 201 + }, + "overall": { + "week_idle_ms": 4827, + "week_ms": 201 + } + } + ], + "memory": { + "endpoint": { + "private": { + "mean": 118266395, + "latest": 135163904 + } + } + }, + "disks": [ + { + "total": 0, + "free": 0, + "device": "sysfs", + "mount": "/sys", + "fstype": "sysfs" + }, + { + "total": 0, + "free": 0, + "device": "proc", + "mount": "/proc", + "fstype": "proc" + }, + { + "total": 4157120512, + "free": 4157120512, + "device": "devtmpfs", + "mount": "/dev", + "fstype": "devtmpfs" + }, + { + "total": 0, + "free": 0, + "device": "securityfs", + "mount": "/sys/kernel/security", + "fstype": "securityfs" + }, + { + "total": 4166328320, + "free": 4166328320, + "device": "tmpfs", + "mount": "/dev/shm", + "fstype": "tmpfs" + }, + { + "total": 0, + "free": 0, + "device": "devpts", + "mount": "/dev/pts", + "fstype": "devpts" + }, + { + "total": 4166328320, + "free": 4165955584, + "device": "tmpfs", + "mount": "/run", + "fstype": "tmpfs" + }, + { + "total": 4166328320, + "free": 4166328320, + "device": "tmpfs", + "mount": "/sys/fs/cgroup", + "fstype": "tmpfs" + }, + { + "total": 0, + "free": 0, + "device": "cgroup", + "mount": "/sys/fs/cgroup/systemd", + "fstype": "cgroup" + }, + { + "total": 0, + "free": 0, + "device": "pstore", + "mount": "/sys/fs/pstore", + "fstype": "pstore" + }, + { + "total": 0, + "free": 0, + "device": "cgroup", + "mount": "/sys/fs/cgroup/blkio", + "fstype": "cgroup" + }, + { + "total": 0, + "free": 0, + "device": "cgroup", + "mount": "/sys/fs/cgroup/cpu,cpuacct", + "fstype": "cgroup" + }, + { + "total": 0, + "free": 0, + "device": "cgroup", + "mount": "/sys/fs/cgroup/net_cls,net_prio", + "fstype": "cgroup" + }, + { + "total": 0, + "free": 0, + "device": "cgroup", + "mount": "/sys/fs/cgroup/devices", + "fstype": "cgroup" + }, + { + "total": 0, + "free": 0, + "device": "cgroup", + "mount": "/sys/fs/cgroup/cpuset", + "fstype": "cgroup" + }, + { + "total": 0, + "free": 0, + "device": "cgroup", + "mount": "/sys/fs/cgroup/hugetlb", + "fstype": "cgroup" + }, + { + "total": 0, + "free": 0, + "device": "cgroup", + "mount": "/sys/fs/cgroup/pids", + "fstype": "cgroup" + }, + { + "total": 0, + "free": 0, + "device": "cgroup", + "mount": "/sys/fs/cgroup/freezer", + "fstype": "cgroup" + }, + { + "total": 0, + "free": 0, + "device": "cgroup", + "mount": "/sys/fs/cgroup/memory", + "fstype": "cgroup" + }, + { + "total": 0, + "free": 0, + "device": "cgroup", + "mount": "/sys/fs/cgroup/perf_event", + "fstype": "cgroup" + }, + { + "endpoint_drive": true, + "total": 8577331200, + "free": 1908756480, + "device": "/dev/xvda1", + "mount": "/", + "fstype": "xfs" + }, + { + "total": 0, + "free": 0, + "device": "hugetlbfs", + "mount": "/dev/hugepages", + "fstype": "hugetlbfs" + }, + { + "total": 0, + "free": 0, + "device": "mqueue", + "mount": "/dev/mqueue", + "fstype": "mqueue" + }, + { + "total": 0, + "free": 0, + "device": "debugfs", + "mount": "/sys/kernel/debug", + "fstype": "debugfs" + }, + { + "total": 0, + "free": 0, + "device": "sunrpc", + "mount": "/var/lib/nfs/rpc_pipefs", + "fstype": "rpc_pipefs" + }, + { + "total": 0, + "free": 0, + "device": "systemd-1", + "mount": "/proc/sys/fs/binfmt_misc", + "fstype": "autofs" + }, + { + "total": 0, + "free": 0, + "device": "binfmt_misc", + "mount": "/proc/sys/fs/binfmt_misc", + "fstype": "binfmt_misc" + }, + { + "total": 0, + "free": 0, + "device": "tracefs", + "mount": "/sys/kernel/debug/tracing", + "fstype": "tracefs" + }, + { + "total": 0, + "free": 0, + "device": "none", + "mount": "/sys/fs/bpf", + "fstype": "bpf" + } + ], + "documents_volume": { + "file_events": { + "suppressed_count": 3926292, + "suppressed_bytes": 0, + "sent_count": 0, + "sent_bytes": 0 + }, + "process_events": { + "suppressed_count": 0, + "suppressed_bytes": 0, + "sent_count": 5737776, + "sent_bytes": 12992145486 + }, + "network_events": { + "suppressed_count": 24813200, + "suppressed_bytes": 0, + "sent_count": 0, + "sent_bytes": 0 + }, + "overall": { + "suppressed_count": 28739492, + "suppressed_bytes": 0, + "sent_count": 5737776, + "sent_bytes": 12992145486 + } + }, + "malicious_behavior_rules": [ + { + "id": "123", + "endpoint_uptime_percent": 0 + } + ], + "cpu": { + "endpoint": { + "histogram": { + "counts": [ + 3147842, + 87, + 11, + 10, + 9, + 5, + 10, + 6, + 1, + 2, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "values": [ + 5, + 10, + 15, + 20, + 25, + 30, + 35, + 40, + 45, + 50, + 55, + 60, + 65, + 70, + 75, + 80, + 85, + 90, + 95, + 100 + ] + }, + "mean": 0.2338436071605115, + "latest": 0.9000000000000001 + } + }, + "threads": [ + { + "name": "Cron", + "cpu": { + "mean": 0.002129690752013577 + } + }, + { + "name": "FileLogThread", + "cpu": { + "mean": 0.0004778416387496615 + } + }, + { + "name": "LoggingLimitThread", + "cpu": { + "mean": 0.0002965913619825484 + } + }, + { + "name": "DocumentLoggingThread", + "cpu": { + "mean": 0.001116336931906537 + } + }, + { + "name": "DocumentLoggingMaintenance", + "cpu": { + "mean": 0.00009062513838355648 + } + }, + { + "name": "BulkConsumerThread", + "cpu": { + "mean": 0.008971888699972091 + } + }, + { + "name": "DocumentLoggingConsumerThread", + "cpu": { + "mean": 0.1281974968924846 + } + }, + { + "name": "DocumentLoggingLimitThread", + "cpu": { + "mean": 0.000205966223598992 + } + }, + { + "name": "ArtifactManifestDownload", + "cpu": { + "mean": 0.001515911405688581 + } + }, + { + "name": "PolicyReloadThread", + "cpu": { + "mean": 0.00000823864894395968 + } + }, + { + "name": "PerformanceMonitorWorkerThread", + "cpu": { + "mean": 0.003917477572852828 + } + }, + { + "name": "MetadataThread", + "cpu": { + "mean": 0.00000411932447197984 + } + }, + { + "name": "EventsQueueThread", + "cpu": { + "mean": 0.04977791691940438 + } + }, + { + "name": "DelayedAlertEnrichment", + "cpu": { + "mean": 0 + } + }, + { + "name": "MaintainProcessMap", + "cpu": { + "mean": 0.001837218714503009 + } + }, + { + "name": "FileScoreThread", + "cpu": { + "mean": 0.001923724528414585 + } + }, + { + "name": "DiagnosticMalwareThread", + "cpu": { + "mean": 0 + } + }, + { + "name": "QuarantineManagerWorkerThread", + "cpu": { + "mean": 0.001557104650408379 + } + }, + { + "name": "EventProcessingThread", + "cpu": { + "mean": 0.00591534994176305 + } + }, + { + "name": "HostIsolationMonitorThread", + "cpu": { + "mean": 0.0006508532665728148 + } + }, + { + "name": "MountMonitor", + "cpu": { + "mean": 0.002533384550267602 + } + }, + { + "name": "responseActionsUploadThread", + "cpu": { + "mean": 0.001771309522951331 + } + }, + { + "name": "responseActionsProcessThread", + "cpu": { + "mean": 0.001783667496367271 + } + }, + { + "name": "serviceCommsThread", + "cpu": { + "mean": 0 + } + }, + { + "name": "grpcConnectionManagerThread", + "cpu": { + "mean": 0.001956679124190424 + } + }, + { + "name": "checkinAPIThread", + "cpu": { + "mean": 0.0002389210358973501 + } + }, + { + "name": "actionsAPIThread", + "cpu": { + "mean": 0.0003913361794870389 + } + }, + { + "name": "stateReportThread", + "cpu": { + "mean": 0.003550860912819238 + } + }, + { + "name": "EventsLoopThread", + "cpu": { + "mean": 0.0077690568352644 + } + }, + { + "name": "FanotifyWatchdog", + "cpu": { + "mean": 0.0001647732096556607 + } + }, + { + "name": "FanotifySyncConsumer", + "cpu": { + "mean": 0.001103980504692926 + } + }, + { + "name": "FanotifyAsyncConsumer", + "cpu": { + "mean": 0 + } + }, + { + "name": "FanotifyConsumer", + "cpu": { + "mean": 0.02717934093270123 + } + }, + { + "name": "RulesEngineThread", + "cpu": { + "mean": 0.03744471189424889 + } + } + ], + "event_filter": { + "active_global_count": 0, + "active_user_count": 0 + }, + "uptime": { + "endpoint": 24275825, + "system": 29714870 + } + } + }, + "ecs": { + "version": "1.11.0" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "endpoint.metrics" + }, + "elastic": { + "agent": { + "id": "123" + } + }, + "host": { + "hostname": "123", + "os": { + "Ext": { + "variant": "Amazon Linux" + }, + "kernel": "5.10.102-99.473.amzn2.x86_64 #1 SMP Wed Mar 2 19:14:12 UTC 2022", + "name": "Linux", + "family": "amazon linux", + "type": "linux", + "version": "2", + "platform": "amazon linux", + "full": "Amazon Linux 2" + }, + "ip": [ + "127.0.0.1", + "::1" + ], + "name": "123", + "id": "123", + "mac": [ + "aa:aa:aa:aa:aa:aa" + ], + "architecture": "x86_64" + }, + "event": { + "agent_id_status": "verified", + "sequence": 34569822, + "ingested": "2024-01-26T14:35:45Z", + "created": "2024-01-26T14:35:43.564911752Z", + "kind": "metric", + "module": "endpoint", + "action": "endpoint_metrics", + "id": "123", + "category": [ + "host" + ], + "type": [ + "info" + ], + "dataset": "endpoint.metrics" + }, + "message": "Endpoint metrics" + } +] diff --git a/x-pack/plugins/security_solution/server/integration_tests/__mocks__/endpoint-policy.json b/x-pack/plugins/security_solution/server/integration_tests/__mocks__/endpoint-policy.json new file mode 100644 index 0000000000000..7085e3efa6146 --- /dev/null +++ b/x-pack/plugins/security_solution/server/integration_tests/__mocks__/endpoint-policy.json @@ -0,0 +1,1205 @@ +[ + { + "agent": { + "build": { + "original": "version: 8.6.2, compiled: Fri Feb 10 14:00:00 2023, branch: 8.6, commit: eaee25e73cc58a52387e50d5bce542ec8965f638" + }, + "id": "123", + "type": "endpoint", + "version": "8.6.2" + }, + "@timestamp": "2024-01-26T15:06:50.25217928Z", + "Endpoint": { + "configuration": { + "isolation": false + }, + "state": { + "isolation": false + }, + "policy": { + "applied": { + "response": { + "configurations": { + "behavior_protection": { + "concerned_actions": [ + "agent_connectivity", + "load_config", + "workflow", + "download_global_artifacts", + "download_user_artifacts", + "configure_file_events", + "configure_network_events", + "configure_process_events", + "configure_malicious_behavior" + ], + "status": "success" + }, + "streaming": { + "concerned_actions": [ + "agent_connectivity", + "load_config", + "configure_output", + "workflow" + ], + "status": "success" + }, + "malware": { + "concerned_actions": [ + "agent_connectivity", + "load_config", + "workflow", + "download_global_artifacts", + "download_user_artifacts", + "configure_malware", + "detect_process_events", + "detect_file_write_events", + "configure_user_notification" + ], + "status": "success" + }, + "logging": { + "concerned_actions": [ + "agent_connectivity", + "load_config", + "configure_logging", + "workflow" + ], + "status": "success" + }, + "host_isolation": { + "concerned_actions": [ + "agent_connectivity", + "configure_host_isolation", + "load_config", + "workflow" + ], + "status": "success" + }, + "events": { + "concerned_actions": [ + "agent_connectivity", + "load_config", + "workflow", + "download_global_artifacts", + "download_user_artifacts", + "detect_process_events", + "detect_file_write_events", + "detect_network_events", + "configure_file_events", + "configure_network_events", + "configure_process_events" + ], + "status": "success" + }, + "memory_protection": { + "concerned_actions": [ + "agent_connectivity", + "configure_memory_threat", + "configure_process_events", + "download_global_artifacts", + "download_user_artifacts", + "workflow", + "load_config", + "detect_process_events" + ], + "status": "success" + } + }, + "diagnostic": { + "behavior_protection": { + "concerned_actions": [ + "load_config", + "workflow", + "download_global_artifacts", + "download_user_artifacts", + "configure_file_events", + "configure_network_events", + "configure_process_events", + "configure_diagnostic_malicious_behavior" + ], + "status": "success" + }, + "malware": { + "concerned_actions": [ + "load_config", + "workflow", + "download_global_artifacts", + "download_user_artifacts", + "configure_diagnostic_malware", + "detect_process_events", + "detect_file_write_events" + ], + "status": "success" + }, + "memory_protection": { + "concerned_actions": [ + "load_config", + "workflow", + "download_global_artifacts", + "download_user_artifacts", + "detect_process_events", + "configure_process_events", + "configure_diagnostic_memory_threat" + ], + "status": "success" + } + } + }, + "name": "123", + "id": "123", + "actions": [ + { + "name": "configure_memory_threat", + "message": "Successfully enabled memory threat prevention with memory scanning enabled", + "status": "success" + }, + { + "name": "configure_diagnostic_memory_threat", + "message": "Successfully enabled memory threat detection with memory scanning enabled", + "status": "success" + }, + { + "name": "configure_host_isolation", + "message": "Host is not isolated", + "status": "success" + }, + { + "name": "configure_malicious_behavior", + "message": "Enabled 34 out of 34 malicious behavior rules", + "status": "success" + }, + { + "name": "configure_diagnostic_malicious_behavior", + "message": "Enabled 65 out of 65 diagnostic malicious behavior rules", + "status": "success" + }, + { + "name": "configure_user_notification", + "message": "Successfully configured user notification", + "status": "success" + }, + { + "name": "configure_malware", + "message": "Successfully enabled malware prevention", + "status": "success" + }, + { + "name": "configure_diagnostic_malware", + "message": "Successfully enabled malware detection", + "status": "success" + }, + { + "name": "configure_output", + "message": "Successfully configured output connection", + "status": "success" + }, + { + "name": "configure_logging", + "message": "Successfully configured logging", + "status": "success" + }, + { + "name": "load_config", + "message": "Successfully parsed configuration", + "status": "success" + }, + { + "name": "download_user_artifacts", + "message": "Successfully downloaded user artifacts", + "status": "success" + }, + { + "name": "download_global_artifacts", + "message": "Global artifacts are available for use", + "status": "success" + }, + { + "name": "detect_process_events", + "message": "Success enabling process events; current state is enabled", + "status": "success" + }, + { + "name": "detect_network_events", + "message": "Success enabling network events; current state is enabled", + "status": "success" + }, + { + "name": "detect_file_write_events", + "message": "Success enabling file events; current state is enabled", + "status": "success" + }, + { + "name": "configure_file_events", + "message": "Success enabling file events; current state is enabled", + "status": "success" + }, + { + "name": "configure_network_events", + "message": "Success enabling network events; current state is enabled", + "status": "success" + }, + { + "name": "configure_process_events", + "message": "Success enabling process events; current state is enabled", + "status": "success" + }, + { + "name": "agent_connectivity", + "message": "Successfully connected to Agent", + "status": "success" + }, + { + "name": "workflow", + "message": "Successfully executed all workflows", + "status": "success" + } + ], + "endpoint_policy_version": "1", + "version": "51", + "artifacts": { + "global": { + "identifiers": [ + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "diagnostic-configuration-v1" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "diagnostic-endpointelf-v1-blocklist" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "diagnostic-endpointelf-v1-exceptionlist" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "diagnostic-endpointelf-v1-model" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "diagnostic-malware-signature-v1-linux" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "diagnostic-rules-linux-v1" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "endpointelf-v1-blocklist" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "endpointelf-v1-exceptionlist" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "endpointelf-v1-model" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "global-configuration-v1" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "global-eventfilterlist-linux-v1" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "global-exceptionlist-linux" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "global-trustlist-linux-v1" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "production-malware-signature-v1-linux" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "production-rules-linux-v1" + } + ], + "version": "1.0.874" + }, + "user": { + "identifiers": [ + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "endpoint-blocklist-linux-v1" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "endpoint-eventfilterlist-linux-v1" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "endpoint-exceptionlist-linux-v1" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "endpoint-hostisolationexceptionlist-linux-v1" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b" + } + ], + "version": "1.0.9" + } + }, + "status": "success" + } + } + }, + "ecs": { + "version": "1.11.0" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "endpoint.policy" + }, + "elastic": { + "agent": { + "id": "123" + } + }, + "host": { + "hostname": "123", + "os": { + "Ext": { + "variant": "Debian" + }, + "kernel": "5.10.0-21-cloud-amd64 #1 SMP Debian 5.10.162-1 (2023-01-21)", + "name": "Linux", + "family": "debian", + "type": "linux", + "version": "11.8", + "platform": "debian", + "full": "Debian 11.8" + }, + "ip": ["127.0.0.1", "::1"], + "name": "123", + "id": "123", + "mac": ["aa:aa:aa:aa:aa:aa"], + "architecture": "x86_64" + }, + "event": { + "agent_id_status": "verified", + "sequence": 1, + "ingested": "2024-01-26T15:06:52Z", + "created": "2024-01-26T15:06:50.25217928Z", + "kind": "state", + "module": "endpoint", + "action": "endpoint_policy_response", + "id": "123", + "category": ["host"], + "type": ["change"], + "dataset": "endpoint.policy" + }, + "message": "Endpoint policy change" + }, + { + "agent": { + "build": { + "original": "version: 8.6.0, compiled: Mon Jan 2 23:00:00 2023, branch: 8.6, commit: e2d09ff1b8e49bfb5f8940d317eb4ac96672d956" + }, + "id": "123", + "type": "endpoint", + "version": "8.6.0" + }, + "@timestamp": "2024-01-26T14:38:30.627297556Z", + "Endpoint": { + "configuration": { + "isolation": false + }, + "state": { + "isolation": false + }, + "policy": { + "applied": { + "response": { + "configurations": { + "behavior_protection": { + "concerned_actions": [ + "agent_connectivity", + "load_config", + "workflow", + "download_global_artifacts", + "download_user_artifacts", + "configure_file_events", + "configure_network_events", + "configure_process_events", + "configure_malicious_behavior" + ], + "status": "success" + }, + "streaming": { + "concerned_actions": [ + "agent_connectivity", + "load_config", + "configure_output", + "workflow" + ], + "status": "success" + }, + "malware": { + "concerned_actions": [ + "agent_connectivity", + "load_config", + "workflow", + "download_global_artifacts", + "download_user_artifacts", + "configure_malware", + "detect_process_events", + "detect_file_write_events", + "configure_user_notification" + ], + "status": "success" + }, + "logging": { + "concerned_actions": [ + "agent_connectivity", + "load_config", + "configure_logging", + "workflow" + ], + "status": "success" + }, + "host_isolation": { + "concerned_actions": [ + "agent_connectivity", + "configure_host_isolation", + "load_config", + "workflow" + ], + "status": "success" + }, + "events": { + "concerned_actions": [ + "agent_connectivity", + "load_config", + "workflow", + "download_global_artifacts", + "download_user_artifacts", + "detect_process_events", + "detect_file_write_events", + "detect_network_events", + "configure_file_events", + "configure_network_events", + "configure_process_events" + ], + "status": "success" + }, + "memory_protection": { + "concerned_actions": [ + "agent_connectivity", + "configure_memory_threat", + "configure_process_events", + "download_global_artifacts", + "download_user_artifacts", + "workflow", + "load_config", + "detect_process_events" + ], + "status": "success" + } + }, + "diagnostic": { + "behavior_protection": { + "concerned_actions": [ + "load_config", + "workflow", + "download_global_artifacts", + "download_user_artifacts", + "configure_file_events", + "configure_network_events", + "configure_process_events", + "configure_diagnostic_malicious_behavior" + ], + "status": "success" + }, + "malware": { + "concerned_actions": [ + "load_config", + "workflow", + "download_global_artifacts", + "download_user_artifacts", + "configure_diagnostic_malware", + "detect_process_events", + "detect_file_write_events" + ], + "status": "success" + }, + "memory_protection": { + "concerned_actions": [ + "load_config", + "workflow", + "download_global_artifacts", + "download_user_artifacts", + "detect_process_events", + "configure_process_events", + "configure_diagnostic_memory_threat" + ], + "status": "success" + } + } + }, + "name": "123", + "id": "123", + "actions": [ + { + "name": "configure_memory_threat", + "message": "Successfully enabled memory threat prevention with memory scanning enabled", + "status": "success" + }, + { + "name": "configure_diagnostic_memory_threat", + "message": "Successfully enabled memory threat detection with memory scanning enabled", + "status": "success" + }, + { + "name": "configure_host_isolation", + "message": "Host is not isolated", + "status": "success" + }, + { + "name": "configure_malicious_behavior", + "message": "Enabled 34 out of 34 malicious behavior rules", + "status": "success" + }, + { + "name": "configure_diagnostic_malicious_behavior", + "message": "Enabled 65 out of 65 diagnostic malicious behavior rules", + "status": "success" + }, + { + "name": "configure_user_notification", + "message": "Successfully configured user notification", + "status": "success" + }, + { + "name": "configure_malware", + "message": "Successfully enabled malware prevention", + "status": "success" + }, + { + "name": "configure_diagnostic_malware", + "message": "Successfully enabled malware detection", + "status": "success" + }, + { + "name": "configure_output", + "message": "Successfully configured output connection", + "status": "success" + }, + { + "name": "configure_logging", + "message": "Successfully configured logging", + "status": "success" + }, + { + "name": "load_config", + "message": "Successfully parsed configuration", + "status": "success" + }, + { + "name": "download_user_artifacts", + "message": "Successfully downloaded user artifacts", + "status": "success" + }, + { + "name": "download_global_artifacts", + "message": "Global artifacts are available for use", + "status": "success" + }, + { + "name": "detect_process_events", + "message": "Success enabling process events; current state is enabled", + "status": "success" + }, + { + "name": "detect_network_events", + "message": "Success enabling network events; current state is enabled", + "status": "success" + }, + { + "name": "detect_file_write_events", + "message": "Success enabling file events; current state is enabled", + "status": "success" + }, + { + "name": "configure_file_events", + "message": "Success enabling file events; current state is enabled", + "status": "success" + }, + { + "name": "configure_network_events", + "message": "Success enabling network events; current state is enabled", + "status": "success" + }, + { + "name": "configure_process_events", + "message": "Success enabling process events; current state is enabled", + "status": "success" + }, + { + "name": "agent_connectivity", + "message": "Successfully connected to Agent", + "status": "success" + }, + { + "name": "workflow", + "message": "Successfully executed all workflows", + "status": "success" + } + ], + "endpoint_policy_version": "1", + "version": "26", + "artifacts": { + "global": { + "identifiers": [ + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "diagnostic-configuration-v1" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "diagnostic-endpointelf-v1-blocklist" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "diagnostic-endpointelf-v1-exceptionlist" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "diagnostic-endpointelf-v1-model" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "diagnostic-malware-signature-v1-linux" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "diagnostic-rules-linux-v1" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "endpointelf-v1-blocklist" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "endpointelf-v1-exceptionlist" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "endpointelf-v1-model" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "global-configuration-v1" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "global-eventfilterlist-linux-v1" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "global-exceptionlist-linux" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "global-trustlist-linux-v1" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "production-malware-signature-v1-linux" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "production-rules-linux-v1" + } + ], + "version": "1.0.874" + }, + "user": { + "identifiers": [ + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "endpoint-blocklist-linux-v1" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "endpoint-eventfilterlist-linux-v1" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "endpoint-exceptionlist-linux-v1" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "endpoint-hostisolationexceptionlist-linux-v1" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "endpoint-trustlist-linux-v1" + } + ], + "version": "1.0.12" + } + }, + "status": "success" + } + } + }, + "ecs": { + "version": "1.11.0" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "endpoint.policy" + }, + "elastic": { + "agent": { + "id": "123" + } + }, + "host": { + "hostname": "123", + "os": { + "Ext": { + "variant": "Amazon Linux" + }, + "kernel": "5.10.102-99.473.amzn2.x86_64 #1 SMP Wed Mar 2 19:14:12 UTC 2022", + "name": "Linux", + "family": "amazon linux", + "type": "linux", + "version": "2", + "platform": "amazon linux", + "full": "Amazon Linux 2" + }, + "ip": ["127.0.0.1", "::1"], + "name": "123", + "id": "123", + "mac": ["aa:aa:aa:aa:aa:aa"], + "architecture": "x86_64" + }, + "event": { + "agent_id_status": "verified", + "sequence": 1, + "ingested": "2024-01-26T14:38:32Z", + "created": "2024-01-26T14:38:30.627297556Z", + "kind": "state", + "module": "endpoint", + "action": "endpoint_policy_response", + "id": "123", + "category": ["host"], + "type": ["change"], + "dataset": "endpoint.policy" + }, + "message": "Endpoint policy change" + }, + { + "agent": { + "build": { + "original": "version: 8.6.0, compiled: Mon Jan 2 23:00:00 2023, branch: 8.6, commit: e2d09ff1b8e49bfb5f8940d317eb4ac96672d956" + }, + "id": "123", + "type": "endpoint", + "version": "8.6.0" + }, + "@timestamp": "2024-01-26T14:35:43.563903627Z", + "Endpoint": { + "configuration": { + "isolation": false + }, + "state": { + "isolation": false + }, + "policy": { + "applied": { + "response": { + "configurations": { + "behavior_protection": { + "concerned_actions": [ + "agent_connectivity", + "load_config", + "workflow", + "download_global_artifacts", + "download_user_artifacts", + "configure_file_events", + "configure_network_events", + "configure_process_events", + "configure_malicious_behavior" + ], + "status": "success" + }, + "streaming": { + "concerned_actions": [ + "agent_connectivity", + "load_config", + "configure_output", + "workflow" + ], + "status": "success" + }, + "malware": { + "concerned_actions": [ + "agent_connectivity", + "load_config", + "workflow", + "download_global_artifacts", + "download_user_artifacts", + "configure_malware", + "detect_process_events", + "detect_file_write_events", + "configure_user_notification" + ], + "status": "success" + }, + "logging": { + "concerned_actions": [ + "agent_connectivity", + "load_config", + "configure_logging", + "workflow" + ], + "status": "success" + }, + "host_isolation": { + "concerned_actions": [ + "agent_connectivity", + "configure_host_isolation", + "load_config", + "workflow" + ], + "status": "success" + }, + "events": { + "concerned_actions": [ + "agent_connectivity", + "load_config", + "workflow", + "download_global_artifacts", + "download_user_artifacts", + "detect_process_events", + "detect_file_write_events", + "detect_network_events", + "configure_file_events", + "configure_network_events", + "configure_process_events" + ], + "status": "success" + }, + "memory_protection": { + "concerned_actions": [ + "agent_connectivity", + "configure_memory_threat", + "configure_process_events", + "download_global_artifacts", + "download_user_artifacts", + "workflow", + "load_config", + "detect_process_events" + ], + "status": "success" + } + }, + "diagnostic": { + "behavior_protection": { + "concerned_actions": [ + "load_config", + "workflow", + "download_global_artifacts", + "download_user_artifacts", + "configure_file_events", + "configure_network_events", + "configure_process_events", + "configure_diagnostic_malicious_behavior" + ], + "status": "success" + }, + "malware": { + "concerned_actions": [ + "load_config", + "workflow", + "download_global_artifacts", + "download_user_artifacts", + "configure_diagnostic_malware", + "detect_process_events", + "detect_file_write_events" + ], + "status": "success" + }, + "memory_protection": { + "concerned_actions": [ + "load_config", + "workflow", + "download_global_artifacts", + "download_user_artifacts", + "detect_process_events", + "configure_process_events", + "configure_diagnostic_memory_threat" + ], + "status": "success" + } + } + }, + "name": "123", + "id": "123", + "actions": [ + { + "name": "configure_memory_threat", + "message": "Successfully enabled memory threat prevention with memory scanning enabled", + "status": "success" + }, + { + "name": "configure_diagnostic_memory_threat", + "message": "Successfully enabled memory threat detection with memory scanning enabled", + "status": "success" + }, + { + "name": "configure_host_isolation", + "message": "Host is not isolated", + "status": "success" + }, + { + "name": "configure_malicious_behavior", + "message": "Enabled 34 out of 34 malicious behavior rules", + "status": "success" + }, + { + "name": "configure_diagnostic_malicious_behavior", + "message": "Enabled 65 out of 65 diagnostic malicious behavior rules", + "status": "success" + }, + { + "name": "configure_user_notification", + "message": "Successfully configured user notification", + "status": "success" + }, + { + "name": "configure_malware", + "message": "Successfully enabled malware prevention", + "status": "success" + }, + { + "name": "configure_diagnostic_malware", + "message": "Successfully enabled malware detection", + "status": "success" + }, + { + "name": "configure_output", + "message": "Successfully configured output connection", + "status": "success" + }, + { + "name": "configure_logging", + "message": "Successfully configured logging", + "status": "success" + }, + { + "name": "load_config", + "message": "Successfully parsed configuration", + "status": "success" + }, + { + "name": "download_user_artifacts", + "message": "Successfully downloaded user artifacts", + "status": "success" + }, + { + "name": "download_global_artifacts", + "message": "Global artifacts are available for use", + "status": "success" + }, + { + "name": "detect_process_events", + "message": "Success enabling process events; current state is enabled", + "status": "success" + }, + { + "name": "detect_network_events", + "message": "Success enabling network events; current state is enabled", + "status": "success" + }, + { + "name": "detect_file_write_events", + "message": "Success enabling file events; current state is enabled", + "status": "success" + }, + { + "name": "configure_file_events", + "message": "Success enabling file events; current state is enabled", + "status": "success" + }, + { + "name": "configure_network_events", + "message": "Success enabling network events; current state is enabled", + "status": "success" + }, + { + "name": "configure_process_events", + "message": "Success enabling process events; current state is enabled", + "status": "success" + }, + { + "name": "agent_connectivity", + "message": "Successfully connected to Agent", + "status": "success" + }, + { + "name": "workflow", + "message": "Successfully executed all workflows", + "status": "success" + } + ], + "endpoint_policy_version": "1", + "version": "55", + "artifacts": { + "global": { + "identifiers": [ + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "diagnostic-configuration-v1" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "diagnostic-endpointelf-v1-blocklist" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "diagnostic-endpointelf-v1-exceptionlist" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "diagnostic-endpointelf-v1-model" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "diagnostic-malware-signature-v1-linux" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "diagnostic-rules-linux-v1" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "endpointelf-v1-blocklist" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "endpointelf-v1-exceptionlist" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "endpointelf-v1-model" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "global-configuration-v1" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "global-eventfilterlist-linux-v1" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "global-exceptionlist-linux" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "global-trustlist-linux-v1" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "production-malware-signature-v1-linux" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "production-rules-linux-v1" + } + ], + "version": "1.0.874" + }, + "user": { + "identifiers": [ + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "endpoint-blocklist-linux-v1" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "endpoint-eventfilterlist-linux-v1" + }, + { + "sha256": "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b", + "name": "endpoint-exceptionlist-linux-v1" + }, + { + "sha256": "d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658", + "name": "endpoint-hostisolationexceptionlist-linux-v1" + }, + { + "sha256": "d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658", + "name": "endpoint-trustlist-linux-v1" + } + ], + "version": "1.0.10" + } + }, + "status": "success" + } + } + }, + "ecs": { + "version": "1.11.0" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "endpoint.policy" + }, + "elastic": { + "agent": { + "id": "123" + } + }, + "host": { + "hostname": "123", + "os": { + "Ext": { + "variant": "Amazon Linux" + }, + "kernel": "5.10.102-99.473.amzn2.x86_64 #1 SMP Wed Mar 2 19:14:12 UTC 2022", + "name": "Linux", + "family": "amazon linux", + "type": "linux", + "version": "2", + "platform": "amazon linux", + "full": "Amazon Linux 2" + }, + "ip": ["127.0.0.1", "::1"], + "name": "123", + "id": "123", + "mac": ["aa:aa:aa:aa:aa:aa"], + "architecture": "x86_64" + }, + "event": { + "agent_id_status": "verified", + "sequence": 1, + "ingested": "2024-01-26T14:35:45Z", + "created": "2024-01-26T14:35:43.563903627Z", + "kind": "state", + "module": "endpoint", + "action": "endpoint_policy_response", + "id": "123", + "category": ["host"], + "type": ["change"], + "dataset": "endpoint.policy" + }, + "message": "Endpoint policy change" + } +] diff --git a/x-pack/plugins/security_solution/server/integration_tests/__mocks__/fleet-agents.json b/x-pack/plugins/security_solution/server/integration_tests/__mocks__/fleet-agents.json new file mode 100644 index 0000000000000..35a68ffd7fb14 --- /dev/null +++ b/x-pack/plugins/security_solution/server/integration_tests/__mocks__/fleet-agents.json @@ -0,0 +1,146 @@ +[ + { + "policy_coordinator_idx": 1, + "outputs": { + "default": { + "api_key": "", + "permissions_hash": "123", + "type": "elasticsearch", + "to_retire_api_key_ids": [ + { + "retired_at": "2022-11-10T17:22:38Z", + "id": "123" + } + ], + "api_key_id": "" + } + }, + "default_api_key": null, + "agent": { + "id": "123", + "version": "8.0.0" + }, + "enrolled_at": "2022-06-02T18:49:23Z", + "policy_id": "policy-elastic-agent-on-cloud", + "last_checkin_status": "online", + "active": false, + "default_api_key_history": null, + "local_metadata": { + "os": { + "kernel": "4.15.0-1027-aws", + "name": "Ubuntu", + "family": "debian", + "version": "20.04.3 LTS (Focal Fossa)", + "platform": "ubuntu", + "full": "Ubuntu focal(20.04.3 LTS (Focal Fossa))" + }, + "elastic": { + "agent": { + "build.original": "8.0.0 (build: 2ab3a7334016f570e0bfc7e9a577a35a22e02df5 at 2022-02-03 22:52:29 +0000 UTC)", + "upgradeable": false, + "log_level": "info", + "id": "123", + "version": "8.0.0", + "snapshot": false + } + }, + "host": { + "hostname": "123", + "ip": [ + "127.0.0.1/8", + "172.17.0.85/16" + ], + "name": "123", + "id": "", + "mac": [ + "02:42:ac:11:00:55" + ], + "architecture": "x86_64" + } + }, + "unenrolled_reason": "timeout", + "type": "PERMANENT", + "last_checkin": "2022-06-06T20:08:36Z", + "access_api_key_id": "123", + "default_api_key_id": null, + "unenrolled_at": "2022-06-07T20:09:32Z", + "updated_at": "2022-06-07T20:09:32Z", + "policy_revision_idx": 3, + "policy_output_permissions_hash": null, + "action_seq_no": [ + -1 + ] + }, + { + "policy_coordinator_idx": 1, + "outputs": { + "default": { + "api_key": "", + "permissions_hash": "123", + "type": "elasticsearch", + "to_retire_api_key_ids": [ + { + "retired_at": "2022-11-10T17:22:38Z", + "id": "123" + } + ], + "api_key_id": "" + } + }, + "default_api_key": null, + "agent": { + "id": "123", + "version": "8.1.2" + }, + "enrolled_at": "2022-06-06T20:08:27Z", + "policy_id": "policy-elastic-agent-on-cloud", + "last_checkin_status": "online", + "active": false, + "default_api_key_history": null, + "local_metadata": { + "os": { + "kernel": "5.4.0-1032-aws", + "name": "Ubuntu", + "family": "debian", + "version": "20.04.4 LTS (Focal Fossa)", + "platform": "ubuntu", + "full": "Ubuntu focal(20.04.4 LTS (Focal Fossa))" + }, + "elastic": { + "agent": { + "build.original": "8.1.2 (build: 6118f25235a52a7f0c4937a0a309e380c92d8119 at 2022-03-30 00:39:00 +0000 UTC)", + "upgradeable": false, + "log_level": "info", + "id": "123", + "version": "8.1.2", + "snapshot": false + } + }, + "host": { + "hostname": "123", + "ip": [ + "127.0.0.1/8", + "172.17.0.69/16" + ], + "name": "123", + "id": "", + "mac": [ + "02:42:ac:11:00:45" + ], + "architecture": "x86_64" + } + }, + "unenrolled_reason": "timeout", + "type": "PERMANENT", + "last_checkin": "2022-06-14T18:37:15Z", + "access_api_key_id": "123", + "default_api_key_id": null, + "unenrolled_at": "2022-06-15T18:38:08Z", + "updated_at": "2022-06-15T18:38:08Z", + "policy_revision_idx": 4, + "policy_output_permissions_hash": null, + "action_seq_no": [ + -1 + ] + } +] diff --git a/x-pack/plugins/security_solution/server/integration_tests/lib/helpers.ts b/x-pack/plugins/security_solution/server/integration_tests/lib/helpers.ts index db9ab8c656c3c..3697d9d2e4163 100644 --- a/x-pack/plugins/security_solution/server/integration_tests/lib/helpers.ts +++ b/x-pack/plugins/security_solution/server/integration_tests/lib/helpers.ts @@ -6,6 +6,7 @@ */ import Fs from 'fs'; import Util from 'util'; +import type { ElasticsearchClient } from '@kbn/core/server'; import deepmerge from 'deepmerge'; import { createTestServers, createRootWithCorePlugins } from '@kbn/core-test-helpers-kbn-server'; const asyncUnlink = Util.promisify(Fs.unlink); @@ -104,3 +105,16 @@ export async function setupTestServers(logFilePath: string, settings = {}) { export async function removeFile(path: string) { await asyncUnlink(path).catch(() => void 0); } + +export async function bulkInsert( + index: string, + data: unknown[], + esClient: ElasticsearchClient +): Promise { + const bulk = data.flatMap((d) => [{ index: { _index: index } }, d]); + await esClient.bulk({ body: bulk, refresh: 'wait_for' }).catch(() => {}); +} + +export function updateTimestamps(data: object[]): object[] { + return data.map((d) => ({ ...d, '@timestamp': new Date().toISOString() })); +} diff --git a/x-pack/plugins/security_solution/server/integration_tests/lib/telemetry_helpers.ts b/x-pack/plugins/security_solution/server/integration_tests/lib/telemetry_helpers.ts index abf6d1a5ef9d0..060acd47f9a05 100644 --- a/x-pack/plugins/security_solution/server/integration_tests/lib/telemetry_helpers.ts +++ b/x-pack/plugins/security_solution/server/integration_tests/lib/telemetry_helpers.ts @@ -23,8 +23,11 @@ import { deleteExceptionList, deleteExceptionListItem, } from '@kbn/lists-plugin/server/services/exception_lists'; + +import { createAgentPolicyWithPackages } from '@kbn/fleet-plugin/server/services/agent_policy_create'; import { ENDPOINT_ARTIFACT_LISTS } from '@kbn/securitysolution-list-constants'; import { DETECTION_TYPE, NAMESPACE_TYPE } from '@kbn/lists-plugin/common/constants.mock'; +import { bulkInsert, updateTimestamps } from './helpers'; import { TelemetryEventsSender } from '../../lib/telemetry/sender'; import type { SecuritySolutionPluginStart, @@ -37,6 +40,15 @@ import { type ITelemetryReceiver, TelemetryReceiver } from '../../lib/telemetry/ import { DEFAULT_DIAGNOSTIC_INDEX } from '../../lib/telemetry/constants'; import mockEndpointAlert from '../__mocks__/endpoint-alert.json'; import mockedRule from '../__mocks__/rule.json'; +import fleetAgents from '../__mocks__/fleet-agents.json'; +import endpointMetrics from '../__mocks__/endpoint-metrics.json'; +import endpointMetadata from '../__mocks__/endpoint-metadata.json'; +import endpointPolicy from '../__mocks__/endpoint-policy.json'; + +const fleetIndex = '.fleet-agents'; +const endpointMetricsIndex = '.ds-metrics-endpoint.metrics-1'; +const endpointMetricsMetadataIndex = '.ds-metrics-endpoint.metadata-1'; +const endpointMetricsPolicyIndex = '.ds-metrics-endpoint.policy-1'; export function getTelemetryTasks( spy: jest.SpyInstance< @@ -157,6 +169,45 @@ export async function createMockedAlert( }); } +export async function mockEndpointData( + esClient: ElasticsearchClient, + so: SavedObjectsServiceStart +) { + await createAgentPolicy('policy-elastic-agent-on-cloud', esClient, so); + await bulkInsert(fleetIndex, fleetAgents, esClient); + await bulkInsert(endpointMetricsIndex, updateTimestamps(endpointMetrics), esClient); + await bulkInsert(endpointMetricsMetadataIndex, updateTimestamps(endpointMetadata), esClient); + await bulkInsert(endpointMetricsPolicyIndex, updateTimestamps(endpointPolicy), esClient); +} + +export async function initEndpointIndices(esClient: ElasticsearchClient) { + const mappings: object = { + dynamic: false, + properties: { + '@timestamp': { + type: 'date', + }, + agent: { + properties: { + id: { + type: 'keyword', + }, + }, + }, + }, + }; + + await esClient.indices.create({ index: endpointMetricsIndex, mappings }).catch(() => {}); + await esClient.indices.create({ index: endpointMetricsMetadataIndex, mappings }).catch(() => {}); + await esClient.indices.create({ index: endpointMetricsPolicyIndex, mappings }).catch(() => {}); +} + +export async function dropEndpointIndices(esClient: ElasticsearchClient) { + await esClient.indices.delete({ index: endpointMetricsIndex }).catch(() => {}); + await esClient.indices.delete({ index: endpointMetricsMetadataIndex }).catch(() => {}); + await esClient.indices.delete({ index: endpointMetricsPolicyIndex }).catch(() => {}); +} + export async function cleanupMockedEndpointAlerts(esClient: ElasticsearchClient) { const index = `${DEFAULT_DIAGNOSTIC_INDEX.replace('-*', '')}-001`; @@ -187,6 +238,21 @@ export async function cleanupMockedAlerts( }); } +export async function createAgentPolicy( + id: string, + esClient: ElasticsearchClient, + so: SavedObjectsServiceStart +) { + const soClient = so.getScopedClient(fakeKibanaRequest); + await createAgentPolicyWithPackages({ + esClient, + soClient, + newPolicy: { id, name: 'Agent policy 1', namespace: 'default' }, + withSysMonitoring: true, + spaceId: 'default', + }); +} + export async function createMockedExceptionList(so: SavedObjectsServiceStart) { const type = DETECTION_TYPE; const listId = ENDPOINT_ARTIFACT_LISTS.trustedApps.id; diff --git a/x-pack/plugins/security_solution/server/integration_tests/receiver.test.ts b/x-pack/plugins/security_solution/server/integration_tests/receiver.test.ts index ad366e1847654..bc3ec2792ebf8 100644 --- a/x-pack/plugins/security_solution/server/integration_tests/receiver.test.ts +++ b/x-pack/plugins/security_solution/server/integration_tests/receiver.test.ts @@ -7,7 +7,7 @@ import Path from 'path'; import type { ElasticsearchClient } from '@kbn/core/server'; -import { setupTestServers, removeFile } from './lib/helpers'; +import { bulkInsert, setupTestServers, removeFile } from './lib/helpers'; import { getTelemetryReceiver } from './lib/telemetry_helpers'; import type { ITelemetryReceiver } from '../lib/telemetry/receiver'; @@ -80,7 +80,7 @@ describe('ITelemetryReceiver', () => { it('should paginate queries', async () => { const docs = mockedDocs(numOfDocs); - await bulkInsert(docs); + await bulkInsert(TEST_INDEX, docs, esClient); const results = telemetryReceiver.paginate(TEST_INDEX, testQuery()); @@ -100,8 +100,8 @@ describe('ITelemetryReceiver', () => { await new Promise((resolve) => setTimeout(resolve, 500)); const batchTwo = mockedDocs(numOfDocs, 'batchTwo'); - await bulkInsert(batchOne); - await bulkInsert(batchTwo); + await bulkInsert(TEST_INDEX, batchOne, esClient); + await bulkInsert(TEST_INDEX, batchTwo, esClient); const results = telemetryReceiver.paginate(TEST_INDEX, testQuery(from, to)); @@ -113,7 +113,7 @@ describe('ITelemetryReceiver', () => { }); it('should manage empty response', async () => { - await bulkInsert(mockedDocs(numOfDocs)); + await bulkInsert(TEST_INDEX, mockedDocs(numOfDocs), esClient); const results = telemetryReceiver.paginate(TEST_INDEX, testQuery('now-2d', 'now-1d')); @@ -155,11 +155,6 @@ describe('ITelemetryReceiver', () => { }; } - async function bulkInsert(data: unknown[]): Promise { - const bulk = data.flatMap((d) => [{ index: { _index: TEST_INDEX } }, d]); - await esClient.bulk({ body: bulk, refresh: 'wait_for' }).catch((_) => {}); - } - async function getPages(it: AsyncGenerator): Promise { const pages = []; for await (const doc of it) { diff --git a/x-pack/plugins/security_solution/server/integration_tests/telemetry.test.ts b/x-pack/plugins/security_solution/server/integration_tests/telemetry.test.ts index 67034b45e08c9..84e7aa7f65f15 100644 --- a/x-pack/plugins/security_solution/server/integration_tests/telemetry.test.ts +++ b/x-pack/plugins/security_solution/server/integration_tests/telemetry.test.ts @@ -7,12 +7,15 @@ import Path from 'path'; import axios, { type AxiosRequestConfig } from 'axios'; +import type { ElasticsearchClient } from '@kbn/core/server'; + import type { ExceptionListItemSchema, ExceptionListSchema, } from '@kbn/securitysolution-io-ts-list-types'; import { ENDPOINT_STAGING } from '@kbn/telemetry-plugin/common/constants'; +import { TELEMETRY_CHANNEL_ENDPOINT_META } from '../lib/telemetry/constants'; import { eventually, setupTestServers, removeFile } from './lib/helpers'; import { @@ -26,6 +29,9 @@ import { getTelemetryTask, getTelemetryTaskType, getTelemetryTasks, + initEndpointIndices, + dropEndpointIndices, + mockEndpointData, } from './lib/telemetry_helpers'; import { @@ -58,6 +64,7 @@ describe('telemetry tasks', () => { let asyncTelemetryEventSender: AsyncTelemetryEventsSender; let exceptionsList: ExceptionListSchema[] = []; let exceptionsListItem: ExceptionListItemSchema[] = []; + let esClient: ElasticsearchClient; beforeAll(async () => { await removeFile(logFilePath); @@ -80,6 +87,8 @@ describe('telemetry tasks', () => { inflightEventsThreshold: 1_000, maxPayloadSizeBytes: 1024 * 1024, }); + + esClient = kibanaServer.coreStart.elasticsearch.client.asInternalUser; }); afterAll(async () => { @@ -238,6 +247,34 @@ describe('telemetry tasks', () => { }); }); + describe('endpoint-meta-telemetry', () => { + beforeEach(async () => { + await initEndpointIndices(esClient); + }); + + afterEach(async () => { + await dropEndpointIndices(esClient); + }); + + it('should execute when scheduled', async () => { + await mockAndScheduleEndpointTask(); + + const body = await eventually(async () => { + const found = mockedAxiosPost.mock.calls.find(([url]) => { + return url.startsWith(ENDPOINT_STAGING) && url.endsWith(TELEMETRY_CHANNEL_ENDPOINT_META); + }); + + expect(found).not.toBeFalsy(); + + return JSON.parse((found ? found[1] : '{}') as string); + }); + + expect(body).not.toBeFalsy(); + expect(body.endpoint_metrics).not.toBeFalsy(); + expect(body.endpoint_meta).not.toBeFalsy(); + }); + }); + async function mockAndScheduleDetectionRulesTask(): Promise { const task = getTelemetryTask(tasks, 'security:telemetry-detection-rules'); @@ -261,6 +298,19 @@ describe('telemetry tasks', () => { return task; } + async function mockAndScheduleEndpointTask(): Promise { + const task = getTelemetryTask(tasks, 'security:endpoint-meta-telemetry'); + + await mockEndpointData(esClient, kibanaServer.coreStart.savedObjects); + + // schedule task to run ASAP + await eventually(async () => { + await taskManagerPlugin.runSoon(task.getTaskId()); + }); + + return task; + } + async function mockAndScheduleEndpointDiagnosticsTask(): Promise { const task = getTelemetryTask(tasks, 'security:endpoint-diagnostics'); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts index 348146361e8f7..1b6e6e4dc30d9 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts @@ -114,7 +114,7 @@ export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) { }; if (endpointMetricsResponse.aggregations === undefined) { - log(`no endpoint metrics to report`); + log(`no endpoint metrics response to report`); taskMetricsService.end(trace); return 0; } @@ -348,6 +348,8 @@ export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) { }; }); + log(`sending ${telemetryPayloads.length} endpoint telemetry records`); + /** * STAGE 6 - Send the documents * From 0a727130070b96b6dfa72aa45d275df1cec4ccff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Fri, 23 Feb 2024 10:09:04 +0100 Subject: [PATCH 09/19] Add more assertions --- .../endpoint-meta-telemetry-request.json | 683 ++++++++++++++++++ .../lib/telemetry_helpers.ts | 2 +- .../integration_tests/telemetry.test.ts | 7 +- 3 files changed, 688 insertions(+), 4 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/integration_tests/__mocks__/endpoint-meta-telemetry-request.json diff --git a/x-pack/plugins/security_solution/server/integration_tests/__mocks__/endpoint-meta-telemetry-request.json b/x-pack/plugins/security_solution/server/integration_tests/__mocks__/endpoint-meta-telemetry-request.json new file mode 100644 index 0000000000000..157b0346850db --- /dev/null +++ b/x-pack/plugins/security_solution/server/integration_tests/__mocks__/endpoint-meta-telemetry-request.json @@ -0,0 +1,683 @@ +{ + "@timestamp": "2024-02-23T08:47:03.036Z", + "cluster_uuid": "PanARIslSNC5x-B-ttgunw", + "cluster_name": "es-test-cluster", + "license_id": "b15da743-9adb-45fa-9258-dcecad41d1e4", + "endpoint_id": "123", + "endpoint_version": "8.6.2", + "endpoint_package_version": null, + "endpoint_metrics": { + "cpu": { + "histogram": { + "counts": [ + 3934326, + 520, + 140, + 110, + 212, + 78, + 41, + 18, + 12, + 25, + 9, + 5, + 5, + 2, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "values": [ + 5, + 10, + 15, + 20, + 25, + 30, + 35, + 40, + 45, + 50, + 55, + 60, + 65, + 70, + 75, + 80, + 85, + 90, + 95, + 100 + ] + }, + "mean": 0.2697424700222798, + "latest": 0.1 + }, + "memory": { + "mean": 373780821, + "latest": 411860992 + }, + "uptime": { + "endpoint": 26882599, + "system": 26883343 + }, + "documentsVolume": { + "file_events": { + "suppressed_count": 1651293, + "suppressed_bytes": 0, + "sent_count": 0, + "sent_bytes": 0 + }, + "process_events": { + "suppressed_count": 0, + "suppressed_bytes": 0, + "sent_count": 11026887, + "sent_bytes": 23412176619 + }, + "network_events": { + "suppressed_count": 3270009, + "suppressed_bytes": 0, + "sent_count": 0, + "sent_bytes": 0 + }, + "overall": { + "suppressed_count": 4921302, + "suppressed_bytes": 0, + "sent_count": 11026887, + "sent_bytes": 23412176619 + } + }, + "maliciousBehaviorRules": [ + { + "id": "123", + "endpoint_uptime_percent": 0 + } + ], + "systemImpact": [ + { + "process": { + "executable": "/usr/sbin/sshd" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 304 + }, + "process_events": { + "week_idle_ms": 31791, + "week_ms": 110821 + }, + "network_events": { + "week_idle_ms": 2030, + "week_ms": 53 + }, + "overall": { + "week_idle_ms": 33821, + "week_ms": 111178 + } + }, + { + "process": { + "executable": "unknown" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 4552 + }, + "overall": { + "week_idle_ms": 0, + "week_ms": 4552 + } + }, + { + "file_events": { + "week_idle_ms": 130, + "week_ms": 7 + }, + "process": { + "executable": "/lib/systemd/systemd" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 1920 + }, + "process_events": { + "week_idle_ms": 1273, + "week_ms": 1750 + }, + "overall": { + "week_idle_ms": 1403, + "week_ms": 3677 + } + }, + { + "file_events": { + "week_idle_ms": 7, + "week_ms": 0 + }, + "process": { + "executable": "/bin/systemctl" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 3055 + }, + "process_events": { + "week_idle_ms": 0, + "week_ms": 9 + }, + "overall": { + "week_idle_ms": 7, + "week_ms": 3064 + } + }, + { + "file_events": { + "week_idle_ms": 3104, + "week_ms": 1 + }, + "process": { + "executable": "/usr/bin/apt-key" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 61 + }, + "process_events": { + "week_idle_ms": 178857, + "week_ms": 2062 + }, + "overall": { + "week_idle_ms": 181961, + "week_ms": 2124 + } + }, + { + "process": { + "executable": "/usr/bin/gce_workload_cert_refresh" + }, + "process_events": { + "week_idle_ms": 230, + "week_ms": 1405 + }, + "network_events": { + "week_idle_ms": 135, + "week_ms": 21 + }, + "overall": { + "week_idle_ms": 365, + "week_ms": 1426 + } + }, + { + "process": { + "executable": "/usr/bin/apt-config" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 65 + }, + "process_events": { + "week_idle_ms": 18708, + "week_ms": 840 + }, + "overall": { + "week_idle_ms": 18708, + "week_ms": 905 + } + }, + { + "process": { + "executable": "/usr/lib/apt/apt.systemd.daily" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 273 + }, + "process_events": { + "week_idle_ms": 1064, + "week_ms": 352 + }, + "overall": { + "week_idle_ms": 1064, + "week_ms": 625 + } + }, + { + "process": { + "executable": "/usr/bin/dpkg" + }, + "process_events": { + "week_idle_ms": 11479, + "week_ms": 558 + }, + "overall": { + "week_idle_ms": 11479, + "week_ms": 558 + } + }, + { + "process": { + "executable": "/usr/sbin/cron" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 80 + }, + "process_events": { + "week_idle_ms": 270, + "week_ms": 451 + }, + "overall": { + "week_idle_ms": 270, + "week_ms": 531 + } + }, + { + "process": { + "executable": "/usr/bin/cmp" + }, + "process_events": { + "week_idle_ms": 39452, + "week_ms": 513 + }, + "overall": { + "week_idle_ms": 39452, + "week_ms": 513 + } + }, + { + "file_events": { + "week_idle_ms": 144, + "week_ms": 0 + }, + "process": { + "executable": "/sbin/dhclient-script" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 251 + }, + "process_events": { + "week_idle_ms": 2412, + "week_ms": 186 + }, + "overall": { + "week_idle_ms": 2556, + "week_ms": 437 + } + }, + { + "process": { + "executable": "/usr/bin/env" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 413 + }, + "process_events": { + "week_idle_ms": 45, + "week_ms": 4 + }, + "overall": { + "week_idle_ms": 45, + "week_ms": 417 + } + }, + { + "process": { + "executable": "/usr/bin/cat" + }, + "process_events": { + "week_idle_ms": 39679, + "week_ms": 359 + }, + "overall": { + "week_idle_ms": 39679, + "week_ms": 359 + } + }, + { + "process": { + "executable": "/bin/sh" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 35 + }, + "process_events": { + "week_idle_ms": 444, + "week_ms": 197 + }, + "overall": { + "week_idle_ms": 444, + "week_ms": 232 + } + }, + { + "process": { + "executable": "/usr/lib/apt/apt-helper" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 100 + }, + "process_events": { + "week_idle_ms": 8, + "week_ms": 62 + }, + "overall": { + "week_idle_ms": 8, + "week_ms": 162 + } + }, + { + "process": { + "executable": "/usr/bin/date" + }, + "process_events": { + "week_idle_ms": 1024, + "week_ms": 143 + }, + "overall": { + "week_idle_ms": 1024, + "week_ms": 143 + } + }, + { + "file_events": { + "week_idle_ms": 5315, + "week_ms": 2 + }, + "process": { + "executable": "/usr/bin/apt-get" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 53 + }, + "process_events": { + "week_idle_ms": 576, + "week_ms": 76 + }, + "overall": { + "week_idle_ms": 5891, + "week_ms": 131 + } + }, + { + "process": { + "executable": "/usr/bin/unattended-upgrade" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 27 + }, + "process_events": { + "week_idle_ms": 139, + "week_ms": 95 + }, + "overall": { + "week_idle_ms": 139, + "week_ms": 122 + } + }, + { + "process": { + "executable": "/usr/bin/gpgconf" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 26 + }, + "process_events": { + "week_idle_ms": 12075, + "week_ms": 93 + }, + "overall": { + "week_idle_ms": 12075, + "week_ms": 119 + } + } + ], + "threads": [ + { + "name": "Cron", + "cpu": { + "mean": 0.005330585583239717 + } + }, + { + "name": "FileLogThread", + "cpu": { + "mean": 0.01387886588350829 + } + }, + { + "name": "LoggingLimitThread", + "cpu": { + "mean": 0.0003385089239880071 + } + }, + { + "name": "DocumentLoggingThread", + "cpu": { + "mean": 0.002019893909071295 + } + }, + { + "name": "DocumentLoggingMaintenance", + "cpu": { + "mean": 0.0001711144011367948 + } + }, + { + "name": "BulkConsumerThread", + "cpu": { + "mean": 0.01437360969549076 + } + }, + { + "name": "DocumentLoggingConsumerThread", + "cpu": { + "mean": 0.1566068758230231 + } + }, + { + "name": "DocumentLoggingLimitThread", + "cpu": { + "mean": 0.000383147463414997 + } + }, + { + "name": "ArtifactManifestDownload", + "cpu": { + "mean": 0.00202733366564246 + } + }, + { + "name": "PolicyReloadThread", + "cpu": { + "mean": 0.00001487951314232998 + } + }, + { + "name": "PerformanceMonitorWorkerThread", + "cpu": { + "mean": 0.005423582540379278 + } + }, + { + "name": "MetadataThread", + "cpu": { + "mean": 0.00001115963485674749 + } + }, + { + "name": "EventsQueueThread", + "cpu": { + "mean": 0.05409447002894065 + } + }, + { + "name": "DelayedAlertEnrichment", + "cpu": { + "mean": 0 + } + }, + { + "name": "MaintainProcessMap", + "cpu": { + "mean": 0.002905224941039929 + } + }, + { + "name": "FileScoreThread", + "cpu": { + "mean": 0.003567363275873613 + } + }, + { + "name": "DiagnosticMalwareThread", + "cpu": { + "mean": 0 + } + }, + { + "name": "QuarantineManagerWorkerThread", + "cpu": { + "mean": 0.002083131839926198 + } + }, + { + "name": "EventProcessingThread", + "cpu": { + "mean": 0.01754294664738331 + } + }, + { + "name": "HostIsolationMonitorThread", + "cpu": { + "mean": 0.0008481322806622976 + } + }, + { + "name": "MountMonitor", + "cpu": { + "mean": 0.003463206812704382 + } + }, + { + "name": "responseActionsUploadThread", + "cpu": { + "mean": 0.00270807149264102 + } + }, + { + "name": "responseActionsProcessThread", + "cpu": { + "mean": 0.002689472100521233 + } + }, + { + "name": "serviceCommsThread", + "cpu": { + "mean": 0 + } + }, + { + "name": "grpcConnectionManagerThread", + "cpu": { + "mean": 0.0110926774602411 + } + }, + { + "name": "checkinAPIThread", + "cpu": { + "mean": 0.0003124700433295513 + } + }, + { + "name": "actionsAPIThread", + "cpu": { + "mean": 0.0004910243538035807 + } + }, + { + "name": "stateReportThread", + "cpu": { + "mean": 0.00539010824743476 + } + }, + { + "name": "EventsLoopThread", + "cpu": { + "mean": 0.003266057265515501 + } + }, + { + "name": "FanotifyWatchdog", + "cpu": { + "mean": 0.0002231929794201937 + } + }, + { + "name": "FanotifySyncConsumer", + "cpu": { + "mean": 0.001885980676100637 + } + }, + { + "name": "FanotifyAsyncConsumer", + "cpu": { + "mean": 0.0002492321603525497 + } + }, + { + "name": "FanotifyConsumer", + "cpu": { + "mean": 0.03793164685246193 + } + }, + { + "name": "RulesEngineThread", + "cpu": { + "mean": 0.02721466395730229 + } + } + ], + "eventFilter": { + "active_global_count": 0, + "active_user_count": 0 + } + }, + "endpoint_meta": { + "os": { + "Ext": { + "variant": "Debian" + }, + "kernel": "5.10.0-21-cloud-amd64 #1 SMP Debian 5.10.162-1 (2023-01-21)", + "name": "Linux", + "family": "debian", + "type": "linux", + "version": "11.8", + "platform": "debian", + "full": "Debian 11.8" + }, + "capabilities": [ + "kill_process", + "suspend_process", + "running_processes", + "isolation", + "ebpf(auto)" + ] + }, + "policy_config": {}, + "policy_response": {}, + "telemetry_meta": { + "metrics_timestamp": "2024-02-23T08:46:59.303Z" + } +} \ No newline at end of file diff --git a/x-pack/plugins/security_solution/server/integration_tests/lib/telemetry_helpers.ts b/x-pack/plugins/security_solution/server/integration_tests/lib/telemetry_helpers.ts index 060acd47f9a05..56a9e3c2a378e 100644 --- a/x-pack/plugins/security_solution/server/integration_tests/lib/telemetry_helpers.ts +++ b/x-pack/plugins/security_solution/server/integration_tests/lib/telemetry_helpers.ts @@ -198,8 +198,8 @@ export async function initEndpointIndices(esClient: ElasticsearchClient) { }; await esClient.indices.create({ index: endpointMetricsIndex, mappings }).catch(() => {}); - await esClient.indices.create({ index: endpointMetricsMetadataIndex, mappings }).catch(() => {}); await esClient.indices.create({ index: endpointMetricsPolicyIndex, mappings }).catch(() => {}); + await esClient.indices.create({ index: endpointMetricsMetadataIndex, mappings }).catch(() => {}); } export async function dropEndpointIndices(esClient: ElasticsearchClient) { diff --git a/x-pack/plugins/security_solution/server/integration_tests/telemetry.test.ts b/x-pack/plugins/security_solution/server/integration_tests/telemetry.test.ts index 84e7aa7f65f15..f666e292b50bd 100644 --- a/x-pack/plugins/security_solution/server/integration_tests/telemetry.test.ts +++ b/x-pack/plugins/security_solution/server/integration_tests/telemetry.test.ts @@ -46,6 +46,7 @@ import { import type { SecurityTelemetryTask } from '../lib/telemetry/task'; import { TelemetryChannel } from '../lib/telemetry/types'; import type { AsyncTelemetryEventsSender } from '../lib/telemetry/async_sender'; +import endpointMetaTelemetryRequest from './__mocks__/endpoint-meta-telemetry-request.json'; jest.mock('axios'); @@ -269,9 +270,9 @@ describe('telemetry tasks', () => { return JSON.parse((found ? found[1] : '{}') as string); }); - expect(body).not.toBeFalsy(); - expect(body.endpoint_metrics).not.toBeFalsy(); - expect(body.endpoint_meta).not.toBeFalsy(); + // save body in a file for debugging + expect(body.endpoint_metrics).toStrictEqual(endpointMetaTelemetryRequest.endpoint_metrics); + expect(body.endpoint_meta).toStrictEqual(endpointMetaTelemetryRequest.endpoint_meta); }); }); From 4935abffb970ff08e9f68978c85dd4ea3261bf49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Fri, 23 Feb 2024 11:22:07 +0100 Subject: [PATCH 10/19] Improve endpoint-meta queries --- .../endpoint-meta-telemetry-request.json | 285 ++- .../__mocks__/endpoint-metrics.json | 1683 ++++++++--------- .../__mocks__/fleet-agents.json | 26 +- .../server/integration_tests/lib/helpers.ts | 15 +- .../lib/telemetry_helpers.ts | 56 +- .../server/integration_tests/receiver.test.ts | 8 +- .../integration_tests/telemetry.test.ts | 3 +- .../server/lib/telemetry/__mocks__/index.ts | 5 +- .../server/lib/telemetry/__mocks__/metrics.ts | 163 +- .../server/lib/telemetry/receiver.ts | 91 +- .../lib/telemetry/tasks/endpoint.test.ts | 2 +- .../server/lib/telemetry/tasks/endpoint.ts | 255 ++- .../server/lib/telemetry/types.ts | 87 +- 13 files changed, 1409 insertions(+), 1270 deletions(-) diff --git a/x-pack/plugins/security_solution/server/integration_tests/__mocks__/endpoint-meta-telemetry-request.json b/x-pack/plugins/security_solution/server/integration_tests/__mocks__/endpoint-meta-telemetry-request.json index 157b0346850db..2d9951d1c1dc2 100644 --- a/x-pack/plugins/security_solution/server/integration_tests/__mocks__/endpoint-meta-telemetry-request.json +++ b/x-pack/plugins/security_solution/server/integration_tests/__mocks__/endpoint-meta-telemetry-request.json @@ -9,50 +9,8 @@ "endpoint_metrics": { "cpu": { "histogram": { - "counts": [ - 3934326, - 520, - 140, - 110, - 212, - 78, - 41, - 18, - 12, - 25, - 9, - 5, - 5, - 2, - 0, - 0, - 0, - 0, - 0, - 0 - ], - "values": [ - 5, - 10, - 15, - 20, - 25, - 30, - 35, - 40, - 45, - 50, - 55, - 60, - 65, - 70, - 75, - 80, - 85, - 90, - 95, - 100 - ] + "counts": [3934326, 520, 140, 110, 212, 78, 41, 18, 12, 25, 9, 5, 5, 2, 0, 0, 0, 0, 0, 0], + "values": [5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 85, 90, 95, 100] }, "mean": 0.2697424700222798, "latest": 0.1 @@ -675,9 +633,242 @@ "ebpf(auto)" ] }, - "policy_config": {}, - "policy_response": {}, + "policy_config": { + "value": { + "linux": { + "advanced": { + "agent": { + "connection_delay": 60 + }, + "alerts": { + "require_user_artifacts": null + }, + "artifacts": { + "global": { + "base_url": null, + "manifest_relative_url": null, + "public_key": null, + "interval": null, + "ca_cert": null + }, + "user": { + "public_key": null, + "ca_cert": null, + "base_url": null, + "interval": null + } + }, + "elasticsearch": { + "delay": null, + "tls": { + "verify_peer": null, + "verify_hostname": null, + "ca_cert": null + } + }, + "fanotify": { + "ignore_unknown_filesystems": null, + "monitored_filesystems": null, + "ignored_filesystems": null + }, + "logging": { + "file": null, + "stdout": null, + "stderr": null, + "syslog": null + }, + "diagnostic": { + "enabled": null + }, + "malware": { + "quarantine": null + }, + "memory_protection": { + "memory_scan_collect_sample": null, + "memory_scan": null + }, + "kernel": { + "capture_mode": null + }, + "event_filter": { + "default": null + }, + "utilization_limits": { + "cpu": null + }, + "logstash": { + "delay": null + } + } + }, + "mac": { + "advanced": { + "agent": { + "connection_delay": null + }, + "artifacts": { + "global": { + "base_url": null, + "manifest_relative_url": null, + "public_key": null, + "interval": null, + "ca_cert": null + }, + "user": { + "public_key": null, + "ca_cert": null, + "base_url": null, + "interval": null + } + }, + "elasticsearch": { + "delay": null, + "tls": { + "verify_peer": null, + "verify_hostname": null, + "ca_cert": null + } + }, + "logging": { + "file": null, + "stdout": null, + "stderr": null, + "syslog": null + }, + "logstash": { + "delay": null + }, + "malware": { + "quarantine": null, + "threshold": null + }, + "kernel": { + "connect": null, + "harden": null, + "process": null, + "filewrite": null, + "network": null, + "network_extension": { + "enable_content_filtering": null, + "enable_packet_filtering": null + } + }, + "harden": { + "self_protect": null + }, + "diagnostic": { + "enabled": null + }, + "alerts": { + "cloud_lookup": null, + "cloud_lookup_url": null + }, + "memory_protection": { + "memory_scan_collect_sample": false, + "memory_scan": null + }, + "event_filter": { + "default": null + } + } + }, + "windows": { + "advanced": { + "agent": { + "connection_delay": null + }, + "artifacts": { + "global": { + "base_url": null, + "manifest_relative_url": null, + "public_key": null, + "interval": null, + "ca_cert": null + }, + "user": { + "public_key": null, + "ca_cert": null, + "base_url": null, + "interval": null + } + }, + "elasticsearch": { + "delay": null, + "tls": { + "verify_peer": null, + "verify_hostname": null, + "ca_cert": null + } + }, + "logging": { + "file": null, + "stdout": null, + "stderr": null, + "syslog": null + }, + "malware": { + "quarantine": null, + "threshold": null + }, + "kernel": { + "connect": null, + "harden": null, + "process": null, + "filewrite": null, + "network": null, + "fileopen": null, + "asyncimageload": null, + "syncimageload": null, + "registry": null, + "fileaccess": null, + "registryaccess": null, + "process_handle": null + }, + "diagnostic": { + "enabled": null, + "rollback_telemetry_enabled": null + }, + "alerts": { + "cloud_lookup": null, + "cloud_lookup_url": null, + "require_user_artifacts": null + }, + "ransomware": { + "mbr": null, + "canary": null + }, + "memory_protection": { + "context_manipulation_detection": null, + "shellcode": null, + "memory_scan": null, + "shellcode_collect_sample": null, + "memory_scan_collect_sample": null, + "shellcode_enhanced_pe_parsing": null, + "shellcode_trampoline_detection": null + }, + "event_filter": { + "default": null + }, + "utilization_limits": { + "cpu": null + } + } + } + } + }, + "policy_response": { + "agent_policy_status": "verified", + "manifest_version": "1.0.874", + "status": "success", + "actions": [], + "configuration": { + "isolation": false + }, + "state": { + "isolation": false + } + }, "telemetry_meta": { "metrics_timestamp": "2024-02-23T08:46:59.303Z" } -} \ No newline at end of file +} diff --git a/x-pack/plugins/security_solution/server/integration_tests/__mocks__/endpoint-metrics.json b/x-pack/plugins/security_solution/server/integration_tests/__mocks__/endpoint-metrics.json index a5af045d2f4c3..cc3e0b03af31c 100644 --- a/x-pack/plugins/security_solution/server/integration_tests/__mocks__/endpoint-metrics.json +++ b/x-pack/plugins/security_solution/server/integration_tests/__mocks__/endpoint-metrics.json @@ -2,366 +2,330 @@ { "agent": { "build": { - "original": "version: 8.6.2, compiled: Fri Feb 10 14:00:00 2023, branch: 8.6, commit: eaee25e73cc58a52387e50d5bce542ec8965f638" + "original": "version: 8.6.0, compiled: Mon Jan 2 23:00:00 2023, branch: 8.6, commit: e2d09ff1b8e49bfb5f8940d317eb4ac96672d956" }, "id": "123", "type": "endpoint", - "version": "8.6.2" + "version": "8.6.0" }, - "@timestamp": "2024-01-26T15:06:50.254822749Z", + "@timestamp": "2024-01-26T14:38:30.628421712Z", "Endpoint": { "metrics": { "system_impact": [ { "process": { - "executable": "/usr/sbin/sshd" + "executable": "/usr/bin/amazon-ssm-agent" }, "malware": { "week_idle_ms": 0, - "week_ms": 304 + "week_ms": 36 }, "process_events": { - "week_idle_ms": 31791, - "week_ms": 110821 - }, - "network_events": { - "week_idle_ms": 2030, - "week_ms": 53 + "week_idle_ms": 5, + "week_ms": 12487 }, "overall": { - "week_idle_ms": 33821, - "week_ms": 111178 + "week_idle_ms": 5, + "week_ms": 12523 } }, { "process": { - "executable": "unknown" + "executable": "/usr/bin/ps" }, - "malware": { - "week_idle_ms": 0, - "week_ms": 4552 + "process_events": { + "week_idle_ms": 8692, + "week_ms": 10288 }, "overall": { - "week_idle_ms": 0, - "week_ms": 4552 + "week_idle_ms": 8692, + "week_ms": 10288 } }, { - "file_events": { - "week_idle_ms": 130, - "week_ms": 7 - }, "process": { - "executable": "/lib/systemd/systemd" + "executable": "/usr/sbin/sshd" }, "malware": { "week_idle_ms": 0, - "week_ms": 1920 + "week_ms": 116 }, "process_events": { - "week_idle_ms": 1273, - "week_ms": 1750 + "week_idle_ms": 3621, + "week_ms": 7204 + }, + "network_events": { + "week_idle_ms": 97, + "week_ms": 0 }, "overall": { - "week_idle_ms": 1403, - "week_ms": 3677 + "week_idle_ms": 3718, + "week_ms": 7320 } }, { - "file_events": { - "week_idle_ms": 7, - "week_ms": 0 - }, "process": { - "executable": "/bin/systemctl" + "executable": "/usr/sbin/dhclient-script" }, "malware": { "week_idle_ms": 0, - "week_ms": 3055 + "week_ms": 430 }, "process_events": { - "week_idle_ms": 0, - "week_ms": 9 + "week_idle_ms": 152054, + "week_ms": 6229 }, "overall": { - "week_idle_ms": 7, - "week_ms": 3064 + "week_idle_ms": 152054, + "week_ms": 6659 } }, { - "file_events": { - "week_idle_ms": 3104, - "week_ms": 1 - }, "process": { - "executable": "/usr/bin/apt-key" + "executable": "/usr/sbin/crond" }, "malware": { "week_idle_ms": 0, - "week_ms": 61 + "week_ms": 73 }, "process_events": { - "week_idle_ms": 178857, - "week_ms": 2062 + "week_idle_ms": 9491, + "week_ms": 2060 }, "overall": { - "week_idle_ms": 181961, - "week_ms": 2124 + "week_idle_ms": 9491, + "week_ms": 2133 } }, { - "process": { - "executable": "/usr/bin/gce_workload_cert_refresh" - }, - "process_events": { - "week_idle_ms": 230, - "week_ms": 1405 - }, - "network_events": { - "week_idle_ms": 135, - "week_ms": 21 + "file_events": { + "week_idle_ms": 80958, + "week_ms": 1963 }, - "overall": { - "week_idle_ms": 365, - "week_ms": 1426 - } - }, - { "process": { - "executable": "/usr/bin/apt-config" + "executable": "/usr/lib/systemd/systemd" }, "malware": { "week_idle_ms": 0, - "week_ms": 65 + "week_ms": 23 }, "process_events": { - "week_idle_ms": 18708, - "week_ms": 840 + "week_idle_ms": 6, + "week_ms": 7 }, "overall": { - "week_idle_ms": 18708, - "week_ms": 905 + "week_idle_ms": 80964, + "week_ms": 1993 } }, { "process": { - "executable": "/usr/lib/apt/apt.systemd.daily" - }, - "malware": { - "week_idle_ms": 0, - "week_ms": 273 + "executable": "/bin/logger" }, "process_events": { - "week_idle_ms": 1064, - "week_ms": 352 + "week_idle_ms": 25156, + "week_ms": 1097 }, "overall": { - "week_idle_ms": 1064, - "week_ms": 625 + "week_idle_ms": 25156, + "week_ms": 1097 } }, { + "file_events": { + "week_idle_ms": 12, + "week_ms": 0 + }, "process": { - "executable": "/usr/bin/dpkg" + "executable": "/usr/lib64/sa/sadc" }, "process_events": { - "week_idle_ms": 11479, - "week_ms": 558 + "week_idle_ms": 4182, + "week_ms": 1005 }, "overall": { - "week_idle_ms": 11479, - "week_ms": 558 + "week_idle_ms": 4194, + "week_ms": 1005 } }, { "process": { - "executable": "/usr/sbin/cron" + "executable": "unknown" }, "malware": { "week_idle_ms": 0, - "week_ms": 80 - }, - "process_events": { - "week_idle_ms": 270, - "week_ms": 451 + "week_ms": 981 }, "overall": { - "week_idle_ms": 270, - "week_ms": 531 + "week_idle_ms": 0, + "week_ms": 981 } }, { "process": { - "executable": "/usr/bin/cmp" + "executable": "/usr/bin/date" }, "process_events": { - "week_idle_ms": 39452, - "week_ms": 513 + "week_idle_ms": 8682, + "week_ms": 856 }, "overall": { - "week_idle_ms": 39452, - "week_ms": 513 + "week_idle_ms": 8682, + "week_ms": 856 } }, { "file_events": { - "week_idle_ms": 144, - "week_ms": 0 + "week_idle_ms": 89949, + "week_ms": 835 }, "process": { - "executable": "/sbin/dhclient-script" + "executable": "/usr/lib/systemd/systemd-logind (deleted)" }, - "malware": { - "week_idle_ms": 0, - "week_ms": 251 + "overall": { + "week_idle_ms": 89949, + "week_ms": 835 + } + }, + { + "process": { + "executable": "/bin/cat" }, "process_events": { - "week_idle_ms": 2412, - "week_ms": 186 + "week_idle_ms": 28801, + "week_ms": 754 }, "overall": { - "week_idle_ms": 2556, - "week_ms": 437 + "week_idle_ms": 28801, + "week_ms": 754 } }, { "process": { - "executable": "/usr/bin/env" + "executable": "/usr/lib64/sa/sa1" }, "malware": { "week_idle_ms": 0, - "week_ms": 413 + "week_ms": 69 }, "process_events": { - "week_idle_ms": 45, - "week_ms": 4 + "week_idle_ms": 7188, + "week_ms": 637 }, "overall": { - "week_idle_ms": 45, - "week_ms": 417 + "week_idle_ms": 7188, + "week_ms": 706 } }, { "process": { - "executable": "/usr/bin/cat" + "executable": "/bin/cut" }, "process_events": { - "week_idle_ms": 39679, - "week_ms": 359 + "week_idle_ms": 7258, + "week_ms": 660 }, "overall": { - "week_idle_ms": 39679, - "week_ms": 359 + "week_idle_ms": 7258, + "week_ms": 660 } }, { "process": { - "executable": "/bin/sh" - }, - "malware": { - "week_idle_ms": 0, - "week_ms": 35 + "executable": "/bin/ipcalc" }, "process_events": { - "week_idle_ms": 444, - "week_ms": 197 + "week_idle_ms": 6603, + "week_ms": 477 }, "overall": { - "week_idle_ms": 444, - "week_ms": 232 + "week_idle_ms": 6603, + "week_ms": 477 } }, { "process": { - "executable": "/usr/lib/apt/apt-helper" - }, - "malware": { - "week_idle_ms": 0, - "week_ms": 100 + "executable": "/sbin/ip" }, "process_events": { - "week_idle_ms": 8, - "week_ms": 62 + "week_idle_ms": 14654, + "week_ms": 470 }, "overall": { - "week_idle_ms": 8, - "week_ms": 162 + "week_idle_ms": 14654, + "week_ms": 470 } }, { "process": { - "executable": "/usr/bin/date" + "executable": "/bin/curl" }, "process_events": { - "week_idle_ms": 1024, - "week_ms": 143 + "week_idle_ms": 4696, + "week_ms": 432 + }, + "network_events": { + "week_idle_ms": 3120, + "week_ms": 20 }, "overall": { - "week_idle_ms": 1024, - "week_ms": 143 + "week_idle_ms": 7816, + "week_ms": 452 } }, { - "file_events": { - "week_idle_ms": 5315, - "week_ms": 2 - }, "process": { - "executable": "/usr/bin/apt-get" + "executable": "/bin/run-parts" }, "malware": { "week_idle_ms": 0, - "week_ms": 53 + "week_ms": 66 }, "process_events": { - "week_idle_ms": 576, - "week_ms": 76 + "week_idle_ms": 4219, + "week_ms": 354 }, "overall": { - "week_idle_ms": 5891, - "week_ms": 131 + "week_idle_ms": 4219, + "week_ms": 420 } }, { "process": { - "executable": "/usr/bin/unattended-upgrade" - }, - "malware": { - "week_idle_ms": 0, - "week_ms": 27 + "executable": "/usr/sbin/dhclient" }, "process_events": { - "week_idle_ms": 139, - "week_ms": 95 + "week_idle_ms": 3, + "week_ms": 326 }, "overall": { - "week_idle_ms": 139, - "week_ms": 122 + "week_idle_ms": 3, + "week_ms": 326 } }, { "process": { - "executable": "/usr/bin/gpgconf" + "executable": "/bin/sh" }, "malware": { "week_idle_ms": 0, - "week_ms": 26 + "week_ms": 51 }, "process_events": { - "week_idle_ms": 12075, - "week_ms": 93 + "week_idle_ms": 2569, + "week_ms": 239 }, "overall": { - "week_idle_ms": 12075, - "week_ms": 119 + "week_idle_ms": 2569, + "week_ms": 290 } } ], "memory": { "endpoint": { "private": { - "mean": 373780821, - "latest": 411860992 + "mean": 119318921, + "latest": 136073216 } } }, @@ -381,61 +345,53 @@ "fstype": "proc" }, { - "total": 2049777664, - "free": 2049777664, - "device": "udev", + "total": 4157120512, + "free": 4157120512, + "device": "devtmpfs", "mount": "/dev", "fstype": "devtmpfs" }, { "total": 0, "free": 0, - "device": "devpts", - "mount": "/dev/pts", - "fstype": "devpts" + "device": "securityfs", + "mount": "/sys/kernel/security", + "fstype": "securityfs" }, { - "total": 412172288, - "free": 411795456, + "total": 4166328320, + "free": 4166328320, "device": "tmpfs", - "mount": "/run", + "mount": "/dev/shm", "fstype": "tmpfs" }, - { - "endpoint_drive": true, - "total": 10331889664, - "free": 4091109376, - "device": "/dev/sda1", - "mount": "/", - "fstype": "ext4" - }, { "total": 0, "free": 0, - "device": "securityfs", - "mount": "/sys/kernel/security", - "fstype": "securityfs" + "device": "devpts", + "mount": "/dev/pts", + "fstype": "devpts" }, { - "total": 2060849152, - "free": 2060849152, + "total": 4166328320, + "free": 4165955584, "device": "tmpfs", - "mount": "/dev/shm", + "mount": "/run", "fstype": "tmpfs" }, { - "total": 5242880, - "free": 5242880, + "total": 4166328320, + "free": 4166328320, "device": "tmpfs", - "mount": "/run/lock", + "mount": "/sys/fs/cgroup", "fstype": "tmpfs" }, { "total": 0, "free": 0, - "device": "cgroup2", - "mount": "/sys/fs/cgroup", - "fstype": "cgroup2" + "device": "cgroup", + "mount": "/sys/fs/cgroup/systemd", + "fstype": "cgroup" }, { "total": 0, @@ -447,30 +403,80 @@ { "total": 0, "free": 0, - "device": "efivarfs", - "mount": "/sys/firmware/efi/efivars", - "fstype": "efivarfs" + "device": "cgroup", + "mount": "/sys/fs/cgroup/devices", + "fstype": "cgroup" }, { "total": 0, "free": 0, - "device": "none", - "mount": "/sys/fs/bpf", - "fstype": "bpf" + "device": "cgroup", + "mount": "/sys/fs/cgroup/cpu,cpuacct", + "fstype": "cgroup" }, { "total": 0, "free": 0, - "device": "systemd-1", - "mount": "/proc/sys/fs/binfmt_misc", - "fstype": "autofs" + "device": "cgroup", + "mount": "/sys/fs/cgroup/perf_event", + "fstype": "cgroup" }, { "total": 0, "free": 0, - "device": "hugetlbfs", - "mount": "/dev/hugepages", - "fstype": "hugetlbfs" + "device": "cgroup", + "mount": "/sys/fs/cgroup/pids", + "fstype": "cgroup" + }, + { + "total": 0, + "free": 0, + "device": "cgroup", + "mount": "/sys/fs/cgroup/blkio", + "fstype": "cgroup" + }, + { + "total": 0, + "free": 0, + "device": "cgroup", + "mount": "/sys/fs/cgroup/net_cls,net_prio", + "fstype": "cgroup" + }, + { + "total": 0, + "free": 0, + "device": "cgroup", + "mount": "/sys/fs/cgroup/freezer", + "fstype": "cgroup" + }, + { + "total": 0, + "free": 0, + "device": "cgroup", + "mount": "/sys/fs/cgroup/memory", + "fstype": "cgroup" + }, + { + "total": 0, + "free": 0, + "device": "cgroup", + "mount": "/sys/fs/cgroup/cpuset", + "fstype": "cgroup" + }, + { + "total": 0, + "free": 0, + "device": "cgroup", + "mount": "/sys/fs/cgroup/hugetlb", + "fstype": "cgroup" + }, + { + "endpoint_drive": true, + "total": 8577331200, + "free": 1841799168, + "device": "/dev/xvda1", + "mount": "/", + "fstype": "xfs" }, { "total": 0, @@ -482,37 +488,30 @@ { "total": 0, "free": 0, - "device": "debugfs", - "mount": "/sys/kernel/debug", - "fstype": "debugfs" + "device": "hugetlbfs", + "mount": "/dev/hugepages", + "fstype": "hugetlbfs" }, { "total": 0, "free": 0, - "device": "tracefs", - "mount": "/sys/kernel/tracing", - "fstype": "tracefs" + "device": "debugfs", + "mount": "/sys/kernel/debug", + "fstype": "debugfs" }, { "total": 0, "free": 0, - "device": "fusectl", - "mount": "/sys/fs/fuse/connections", - "fstype": "fusectl" + "device": "sunrpc", + "mount": "/var/lib/nfs/rpc_pipefs", + "fstype": "rpc_pipefs" }, { "total": 0, "free": 0, - "device": "configfs", - "mount": "/sys/kernel/config", - "fstype": "configfs" - }, - { - "total": 129751040, - "free": 118589440, - "device": "/dev/sda15", - "mount": "/boot/efi", - "fstype": "vfat" + "device": "systemd-1", + "mount": "/proc/sys/fs/binfmt_misc", + "fstype": "autofs" }, { "total": 0, @@ -527,11 +526,18 @@ "device": "tracefs", "mount": "/sys/kernel/debug/tracing", "fstype": "tracefs" + }, + { + "total": 0, + "free": 0, + "device": "none", + "mount": "/sys/fs/bpf", + "fstype": "bpf" } ], "documents_volume": { "file_events": { - "suppressed_count": 1651293, + "suppressed_count": 3928398, "suppressed_bytes": 0, "sent_count": 0, "sent_bytes": 0 @@ -539,76 +545,37 @@ "process_events": { "suppressed_count": 0, "suppressed_bytes": 0, - "sent_count": 11026887, - "sent_bytes": 23412176619 + "sent_count": 5689500, + "sent_bytes": 12905842838 }, "network_events": { - "suppressed_count": 3270009, + "suppressed_count": 23529207, "suppressed_bytes": 0, "sent_count": 0, "sent_bytes": 0 }, "overall": { - "suppressed_count": 4921302, + "suppressed_count": 27457605, "suppressed_bytes": 0, - "sent_count": 11026887, - "sent_bytes": 23412176619 + "sent_count": 5689500, + "sent_bytes": 12905842838 } }, "malicious_behavior_rules": [ { "id": "123", "endpoint_uptime_percent": 0 - } ], + } + ], "cpu": { "endpoint": { "histogram": { - "counts": [ - 3934326, - 520, - 140, - 110, - 212, - 78, - 41, - 18, - 12, - 25, - 9, - 5, - 5, - 2, - 0, - 0, - 0, - 0, - 0, - 0 - ], + "counts": [3439059, 129, 15, 0, 5, 6, 4, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "values": [ - 5, - 10, - 15, - 20, - 25, - 30, - 35, - 40, - 45, - 50, - 55, - 60, - 65, - 70, - 75, - 80, - 85, - 90, - 95, - 100 + 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 85, 90, 95, 100 ] }, - "mean": 0.2697424700222798, + "mean": 0.2374646952923115, "latest": 0.1 } }, @@ -616,79 +583,79 @@ { "name": "Cron", "cpu": { - "mean": 0.005330585583239717 + "mean": 0.00252963994365742 } }, { "name": "FileLogThread", "cpu": { - "mean": 0.01387886588350829 + "mean": 0.0006221101490102125 } }, { "name": "LoggingLimitThread", "cpu": { - "mean": 0.0003385089239880071 + "mean": 0.0003625542590258192 } }, { "name": "DocumentLoggingThread", "cpu": { - "mean": 0.002019893909071295 + "mean": 0.001359578471346822 } }, { "name": "DocumentLoggingMaintenance", "cpu": { - "mean": 0.0001711144011367948 + "mean": 0.0001029983690414259 } }, { "name": "BulkConsumerThread", "cpu": { - "mean": 0.01437360969549076 + "mean": 0.009533529038474382 } }, { "name": "DocumentLoggingConsumerThread", "cpu": { - "mean": 0.1566068758230231 + "mean": 0.1315907162873257 } }, { "name": "DocumentLoggingLimitThread", "cpu": { - "mean": 0.000383147463414997 + "mean": 0.0002471960856994222 } }, { "name": "ArtifactManifestDownload", "cpu": { - "mean": 0.00202733366564246 + "mean": 0.001561455274668017 } }, { "name": "PolicyReloadThread", "cpu": { - "mean": 0.00001487951314232998 + "mean": 0.00001235980428497111 } }, { "name": "PerformanceMonitorWorkerThread", "cpu": { - "mean": 0.005423582540379278 + "mean": 0.004639046541625823 } }, { "name": "MetadataThread", "cpu": { - "mean": 0.00001115963485674749 + "mean": 0 } }, { "name": "EventsQueueThread", "cpu": { - "mean": 0.05409447002894065 + "mean": 0.05644722616946305 } }, { @@ -700,13 +667,13 @@ { "name": "MaintainProcessMap", "cpu": { - "mean": 0.002905224941039929 + "mean": 0.002167085684631601 } }, { "name": "FileScoreThread", "cpu": { - "mean": 0.003567363275873613 + "mean": 0.002241244510341427 } }, { @@ -718,37 +685,37 @@ { "name": "QuarantineManagerWorkerThread", "cpu": { - "mean": 0.002083131839926198 + "mean": 0.002014648098450291 } }, { "name": "EventProcessingThread", "cpu": { - "mean": 0.01754294664738331 + "mean": 0.00583794755726802 } }, { "name": "HostIsolationMonitorThread", "cpu": { - "mean": 0.0008481322806622976 + "mean": 0.0008281068870930642 } }, { "name": "MountMonitor", "cpu": { - "mean": 0.003463206812704382 + "mean": 0.003407186047890369 } }, { "name": "responseActionsUploadThread", "cpu": { - "mean": 0.00270807149264102 + "mean": 0.00208468698939846 } }, { "name": "responseActionsProcessThread", "cpu": { - "mean": 0.002689472100521233 + "mean": 0.002105286663206745 } }, { @@ -760,61 +727,61 @@ { "name": "grpcConnectionManagerThread", "cpu": { - "mean": 0.0110926774602411 + "mean": 0.002344242879382854 } }, { "name": "checkinAPIThread", "cpu": { - "mean": 0.0003124700433295513 + "mean": 0.0002513162482505196 } }, { "name": "actionsAPIThread", "cpu": { - "mean": 0.0004910243538035807 + "mean": 0.0004985125580051291 } }, { "name": "stateReportThread", "cpu": { - "mean": 0.00539010824743476 + "mean": 0.004400094313632049 } }, { "name": "EventsLoopThread", "cpu": { - "mean": 0.003266057265515501 + "mean": 0.008866112391817567 } }, { "name": "FanotifyWatchdog", "cpu": { - "mean": 0.0002231929794201937 + "mean": 0.0002142369165309078 } }, { "name": "FanotifySyncConsumer", "cpu": { - "mean": 0.001885980676100637 + "mean": 0.001042344997736917 } }, { "name": "FanotifyAsyncConsumer", "cpu": { - "mean": 0.0002492321603525497 + "mean": 0 } }, { "name": "FanotifyConsumer", "cpu": { - "mean": 0.03793164685246193 + "mean": 0.03632551717409642 } }, { "name": "RulesEngineThread", "cpu": { - "mean": 0.02721466395730229 + "mean": 0.03752853985923151 } } ], @@ -823,8 +790,8 @@ "active_user_count": 0 }, "uptime": { - "endpoint": 26882599, - "system": 26883343 + "endpoint": 24272228, + "system": 29719735 } } }, @@ -845,42 +812,33 @@ "hostname": "123", "os": { "Ext": { - "variant": "Debian" + "variant": "Amazon Linux" }, - "kernel": "5.10.0-21-cloud-amd64 #1 SMP Debian 5.10.162-1 (2023-01-21)", + "kernel": "5.10.102-99.473.amzn2.x86_64 #1 SMP Wed Mar 2 19:14:12 UTC 2022", "name": "Linux", - "family": "debian", + "family": "amazon linux", "type": "linux", - "version": "11.8", - "platform": "debian", - "full": "Debian 11.8" + "version": "2", + "platform": "amazon linux", + "full": "Amazon Linux 2" }, - "ip": [ - "127.0.0.1", - "::1" - ], + "ip": ["127.0.0.1", "::1"], "name": "123", "id": "123", - "mac": [ - "aa:aa:aa:aa:aa:aa" - ], + "mac": ["aa:aa:aa:aa:aa:aa"], "architecture": "x86_64" }, "event": { "agent_id_status": "verified", - "sequence": 16194584, - "ingested": "2024-01-26T15:06:52Z", - "created": "2024-01-26T15:06:50.254822749Z", + "sequence": 33235745, + "ingested": "2024-01-26T14:38:32Z", + "created": "2024-01-26T14:38:30.628421712Z", "kind": "metric", "module": "endpoint", "action": "endpoint_metrics", "id": "123", - "category": [ - "host" - ], - "type": [ - "info" - ], + "category": ["host"], + "type": ["info"], "dataset": "endpoint.metrics" }, "message": "Endpoint metrics" @@ -894,7 +852,7 @@ "type": "endpoint", "version": "8.6.0" }, - "@timestamp": "2024-01-26T14:38:30.628421712Z", + "@timestamp": "2024-01-26T14:35:43.564911752Z", "Endpoint": { "metrics": { "system_impact": [ @@ -904,15 +862,15 @@ }, "malware": { "week_idle_ms": 0, - "week_ms": 36 + "week_ms": 51 }, "process_events": { - "week_idle_ms": 5, - "week_ms": 12487 + "week_idle_ms": 2, + "week_ms": 8914 }, "overall": { - "week_idle_ms": 5, - "week_ms": 12523 + "week_idle_ms": 2, + "week_ms": 8965 } }, { @@ -920,12 +878,12 @@ "executable": "/usr/bin/ps" }, "process_events": { - "week_idle_ms": 8692, - "week_ms": 10288 + "week_idle_ms": 7543, + "week_ms": 6667 }, "overall": { - "week_idle_ms": 8692, - "week_ms": 10288 + "week_idle_ms": 7543, + "week_ms": 6667 } }, { @@ -934,19 +892,19 @@ }, "malware": { "week_idle_ms": 0, - "week_ms": 116 + "week_ms": 51 }, "process_events": { - "week_idle_ms": 3621, - "week_ms": 7204 + "week_idle_ms": 3596, + "week_ms": 4596 }, "network_events": { - "week_idle_ms": 97, - "week_ms": 0 + "week_idle_ms": 85, + "week_ms": 5 }, "overall": { - "week_idle_ms": 3718, - "week_ms": 7320 + "week_idle_ms": 3681, + "week_ms": 4652 } }, { @@ -955,15 +913,15 @@ }, "malware": { "week_idle_ms": 0, - "week_ms": 430 + "week_ms": 380 }, "process_events": { - "week_idle_ms": 152054, - "week_ms": 6229 + "week_idle_ms": 105790, + "week_ms": 3877 }, "overall": { - "week_idle_ms": 152054, - "week_ms": 6659 + "week_idle_ms": 105790, + "week_ms": 4257 } }, { @@ -972,105 +930,105 @@ }, "malware": { "week_idle_ms": 0, - "week_ms": 73 + "week_ms": 129 }, "process_events": { - "week_idle_ms": 9491, - "week_ms": 2060 + "week_idle_ms": 8062, + "week_ms": 1903 }, "overall": { - "week_idle_ms": 9491, - "week_ms": 2133 + "week_idle_ms": 8062, + "week_ms": 2032 } }, { "file_events": { - "week_idle_ms": 80958, - "week_ms": 1963 + "week_idle_ms": 69051, + "week_ms": 1544 }, "process": { "executable": "/usr/lib/systemd/systemd" }, "malware": { "week_idle_ms": 0, - "week_ms": 23 + "week_ms": 15 }, "process_events": { - "week_idle_ms": 6, - "week_ms": 7 + "week_idle_ms": 31, + "week_ms": 9 }, "overall": { - "week_idle_ms": 80964, - "week_ms": 1993 + "week_idle_ms": 69082, + "week_ms": 1568 } }, { "process": { - "executable": "/bin/logger" + "executable": "unknown" }, - "process_events": { - "week_idle_ms": 25156, - "week_ms": 1097 + "malware": { + "week_idle_ms": 0, + "week_ms": 979 }, "overall": { - "week_idle_ms": 25156, - "week_ms": 1097 + "week_idle_ms": 0, + "week_ms": 979 } }, { "file_events": { - "week_idle_ms": 12, + "week_idle_ms": 29, "week_ms": 0 }, "process": { "executable": "/usr/lib64/sa/sadc" }, "process_events": { - "week_idle_ms": 4182, - "week_ms": 1005 + "week_idle_ms": 5107, + "week_ms": 917 }, "overall": { - "week_idle_ms": 4194, - "week_ms": 1005 + "week_idle_ms": 5136, + "week_ms": 917 } }, { "process": { - "executable": "unknown" + "executable": "/bin/logger" }, - "malware": { - "week_idle_ms": 0, - "week_ms": 981 + "process_events": { + "week_idle_ms": 20585, + "week_ms": 880 }, "overall": { - "week_idle_ms": 0, - "week_ms": 981 + "week_idle_ms": 20585, + "week_ms": 880 } }, { - "process": { - "executable": "/usr/bin/date" + "file_events": { + "week_idle_ms": 72520, + "week_ms": 852 }, - "process_events": { - "week_idle_ms": 8682, - "week_ms": 856 + "process": { + "executable": "/usr/lib/systemd/systemd-logind (deleted)" }, "overall": { - "week_idle_ms": 8682, - "week_ms": 856 + "week_idle_ms": 72520, + "week_ms": 852 } }, { - "file_events": { - "week_idle_ms": 89949, - "week_ms": 835 - }, "process": { - "executable": "/usr/lib/systemd/systemd-logind (deleted)" + "executable": "/usr/bin/date" + }, + "process_events": { + "week_idle_ms": 9862, + "week_ms": 808 }, "overall": { - "week_idle_ms": 89949, - "week_ms": 835 + "week_idle_ms": 9862, + "week_ms": 808 } }, { @@ -1078,12 +1036,12 @@ "executable": "/bin/cat" }, "process_events": { - "week_idle_ms": 28801, - "week_ms": 754 + "week_idle_ms": 17468, + "week_ms": 701 }, "overall": { - "week_idle_ms": 28801, - "week_ms": 754 + "week_idle_ms": 17468, + "week_ms": 701 } }, { @@ -1092,126 +1050,130 @@ }, "malware": { "week_idle_ms": 0, - "week_ms": 69 + "week_ms": 67 }, "process_events": { - "week_idle_ms": 7188, - "week_ms": 637 + "week_idle_ms": 7451, + "week_ms": 618 }, "overall": { - "week_idle_ms": 7188, - "week_ms": 706 + "week_idle_ms": 7451, + "week_ms": 685 } }, { "process": { - "executable": "/bin/cut" + "executable": "/bin/curl" }, "process_events": { - "week_idle_ms": 7258, - "week_ms": 660 + "week_idle_ms": 5615, + "week_ms": 403 + }, + "network_events": { + "week_idle_ms": 4128, + "week_ms": 6 }, "overall": { - "week_idle_ms": 7258, - "week_ms": 660 + "week_idle_ms": 9743, + "week_ms": 409 } }, { "process": { - "executable": "/bin/ipcalc" + "executable": "/bin/run-parts" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 116 }, "process_events": { - "week_idle_ms": 6603, - "week_ms": 477 + "week_idle_ms": 3770, + "week_ms": 260 }, "overall": { - "week_idle_ms": 6603, - "week_ms": 477 + "week_idle_ms": 3770, + "week_ms": 376 } }, { "process": { - "executable": "/sbin/ip" + "executable": "/usr/sbin/dhclient" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 36 }, "process_events": { - "week_idle_ms": 14654, - "week_ms": 470 + "week_idle_ms": 1, + "week_ms": 278 }, "overall": { - "week_idle_ms": 14654, - "week_ms": 470 + "week_idle_ms": 1, + "week_ms": 314 } }, { "process": { - "executable": "/bin/curl" + "executable": "/bin/cut" }, "process_events": { - "week_idle_ms": 4696, - "week_ms": 432 - }, - "network_events": { - "week_idle_ms": 3120, - "week_ms": 20 + "week_idle_ms": 5067, + "week_ms": 313 }, "overall": { - "week_idle_ms": 7816, - "week_ms": 452 + "week_idle_ms": 5067, + "week_ms": 313 } }, { "process": { - "executable": "/bin/run-parts" - }, - "malware": { - "week_idle_ms": 0, - "week_ms": 66 + "executable": "/sbin/ip" }, "process_events": { - "week_idle_ms": 4219, - "week_ms": 354 + "week_idle_ms": 10369, + "week_ms": 286 }, "overall": { - "week_idle_ms": 4219, - "week_ms": 420 + "week_idle_ms": 10369, + "week_ms": 286 } }, { "process": { - "executable": "/usr/sbin/dhclient" + "executable": "/bin/sh" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 20 }, "process_events": { - "week_idle_ms": 3, - "week_ms": 326 + "week_idle_ms": 2397, + "week_ms": 208 }, "overall": { - "week_idle_ms": 3, - "week_ms": 326 + "week_idle_ms": 2397, + "week_ms": 228 } }, { "process": { - "executable": "/bin/sh" - }, - "malware": { - "week_idle_ms": 0, - "week_ms": 51 + "executable": "/bin/ipcalc" }, "process_events": { - "week_idle_ms": 2569, - "week_ms": 239 + "week_idle_ms": 4827, + "week_ms": 201 }, "overall": { - "week_idle_ms": 2569, - "week_ms": 290 + "week_idle_ms": 4827, + "week_ms": 201 } } ], "memory": { "endpoint": { "private": { - "mean": 119318921, - "latest": 136073216 + "mean": 118266395, + "latest": 135163904 } } }, @@ -1290,7 +1252,7 @@ "total": 0, "free": 0, "device": "cgroup", - "mount": "/sys/fs/cgroup/devices", + "mount": "/sys/fs/cgroup/blkio", "fstype": "cgroup" }, { @@ -1304,62 +1266,62 @@ "total": 0, "free": 0, "device": "cgroup", - "mount": "/sys/fs/cgroup/perf_event", + "mount": "/sys/fs/cgroup/net_cls,net_prio", "fstype": "cgroup" }, { "total": 0, "free": 0, "device": "cgroup", - "mount": "/sys/fs/cgroup/pids", + "mount": "/sys/fs/cgroup/devices", "fstype": "cgroup" }, { "total": 0, "free": 0, "device": "cgroup", - "mount": "/sys/fs/cgroup/blkio", + "mount": "/sys/fs/cgroup/cpuset", "fstype": "cgroup" }, { "total": 0, "free": 0, "device": "cgroup", - "mount": "/sys/fs/cgroup/net_cls,net_prio", + "mount": "/sys/fs/cgroup/hugetlb", "fstype": "cgroup" }, { "total": 0, "free": 0, "device": "cgroup", - "mount": "/sys/fs/cgroup/freezer", + "mount": "/sys/fs/cgroup/pids", "fstype": "cgroup" }, { "total": 0, "free": 0, "device": "cgroup", - "mount": "/sys/fs/cgroup/memory", + "mount": "/sys/fs/cgroup/freezer", "fstype": "cgroup" }, { "total": 0, "free": 0, "device": "cgroup", - "mount": "/sys/fs/cgroup/cpuset", + "mount": "/sys/fs/cgroup/memory", "fstype": "cgroup" }, { "total": 0, "free": 0, "device": "cgroup", - "mount": "/sys/fs/cgroup/hugetlb", + "mount": "/sys/fs/cgroup/perf_event", "fstype": "cgroup" }, { "endpoint_drive": true, "total": 8577331200, - "free": 1841799168, + "free": 1908756480, "device": "/dev/xvda1", "mount": "/", "fstype": "xfs" @@ -1367,16 +1329,16 @@ { "total": 0, "free": 0, - "device": "mqueue", - "mount": "/dev/mqueue", - "fstype": "mqueue" + "device": "hugetlbfs", + "mount": "/dev/hugepages", + "fstype": "hugetlbfs" }, { "total": 0, "free": 0, - "device": "hugetlbfs", - "mount": "/dev/hugepages", - "fstype": "hugetlbfs" + "device": "mqueue", + "mount": "/dev/mqueue", + "fstype": "mqueue" }, { "total": 0, @@ -1423,7 +1385,7 @@ ], "documents_volume": { "file_events": { - "suppressed_count": 3928398, + "suppressed_count": 3926292, "suppressed_bytes": 0, "sent_count": 0, "sent_bytes": 0 @@ -1431,156 +1393,117 @@ "process_events": { "suppressed_count": 0, "suppressed_bytes": 0, - "sent_count": 5689500, - "sent_bytes": 12905842838 + "sent_count": 5737776, + "sent_bytes": 12992145486 }, "network_events": { - "suppressed_count": 23529207, + "suppressed_count": 24813200, "suppressed_bytes": 0, "sent_count": 0, "sent_bytes": 0 }, "overall": { - "suppressed_count": 27457605, + "suppressed_count": 28739492, "suppressed_bytes": 0, - "sent_count": 5689500, - "sent_bytes": 12905842838 + "sent_count": 5737776, + "sent_bytes": 12992145486 } }, "malicious_behavior_rules": [ { "id": "123", "endpoint_uptime_percent": 0 - } ], + } + ], "cpu": { "endpoint": { "histogram": { - "counts": [ - 3439059, - 129, - 15, - 0, - 5, - 6, - 4, - 0, - 3, - 4, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0 - ], + "counts": [3147842, 87, 11, 10, 9, 5, 10, 6, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "values": [ - 5, - 10, - 15, - 20, - 25, - 30, - 35, - 40, - 45, - 50, - 55, - 60, - 65, - 70, - 75, - 80, - 85, - 90, - 95, - 100 + 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 85, 90, 95, 100 ] }, - "mean": 0.2374646952923115, - "latest": 0.1 + "mean": 0.2338436071605115, + "latest": 0.9000000000000001 } }, "threads": [ { "name": "Cron", "cpu": { - "mean": 0.00252963994365742 + "mean": 0.002129690752013577 } }, { "name": "FileLogThread", "cpu": { - "mean": 0.0006221101490102125 + "mean": 0.0004778416387496615 } }, { "name": "LoggingLimitThread", "cpu": { - "mean": 0.0003625542590258192 + "mean": 0.0002965913619825484 } }, { "name": "DocumentLoggingThread", "cpu": { - "mean": 0.001359578471346822 + "mean": 0.001116336931906537 } }, { "name": "DocumentLoggingMaintenance", "cpu": { - "mean": 0.0001029983690414259 + "mean": 0.00009062513838355648 } }, { "name": "BulkConsumerThread", "cpu": { - "mean": 0.009533529038474382 + "mean": 0.008971888699972091 } }, { "name": "DocumentLoggingConsumerThread", "cpu": { - "mean": 0.1315907162873257 + "mean": 0.1281974968924846 } }, { "name": "DocumentLoggingLimitThread", "cpu": { - "mean": 0.0002471960856994222 + "mean": 0.000205966223598992 } }, { "name": "ArtifactManifestDownload", "cpu": { - "mean": 0.001561455274668017 + "mean": 0.001515911405688581 } }, { "name": "PolicyReloadThread", "cpu": { - "mean": 0.00001235980428497111 + "mean": 0.00000823864894395968 } }, { "name": "PerformanceMonitorWorkerThread", "cpu": { - "mean": 0.004639046541625823 + "mean": 0.003917477572852828 } }, { "name": "MetadataThread", "cpu": { - "mean": 0 + "mean": 0.00000411932447197984 } }, { "name": "EventsQueueThread", "cpu": { - "mean": 0.05644722616946305 + "mean": 0.04977791691940438 } }, { @@ -1592,13 +1515,13 @@ { "name": "MaintainProcessMap", "cpu": { - "mean": 0.002167085684631601 + "mean": 0.001837218714503009 } }, { "name": "FileScoreThread", "cpu": { - "mean": 0.002241244510341427 + "mean": 0.001923724528414585 } }, { @@ -1610,37 +1533,37 @@ { "name": "QuarantineManagerWorkerThread", "cpu": { - "mean": 0.002014648098450291 + "mean": 0.001557104650408379 } }, { "name": "EventProcessingThread", "cpu": { - "mean": 0.00583794755726802 + "mean": 0.00591534994176305 } }, { "name": "HostIsolationMonitorThread", "cpu": { - "mean": 0.0008281068870930642 + "mean": 0.0006508532665728148 } }, { "name": "MountMonitor", "cpu": { - "mean": 0.003407186047890369 + "mean": 0.002533384550267602 } }, { "name": "responseActionsUploadThread", "cpu": { - "mean": 0.00208468698939846 + "mean": 0.001771309522951331 } }, { "name": "responseActionsProcessThread", "cpu": { - "mean": 0.002105286663206745 + "mean": 0.001783667496367271 } }, { @@ -1652,43 +1575,43 @@ { "name": "grpcConnectionManagerThread", "cpu": { - "mean": 0.002344242879382854 + "mean": 0.001956679124190424 } }, { "name": "checkinAPIThread", "cpu": { - "mean": 0.0002513162482505196 + "mean": 0.0002389210358973501 } }, { "name": "actionsAPIThread", "cpu": { - "mean": 0.0004985125580051291 + "mean": 0.0003913361794870389 } }, { "name": "stateReportThread", "cpu": { - "mean": 0.004400094313632049 + "mean": 0.003550860912819238 } }, { "name": "EventsLoopThread", "cpu": { - "mean": 0.008866112391817567 + "mean": 0.0077690568352644 } }, { "name": "FanotifyWatchdog", "cpu": { - "mean": 0.0002142369165309078 + "mean": 0.0001647732096556607 } }, { "name": "FanotifySyncConsumer", "cpu": { - "mean": 0.001042344997736917 + "mean": 0.001103980504692926 } }, { @@ -1700,13 +1623,13 @@ { "name": "FanotifyConsumer", "cpu": { - "mean": 0.03632551717409642 + "mean": 0.02717934093270123 } }, { "name": "RulesEngineThread", "cpu": { - "mean": 0.03752853985923151 + "mean": 0.03744471189424889 } } ], @@ -1715,8 +1638,8 @@ "active_user_count": 0 }, "uptime": { - "endpoint": 24272228, - "system": 29719735 + "endpoint": 24275825, + "system": 29714870 } } }, @@ -1747,32 +1670,23 @@ "platform": "amazon linux", "full": "Amazon Linux 2" }, - "ip": [ - "127.0.0.1", - "::1" - ], + "ip": ["127.0.0.1", "::1"], "name": "123", "id": "123", - "mac": [ - "aa:aa:aa:aa:aa:aa" - ], + "mac": ["aa:aa:aa:aa:aa:aa"], "architecture": "x86_64" }, "event": { "agent_id_status": "verified", - "sequence": 33235745, - "ingested": "2024-01-26T14:38:32Z", - "created": "2024-01-26T14:38:30.628421712Z", + "sequence": 34569822, + "ingested": "2024-01-26T14:35:45Z", + "created": "2024-01-26T14:35:43.564911752Z", "kind": "metric", "module": "endpoint", "action": "endpoint_metrics", "id": "123", - "category": [ - "host" - ], - "type": [ - "info" - ], + "category": ["host"], + "type": ["info"], "dataset": "endpoint.metrics" }, "message": "Endpoint metrics" @@ -1780,334 +1694,366 @@ { "agent": { "build": { - "original": "version: 8.6.0, compiled: Mon Jan 2 23:00:00 2023, branch: 8.6, commit: e2d09ff1b8e49bfb5f8940d317eb4ac96672d956" + "original": "version: 8.6.2, compiled: Fri Feb 10 14:00:00 2023, branch: 8.6, commit: eaee25e73cc58a52387e50d5bce542ec8965f638" }, "id": "123", "type": "endpoint", - "version": "8.6.0" + "version": "8.6.2" }, - "@timestamp": "2024-01-26T14:35:43.564911752Z", + "@timestamp": "2024-01-26T15:06:50.254822749Z", "Endpoint": { "metrics": { "system_impact": [ { "process": { - "executable": "/usr/bin/amazon-ssm-agent" + "executable": "/usr/sbin/sshd" }, "malware": { "week_idle_ms": 0, - "week_ms": 51 + "week_ms": 304 }, "process_events": { - "week_idle_ms": 2, - "week_ms": 8914 + "week_idle_ms": 31791, + "week_ms": 110821 + }, + "network_events": { + "week_idle_ms": 2030, + "week_ms": 53 }, "overall": { - "week_idle_ms": 2, - "week_ms": 8965 + "week_idle_ms": 33821, + "week_ms": 111178 } }, { "process": { - "executable": "/usr/bin/ps" + "executable": "unknown" }, - "process_events": { - "week_idle_ms": 7543, - "week_ms": 6667 + "malware": { + "week_idle_ms": 0, + "week_ms": 4552 }, "overall": { - "week_idle_ms": 7543, - "week_ms": 6667 + "week_idle_ms": 0, + "week_ms": 4552 } }, { + "file_events": { + "week_idle_ms": 130, + "week_ms": 7 + }, "process": { - "executable": "/usr/sbin/sshd" + "executable": "/lib/systemd/systemd" }, "malware": { "week_idle_ms": 0, - "week_ms": 51 + "week_ms": 1920 }, "process_events": { - "week_idle_ms": 3596, - "week_ms": 4596 - }, - "network_events": { - "week_idle_ms": 85, - "week_ms": 5 + "week_idle_ms": 1273, + "week_ms": 1750 }, "overall": { - "week_idle_ms": 3681, - "week_ms": 4652 + "week_idle_ms": 1403, + "week_ms": 3677 } }, { + "file_events": { + "week_idle_ms": 7, + "week_ms": 0 + }, "process": { - "executable": "/usr/sbin/dhclient-script" + "executable": "/bin/systemctl" }, "malware": { "week_idle_ms": 0, - "week_ms": 380 + "week_ms": 3055 }, "process_events": { - "week_idle_ms": 105790, - "week_ms": 3877 + "week_idle_ms": 0, + "week_ms": 9 }, "overall": { - "week_idle_ms": 105790, - "week_ms": 4257 + "week_idle_ms": 7, + "week_ms": 3064 } }, { + "file_events": { + "week_idle_ms": 3104, + "week_ms": 1 + }, "process": { - "executable": "/usr/sbin/crond" + "executable": "/usr/bin/apt-key" }, "malware": { "week_idle_ms": 0, - "week_ms": 129 + "week_ms": 61 }, "process_events": { - "week_idle_ms": 8062, - "week_ms": 1903 + "week_idle_ms": 178857, + "week_ms": 2062 }, "overall": { - "week_idle_ms": 8062, - "week_ms": 2032 + "week_idle_ms": 181961, + "week_ms": 2124 } }, { - "file_events": { - "week_idle_ms": 69051, - "week_ms": 1544 - }, "process": { - "executable": "/usr/lib/systemd/systemd" - }, - "malware": { - "week_idle_ms": 0, - "week_ms": 15 + "executable": "/usr/bin/gce_workload_cert_refresh" }, "process_events": { - "week_idle_ms": 31, - "week_ms": 9 + "week_idle_ms": 230, + "week_ms": 1405 + }, + "network_events": { + "week_idle_ms": 135, + "week_ms": 21 }, "overall": { - "week_idle_ms": 69082, - "week_ms": 1568 + "week_idle_ms": 365, + "week_ms": 1426 } }, { "process": { - "executable": "unknown" + "executable": "/usr/bin/apt-config" }, "malware": { "week_idle_ms": 0, - "week_ms": 979 + "week_ms": 65 + }, + "process_events": { + "week_idle_ms": 18708, + "week_ms": 840 }, "overall": { - "week_idle_ms": 0, - "week_ms": 979 + "week_idle_ms": 18708, + "week_ms": 905 } }, { - "file_events": { - "week_idle_ms": 29, - "week_ms": 0 - }, "process": { - "executable": "/usr/lib64/sa/sadc" + "executable": "/usr/lib/apt/apt.systemd.daily" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 273 }, "process_events": { - "week_idle_ms": 5107, - "week_ms": 917 + "week_idle_ms": 1064, + "week_ms": 352 }, "overall": { - "week_idle_ms": 5136, - "week_ms": 917 + "week_idle_ms": 1064, + "week_ms": 625 } }, { "process": { - "executable": "/bin/logger" + "executable": "/usr/bin/dpkg" }, "process_events": { - "week_idle_ms": 20585, - "week_ms": 880 + "week_idle_ms": 11479, + "week_ms": 558 }, "overall": { - "week_idle_ms": 20585, - "week_ms": 880 + "week_idle_ms": 11479, + "week_ms": 558 } }, { - "file_events": { - "week_idle_ms": 72520, - "week_ms": 852 - }, "process": { - "executable": "/usr/lib/systemd/systemd-logind (deleted)" + "executable": "/usr/sbin/cron" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 80 + }, + "process_events": { + "week_idle_ms": 270, + "week_ms": 451 }, "overall": { - "week_idle_ms": 72520, - "week_ms": 852 + "week_idle_ms": 270, + "week_ms": 531 } }, { "process": { - "executable": "/usr/bin/date" + "executable": "/usr/bin/cmp" }, "process_events": { - "week_idle_ms": 9862, - "week_ms": 808 + "week_idle_ms": 39452, + "week_ms": 513 }, "overall": { - "week_idle_ms": 9862, - "week_ms": 808 + "week_idle_ms": 39452, + "week_ms": 513 } }, { + "file_events": { + "week_idle_ms": 144, + "week_ms": 0 + }, "process": { - "executable": "/bin/cat" + "executable": "/sbin/dhclient-script" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 251 }, "process_events": { - "week_idle_ms": 17468, - "week_ms": 701 + "week_idle_ms": 2412, + "week_ms": 186 }, "overall": { - "week_idle_ms": 17468, - "week_ms": 701 + "week_idle_ms": 2556, + "week_ms": 437 } }, { "process": { - "executable": "/usr/lib64/sa/sa1" + "executable": "/usr/bin/env" }, "malware": { "week_idle_ms": 0, - "week_ms": 67 + "week_ms": 413 }, "process_events": { - "week_idle_ms": 7451, - "week_ms": 618 + "week_idle_ms": 45, + "week_ms": 4 }, "overall": { - "week_idle_ms": 7451, - "week_ms": 685 + "week_idle_ms": 45, + "week_ms": 417 } }, { "process": { - "executable": "/bin/curl" + "executable": "/usr/bin/cat" }, "process_events": { - "week_idle_ms": 5615, - "week_ms": 403 - }, - "network_events": { - "week_idle_ms": 4128, - "week_ms": 6 + "week_idle_ms": 39679, + "week_ms": 359 }, "overall": { - "week_idle_ms": 9743, - "week_ms": 409 + "week_idle_ms": 39679, + "week_ms": 359 } }, { "process": { - "executable": "/bin/run-parts" + "executable": "/bin/sh" }, "malware": { "week_idle_ms": 0, - "week_ms": 116 + "week_ms": 35 }, "process_events": { - "week_idle_ms": 3770, - "week_ms": 260 + "week_idle_ms": 444, + "week_ms": 197 }, "overall": { - "week_idle_ms": 3770, - "week_ms": 376 + "week_idle_ms": 444, + "week_ms": 232 } }, { "process": { - "executable": "/usr/sbin/dhclient" + "executable": "/usr/lib/apt/apt-helper" }, "malware": { "week_idle_ms": 0, - "week_ms": 36 + "week_ms": 100 }, "process_events": { - "week_idle_ms": 1, - "week_ms": 278 + "week_idle_ms": 8, + "week_ms": 62 }, "overall": { - "week_idle_ms": 1, - "week_ms": 314 + "week_idle_ms": 8, + "week_ms": 162 } }, { "process": { - "executable": "/bin/cut" + "executable": "/usr/bin/date" }, "process_events": { - "week_idle_ms": 5067, - "week_ms": 313 + "week_idle_ms": 1024, + "week_ms": 143 }, "overall": { - "week_idle_ms": 5067, - "week_ms": 313 + "week_idle_ms": 1024, + "week_ms": 143 } }, { + "file_events": { + "week_idle_ms": 5315, + "week_ms": 2 + }, "process": { - "executable": "/sbin/ip" + "executable": "/usr/bin/apt-get" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 53 }, "process_events": { - "week_idle_ms": 10369, - "week_ms": 286 + "week_idle_ms": 576, + "week_ms": 76 }, "overall": { - "week_idle_ms": 10369, - "week_ms": 286 + "week_idle_ms": 5891, + "week_ms": 131 } }, { "process": { - "executable": "/bin/sh" + "executable": "/usr/bin/unattended-upgrade" }, "malware": { "week_idle_ms": 0, - "week_ms": 20 + "week_ms": 27 }, "process_events": { - "week_idle_ms": 2397, - "week_ms": 208 + "week_idle_ms": 139, + "week_ms": 95 }, "overall": { - "week_idle_ms": 2397, - "week_ms": 228 + "week_idle_ms": 139, + "week_ms": 122 } }, { "process": { - "executable": "/bin/ipcalc" + "executable": "/usr/bin/gpgconf" + }, + "malware": { + "week_idle_ms": 0, + "week_ms": 26 }, "process_events": { - "week_idle_ms": 4827, - "week_ms": 201 + "week_idle_ms": 12075, + "week_ms": 93 }, "overall": { - "week_idle_ms": 4827, - "week_ms": 201 + "week_idle_ms": 12075, + "week_ms": 119 } } ], "memory": { "endpoint": { "private": { - "mean": 118266395, - "latest": 135163904 + "mean": 373780821, + "latest": 411860992 } } }, @@ -2127,53 +2073,61 @@ "fstype": "proc" }, { - "total": 4157120512, - "free": 4157120512, - "device": "devtmpfs", + "total": 2049777664, + "free": 2049777664, + "device": "udev", "mount": "/dev", "fstype": "devtmpfs" }, { "total": 0, "free": 0, - "device": "securityfs", - "mount": "/sys/kernel/security", - "fstype": "securityfs" + "device": "devpts", + "mount": "/dev/pts", + "fstype": "devpts" }, { - "total": 4166328320, - "free": 4166328320, + "total": 412172288, + "free": 411795456, "device": "tmpfs", - "mount": "/dev/shm", + "mount": "/run", "fstype": "tmpfs" }, + { + "endpoint_drive": true, + "total": 10331889664, + "free": 4091109376, + "device": "/dev/sda1", + "mount": "/", + "fstype": "ext4" + }, { "total": 0, "free": 0, - "device": "devpts", - "mount": "/dev/pts", - "fstype": "devpts" + "device": "securityfs", + "mount": "/sys/kernel/security", + "fstype": "securityfs" }, { - "total": 4166328320, - "free": 4165955584, + "total": 2060849152, + "free": 2060849152, "device": "tmpfs", - "mount": "/run", + "mount": "/dev/shm", "fstype": "tmpfs" }, { - "total": 4166328320, - "free": 4166328320, + "total": 5242880, + "free": 5242880, "device": "tmpfs", - "mount": "/sys/fs/cgroup", + "mount": "/run/lock", "fstype": "tmpfs" }, { "total": 0, "free": 0, - "device": "cgroup", - "mount": "/sys/fs/cgroup/systemd", - "fstype": "cgroup" + "device": "cgroup2", + "mount": "/sys/fs/cgroup", + "fstype": "cgroup2" }, { "total": 0, @@ -2185,80 +2139,23 @@ { "total": 0, "free": 0, - "device": "cgroup", - "mount": "/sys/fs/cgroup/blkio", - "fstype": "cgroup" - }, - { - "total": 0, - "free": 0, - "device": "cgroup", - "mount": "/sys/fs/cgroup/cpu,cpuacct", - "fstype": "cgroup" - }, - { - "total": 0, - "free": 0, - "device": "cgroup", - "mount": "/sys/fs/cgroup/net_cls,net_prio", - "fstype": "cgroup" - }, - { - "total": 0, - "free": 0, - "device": "cgroup", - "mount": "/sys/fs/cgroup/devices", - "fstype": "cgroup" - }, - { - "total": 0, - "free": 0, - "device": "cgroup", - "mount": "/sys/fs/cgroup/cpuset", - "fstype": "cgroup" - }, - { - "total": 0, - "free": 0, - "device": "cgroup", - "mount": "/sys/fs/cgroup/hugetlb", - "fstype": "cgroup" - }, - { - "total": 0, - "free": 0, - "device": "cgroup", - "mount": "/sys/fs/cgroup/pids", - "fstype": "cgroup" - }, - { - "total": 0, - "free": 0, - "device": "cgroup", - "mount": "/sys/fs/cgroup/freezer", - "fstype": "cgroup" + "device": "efivarfs", + "mount": "/sys/firmware/efi/efivars", + "fstype": "efivarfs" }, { "total": 0, "free": 0, - "device": "cgroup", - "mount": "/sys/fs/cgroup/memory", - "fstype": "cgroup" + "device": "none", + "mount": "/sys/fs/bpf", + "fstype": "bpf" }, { "total": 0, "free": 0, - "device": "cgroup", - "mount": "/sys/fs/cgroup/perf_event", - "fstype": "cgroup" - }, - { - "endpoint_drive": true, - "total": 8577331200, - "free": 1908756480, - "device": "/dev/xvda1", - "mount": "/", - "fstype": "xfs" + "device": "systemd-1", + "mount": "/proc/sys/fs/binfmt_misc", + "fstype": "autofs" }, { "total": 0, @@ -2284,16 +2181,30 @@ { "total": 0, "free": 0, - "device": "sunrpc", - "mount": "/var/lib/nfs/rpc_pipefs", - "fstype": "rpc_pipefs" + "device": "tracefs", + "mount": "/sys/kernel/tracing", + "fstype": "tracefs" }, { "total": 0, "free": 0, - "device": "systemd-1", - "mount": "/proc/sys/fs/binfmt_misc", - "fstype": "autofs" + "device": "fusectl", + "mount": "/sys/fs/fuse/connections", + "fstype": "fusectl" + }, + { + "total": 0, + "free": 0, + "device": "configfs", + "mount": "/sys/kernel/config", + "fstype": "configfs" + }, + { + "total": 129751040, + "free": 118589440, + "device": "/dev/sda15", + "mount": "/boot/efi", + "fstype": "vfat" }, { "total": 0, @@ -2308,18 +2219,11 @@ "device": "tracefs", "mount": "/sys/kernel/debug/tracing", "fstype": "tracefs" - }, - { - "total": 0, - "free": 0, - "device": "none", - "mount": "/sys/fs/bpf", - "fstype": "bpf" } ], "documents_volume": { "file_events": { - "suppressed_count": 3926292, + "suppressed_count": 1651293, "suppressed_bytes": 0, "sent_count": 0, "sent_bytes": 0 @@ -2327,20 +2231,20 @@ "process_events": { "suppressed_count": 0, "suppressed_bytes": 0, - "sent_count": 5737776, - "sent_bytes": 12992145486 + "sent_count": 11026887, + "sent_bytes": 23412176619 }, "network_events": { - "suppressed_count": 24813200, + "suppressed_count": 3270009, "suppressed_bytes": 0, "sent_count": 0, "sent_bytes": 0 }, "overall": { - "suppressed_count": 28739492, + "suppressed_count": 4921302, "suppressed_bytes": 0, - "sent_count": 5737776, - "sent_bytes": 12992145486 + "sent_count": 11026887, + "sent_bytes": 23412176619 } }, "malicious_behavior_rules": [ @@ -2353,131 +2257,93 @@ "endpoint": { "histogram": { "counts": [ - 3147842, - 87, - 11, - 10, - 9, - 5, - 10, - 6, - 1, - 2, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0 + 3934326, 520, 140, 110, 212, 78, 41, 18, 12, 25, 9, 5, 5, 2, 0, 0, 0, 0, 0, 0 ], "values": [ - 5, - 10, - 15, - 20, - 25, - 30, - 35, - 40, - 45, - 50, - 55, - 60, - 65, - 70, - 75, - 80, - 85, - 90, - 95, - 100 + 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 85, 90, 95, 100 ] }, - "mean": 0.2338436071605115, - "latest": 0.9000000000000001 + "mean": 0.2697424700222798, + "latest": 0.1 } }, "threads": [ { "name": "Cron", "cpu": { - "mean": 0.002129690752013577 + "mean": 0.005330585583239717 } }, { "name": "FileLogThread", "cpu": { - "mean": 0.0004778416387496615 + "mean": 0.01387886588350829 } }, { "name": "LoggingLimitThread", "cpu": { - "mean": 0.0002965913619825484 + "mean": 0.0003385089239880071 } }, { "name": "DocumentLoggingThread", "cpu": { - "mean": 0.001116336931906537 + "mean": 0.002019893909071295 } }, { "name": "DocumentLoggingMaintenance", "cpu": { - "mean": 0.00009062513838355648 + "mean": 0.0001711144011367948 } }, { "name": "BulkConsumerThread", "cpu": { - "mean": 0.008971888699972091 + "mean": 0.01437360969549076 } }, { "name": "DocumentLoggingConsumerThread", "cpu": { - "mean": 0.1281974968924846 + "mean": 0.1566068758230231 } }, { "name": "DocumentLoggingLimitThread", "cpu": { - "mean": 0.000205966223598992 + "mean": 0.000383147463414997 } }, { "name": "ArtifactManifestDownload", "cpu": { - "mean": 0.001515911405688581 + "mean": 0.00202733366564246 } }, { "name": "PolicyReloadThread", "cpu": { - "mean": 0.00000823864894395968 + "mean": 0.00001487951314232998 } }, { "name": "PerformanceMonitorWorkerThread", "cpu": { - "mean": 0.003917477572852828 + "mean": 0.005423582540379278 } }, { "name": "MetadataThread", "cpu": { - "mean": 0.00000411932447197984 + "mean": 0.00001115963485674749 } }, { "name": "EventsQueueThread", "cpu": { - "mean": 0.04977791691940438 + "mean": 0.05409447002894065 } }, { @@ -2489,13 +2355,13 @@ { "name": "MaintainProcessMap", "cpu": { - "mean": 0.001837218714503009 + "mean": 0.002905224941039929 } }, { "name": "FileScoreThread", "cpu": { - "mean": 0.001923724528414585 + "mean": 0.003567363275873613 } }, { @@ -2507,37 +2373,37 @@ { "name": "QuarantineManagerWorkerThread", "cpu": { - "mean": 0.001557104650408379 + "mean": 0.002083131839926198 } }, { "name": "EventProcessingThread", "cpu": { - "mean": 0.00591534994176305 + "mean": 0.01754294664738331 } }, { "name": "HostIsolationMonitorThread", "cpu": { - "mean": 0.0006508532665728148 + "mean": 0.0008481322806622976 } }, { "name": "MountMonitor", "cpu": { - "mean": 0.002533384550267602 + "mean": 0.003463206812704382 } }, { "name": "responseActionsUploadThread", "cpu": { - "mean": 0.001771309522951331 + "mean": 0.00270807149264102 } }, { "name": "responseActionsProcessThread", "cpu": { - "mean": 0.001783667496367271 + "mean": 0.002689472100521233 } }, { @@ -2549,61 +2415,61 @@ { "name": "grpcConnectionManagerThread", "cpu": { - "mean": 0.001956679124190424 + "mean": 0.0110926774602411 } }, { "name": "checkinAPIThread", "cpu": { - "mean": 0.0002389210358973501 + "mean": 0.0003124700433295513 } }, { "name": "actionsAPIThread", "cpu": { - "mean": 0.0003913361794870389 + "mean": 0.0004910243538035807 } }, { "name": "stateReportThread", "cpu": { - "mean": 0.003550860912819238 + "mean": 0.00539010824743476 } }, { "name": "EventsLoopThread", "cpu": { - "mean": 0.0077690568352644 + "mean": 0.003266057265515501 } }, { "name": "FanotifyWatchdog", "cpu": { - "mean": 0.0001647732096556607 + "mean": 0.0002231929794201937 } }, { "name": "FanotifySyncConsumer", "cpu": { - "mean": 0.001103980504692926 + "mean": 0.001885980676100637 } }, { "name": "FanotifyAsyncConsumer", "cpu": { - "mean": 0 + "mean": 0.0002492321603525497 } }, { "name": "FanotifyConsumer", "cpu": { - "mean": 0.02717934093270123 + "mean": 0.03793164685246193 } }, { "name": "RulesEngineThread", "cpu": { - "mean": 0.03744471189424889 + "mean": 0.02721466395730229 } } ], @@ -2612,8 +2478,8 @@ "active_user_count": 0 }, "uptime": { - "endpoint": 24275825, - "system": 29714870 + "endpoint": 26882599, + "system": 26883343 } } }, @@ -2634,42 +2500,33 @@ "hostname": "123", "os": { "Ext": { - "variant": "Amazon Linux" + "variant": "Debian" }, - "kernel": "5.10.102-99.473.amzn2.x86_64 #1 SMP Wed Mar 2 19:14:12 UTC 2022", + "kernel": "5.10.0-21-cloud-amd64 #1 SMP Debian 5.10.162-1 (2023-01-21)", "name": "Linux", - "family": "amazon linux", + "family": "debian", "type": "linux", - "version": "2", - "platform": "amazon linux", - "full": "Amazon Linux 2" + "version": "11.8", + "platform": "debian", + "full": "Debian 11.8" }, - "ip": [ - "127.0.0.1", - "::1" - ], + "ip": ["127.0.0.1", "::1"], "name": "123", "id": "123", - "mac": [ - "aa:aa:aa:aa:aa:aa" - ], + "mac": ["aa:aa:aa:aa:aa:aa"], "architecture": "x86_64" }, "event": { "agent_id_status": "verified", - "sequence": 34569822, - "ingested": "2024-01-26T14:35:45Z", - "created": "2024-01-26T14:35:43.564911752Z", + "sequence": 16194584, + "ingested": "2024-01-26T15:06:52Z", + "created": "2024-01-26T15:06:50.254822749Z", "kind": "metric", "module": "endpoint", "action": "endpoint_metrics", "id": "123", - "category": [ - "host" - ], - "type": [ - "info" - ], + "category": ["host"], + "type": ["info"], "dataset": "endpoint.metrics" }, "message": "Endpoint metrics" diff --git a/x-pack/plugins/security_solution/server/integration_tests/__mocks__/fleet-agents.json b/x-pack/plugins/security_solution/server/integration_tests/__mocks__/fleet-agents.json index 35a68ffd7fb14..a1c8c7731024a 100644 --- a/x-pack/plugins/security_solution/server/integration_tests/__mocks__/fleet-agents.json +++ b/x-pack/plugins/security_solution/server/integration_tests/__mocks__/fleet-agents.json @@ -46,15 +46,10 @@ }, "host": { "hostname": "123", - "ip": [ - "127.0.0.1/8", - "172.17.0.85/16" - ], + "ip": ["127.0.0.1/8", "172.17.0.85/16"], "name": "123", "id": "", - "mac": [ - "02:42:ac:11:00:55" - ], + "mac": ["02:42:ac:11:00:55"], "architecture": "x86_64" } }, @@ -67,9 +62,7 @@ "updated_at": "2022-06-07T20:09:32Z", "policy_revision_idx": 3, "policy_output_permissions_hash": null, - "action_seq_no": [ - -1 - ] + "action_seq_no": [-1] }, { "policy_coordinator_idx": 1, @@ -118,15 +111,10 @@ }, "host": { "hostname": "123", - "ip": [ - "127.0.0.1/8", - "172.17.0.69/16" - ], + "ip": ["127.0.0.1/8", "172.17.0.69/16"], "name": "123", "id": "", - "mac": [ - "02:42:ac:11:00:45" - ], + "mac": ["02:42:ac:11:00:45"], "architecture": "x86_64" } }, @@ -139,8 +127,6 @@ "updated_at": "2022-06-15T18:38:08Z", "policy_revision_idx": 4, "policy_output_permissions_hash": null, - "action_seq_no": [ - -1 - ] + "action_seq_no": [-1] } ] diff --git a/x-pack/plugins/security_solution/server/integration_tests/lib/helpers.ts b/x-pack/plugins/security_solution/server/integration_tests/lib/helpers.ts index 3697d9d2e4163..9d2d10b97e1cf 100644 --- a/x-pack/plugins/security_solution/server/integration_tests/lib/helpers.ts +++ b/x-pack/plugins/security_solution/server/integration_tests/lib/helpers.ts @@ -4,6 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import { v4 as uuidGen } from 'uuid'; import Fs from 'fs'; import Util from 'util'; import type { ElasticsearchClient } from '@kbn/core/server'; @@ -107,14 +108,22 @@ export async function removeFile(path: string) { } export async function bulkInsert( + esClient: ElasticsearchClient, index: string, data: unknown[], - esClient: ElasticsearchClient + ids: string[] = [] ): Promise { - const bulk = data.flatMap((d) => [{ index: { _index: index } }, d]); + const bulk = data.flatMap((d, i) => { + const _id = ids[i] ?? uuidGen(); + return [{ create: { _index: index, _id } }, d]; + }); await esClient.bulk({ body: bulk, refresh: 'wait_for' }).catch(() => {}); } export function updateTimestamps(data: object[]): object[] { - return data.map((d) => ({ ...d, '@timestamp': new Date().toISOString() })); + const currentTimeMillis = new Date().getTime(); + return data.map((d, i) => { + // wait a couple of millisecs to not make timestamps overlap + return { ...d, '@timestamp': new Date(currentTimeMillis + i * 50) }; + }); } diff --git a/x-pack/plugins/security_solution/server/integration_tests/lib/telemetry_helpers.ts b/x-pack/plugins/security_solution/server/integration_tests/lib/telemetry_helpers.ts index 56a9e3c2a378e..d4656a1192b34 100644 --- a/x-pack/plugins/security_solution/server/integration_tests/lib/telemetry_helpers.ts +++ b/x-pack/plugins/security_solution/server/integration_tests/lib/telemetry_helpers.ts @@ -23,8 +23,10 @@ import { deleteExceptionList, deleteExceptionListItem, } from '@kbn/lists-plugin/server/services/exception_lists'; +import { AGENT_POLICY_SAVED_OBJECT_TYPE } from '@kbn/fleet-plugin/common/constants'; + +import { packagePolicyService } from '@kbn/fleet-plugin/server/services'; -import { createAgentPolicyWithPackages } from '@kbn/fleet-plugin/server/services/agent_policy_create'; import { ENDPOINT_ARTIFACT_LISTS } from '@kbn/securitysolution-list-constants'; import { DETECTION_TYPE, NAMESPACE_TYPE } from '@kbn/lists-plugin/common/constants.mock'; import { bulkInsert, updateTimestamps } from './helpers'; @@ -174,10 +176,10 @@ export async function mockEndpointData( so: SavedObjectsServiceStart ) { await createAgentPolicy('policy-elastic-agent-on-cloud', esClient, so); - await bulkInsert(fleetIndex, fleetAgents, esClient); - await bulkInsert(endpointMetricsIndex, updateTimestamps(endpointMetrics), esClient); - await bulkInsert(endpointMetricsMetadataIndex, updateTimestamps(endpointMetadata), esClient); - await bulkInsert(endpointMetricsPolicyIndex, updateTimestamps(endpointPolicy), esClient); + await bulkInsert(esClient, fleetIndex, fleetAgents, ['123', '456']); + await bulkInsert(esClient, endpointMetricsIndex, updateTimestamps(endpointMetrics)); + await bulkInsert(esClient, endpointMetricsMetadataIndex, updateTimestamps(endpointMetadata)); + await bulkInsert(esClient, endpointMetricsPolicyIndex, updateTimestamps(endpointPolicy)); } export async function initEndpointIndices(esClient: ElasticsearchClient) { @@ -244,13 +246,43 @@ export async function createAgentPolicy( so: SavedObjectsServiceStart ) { const soClient = so.getScopedClient(fakeKibanaRequest); - await createAgentPolicyWithPackages({ - esClient, - soClient, - newPolicy: { id, name: 'Agent policy 1', namespace: 'default' }, - withSysMonitoring: true, - spaceId: 'default', - }); + const packagePolicy = { + name: 'Endpoint Policy 1', + description: 'Endpoint policy 1', + namespace: 'default', + enabled: true, + policy_id: 'policy-elastic-agent-on-cloud', + package: { name: 'endpoint', title: 'Elastic Endpoint', version: '8.11.1' }, + inputs: [ + { + config: { + policy: { + value: { + linux: { + advanced: { + agent: { + connection_delay: 60, + }, + }, + }, + }, + }, + }, + enabled: true, + type: 'endpoint', + streams: [], + }, + ], + }; + + await soClient.create(AGENT_POLICY_SAVED_OBJECT_TYPE, {}, { id }).catch(() => {}); + await packagePolicyService + .create(soClient, esClient, packagePolicy, { + id, + spaceId: 'default', + bumpRevision: false, + }) + .catch(() => {}); } export async function createMockedExceptionList(so: SavedObjectsServiceStart) { diff --git a/x-pack/plugins/security_solution/server/integration_tests/receiver.test.ts b/x-pack/plugins/security_solution/server/integration_tests/receiver.test.ts index bc3ec2792ebf8..d78a8b1b5d293 100644 --- a/x-pack/plugins/security_solution/server/integration_tests/receiver.test.ts +++ b/x-pack/plugins/security_solution/server/integration_tests/receiver.test.ts @@ -80,7 +80,7 @@ describe('ITelemetryReceiver', () => { it('should paginate queries', async () => { const docs = mockedDocs(numOfDocs); - await bulkInsert(TEST_INDEX, docs, esClient); + await bulkInsert(esClient, TEST_INDEX, docs); const results = telemetryReceiver.paginate(TEST_INDEX, testQuery()); @@ -100,8 +100,8 @@ describe('ITelemetryReceiver', () => { await new Promise((resolve) => setTimeout(resolve, 500)); const batchTwo = mockedDocs(numOfDocs, 'batchTwo'); - await bulkInsert(TEST_INDEX, batchOne, esClient); - await bulkInsert(TEST_INDEX, batchTwo, esClient); + await bulkInsert(esClient, TEST_INDEX, batchTwo); + await bulkInsert(esClient, TEST_INDEX, batchOne); const results = telemetryReceiver.paginate(TEST_INDEX, testQuery(from, to)); @@ -113,7 +113,7 @@ describe('ITelemetryReceiver', () => { }); it('should manage empty response', async () => { - await bulkInsert(TEST_INDEX, mockedDocs(numOfDocs), esClient); + await bulkInsert(esClient, TEST_INDEX, mockedDocs(numOfDocs)); const results = telemetryReceiver.paginate(TEST_INDEX, testQuery('now-2d', 'now-1d')); diff --git a/x-pack/plugins/security_solution/server/integration_tests/telemetry.test.ts b/x-pack/plugins/security_solution/server/integration_tests/telemetry.test.ts index f666e292b50bd..15b25a45c68c2 100644 --- a/x-pack/plugins/security_solution/server/integration_tests/telemetry.test.ts +++ b/x-pack/plugins/security_solution/server/integration_tests/telemetry.test.ts @@ -270,9 +270,10 @@ describe('telemetry tasks', () => { return JSON.parse((found ? found[1] : '{}') as string); }); - // save body in a file for debugging expect(body.endpoint_metrics).toStrictEqual(endpointMetaTelemetryRequest.endpoint_metrics); expect(body.endpoint_meta).toStrictEqual(endpointMetaTelemetryRequest.endpoint_meta); + expect(body.policy_config).toStrictEqual(endpointMetaTelemetryRequest.policy_config); + expect(body.policy_response).toStrictEqual(endpointMetaTelemetryRequest.policy_response); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/index.ts b/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/index.ts index 5f430ca9e41e0..bbdb08e481c48 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/index.ts @@ -18,7 +18,7 @@ import type { ITaskMetricsService } from '../task_metrics.types'; import { type IUsageCounter } from '@kbn/usage-collection-plugin/server/usage_counters/usage_counter'; import { type UsageCounters } from '@kbn/usage-collection-plugin/common/types'; import { stubEndpointAlertResponse, stubProcessTree, stubFetchTimelineEvents } from './timeline'; -import { stubEndpointMetricsResponse } from './metrics'; +import { stubEndpointMetricsAbstractResponse, stubEndpointMetricsByIdResponse } from './metrics'; import { prebuiltRuleAlertsResponse } from './prebuilt_rule_alerts'; import type { ESClusterInfo, ESLicense } from '../types'; import { stubFleetAgentResponse } from './fleet_agent_response'; @@ -122,7 +122,8 @@ export const createMockTelemetryReceiver = ( getAlertsIndex: jest.fn().mockReturnValue('alerts-*'), fetchDiagnosticAlertsBatch: jest.fn().mockReturnValue(diagnosticsAlert ?? jest.fn()), getExperimentalFeatures: jest.fn().mockReturnValue(undefined), - fetchEndpointMetrics: jest.fn().mockReturnValue(stubEndpointMetricsResponse), + fetchEndpointMetricsAbstract: jest.fn().mockReturnValue(stubEndpointMetricsAbstractResponse), + fetchEndpointMetricsById: jest.fn().mockReturnValue(stubEndpointMetricsByIdResponse), fetchEndpointPolicyResponses: jest.fn(), fetchPrebuiltRuleAlertsBatch: jest.fn().mockReturnValue(prebuiltRuleAlertsResponse), fetchDetectionRulesPackageVersion: jest.fn(), diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/metrics.ts b/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/metrics.ts index 2cf94eccb78dd..c025468a6dc67 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/metrics.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/metrics.ts @@ -5,7 +5,7 @@ * 2.0. */ -export const stubEndpointMetricsResponse = { +export const stubEndpointMetricsAbstractResponse = { body: { took: 0, timed_out: false, @@ -47,71 +47,7 @@ export const stubEndpointMetricsResponse = { _id: 'ChGf7YEBj3fALY0Ne9um', _score: null, _source: { - agent: { - id: '7116aa6c-0bad-4edc-b954-860f9d487755', - type: 'endpoint', - version: '7.16.11', - }, - '@timestamp': 1657484259677, - Endpoint: { - capabilities: ['isolation'], - configuration: { - isolation: true, - }, - state: { - isolation: true, - }, - status: 'enrolled', - policy: { - applied: { - name: 'With Eventing', - id: 'C2A9093E-E289-4C0A-AA44-8C32A414FA7A', - endpoint_policy_version: 3, - version: 5, - status: 'failure', - }, - }, - }, - data_stream: { - namespace: 'default', - type: 'metrics', - dataset: 'endpoint.metadata', - }, - elastic: { - agent: { - id: '7116aa6c-0bad-4edc-b954-860f9d487755', - }, - }, - host: { - hostname: 'Host-x46nlluvd1', - os: { - Ext: { - variant: 'Windows Server Release 2', - }, - name: 'Windows', - family: 'windows', - version: '6.3', - platform: 'Windows', - full: 'Windows Server 2012R2', - }, - ip: ['10.198.33.76'], - name: 'Host-x46nlluvd1', - id: 'b85883ad-6f72-4ab5-9794-4e1f0593a15e', - mac: ['b6-f3-d2-3d-b4-95', '33-5c-95-1e-20-17', 'b0-90-8a-57-82-1f'], - architecture: '1dc25ub5gq', - }, - event: { - agent_id_status: 'auth_metadata_missing', - ingested: '2022-07-11T14:17:41Z', - created: 1657484259677, - kind: 'metric', - module: 'endpoint', - action: 'endpoint_metadata', - id: 'c7648bef-b723-4925-b9a1-b3953fbb2a23', - category: ['host'], - type: ['info'], - dataset: 'endpoint.metadata', - }, + id: '7116aa6c-0bad-4edc-b954-860f9d487755', }, sort: [1657484259677], }, @@ -124,3 +60,98 @@ export const stubEndpointMetricsResponse = { }, }, }; + +export const stubEndpointMetricsByIdResponse = { + body: { + took: 0, + timed_out: false, + _shards: { + total: 1, + successful: 1, + skipped: 0, + failed: 0, + }, + hits: { + total: { + value: 2, + relation: 'eq', + }, + max_score: null, + hits: [ + { + _index: '.ds-metrics-endpoint.metadata-default-2022.07.08-000001', + _id: 'ChGf7YEBj3fALY0Ne9um', + _score: null, + _source: { + agent: { + id: '7116aa6c-0bad-4edc-b954-860f9d487755', + type: 'endpoint', + version: '7.16.11', + }, + '@timestamp': 1657484259677, + Endpoint: { + capabilities: ['isolation'], + configuration: { + isolation: true, + }, + state: { + isolation: true, + }, + status: 'enrolled', + policy: { + applied: { + name: 'With Eventing', + id: 'C2A9093E-E289-4C0A-AA44-8C32A414FA7A', + endpoint_policy_version: 3, + version: 5, + status: 'failure', + }, + }, + }, + data_stream: { + namespace: 'default', + type: 'metrics', + dataset: 'endpoint.metadata', + }, + elastic: { + agent: { + id: '7116aa6c-0bad-4edc-b954-860f9d487755', + }, + }, + host: { + hostname: 'Host-x46nlluvd1', + os: { + Ext: { + variant: 'Windows Server Release 2', + }, + name: 'Windows', + family: 'windows', + version: '6.3', + platform: 'Windows', + full: 'Windows Server 2012R2', + }, + ip: ['10.198.33.76'], + name: 'Host-x46nlluvd1', + id: 'b85883ad-6f72-4ab5-9794-4e1f0593a15e', + mac: ['b6-f3-d2-3d-b4-95', '33-5c-95-1e-20-17', 'b0-90-8a-57-82-1f'], + architecture: '1dc25ub5gq', + }, + event: { + agent_id_status: 'auth_metadata_missing', + ingested: '2022-07-11T14:17:41Z', + created: 1657484259677, + kind: 'metric', + module: 'endpoint', + action: 'endpoint_metadata', + id: 'c7648bef-b723-4925-b9a1-b3953fbb2a23', + category: ['host'], + type: ['info'], + dataset: 'endpoint.metadata', + }, + }, + sort: [1657484259677], + }, + ], + }, + }, +}; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts index 07a3b5914816b..bb9caefc5adf5 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -61,6 +61,7 @@ import type { SafeEndpointEvent, ResolverSchema } from '../../../common/endpoint import type { TelemetryEvent, EnhancedAlertEvent, + EndpointMetricDocument, ESLicense, ESClusterInfo, GetEndpointListResponse, @@ -73,6 +74,10 @@ import type { ValueListIndicatorMatchResponseAggregation, Nullable, FleetAgentResponse, + EndpointMetricsAggregation, + EndpointMetricsAbstract, + EndpointPolicyResponseDocument, + EndpointPolicyResponseAggregation, } from './types'; import { telemetryConfiguration } from './configuration'; import { ENDPOINT_METRICS_INDEX } from '../../../common/constants'; @@ -101,19 +106,26 @@ export interface ITelemetryReceiver { fetchFleetAgents(): Promise>; + /** + * Reads Endpoint Agent policy responses out of the `.ds-metrics-endpoint.policy*` data + * stream and creates a local K/V structure that stores the policy response (V) with + * the Endpoint Agent Id (K). A value will only exist if there has been a endpoint + * enrolled in the last 24 hours OR a policy change has occurred. We only send + * non-successful responses. If the field is null, we assume no responses in + * the last 24h or no failures/warnings in the policy applied. + * + */ fetchEndpointPolicyResponses( executeFrom: string, executeTo: string - ): Promise< - TransportResult>, unknown> - >; + ): Promise>; - fetchEndpointMetrics( + fetchEndpointMetricsAbstract( executeFrom: string, executeTo: string - ): Promise< - TransportResult>, unknown> - >; + ): Promise; + + fetchEndpointMetricsById(ids: string[]): AsyncGenerator; fetchEndpointMetadata( executeFrom: string, @@ -303,6 +315,17 @@ export class TelemetryReceiver implements ITelemetryReceiver { latest_response: { top_hits: { size: 1, + _source: { + includes: [ + 'agent', + 'event', + 'Endpoint.policy.applied.status', + 'Endpoint.policy.applied.actions', + 'Endpoint.policy.applied.artifacts.global', + 'Endpoint.configuration', + 'Endpoint.state', + ], + }, sort: [ { '@timestamp': { @@ -318,10 +341,24 @@ export class TelemetryReceiver implements ITelemetryReceiver { }, }; - return this.esClient().search(query, { meta: true }); + const response = await this.esClient().search(query, { meta: true }); + const { body: failedPolicyResponses } = response as unknown as { + body: EndpointPolicyResponseAggregation; + }; + + if (!failedPolicyResponses) { + return new Map(); + } + + // If there is no policy responses in the 24h > now then we will continue + return failedPolicyResponses.aggregations?.policy_responses?.buckets.reduce( + (cache, endpointAgentId) => + cache.set(endpointAgentId.key, endpointAgentId.latest_response.hits.hits[0]._source), + new Map() + ); } - public async fetchEndpointMetrics(executeFrom: string, executeTo: string) { + public async fetchEndpointMetricsAbstract(executeFrom: string, executeTo: string) { const query: SearchRequest = { expand_wildcards: ['open' as const, 'hidden' as const], index: ENDPOINT_METRICS_INDEX, @@ -346,6 +383,9 @@ export class TelemetryReceiver implements ITelemetryReceiver { latest_metrics: { top_hits: { size: 1, + _source: { + excludes: ['*'], + }, sort: [ { '@timestamp': { @@ -366,7 +406,36 @@ export class TelemetryReceiver implements ITelemetryReceiver { }, }; - return this.esClient().search(query, { meta: true }); + return this.esClient() + .search(query, { meta: true }) + .then((response) => { + const { body: endpointMetricsResponse } = response as unknown as { + body: EndpointMetricsAggregation; + }; + + return { + endpointMetricIds: endpointMetricsResponse.aggregations.endpoint_agents.buckets.map( + (epMetrics) => epMetrics.latest_metrics.hits.hits[0]._id + ), + totalEndpoints: endpointMetricsResponse.aggregations.endpoint_count.value, + }; + }); + } + + fetchEndpointMetricsById(ids: string[]): AsyncGenerator { + const query: ESSearchRequest = { + sort: [{ '@timestamp': { order: 'desc' as const } }], + query: { + ids: { + values: ids, + }, + }, + _source: { + includes: ['agent', 'Endpoint.metrics', 'elastic.agent', 'host', 'event'], + }, + }; + + return this.paginate(ENDPOINT_METRICS_INDEX, query); } public async fetchEndpointMetadata(executeFrom: string, executeTo: string) { @@ -1125,7 +1194,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { const esQuery = { ...cloneDeep(query), pit, - size, + size: Math.min(size, 10_000), }; try { do { diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.test.ts index 7b1b59e316c7e..fa1140545640b 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.test.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.test.ts @@ -49,7 +49,7 @@ describe('endpoint telemetry task test', () => { ); expect(mockTelemetryReceiver.fetchFleetAgents).toHaveBeenCalled(); - expect(mockTelemetryReceiver.fetchEndpointMetrics).toHaveBeenCalledWith( + expect(mockTelemetryReceiver.fetchEndpointMetricsAbstract).toHaveBeenCalledWith( testTaskExecutionPeriod.last, testTaskExecutionPeriod.current ); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts index 1b6e6e4dc30d9..e712ed5ffa34b 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts @@ -9,16 +9,14 @@ import type { Logger } from '@kbn/core/server'; import type { AgentPolicy } from '@kbn/fleet-plugin/common'; import { FLEET_ENDPOINT_PACKAGE } from '@kbn/fleet-plugin/common'; import type { ITelemetryEventsSender } from '../sender'; -import type { - EndpointMetricsAggregation, - EndpointPolicyResponseAggregation, - EndpointPolicyResponseDocument, - EndpointMetadataAggregation, - EndpointMetadataDocument, - ESClusterInfo, - ESLicense, - Nullable, - FleetAgentResponse, +import { + type EndpointMetadataAggregation, + type EndpointMetadataDocument, + type Nullable, + type FleetAgentResponse, + TelemetryCounter, + TelemetryChannel, + type EndpointPolicyResponseDocument, } from '../types'; import type { ITelemetryReceiver } from '../receiver'; import type { TaskExecutionPeriod } from '../task'; @@ -31,6 +29,7 @@ import { getPreviousDailyTaskTimestamp, isPackagePolicyList, newTelemetryLogger, + safeValue, } from '../helpers'; import type { PolicyData } from '../../../../common/endpoint/types'; import { TELEMETRY_CHANNEL_ENDPOINT_META } from '../constants'; @@ -48,11 +47,10 @@ const EmptyFleetAgentResponse: FleetAgentResponse = { const usageLabelPrefix: string[] = ['security_telemetry', 'endpoint_task']; export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) { - const taskName = 'Security Solution Telemetry Endpoint Metrics and Info task'; const taskType = 'security:endpoint-meta-telemetry'; return { type: taskType, - title: taskName, + title: 'Security Solution Telemetry Endpoint Metrics and Info task', interval: '24h', timeout: '5m', version: '1.0.0', @@ -65,10 +63,10 @@ export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) { taskMetricsService: ITaskMetricsService, taskExecutionPeriod: TaskExecutionPeriod ) => { - const log = newTelemetryLogger(logger.get('endpoint')).l; + const log = newTelemetryLogger(logger.get('endpoint')); const trace = taskMetricsService.start(taskType); - log( + log.l( `Running task: ${taskId} [last: ${taskExecutionPeriod.last} - current: ${taskExecutionPeriod.current}]` ); @@ -76,19 +74,8 @@ export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) { if (!taskExecutionPeriod.last) { throw new Error('last execution timestamp is required'); } - const [clusterInfoPromise, licenseInfoPromise] = await Promise.allSettled([ - receiver.fetchClusterInfo(), - receiver.fetchLicenseInfo(), - ]); - - const clusterInfo = - clusterInfoPromise.status === 'fulfilled' - ? clusterInfoPromise.value - : ({} as ESClusterInfo); - const licenseInfo = - licenseInfoPromise.status === 'fulfilled' - ? licenseInfoPromise.value - : ({} as ESLicense | undefined); + + const clusterData = await fetchClusterData(receiver); const endpointData = await fetchEndpointData( receiver, @@ -103,18 +90,8 @@ export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) { * report its metrics once per day OR every time a policy change has occured. If * a metric document(s) exists for an EP agent we map to fleet agent and policy */ - if (endpointData.endpointMetrics === undefined) { - log('no endpoint metrics to report'); - taskMetricsService.end(trace); - return 0; - } - - const { body: endpointMetricsResponse } = endpointData.endpointMetrics as unknown as { - body: EndpointMetricsAggregation; - }; - - if (endpointMetricsResponse.aggregations === undefined) { - log(`no endpoint metrics response to report`); + if (endpointData.endpointMetrics.totalEndpoints === 0) { + log.l('no endpoint metrics to report'); taskMetricsService.end(trace); return 0; } @@ -122,22 +99,12 @@ export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) { const telemetryUsageCounter = sender.getTelemetryUsageCluster(); telemetryUsageCounter?.incrementCounter({ counterName: createUsageCounterLabel( - usageLabelPrefix.concat(['payloads', TELEMETRY_CHANNEL_ENDPOINT_META]) + usageLabelPrefix.concat(['payloads', TelemetryChannel.ENDPOINT_META]) ), - counterType: 'num_endpoint', - incrementBy: endpointMetricsResponse.aggregations.endpoint_count.value, + counterType: TelemetryCounter.NUM_ENDPOINT, + incrementBy: endpointData.endpointMetrics.totalEndpoints, }); - const endpointMetrics = endpointMetricsResponse.aggregations.endpoint_agents.buckets.map( - (epMetrics) => { - return { - endpoint_agent: epMetrics.latest_metrics.hits.hits[0]._source.agent.id, - endpoint_version: epMetrics.latest_metrics.hits.hits[0]._source.agent.version, - endpoint_metrics: epMetrics.latest_metrics.hits.hits[0]._source, - }; - } - ); - /** STAGE 2 - Fetch Fleet Agent Config * * As the policy id + policy version does not exist on the Endpoint Metrics document @@ -148,12 +115,12 @@ export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) { const agentsResponse = endpointData.fleetAgentsResponse; if (agentsResponse === undefined || agentsResponse === null) { - log('no fleet agent information available'); + log.l('no fleet agent information available'); taskMetricsService.end(trace); return 0; } - const fleetAgents = agentsResponse.agents.reduce((cache, agent) => { + const policyInfoByAgent = agentsResponse.agents.reduce((cache, agent) => { if (agent.id === DefaultEndpointPolicyIdToIgnore) { return cache; } @@ -165,69 +132,12 @@ export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) { return cache; }, new Map()); - const endpointPolicyCache = new Map(); - for (const policyInfo of fleetAgents.values()) { - if ( - policyInfo !== null && - policyInfo !== undefined && - !endpointPolicyCache.has(policyInfo) - ) { - let agentPolicy: Nullable; - try { - agentPolicy = await receiver.fetchPolicyConfigs(policyInfo); - } catch (err) { - log(`error fetching policy config due to ${err?.message}`); - } - const packagePolicies = agentPolicy?.package_policies; - - if (packagePolicies !== undefined && isPackagePolicyList(packagePolicies)) { - packagePolicies - .map((pPolicy) => pPolicy as PolicyData) - .forEach((pPolicy) => { - if ( - pPolicy.inputs[0]?.config !== undefined && - pPolicy.inputs[0]?.config !== null - ) { - pPolicy.inputs.forEach((input) => { - if ( - input.type === FLEET_ENDPOINT_PACKAGE && - input?.config !== undefined && - policyInfo !== undefined - ) { - endpointPolicyCache.set(policyInfo, pPolicy); - } - }); - } - }); - } - } - } + const endpointPolicyById = await endpointPolicies(policyInfoByAgent.values(), receiver); - /** STAGE 3 - Fetch Endpoint Policy Responses - * - * Reads Endpoint Agent policy responses out of the `.ds-metrics-endpoint.policy*` data - * stream and creates a local K/V structure that stores the policy response (V) with - * the Endpoint Agent Id (K). A value will only exist if there has been a endpoint - * enrolled in the last 24 hours OR a policy change has occurred. We only send - * non-successful responses. If the field is null, we assume no responses in - * the last 24h or no failures/warnings in the policy applied. - * + /** + * STAGE 3 - Fetch Endpoint Policy Responses */ - const { body: failedPolicyResponses } = endpointData.epPolicyResponse as unknown as { - body: EndpointPolicyResponseAggregation; - }; - - // If there is no policy responses in the 24h > now then we will continue - const policyResponses = failedPolicyResponses.aggregations - ? failedPolicyResponses.aggregations.policy_responses.buckets.reduce( - (cache, endpointAgentId) => { - const doc = endpointAgentId.latest_response.hits.hits[0]; - cache.set(endpointAgentId.key, doc); - return cache; - }, - new Map() - ) - : new Map(); + const policyResponses = endpointData.epPolicyResponse; /** STAGE 4 - Fetch Endpoint Agent Metadata * @@ -237,7 +147,7 @@ export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) { * a metadata document(s) exists for an EP agent we map to fleet agent and policy */ if (endpointData.endpointMetadata === undefined) { - log(`no endpoint metadata to report`); + log.l(`no endpoint metadata to report`); } const { body: endpointMetadataResponse } = endpointData.endpointMetadata as unknown as { @@ -245,7 +155,7 @@ export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) { }; if (endpointMetadataResponse.aggregations === undefined) { - log(`no endpoint metadata to report`); + log.l(`no endpoint metadata to report`); } const endpointMetadata = @@ -257,6 +167,7 @@ export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) { }, new Map() ); + /** STAGE 5 - Create the telemetry log records * * Iterates through the endpoint metrics documents at STAGE 1 and joins them together @@ -265,17 +176,24 @@ export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) { * */ try { - const telemetryPayloads = endpointMetrics.map((endpoint) => { + const endpointMetrics = []; + for await (const metrics of receiver.fetchEndpointMetricsById( + endpointData.endpointMetrics.endpointMetricIds + )) { + endpointMetrics.push(...metrics); + } + + const telemetryPayloads = endpointMetrics.map((endpointMetric) => { let policyConfig = null; - let failedPolicy = null; + let failedPolicy: Nullable = null; let endpointMetadataById = null; - const fleetAgentId = endpoint.endpoint_metrics.elastic.agent.id; - const endpointAgentId = endpoint.endpoint_agent; + const fleetAgentId = endpointMetric.elastic.agent.id; + const endpointAgentId = endpointMetric.agent.id; - const policyInformation = fleetAgents.get(fleetAgentId); + const policyInformation = policyInfoByAgent.get(fleetAgentId); if (policyInformation) { - policyConfig = endpointPolicyCache.get(policyInformation) || null; + policyConfig = endpointPolicyById.get(policyInformation) || null; if (policyConfig) { failedPolicy = policyResponses.get(endpointAgentId); @@ -295,7 +213,7 @@ export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) { system_impact: systemImpact, threads, event_filter: eventFilter, - } = endpoint.endpoint_metrics.Endpoint.metrics; + } = endpointMetric.Endpoint.metrics; const endpointPolicyDetail = extractEndpointPolicyConfig(policyConfig); if (endpointPolicyDetail) { endpointPolicyDetail.value = addDefaultAdvancedPolicyConfigSettings( @@ -304,11 +222,11 @@ export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) { } return { '@timestamp': taskExecutionPeriod.current, - cluster_uuid: clusterInfo.cluster_uuid, - cluster_name: clusterInfo.cluster_name, - license_id: licenseInfo?.uid, + cluster_uuid: clusterData.clusterInfo.cluster_uuid, + cluster_name: clusterData.clusterInfo.cluster_name, + license_id: clusterData.licenseInfo?.uid, endpoint_id: endpointAgentId, - endpoint_version: endpoint.endpoint_version, + endpoint_version: endpointMetric.agent.version, endpoint_package_version: policyConfig?.package?.version || null, endpoint_metrics: { cpu: cpu.endpoint, @@ -321,7 +239,7 @@ export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) { eventFilter, }, endpoint_meta: { - os: endpoint.endpoint_metrics.host.os, + os: endpointMetric.host.os, capabilities: endpointMetadataById !== null && endpointMetadataById !== undefined ? endpointMetadataById._source.Endpoint.capabilities @@ -331,24 +249,24 @@ export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) { policy_response: failedPolicy !== null && failedPolicy !== undefined ? { - agent_policy_status: failedPolicy._source.event.agent_id_status, + agent_policy_status: failedPolicy.event.agent_id_status, manifest_version: - failedPolicy._source.Endpoint.policy.applied.artifacts.global.version, - status: failedPolicy._source.Endpoint.policy.applied.status, - actions: failedPolicy._source.Endpoint.policy.applied.actions + failedPolicy.Endpoint.policy.applied.artifacts.global.version, + status: failedPolicy.Endpoint.policy.applied.status, + actions: failedPolicy.Endpoint.policy.applied.actions .map((action) => (action.status !== 'success' ? action : null)) .filter((action) => action !== null), - configuration: failedPolicy._source.Endpoint.configuration, - state: failedPolicy._source.Endpoint.state, + configuration: failedPolicy.Endpoint.configuration, + state: failedPolicy.Endpoint.state, } : {}, telemetry_meta: { - metrics_timestamp: endpoint.endpoint_metrics['@timestamp'], + metrics_timestamp: endpointMetric['@timestamp'], }, }; }); - log(`sending ${telemetryPayloads.length} endpoint telemetry records`); + log.l(`sending ${telemetryPayloads.length} endpoint telemetry records`); /** * STAGE 6 - Send the documents @@ -362,7 +280,7 @@ export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) { taskMetricsService.end(trace); return telemetryPayloads.length; } catch (err) { - logger.warn(`could not complete endpoint alert telemetry task due to ${err?.message}`); + log.warn(`could not complete endpoint alert telemetry task due to ${err?.message}`, err); taskMetricsService.end(trace, err); return 0; } @@ -379,21 +297,64 @@ async function fetchEndpointData( executeFrom: string, executeTo: string ) { - const [fleetAgentsResponse, epMetricsResponse, policyResponse, endpointMetadata] = + const [fleetAgentsResponse, epMetricsAbstractResponse, policyResponse, endpointMetadata] = await Promise.allSettled([ receiver.fetchFleetAgents(), - receiver.fetchEndpointMetrics(executeFrom, executeTo), + receiver.fetchEndpointMetricsAbstract(executeFrom, executeTo), receiver.fetchEndpointPolicyResponses(executeFrom, executeTo), receiver.fetchEndpointMetadata(executeFrom, executeTo), ]); return { - fleetAgentsResponse: - fleetAgentsResponse.status === 'fulfilled' - ? fleetAgentsResponse.value - : EmptyFleetAgentResponse, - endpointMetrics: epMetricsResponse.status === 'fulfilled' ? epMetricsResponse.value : undefined, - epPolicyResponse: policyResponse.status === 'fulfilled' ? policyResponse.value : undefined, - endpointMetadata: endpointMetadata.status === 'fulfilled' ? endpointMetadata.value : undefined, + fleetAgentsResponse: safeValue(fleetAgentsResponse, EmptyFleetAgentResponse), + endpointMetrics: safeValue(epMetricsAbstractResponse), + epPolicyResponse: safeValue(policyResponse), + endpointMetadata: safeValue(endpointMetadata), }; } + +async function fetchClusterData(receiver: ITelemetryReceiver) { + const [clusterInfoPromise, licenseInfoPromise] = await Promise.allSettled([ + receiver.fetchClusterInfo(), + receiver.fetchLicenseInfo(), + ]); + + const clusterInfo = safeValue(clusterInfoPromise); + const licenseInfo = safeValue(licenseInfoPromise); + + return { clusterInfo, licenseInfo }; +} + +async function endpointPolicies(policies: IterableIterator, receiver: ITelemetryReceiver) { + const endpointPolicyCache = new Map(); + for (const policyInfo of policies) { + if (policyInfo !== null && policyInfo !== undefined && !endpointPolicyCache.has(policyInfo)) { + let agentPolicy: Nullable; + try { + agentPolicy = await receiver.fetchPolicyConfigs(policyInfo); + } catch (err) { + // log.error(`error fetching policy config due to ${err?.message}`, err); + } + const packagePolicies = agentPolicy?.package_policies; + + if (packagePolicies !== undefined && isPackagePolicyList(packagePolicies)) { + packagePolicies + .map((pPolicy) => pPolicy as PolicyData) + .forEach((pPolicy) => { + if (pPolicy.inputs[0]?.config !== undefined && pPolicy.inputs[0]?.config !== null) { + pPolicy.inputs.forEach((input) => { + if ( + input.type === FLEET_ENDPOINT_PACKAGE && + input?.config !== undefined && + policyInfo !== undefined + ) { + endpointPolicyCache.set(policyInfo, pPolicy); + } + }); + } + }); + } + } + } + return endpointPolicyCache; +} diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/types.ts b/x-pack/plugins/security_solution/server/lib/telemetry/types.ts index 59306a0a6537c..fa43d72f43342 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/types.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/types.ts @@ -104,6 +104,7 @@ export enum TelemetryCounter { FATAL_ERROR = 'fatal_error', TELEMETRY_OPTED_OUT = 'telemetry_opted_out', TELEMETRY_NOT_REACHABLE = 'telemetry_not_reachable', + NUM_ENDPOINT = 'num_endpoint', } // EP Policy Response @@ -126,7 +127,7 @@ export interface EndpointPolicyResponseAggregation { interface EndpointPolicyResponseHits { hits: { total: { value: number }; - hits: EndpointPolicyResponseDocument[]; + hits: Array<{ _source: EndpointPolicyResponseDocument }>; }; } @@ -135,33 +136,30 @@ interface NonPolicyConfiguration { } export interface EndpointPolicyResponseDocument { - _source: { - '@timestamp': string; - agent: { - id: string; - }; - event: { - agent_id_status: string; - }; - Endpoint: { - policy: { - applied: { - actions: Array<{ - name: string; - message: string; - status: string; - }>; - artifacts: { - global: { - version: string; - }; - }; + agent: { + id: string; + }; + event: { + agent_id_status: string; + }; + Endpoint: { + policy: { + applied: { + actions: Array<{ + name: string; + message: string; status: string; + }>; + artifacts: { + global: { + version: string; + }; }; + status: string; }; - configuration: NonPolicyConfiguration; - state: NonPolicyConfiguration; }; + configuration: NonPolicyConfiguration; + state: NonPolicyConfiguration; }; } @@ -182,32 +180,35 @@ export interface EndpointMetricsAggregation { interface EndpointMetricHits { hits: { total: { value: number }; - hits: EndpointMetricDocument[]; + hits: Array<{ _id: string; _source: EndpointMetricDocument }>; }; } -interface EndpointMetricDocument { - _source: { - '@timestamp': string; +export interface EndpointMetricDocument { + '@timestamp': string; + agent: { + id: string; + version: string; + }; + Endpoint: { + metrics: EndpointMetrics; + }; + elastic: { agent: { id: string; - version: string; - }; - Endpoint: { - metrics: EndpointMetrics; - }; - elastic: { - agent: { - id: string; - }; - }; - host: { - os: EndpointMetricOS; - }; - event: { - agent_id_status: string; }; }; + host: { + os: EndpointMetricOS; + }; + event: { + agent_id_status: string; + }; +} + +export interface EndpointMetricsAbstract { + endpointMetricIds: string[]; + totalEndpoints: number; } interface DocumentsVolumeMetrics { From 324dccc8b10b2fd0f634f1fdc1370e5fc208497f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Mon, 4 Mar 2024 19:18:22 +0100 Subject: [PATCH 11/19] Fix FTR --- .../server/lib/telemetry/async_sender.ts | 3 ++- .../server/lib/telemetry/receiver.ts | 16 ++++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/async_sender.ts b/x-pack/plugins/security_solution/server/lib/telemetry/async_sender.ts index bc6c6ffd7d83e..1edd3cb598521 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/async_sender.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/async_sender.ts @@ -268,7 +268,8 @@ export class AsyncTelemetryEventsSender implements IAsyncTelemetryEventsSender { private enrich(event: Event): Event { const clusterInfo = this.telemetryReceiver?.getClusterInfo(); - if (typeof event.payload === 'object') { + // TODO(szaffarano): does it make sense to include cluster info in task metric payloads? + if (typeof event.payload === 'object' && event.channel !== TelemetryChannel.TASK_METRICS) { event.payload = { ...event.payload, // TODO(szaffarano): include license info if available diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts index bb9caefc5adf5..25e6d49965bf6 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -413,12 +413,16 @@ export class TelemetryReceiver implements ITelemetryReceiver { body: EndpointMetricsAggregation; }; - return { - endpointMetricIds: endpointMetricsResponse.aggregations.endpoint_agents.buckets.map( - (epMetrics) => epMetrics.latest_metrics.hits.hits[0]._id - ), - totalEndpoints: endpointMetricsResponse.aggregations.endpoint_count.value, - }; + if (endpointMetricsResponse.aggregations !== undefined) { + const endpointMetricIds = + endpointMetricsResponse.aggregations.endpoint_agents.buckets.map( + (epMetrics) => epMetrics.latest_metrics.hits.hits[0]._id + ); + const totalEndpoints = endpointMetricsResponse.aggregations.endpoint_count.value; + return { endpointMetricIds, totalEndpoints }; + } + + return { endpointMetricIds: [], totalEndpoints: 0 }; }); } From 4a01c876c71adb185aef1724e0fcbaad5f9367fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Tue, 5 Mar 2024 16:47:28 +0100 Subject: [PATCH 12/19] Refactor endpoint task logic --- .../server/integration_tests/lib/helpers.ts | 2 +- .../__mocks__/fleet_agent_response.ts | 482 +----------------- .../server/lib/telemetry/receiver.ts | 123 +++-- .../server/lib/telemetry/tasks/endpoint.ts | 317 ++++++------ .../server/lib/telemetry/types.ts | 22 +- 5 files changed, 248 insertions(+), 698 deletions(-) diff --git a/x-pack/plugins/security_solution/server/integration_tests/lib/helpers.ts b/x-pack/plugins/security_solution/server/integration_tests/lib/helpers.ts index 9d2d10b97e1cf..88e66f10921ba 100644 --- a/x-pack/plugins/security_solution/server/integration_tests/lib/helpers.ts +++ b/x-pack/plugins/security_solution/server/integration_tests/lib/helpers.ts @@ -124,6 +124,6 @@ export function updateTimestamps(data: object[]): object[] { const currentTimeMillis = new Date().getTime(); return data.map((d, i) => { // wait a couple of millisecs to not make timestamps overlap - return { ...d, '@timestamp': new Date(currentTimeMillis + i * 50) }; + return { ...d, '@timestamp': new Date(currentTimeMillis + (i + 1) * 100) }; }); } diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/fleet_agent_response.ts b/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/fleet_agent_response.ts index 2c9c54f696c4f..742088da6bbb9 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/fleet_agent_response.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/fleet_agent_response.ts @@ -5,478 +5,10 @@ * 2.0. */ -import type { Agent } from '@kbn/fleet-plugin/common'; - -export const stubFleetAgentResponse: { - agents: Agent[]; - total: number; - page: number; - perPage: number; -} = { - agents: [ - { - id: '45112616-62e0-42c5-a8f9-2f8a71a92040', - type: 'PERMANENT', - active: true, - enrolled_at: '2024-01-11T03:39:21.515Z', - upgraded_at: null, - upgrade_started_at: null, - access_api_key_id: 'jY3dWnkBj1tiuAw9pAmq', - default_api_key_id: 'so3dWnkBj1tiuAw9yAm3', - policy_id: '147b2096-bd12-4b7e-a100-061dc11ba799', - last_checkin: '2024-01-11T04:00:35.217Z', - last_checkin_status: 'online', - policy_revision: 2, - packages: [], - sort: [1704944361515, 'Host-roan3tb8c3'], - components: [ - { - id: 'endpoint-0', - type: 'endpoint', - status: 'STARTING', - message: 'Running as external service', - units: [ - { - id: 'endpoint-1', - type: 'input', - status: 'STARTING', - message: 'Protecting machine', - payload: { - extra: 'payload', - }, - }, - { - id: 'shipper', - type: 'output', - status: 'STARTING', - message: 'Connected over GRPC', - payload: { - extra: 'payload', - }, - }, - ], - }, - ], - agent: { - id: '45112616-62e0-42c5-a8f9-2f8a71a92040', - version: '8.13.0', - }, - user_provided_metadata: {}, - local_metadata: { - elastic: { - agent: { - 'build.original': '8.0.0-SNAPSHOT (build: j74oz at 2021-05-07 18:42:49 +0000 UTC)', - id: '45112616-62e0-42c5-a8f9-2f8a71a92040', - log_level: 'info', - snapshot: true, - upgradeable: true, - version: '8.13.0', - }, - }, - host: { - architecture: '4nil08yn9j', - hostname: 'Host-roan3tb8c3', - id: '866c98c0-c323-4f6b-9e4c-8cc4694e4ba7', - ip: ['00.00.000.00'], - mac: ['00-00-00-00-00-00', '00-00-00-00-00-00', '0-00-00-00-00-00'], - name: 'Host-roan3tb8c3', - os: { - name: 'Windows', - full: 'Windows 10', - version: '10.0', - platform: 'Windows', - family: 'windows', - Ext: { - variant: 'Windows Pro', - }, - }, - }, - os: { - family: 'windows', - full: 'Windows 10', - kernel: '10.0.17763.1879 (Build.160101.0800)', - name: 'Windows', - platform: 'Windows', - version: '10.0', - Ext: { - variant: 'Windows Pro', - }, - }, - }, - status: 'online', - }, - { - id: '74550426-040d-4216-a227-599fd3efa91c', - type: 'PERMANENT', - active: true, - enrolled_at: '2024-01-11T03:39:21.512Z', - upgraded_at: null, - upgrade_started_at: null, - access_api_key_id: 'jY3dWnkBj1tiuAw9pAmq', - default_api_key_id: 'so3dWnkBj1tiuAw9yAm3', - policy_id: '16608650-4839-4053-a0eb-6ee9d11ac84d', - last_checkin: '2024-01-11T04:00:35.302Z', - last_checkin_status: 'online', - policy_revision: 2, - packages: [], - sort: [1704944361512, 'Host-vso4lwuc51'], - components: [ - { - id: 'endpoint-0', - type: 'endpoint', - status: 'FAILED', - message: 'Running as external service', - units: [ - { - id: 'endpoint-1', - type: 'input', - status: 'FAILED', - message: 'Protecting machine', - payload: { - error: { - code: -272, - message: 'Unable to connect to Elasticsearch', - }, - }, - }, - { - id: 'shipper', - type: 'output', - status: 'FAILED', - message: 'Connected over GRPC', - payload: { - extra: 'payload', - }, - }, - ], - }, - ], - agent: { - id: '74550426-040d-4216-a227-599fd3efa91c', - version: '8.13.0', - }, - user_provided_metadata: {}, - local_metadata: { - elastic: { - agent: { - 'build.original': '8.0.0-SNAPSHOT (build: 315fp at 2021-05-07 18:42:49 +0000 UTC)', - id: '74550426-040d-4216-a227-599fd3efa91c', - log_level: 'info', - snapshot: true, - upgradeable: true, - version: '8.13.0', - }, - }, - host: { - architecture: '3oem2enr1y', - hostname: 'Host-vso4lwuc51', - id: '3cdfece3-8b4e-4006-a19e-7ab7e953bb38', - ip: ['00.00.000.00'], - mac: ['00-00-00-00-00-00', '00-00-00-00-00-00', '0-00-00-00-00-00'], - name: 'Host-vso4lwuc51', - os: { - name: 'Windows', - full: 'Windows Server 2012', - version: '6.2', - platform: 'Windows', - family: 'windows', - Ext: { - variant: 'Windows Server', - }, - }, - }, - os: { - family: 'windows', - full: 'Windows Server 2012', - kernel: '10.0.17763.1879 (Build.160101.0800)', - name: 'Windows', - platform: 'Windows', - version: '6.2', - Ext: { - variant: 'Windows Server', - }, - }, - }, - status: 'online', - }, - { - id: 'b80bc33e-1c65-41b3-80d6-8f9757552ab1', - type: 'PERMANENT', - active: true, - enrolled_at: '2024-01-11T03:31:22.832Z', - upgraded_at: null, - upgrade_started_at: null, - access_api_key_id: 'jY3dWnkBj1tiuAw9pAmq', - default_api_key_id: 'so3dWnkBj1tiuAw9yAm3', - policy_id: '125f0769-20b4-4604-81ce-f0db812d510b', - last_checkin: '2024-01-11T04:00:36.305Z', - last_checkin_status: 'online', - policy_revision: 2, - packages: [], - sort: [1704943882832, 'Host-y0zwnrnucm'], - components: [ - { - id: 'endpoint-0', - type: 'endpoint', - status: 'STOPPING', - message: 'Running as external service', - units: [ - { - id: 'endpoint-1', - type: 'input', - status: 'STOPPING', - message: 'Protecting machine', - payload: { - extra: 'payload', - }, - }, - { - id: 'shipper', - type: 'output', - status: 'STOPPING', - message: 'Connected over GRPC', - payload: { - extra: 'payload', - }, - }, - ], - }, - ], - agent: { - id: 'b80bc33e-1c65-41b3-80d6-8f9757552ab1', - version: '8.13.0', - }, - user_provided_metadata: {}, - local_metadata: { - elastic: { - agent: { - 'build.original': '8.0.0-SNAPSHOT (build: klkq1 at 2021-05-07 18:42:49 +0000 UTC)', - id: 'b80bc33e-1c65-41b3-80d6-8f9757552ab1', - log_level: 'info', - snapshot: true, - upgradeable: true, - version: '8.13.0', - }, - }, - host: { - architecture: 'ogtqmitmts', - hostname: 'Host-y0zwnrnucm', - id: 'aca58288-ac9b-4ce7-9cef-67692fe10363', - ip: ['00.00.000.00'], - mac: ['00-00-00-00-00-00', '00-00-00-00-00-00', '0-00-00-00-00-00'], - name: 'Host-y0zwnrnucm', - os: { - name: 'macOS', - full: 'macOS Monterey', - version: '12.6.1', - platform: 'macOS', - family: 'Darwin', - Ext: { - variant: 'Darwin', - }, - }, - }, - os: { - family: 'Darwin', - full: 'macOS Monterey', - kernel: '10.0.17763.1879 (Build.160101.0800)', - name: 'macOS', - platform: 'macOS', - version: '12.6.1', - Ext: { - variant: 'Darwin', - }, - }, - }, - status: 'online', - }, - { - id: 'cbd4cda1-3bac-45a7-9914-812d3b9c5f44', - type: 'PERMANENT', - active: true, - enrolled_at: '2024-01-11T03:21:13.662Z', - upgraded_at: null, - upgrade_started_at: null, - access_api_key_id: 'jY3dWnkBj1tiuAw9pAmq', - default_api_key_id: 'so3dWnkBj1tiuAw9yAm3', - policy_id: '004e29f7-3b96-4ce3-8de8-c3024f56eae2', - last_checkin: '2024-01-11T04:00:37.315Z', - last_checkin_status: 'online', - policy_revision: 2, - packages: [], - sort: [1704943273662, 'Host-60j0gd14nc'], - components: [ - { - id: 'endpoint-0', - type: 'endpoint', - status: 'STOPPING', - message: 'Running as external service', - units: [ - { - id: 'endpoint-1', - type: 'input', - status: 'STOPPING', - message: 'Protecting machine', - payload: { - extra: 'payload', - }, - }, - { - id: 'shipper', - type: 'output', - status: 'STOPPING', - message: 'Connected over GRPC', - payload: { - extra: 'payload', - }, - }, - ], - }, - ], - agent: { - id: 'cbd4cda1-3bac-45a7-9914-812d3b9c5f44', - version: '8.13.0', - }, - user_provided_metadata: {}, - local_metadata: { - elastic: { - agent: { - 'build.original': '8.0.0-SNAPSHOT (build: slgsg at 2021-05-07 18:42:49 +0000 UTC)', - id: 'cbd4cda1-3bac-45a7-9914-812d3b9c5f44', - log_level: 'info', - snapshot: true, - upgradeable: true, - version: '8.13.0', - }, - }, - host: { - architecture: '3cgyqy4tx4', - hostname: 'Host-60j0gd14nc', - id: 'e76f684a-1f5c-4082-9a21-145d34c2d901', - ip: ['00.00.000.00'], - mac: ['00-00-00-00-00-00', '00-00-00-00-00-00', '0-00-00-00-00-00'], - name: 'Host-60j0gd14nc', - os: { - name: 'Windows', - full: 'Windows Server 2012', - version: '6.2', - platform: 'Windows', - family: 'windows', - Ext: { - variant: 'Windows Server', - }, - }, - }, - os: { - family: 'windows', - full: 'Windows Server 2012', - kernel: '10.0.17763.1879 (Build.160101.0800)', - name: 'Windows', - platform: 'Windows', - version: '6.2', - Ext: { - variant: 'Windows Server', - }, - }, - }, - status: 'online', - }, - { - id: '9918e050-035a-4764-bdd3-5cd536a7201c', - type: 'PERMANENT', - active: true, - enrolled_at: '2024-01-11T03:20:54.483Z', - upgraded_at: null, - upgrade_started_at: null, - access_api_key_id: 'jY3dWnkBj1tiuAw9pAmq', - default_api_key_id: 'so3dWnkBj1tiuAw9yAm3', - policy_id: '004e29f7-3b96-4ce3-8de8-c3024f56eae2', - last_checkin: '2024-01-11T04:00:38.328Z', - last_checkin_status: 'online', - policy_revision: 2, - packages: [], - sort: [1704943254483, 'Host-nh94b0esjr'], - components: [ - { - id: 'endpoint-0', - type: 'endpoint', - status: 'STARTING', - message: 'Running as external service', - units: [ - { - id: 'endpoint-1', - type: 'input', - status: 'STARTING', - message: 'Protecting machine', - payload: { - extra: 'payload', - }, - }, - { - id: 'shipper', - type: 'output', - status: 'STARTING', - message: 'Connected over GRPC', - payload: { - extra: 'payload', - }, - }, - ], - }, - ], - agent: { - id: '9918e050-035a-4764-bdd3-5cd536a7201c', - version: '8.13.0', - }, - user_provided_metadata: {}, - local_metadata: { - elastic: { - agent: { - 'build.original': '8.0.0-SNAPSHOT (build: pd6rl at 2021-05-07 18:42:49 +0000 UTC)', - id: '9918e050-035a-4764-bdd3-5cd536a7201c', - log_level: 'info', - snapshot: true, - upgradeable: true, - version: '8.13.0', - }, - }, - host: { - architecture: 'q5ni746k3b', - hostname: 'Host-nh94b0esjr', - id: 'd036aae1-8a97-4aa6-988c-2e178665272a', - ip: ['00.00.000.00'], - mac: ['00-00-00-00-00-00', '00-00-00-00-00-00', '0-00-00-00-00-00'], - name: 'Host-nh94b0esjr', - os: { - Ext: { - variant: 'Debian', - }, - kernel: '4.19.0-21-cloud-amd64 #1 SMP Debian 4.19.249-2 (2022-06-30)', - name: 'Linux', - family: 'debian', - type: 'linux', - version: '10.12', - platform: 'debian', - full: 'Debian 10.12', - }, - }, - os: { - family: 'debian', - full: 'Debian 10.12', - kernel: '4.19.0-21-cloud-amd64 #1 SMP Debian 4.19.249-2 (2022-06-30)', - name: 'Linux', - platform: 'debian', - version: '10.12', - Ext: { - variant: 'Debian', - }, - type: 'linux', - }, - }, - status: 'online', - }, - ], - total: 5, - page: 1, - perPage: 10000, -}; +export const stubFleetAgentResponse: Map = new Map([ + ['45112616-62e0-42c5-a8f9-2f8a71a92040', '147b2096-bd12-4b7e-a100-061dc11ba799'], + ['74550426-040d-4216-a227-599fd3efa91c', '16608650-4839-4053-a0eb-6ee9d11ac84d'], + ['b80bc33e-1c65-41b3-80d6-8f9757552ab1', '125f0769-20b4-4604-81ce-f0db812d510b'], + ['cbd4cda1-3bac-45a7-9914-812d3b9c5f44', '004e29f7-3b96-4ce3-8de8-c3024f56eae2'], + ['9918e050-035a-4764-bdd3-5cd536a7201c', '004e29f7-3b96-4ce3-8de8-c3024f56eae2'], +]); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts index 25e6d49965bf6..231329470e9b2 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -73,11 +73,12 @@ import type { ValueListExceptionListResponseAggregation, ValueListIndicatorMatchResponseAggregation, Nullable, - FleetAgentResponse, EndpointMetricsAggregation, EndpointMetricsAbstract, EndpointPolicyResponseDocument, EndpointPolicyResponseAggregation, + EndpointMetadataAggregation, + EndpointMetadataDocument, } from './types'; import { telemetryConfiguration } from './configuration'; import { ENDPOINT_METRICS_INDEX } from '../../../common/constants'; @@ -104,7 +105,12 @@ export interface ITelemetryReceiver { fetchDetectionRulesPackageVersion(): Promise>; - fetchFleetAgents(): Promise>; + /** + * As the policy id + policy version does not exist on the Endpoint Metrics document + * we need to fetch information about the Fleet Agent and sync the metrics document + * with the Agent's policy data. This method maps policy info by agent id. + */ + fetchFleetAgents(): Promise>; /** * Reads Endpoint Agent policy responses out of the `.ds-metrics-endpoint.policy*` data @@ -120,6 +126,12 @@ export interface ITelemetryReceiver { executeTo: string ): Promise>; + /** + * Reads Endpoint Agent metrics out of the `.ds-metrics-endpoint.metrics` data stream + * and buckets them by Endpoint Agent id and sorts by the top hit. The EP agent will + * report its metrics once per day OR every time a policy change has occured. If + * a metric document(s) exists for an EP agent we map to fleet agent and policy. + */ fetchEndpointMetricsAbstract( executeFrom: string, executeTo: string @@ -127,12 +139,16 @@ export interface ITelemetryReceiver { fetchEndpointMetricsById(ids: string[]): AsyncGenerator; + /** + * Reads Endpoint Agent metadata out of the `.ds-metrics-endpoint.metadata` data stream + * and buckets them by Endpoint Agent id and sorts by the top hit. The EP agent will + * report its metadata once per day OR every time a policy change has occured. + * If a metadata document(s) exists for an EP agent we map to fleet agent and policy. + */ fetchEndpointMetadata( executeFrom: string, executeTo: string - ): Promise< - TransportResult>, unknown> - >; + ): Promise>; fetchDiagnosticAlertsBatch( executeFrom: string, @@ -277,17 +293,31 @@ export class TelemetryReceiver implements ITelemetryReceiver { return this.packageService?.asInternalUser.getInstallation(PREBUILT_RULES_PACKAGE_NAME); } - public async fetchFleetAgents(): Promise> { + public async fetchFleetAgents() { if (this.esClient === undefined || this.esClient === null) { throw Error('elasticsearch client is unavailable: cannot retrieve fleet agents'); } - return this.agentClient?.listAgents({ - perPage: this.maxRecords, - showInactive: true, - sortField: 'enrolled_at', - sortOrder: 'desc', - }); + return ( + this.agentClient + ?.listAgents({ + perPage: this.maxRecords, + showInactive: true, + sortField: 'enrolled_at', + sortOrder: 'desc', + }) + .then((response) => { + const agents = response?.agents ?? []; + + return agents.reduce((cache, agent) => { + if (agent.policy_id !== null && agent.policy_id !== undefined) { + cache.set(agent.id, agent.policy_id); + } + + return cache; + }, new Map()); + }) ?? new Map() + ); } public async fetchEndpointPolicyResponses(executeFrom: string, executeTo: string) { @@ -341,21 +371,19 @@ export class TelemetryReceiver implements ITelemetryReceiver { }, }; - const response = await this.esClient().search(query, { meta: true }); - const { body: failedPolicyResponses } = response as unknown as { - body: EndpointPolicyResponseAggregation; - }; - - if (!failedPolicyResponses) { - return new Map(); - } - - // If there is no policy responses in the 24h > now then we will continue - return failedPolicyResponses.aggregations?.policy_responses?.buckets.reduce( - (cache, endpointAgentId) => - cache.set(endpointAgentId.key, endpointAgentId.latest_response.hits.hits[0]._source), - new Map() - ); + return this.esClient() + .search(query, { meta: true }) + .then((response) => response.body as unknown as EndpointPolicyResponseAggregation) + .then((failedPolicyResponses) => { + const buckets = failedPolicyResponses?.aggregations?.policy_responses?.buckets ?? []; + + // If there is no policy responses in the 24h > now then we will continue + return buckets.reduce( + (cache, endpointAgentId) => + cache.set(endpointAgentId.key, endpointAgentId.latest_response.hits.hits[0]._source), + new Map() + ); + }); } public async fetchEndpointMetricsAbstract(executeFrom: string, executeTo: string) { @@ -408,21 +436,16 @@ export class TelemetryReceiver implements ITelemetryReceiver { return this.esClient() .search(query, { meta: true }) - .then((response) => { - const { body: endpointMetricsResponse } = response as unknown as { - body: EndpointMetricsAggregation; - }; - - if (endpointMetricsResponse.aggregations !== undefined) { - const endpointMetricIds = - endpointMetricsResponse.aggregations.endpoint_agents.buckets.map( - (epMetrics) => epMetrics.latest_metrics.hits.hits[0]._id - ); - const totalEndpoints = endpointMetricsResponse.aggregations.endpoint_count.value; - return { endpointMetricIds, totalEndpoints }; - } + .then((response) => response.body as unknown as EndpointMetricsAggregation) + .then((endpointMetricsResponse) => { + const buckets = endpointMetricsResponse?.aggregations?.endpoint_agents?.buckets ?? []; - return { endpointMetricIds: [], totalEndpoints: 0 }; + const endpointMetricIds = buckets.map( + (epMetrics) => epMetrics.latest_metrics.hits.hits[0]._id + ); + const totalEndpoints = buckets.length; + + return { endpointMetricIds, totalEndpoints }; }); } @@ -435,7 +458,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { }, }, _source: { - includes: ['agent', 'Endpoint.metrics', 'elastic.agent', 'host', 'event'], + includes: ['@timestamp', 'agent', 'Endpoint.metrics', 'elastic.agent', 'host', 'event'], }, }; @@ -467,6 +490,9 @@ export class TelemetryReceiver implements ITelemetryReceiver { latest_metadata: { top_hits: { size: 1, + _source: { + includes: ['@timestamp', 'agent', 'Endpoint.capabilities', 'elastic.agent'], + }, sort: [ { '@timestamp': { @@ -482,7 +508,18 @@ export class TelemetryReceiver implements ITelemetryReceiver { }, }; - return this.esClient().search(query, { meta: true }); + return this.esClient() + .search(query, { meta: true }) + .then((response) => response.body as unknown as EndpointMetadataAggregation) + .then((endpointMetadataResponse) => { + const buckets = endpointMetadataResponse?.aggregations?.endpoint_metadata?.buckets ?? []; + + return buckets.reduce((cache, endpointAgentId) => { + const doc = endpointAgentId.latest_metadata.hits.hits[0]._source; + cache.set(endpointAgentId.key, doc); + return cache; + }, new Map()); + }); } public async *fetchDiagnosticAlertsBatch(executeFrom: string, executeTo: string) { diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts index e712ed5ffa34b..eb5584b7f1a0b 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts @@ -10,13 +10,16 @@ import type { AgentPolicy } from '@kbn/fleet-plugin/common'; import { FLEET_ENDPOINT_PACKAGE } from '@kbn/fleet-plugin/common'; import type { ITelemetryEventsSender } from '../sender'; import { - type EndpointMetadataAggregation, - type EndpointMetadataDocument, - type Nullable, - type FleetAgentResponse, - TelemetryCounter, TelemetryChannel, + TelemetryCounter, + type EndpointMetadataDocument, + type EndpointMetricDocument, + type EndpointMetricsAbstract, type EndpointPolicyResponseDocument, + type ESClusterInfo, + type ESLicense, + type FleetAgentResponse, + type Nullable, } from '../types'; import type { ITelemetryReceiver } from '../receiver'; import type { TaskExecutionPeriod } from '../task'; @@ -83,12 +86,9 @@ export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) { taskExecutionPeriod.current ); - /** STAGE 1 - Fetch Endpoint Agent Metrics - * - * Reads Endpoint Agent metrics out of the `.ds-metrics-endpoint.metrics` data stream - * and buckets them by Endpoint Agent id and sorts by the top hit. The EP agent will - * report its metrics once per day OR every time a policy change has occured. If - * a metric document(s) exists for an EP agent we map to fleet agent and policy + /** + * STAGE 1 - Fetch Endpoint Agent Metrics + * If no metrics exist, then abort execution. */ if (endpointData.endpointMetrics.totalEndpoints === 0) { log.l('no endpoint metrics to report'); @@ -105,32 +105,13 @@ export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) { incrementBy: endpointData.endpointMetrics.totalEndpoints, }); - /** STAGE 2 - Fetch Fleet Agent Config - * - * As the policy id + policy version does not exist on the Endpoint Metrics document - * we need to fetch information about the Fleet Agent and sync the metrics document - * with the Agent's policy data. - * + /** + * STAGE 2 - Fetch Fleet Agent Config */ - const agentsResponse = endpointData.fleetAgentsResponse; - - if (agentsResponse === undefined || agentsResponse === null) { - log.l('no fleet agent information available'); - taskMetricsService.end(trace); - return 0; - } - - const policyInfoByAgent = agentsResponse.agents.reduce((cache, agent) => { - if (agent.id === DefaultEndpointPolicyIdToIgnore) { - return cache; - } - - if (agent.policy_id !== null && agent.policy_id !== undefined) { - cache.set(agent.id, agent.policy_id); - } + const policyInfoByAgent = endpointData.fleetAgentsResponse; - return cache; - }, new Map()); + // Ignore policy used while installing the endpoint agent + policyInfoByAgent.delete(DefaultEndpointPolicyIdToIgnore); const endpointPolicyById = await endpointPolicies(policyInfoByAgent.values(), receiver); @@ -139,35 +120,14 @@ export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) { */ const policyResponses = endpointData.epPolicyResponse; - /** STAGE 4 - Fetch Endpoint Agent Metadata - * - * Reads Endpoint Agent metadata out of the `.ds-metrics-endpoint.metadata` data stream - * and buckets them by Endpoint Agent id and sorts by the top hit. The EP agent will - * report its metadata once per day OR every time a policy change has occured. If - * a metadata document(s) exists for an EP agent we map to fleet agent and policy + /** + * STAGE 4 - Fetch Endpoint Agent Metadata */ - if (endpointData.endpointMetadata === undefined) { + const endpointMetadata = endpointData.endpointMetadata; + if (endpointMetadata.size === 0) { log.l(`no endpoint metadata to report`); } - const { body: endpointMetadataResponse } = endpointData.endpointMetadata as unknown as { - body: EndpointMetadataAggregation; - }; - - if (endpointMetadataResponse.aggregations === undefined) { - log.l(`no endpoint metadata to report`); - } - - const endpointMetadata = - endpointMetadataResponse.aggregations.endpoint_metadata.buckets.reduce( - (cache, endpointAgentId) => { - const doc = endpointAgentId.latest_metadata.hits.hits[0]; - cache.set(endpointAgentId.key, doc); - return cache; - }, - new Map() - ); - /** STAGE 5 - Create the telemetry log records * * Iterates through the endpoint metrics documents at STAGE 1 and joins them together @@ -176,96 +136,24 @@ export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) { * */ try { - const endpointMetrics = []; + const telemetryPayloads = []; for await (const metrics of receiver.fetchEndpointMetricsById( endpointData.endpointMetrics.endpointMetricIds )) { - endpointMetrics.push(...metrics); + const payloads = metrics.map((endpointMetric) => + mapEndpointMetric( + endpointMetric, + policyInfoByAgent, + endpointPolicyById, + policyResponses, + endpointMetadata, + taskExecutionPeriod, + clusterData + ) + ); + telemetryPayloads.push(...payloads); } - const telemetryPayloads = endpointMetrics.map((endpointMetric) => { - let policyConfig = null; - let failedPolicy: Nullable = null; - let endpointMetadataById = null; - - const fleetAgentId = endpointMetric.elastic.agent.id; - const endpointAgentId = endpointMetric.agent.id; - - const policyInformation = policyInfoByAgent.get(fleetAgentId); - if (policyInformation) { - policyConfig = endpointPolicyById.get(policyInformation) || null; - - if (policyConfig) { - failedPolicy = policyResponses.get(endpointAgentId); - } - } - - if (endpointMetadata) { - endpointMetadataById = endpointMetadata.get(endpointAgentId); - } - - const { - cpu, - memory, - uptime, - documents_volume: documentsVolume, - malicious_behavior_rules: maliciousBehaviorRules, - system_impact: systemImpact, - threads, - event_filter: eventFilter, - } = endpointMetric.Endpoint.metrics; - const endpointPolicyDetail = extractEndpointPolicyConfig(policyConfig); - if (endpointPolicyDetail) { - endpointPolicyDetail.value = addDefaultAdvancedPolicyConfigSettings( - endpointPolicyDetail.value - ); - } - return { - '@timestamp': taskExecutionPeriod.current, - cluster_uuid: clusterData.clusterInfo.cluster_uuid, - cluster_name: clusterData.clusterInfo.cluster_name, - license_id: clusterData.licenseInfo?.uid, - endpoint_id: endpointAgentId, - endpoint_version: endpointMetric.agent.version, - endpoint_package_version: policyConfig?.package?.version || null, - endpoint_metrics: { - cpu: cpu.endpoint, - memory: memory.endpoint.private, - uptime, - documentsVolume, - maliciousBehaviorRules, - systemImpact, - threads, - eventFilter, - }, - endpoint_meta: { - os: endpointMetric.host.os, - capabilities: - endpointMetadataById !== null && endpointMetadataById !== undefined - ? endpointMetadataById._source.Endpoint.capabilities - : [], - }, - policy_config: endpointPolicyDetail !== null ? endpointPolicyDetail : {}, - policy_response: - failedPolicy !== null && failedPolicy !== undefined - ? { - agent_policy_status: failedPolicy.event.agent_id_status, - manifest_version: - failedPolicy.Endpoint.policy.applied.artifacts.global.version, - status: failedPolicy.Endpoint.policy.applied.status, - actions: failedPolicy.Endpoint.policy.applied.actions - .map((action) => (action.status !== 'success' ? action : null)) - .filter((action) => action !== null), - configuration: failedPolicy.Endpoint.configuration, - state: failedPolicy.Endpoint.state, - } - : {}, - telemetry_meta: { - metrics_timestamp: endpointMetric['@timestamp'], - }, - }; - }); - log.l(`sending ${telemetryPayloads.length} endpoint telemetry records`); /** @@ -296,7 +184,12 @@ async function fetchEndpointData( receiver: ITelemetryReceiver, executeFrom: string, executeTo: string -) { +): Promise<{ + fleetAgentsResponse: Map; + endpointMetrics: EndpointMetricsAbstract; + epPolicyResponse: Map; + endpointMetadata: Map; +}> { const [fleetAgentsResponse, epMetricsAbstractResponse, policyResponse, endpointMetadata] = await Promise.allSettled([ receiver.fetchFleetAgents(), @@ -313,7 +206,9 @@ async function fetchEndpointData( }; } -async function fetchClusterData(receiver: ITelemetryReceiver) { +async function fetchClusterData( + receiver: ITelemetryReceiver +): Promise<{ clusterInfo: ESClusterInfo; licenseInfo: Nullable }> { const [clusterInfoPromise, licenseInfoPromise] = await Promise.allSettled([ receiver.fetchClusterInfo(), receiver.fetchLicenseInfo(), @@ -332,29 +227,117 @@ async function endpointPolicies(policies: IterableIterator, receiver: IT let agentPolicy: Nullable; try { agentPolicy = await receiver.fetchPolicyConfigs(policyInfo); + const packagePolicies = agentPolicy?.package_policies; + + if (packagePolicies !== undefined && isPackagePolicyList(packagePolicies)) { + packagePolicies + .map((pPolicy) => pPolicy as PolicyData) + .forEach((pPolicy) => { + if (pPolicy.inputs[0]?.config !== undefined && pPolicy.inputs[0]?.config !== null) { + pPolicy.inputs.forEach((input) => { + if ( + input.type === FLEET_ENDPOINT_PACKAGE && + input?.config !== undefined && + policyInfo !== undefined + ) { + endpointPolicyCache.set(policyInfo, pPolicy); + } + }); + } + }); + } } catch (err) { - // log.error(`error fetching policy config due to ${err?.message}`, err); - } - const packagePolicies = agentPolicy?.package_policies; - - if (packagePolicies !== undefined && isPackagePolicyList(packagePolicies)) { - packagePolicies - .map((pPolicy) => pPolicy as PolicyData) - .forEach((pPolicy) => { - if (pPolicy.inputs[0]?.config !== undefined && pPolicy.inputs[0]?.config !== null) { - pPolicy.inputs.forEach((input) => { - if ( - input.type === FLEET_ENDPOINT_PACKAGE && - input?.config !== undefined && - policyInfo !== undefined - ) { - endpointPolicyCache.set(policyInfo, pPolicy); - } - }); - } - }); + // just ignore the error and continue } } } return endpointPolicyCache; } + +function mapEndpointMetric( + endpointMetric: EndpointMetricDocument, + policyInfoByAgent: Map, + endpointPolicyById: Map, + policyResponses: Map, + endpointMetadata: Map, + taskExecutionPeriod: TaskExecutionPeriod, + clusterData: { clusterInfo: ESClusterInfo; licenseInfo: Nullable } +) { + let policyConfig = null; + let failedPolicy: Nullable = null; + let endpointMetadataById = null; + + const fleetAgentId = endpointMetric.elastic.agent.id; + const endpointAgentId = endpointMetric.agent.id; + + const policyInformation = policyInfoByAgent.get(fleetAgentId); + if (policyInformation) { + policyConfig = endpointPolicyById.get(policyInformation) || null; + + if (policyConfig) { + failedPolicy = policyResponses.get(endpointAgentId); + } + } + + if (endpointMetadata) { + endpointMetadataById = endpointMetadata.get(endpointAgentId); + } + + const { + cpu, + memory, + uptime, + documents_volume: documentsVolume, + malicious_behavior_rules: maliciousBehaviorRules, + system_impact: systemImpact, + threads, + event_filter: eventFilter, + } = endpointMetric.Endpoint.metrics; + const endpointPolicyDetail = extractEndpointPolicyConfig(policyConfig); + if (endpointPolicyDetail) { + endpointPolicyDetail.value = addDefaultAdvancedPolicyConfigSettings(endpointPolicyDetail.value); + } + return { + '@timestamp': taskExecutionPeriod.current, + cluster_uuid: clusterData.clusterInfo.cluster_uuid, + cluster_name: clusterData.clusterInfo.cluster_name, + license_id: clusterData.licenseInfo?.uid, + endpoint_id: endpointAgentId, + endpoint_version: endpointMetric.agent.version, + endpoint_package_version: policyConfig?.package?.version || null, + endpoint_metrics: { + cpu: cpu.endpoint, + memory: memory.endpoint.private, + uptime, + documentsVolume, + maliciousBehaviorRules, + systemImpact, + threads, + eventFilter, + }, + endpoint_meta: { + os: endpointMetric.host.os, + capabilities: + endpointMetadataById !== null && endpointMetadataById !== undefined + ? endpointMetadataById.Endpoint.capabilities + : [], + }, + policy_config: endpointPolicyDetail !== null ? endpointPolicyDetail : {}, + policy_response: + failedPolicy !== null && failedPolicy !== undefined + ? { + agent_policy_status: failedPolicy.event.agent_id_status, + manifest_version: failedPolicy.Endpoint.policy.applied.artifacts.global.version, + status: failedPolicy.Endpoint.policy.applied.status, + actions: failedPolicy.Endpoint.policy.applied.actions + .map((action) => (action.status !== 'success' ? action : null)) + .filter((action) => action !== null), + configuration: failedPolicy.Endpoint.configuration, + state: failedPolicy.Endpoint.state, + } + : {}, + telemetry_meta: { + metrics_timestamp: endpointMetric['@timestamp'], + }, + }; +} diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/types.ts b/x-pack/plugins/security_solution/server/lib/telemetry/types.ts index fa43d72f43342..90c0cd6d63a63 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/types.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/types.ts @@ -311,24 +311,22 @@ export interface EndpointMetadataAggregation { interface EndpointMetadataHits { hits: { total: { value: number }; - hits: EndpointMetadataDocument[]; + hits: Array<{ _source: EndpointMetadataDocument }>; }; } export interface EndpointMetadataDocument { - _source: { - '@timestamp': string; + '@timestamp': string; + agent: { + id: string; + version: string; + }; + Endpoint: { + capabilities: string[]; + }; + elastic: { agent: { id: string; - version: string; - }; - Endpoint: { - capabilities: string[]; - }; - elastic: { - agent: { - id: string; - }; }; }; } From af38142391e1185a4c3e0fd37afba17a9c337718 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Wed, 6 Mar 2024 12:24:47 +0100 Subject: [PATCH 13/19] Code style changes --- .../server/lib/telemetry/receiver.ts | 4 +- .../server/lib/telemetry/tasks/endpoint.ts | 174 +++++++++--------- 2 files changed, 87 insertions(+), 91 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts index 231329470e9b2..9a6873d5da87e 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -108,7 +108,9 @@ export interface ITelemetryReceiver { /** * As the policy id + policy version does not exist on the Endpoint Metrics document * we need to fetch information about the Fleet Agent and sync the metrics document - * with the Agent's policy data. This method maps policy info by agent id. + * with the Agent's policy data. + * + * @returns Map of agent id to policy id */ fetchFleetAgents(): Promise>; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts index eb5584b7f1a0b..8f7ee00d08e0a 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts @@ -6,7 +6,6 @@ */ import type { Logger } from '@kbn/core/server'; -import type { AgentPolicy } from '@kbn/fleet-plugin/common'; import { FLEET_ENDPOINT_PACKAGE } from '@kbn/fleet-plugin/common'; import type { ITelemetryEventsSender } from '../sender'; import { @@ -88,7 +87,8 @@ export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) { /** * STAGE 1 - Fetch Endpoint Agent Metrics - * If no metrics exist, then abort execution. + * If no metrics exist, then abort execution, otherwise increment + * the usage counter and continue. */ if (endpointData.endpointMetrics.totalEndpoints === 0) { log.l('no endpoint metrics to report'); @@ -106,19 +106,22 @@ export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) { }); /** - * STAGE 2 - Fetch Fleet Agent Config + * STAGE 2 + * - Fetch Fleet Agent Config + * - Ignore policy used while installing the endpoint agent. + * - Fetch Endpoint Policy Configs */ - const policyInfoByAgent = endpointData.fleetAgentsResponse; - - // Ignore policy used while installing the endpoint agent - policyInfoByAgent.delete(DefaultEndpointPolicyIdToIgnore); - - const endpointPolicyById = await endpointPolicies(policyInfoByAgent.values(), receiver); + const policyIdByAgent = endpointData.policyIdByAgent; + endpointData.policyIdByAgent.delete(DefaultEndpointPolicyIdToIgnore); + const endpointPolicyById = await endpointPolicies(policyIdByAgent.values(), receiver); /** * STAGE 3 - Fetch Endpoint Policy Responses */ const policyResponses = endpointData.epPolicyResponse; + if (policyResponses.size === 0) { + log.l('no endpoint policy responses to report'); + } /** * STAGE 4 - Fetch Endpoint Agent Metadata @@ -133,46 +136,40 @@ export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) { * Iterates through the endpoint metrics documents at STAGE 1 and joins them together * to form the telemetry log that is sent back to Elastic Security developers to * make improvements to the product. - * */ - try { - const telemetryPayloads = []; - for await (const metrics of receiver.fetchEndpointMetricsById( - endpointData.endpointMetrics.endpointMetricIds - )) { - const payloads = metrics.map((endpointMetric) => - mapEndpointMetric( - endpointMetric, - policyInfoByAgent, - endpointPolicyById, - policyResponses, - endpointMetadata, - taskExecutionPeriod, - clusterData - ) - ); - telemetryPayloads.push(...payloads); - } + const mappingContext = { + policyIdByAgent, + endpointPolicyById, + policyResponses, + endpointMetadata, + taskExecutionPeriod, + clusterData, + }; + const telemetryPayloads = []; + for await (const metrics of receiver.fetchEndpointMetricsById( + endpointData.endpointMetrics.endpointMetricIds + )) { + const payloads = metrics.map((endpointMetric) => + mapEndpointMetric(endpointMetric, mappingContext) + ); + telemetryPayloads.push(...payloads); + } - log.l(`sending ${telemetryPayloads.length} endpoint telemetry records`); + log.l(`sending ${telemetryPayloads.length} endpoint telemetry records`); - /** - * STAGE 6 - Send the documents - * - * Send the documents in a batches of maxTelemetryBatch - */ - const batches = batchTelemetryRecords(telemetryPayloads, maxTelemetryBatch); - for (const batch of batches) { - await sender.sendOnDemand(TELEMETRY_CHANNEL_ENDPOINT_META, batch); - } - taskMetricsService.end(trace); - return telemetryPayloads.length; - } catch (err) { - log.warn(`could not complete endpoint alert telemetry task due to ${err?.message}`, err); - taskMetricsService.end(trace, err); - return 0; + /** + * STAGE 6 - Send the documents + * + * Send the documents in a batches of maxTelemetryBatch + */ + const batches = batchTelemetryRecords(telemetryPayloads, maxTelemetryBatch); + for (const batch of batches) { + await sender.sendOnDemand(TELEMETRY_CHANNEL_ENDPOINT_META, batch); } + taskMetricsService.end(trace); + return telemetryPayloads.length; } catch (err) { + log.warn(`could not complete endpoint alert telemetry task due to ${err?.message}`, err); taskMetricsService.end(trace, err); return 0; } @@ -185,12 +182,12 @@ async function fetchEndpointData( executeFrom: string, executeTo: string ): Promise<{ - fleetAgentsResponse: Map; + policyIdByAgent: Map; endpointMetrics: EndpointMetricsAbstract; epPolicyResponse: Map; endpointMetadata: Map; }> { - const [fleetAgentsResponse, epMetricsAbstractResponse, policyResponse, endpointMetadata] = + const [policyIdByAgent, epMetricsAbstractResponse, policyResponse, endpointMetadata] = await Promise.allSettled([ receiver.fetchFleetAgents(), receiver.fetchEndpointMetricsAbstract(executeFrom, executeTo), @@ -199,7 +196,7 @@ async function fetchEndpointData( ]); return { - fleetAgentsResponse: safeValue(fleetAgentsResponse, EmptyFleetAgentResponse), + policyIdByAgent: safeValue(policyIdByAgent, EmptyFleetAgentResponse), endpointMetrics: safeValue(epMetricsAbstractResponse), epPolicyResponse: safeValue(policyResponse), endpointMetadata: safeValue(endpointMetadata), @@ -220,34 +217,29 @@ async function fetchClusterData( return { clusterInfo, licenseInfo }; } -async function endpointPolicies(policies: IterableIterator, receiver: ITelemetryReceiver) { +async function endpointPolicies(policyIds: IterableIterator, receiver: ITelemetryReceiver) { const endpointPolicyCache = new Map(); - for (const policyInfo of policies) { - if (policyInfo !== null && policyInfo !== undefined && !endpointPolicyCache.has(policyInfo)) { - let agentPolicy: Nullable; - try { - agentPolicy = await receiver.fetchPolicyConfigs(policyInfo); - const packagePolicies = agentPolicy?.package_policies; - - if (packagePolicies !== undefined && isPackagePolicyList(packagePolicies)) { - packagePolicies - .map((pPolicy) => pPolicy as PolicyData) - .forEach((pPolicy) => { - if (pPolicy.inputs[0]?.config !== undefined && pPolicy.inputs[0]?.config !== null) { - pPolicy.inputs.forEach((input) => { - if ( - input.type === FLEET_ENDPOINT_PACKAGE && - input?.config !== undefined && - policyInfo !== undefined - ) { - endpointPolicyCache.set(policyInfo, pPolicy); - } - }); - } - }); - } - } catch (err) { - // just ignore the error and continue + for (const policyId of policyIds) { + if (policyId !== null && policyId !== undefined && !endpointPolicyCache.has(policyId)) { + const agentPolicy = await receiver.fetchPolicyConfigs(policyId); + const packagePolicies = agentPolicy?.package_policies; + + if (packagePolicies !== undefined && isPackagePolicyList(packagePolicies)) { + packagePolicies + .map((pPolicy) => pPolicy as PolicyData) + .forEach((pPolicy) => { + if (pPolicy.inputs[0]?.config !== undefined && pPolicy.inputs[0]?.config !== null) { + pPolicy.inputs.forEach((input) => { + if ( + input.type === FLEET_ENDPOINT_PACKAGE && + input?.config !== undefined && + policyId !== undefined + ) { + endpointPolicyCache.set(policyId, pPolicy); + } + }); + } + }); } } } @@ -256,12 +248,14 @@ async function endpointPolicies(policies: IterableIterator, receiver: IT function mapEndpointMetric( endpointMetric: EndpointMetricDocument, - policyInfoByAgent: Map, - endpointPolicyById: Map, - policyResponses: Map, - endpointMetadata: Map, - taskExecutionPeriod: TaskExecutionPeriod, - clusterData: { clusterInfo: ESClusterInfo; licenseInfo: Nullable } + ctx: { + policyIdByAgent: Map; + endpointPolicyById: Map; + policyResponses: Map; + endpointMetadata: Map; + taskExecutionPeriod: TaskExecutionPeriod; + clusterData: { clusterInfo: ESClusterInfo; licenseInfo: Nullable }; + } ) { let policyConfig = null; let failedPolicy: Nullable = null; @@ -270,17 +264,17 @@ function mapEndpointMetric( const fleetAgentId = endpointMetric.elastic.agent.id; const endpointAgentId = endpointMetric.agent.id; - const policyInformation = policyInfoByAgent.get(fleetAgentId); - if (policyInformation) { - policyConfig = endpointPolicyById.get(policyInformation) || null; + const policyId = ctx.policyIdByAgent.get(fleetAgentId); + if (policyId) { + policyConfig = ctx.endpointPolicyById.get(policyId) || null; if (policyConfig) { - failedPolicy = policyResponses.get(endpointAgentId); + failedPolicy = ctx.policyResponses.get(endpointAgentId); } } - if (endpointMetadata) { - endpointMetadataById = endpointMetadata.get(endpointAgentId); + if (ctx.endpointMetadata) { + endpointMetadataById = ctx.endpointMetadata.get(endpointAgentId); } const { @@ -298,10 +292,10 @@ function mapEndpointMetric( endpointPolicyDetail.value = addDefaultAdvancedPolicyConfigSettings(endpointPolicyDetail.value); } return { - '@timestamp': taskExecutionPeriod.current, - cluster_uuid: clusterData.clusterInfo.cluster_uuid, - cluster_name: clusterData.clusterInfo.cluster_name, - license_id: clusterData.licenseInfo?.uid, + '@timestamp': ctx.taskExecutionPeriod.current, + cluster_uuid: ctx.clusterData.clusterInfo.cluster_uuid, + cluster_name: ctx.clusterData.clusterInfo.cluster_name, + license_id: ctx.clusterData.licenseInfo?.uid, endpoint_id: endpointAgentId, endpoint_version: endpointMetric.agent.version, endpoint_package_version: policyConfig?.package?.version || null, From dadd112335de3f627de91fe9bb3d5962239622c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Wed, 6 Mar 2024 14:32:19 +0100 Subject: [PATCH 14/19] Update artifact configuration --- .../server/lib/telemetry/configuration.ts | 19 ++++++++++++++++++- .../lib/telemetry/tasks/configuration.ts | 7 +++++++ .../server/lib/telemetry/types.ts | 6 ++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/configuration.ts b/x-pack/plugins/security_solution/server/lib/telemetry/configuration.ts index d5c321ec8bc13..691b0daae8f2c 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/configuration.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/configuration.ts @@ -5,7 +5,8 @@ * 2.0. */ -import type { TelemetrySenderChannelConfiguration } from './types'; +import os from 'os'; +import type { PaginationConfiguration, TelemetrySenderChannelConfiguration } from './types'; class TelemetryConfigurationDTO { private readonly DEFAULT_TELEMETRY_MAX_BUFFER_SIZE = 100; @@ -15,6 +16,11 @@ class TelemetryConfigurationDTO { private readonly DEFAULT_MAX_DETECTION_ALERTS_BATCH = 50; private readonly DEFAULT_ASYNC_SENDER = false; private readonly DEFAULT_SENDER_CHANNELS = {}; + private readonly DEFAULT_PAGINATION_CONFIG = { + // default to 2% of host's total memory or 80MiB, whichever is smaller + max_page_size_bytes: Math.min(os.totalmem() * 0.02, 80 * 1024 * 1024), + num_docs_to_sample: 10, + }; private _telemetry_max_buffer_size = this.DEFAULT_TELEMETRY_MAX_BUFFER_SIZE; private _max_security_list_telemetry_batch = this.DEFAULT_MAX_SECURITY_LIST_TELEMETRY_BATCH; private _max_endpoint_telemetry_batch = this.DEFAULT_MAX_ENDPOINT_TELEMETRY_BATCH; @@ -24,6 +30,7 @@ class TelemetryConfigurationDTO { private _sender_channels: { [key: string]: TelemetrySenderChannelConfiguration; } = this.DEFAULT_SENDER_CHANNELS; + private _pagination_config: PaginationConfiguration = this.DEFAULT_PAGINATION_CONFIG; public get telemetry_max_buffer_size(): number { return this._telemetry_max_buffer_size; @@ -81,12 +88,22 @@ class TelemetryConfigurationDTO { return this._sender_channels; } + public set pagination_config(paginationConfiguration: PaginationConfiguration) { + this._pagination_config = paginationConfiguration; + } + + public get pagination_config(): PaginationConfiguration { + return this._pagination_config; + } + public resetAllToDefault() { this._telemetry_max_buffer_size = this.DEFAULT_TELEMETRY_MAX_BUFFER_SIZE; this._max_security_list_telemetry_batch = this.DEFAULT_MAX_SECURITY_LIST_TELEMETRY_BATCH; this._max_endpoint_telemetry_batch = this.DEFAULT_MAX_ENDPOINT_TELEMETRY_BATCH; this._max_detection_rule_telemetry_batch = this.DEFAULT_MAX_DETECTION_RULE_TELEMETRY_BATCH; this._max_detection_alerts_batch = this.DEFAULT_MAX_DETECTION_ALERTS_BATCH; + this._sender_channels = this.DEFAULT_SENDER_CHANNELS; + this._pagination_config = this.DEFAULT_PAGINATION_CONFIG; } } diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/configuration.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/configuration.ts index fdddc67b84d07..0f621dcde378d 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/configuration.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/configuration.ts @@ -99,6 +99,13 @@ export function createTelemetryConfigurationTaskConfig() { }); } + if (configArtifact.pagination_config) { + log('Updating pagination configuration'); + telemetryConfiguration.pagination_config = configArtifact.pagination_config; + _receiver.setMaxPageSizeBytes(configArtifact.pagination_config.max_page_size_bytes); + _receiver.setNumDocsToSample(configArtifact.pagination_config.num_docs_to_sample); + } + taskMetricsService.end(trace); log(`Updated TelemetryConfiguration: ${JSON.stringify(telemetryConfiguration)}`); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/types.ts b/x-pack/plugins/security_solution/server/lib/telemetry/types.ts index 90c0cd6d63a63..eafe04c96a5ab 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/types.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/types.ts @@ -464,6 +464,12 @@ export interface TelemetryConfiguration { sender_channels?: { [key: string]: TelemetrySenderChannelConfiguration; }; + pagination_config?: PaginationConfiguration; +} + +export interface PaginationConfiguration { + max_page_size_bytes: number; + num_docs_to_sample: number; } export interface TelemetrySenderChannelConfiguration { From a2b1847a640809f0964fb06cd810bcd517ad3731 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Wed, 20 Mar 2024 13:02:36 +0100 Subject: [PATCH 15/19] Manage policy config errors --- .../server/lib/telemetry/tasks/endpoint.ts | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts index 8f7ee00d08e0a..6bcc2e68826a9 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts @@ -32,6 +32,7 @@ import { isPackagePolicyList, newTelemetryLogger, safeValue, + type TelemetryLogger, } from '../helpers'; import type { PolicyData } from '../../../../common/endpoint/types'; import { TELEMETRY_CHANNEL_ENDPOINT_META } from '../constants'; @@ -113,7 +114,7 @@ export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) { */ const policyIdByAgent = endpointData.policyIdByAgent; endpointData.policyIdByAgent.delete(DefaultEndpointPolicyIdToIgnore); - const endpointPolicyById = await endpointPolicies(policyIdByAgent.values(), receiver); + const endpointPolicyById = await endpointPolicies(policyIdByAgent.values(), receiver, log); /** * STAGE 3 - Fetch Endpoint Policy Responses @@ -217,11 +218,19 @@ async function fetchClusterData( return { clusterInfo, licenseInfo }; } -async function endpointPolicies(policyIds: IterableIterator, receiver: ITelemetryReceiver) { +async function endpointPolicies( + policyIds: IterableIterator, + receiver: ITelemetryReceiver, + log: TelemetryLogger +) { const endpointPolicyCache = new Map(); for (const policyId of policyIds) { if (policyId !== null && policyId !== undefined && !endpointPolicyCache.has(policyId)) { - const agentPolicy = await receiver.fetchPolicyConfigs(policyId); + const agentPolicy = await receiver.fetchPolicyConfigs(policyId).catch((e) => { + log.l(`error fetching policy config due to ${e?.message}`); + return null; + }); + const packagePolicies = agentPolicy?.package_policies; if (packagePolicies !== undefined && isPackagePolicyList(packagePolicies)) { From 3d63654c1508c0b4b8f544ba3955702d100ea933 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Wed, 20 Mar 2024 14:08:44 +0100 Subject: [PATCH 16/19] Close PIT --- .../server/lib/telemetry/receiver.ts | 41 ++++++++++--------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts index 9a6873d5da87e..03c0d9d73346b 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -847,8 +847,8 @@ export class TelemetryReceiver implements ITelemetryReceiver { }; let response = null; - while (fetchMore) { - try { + try { + while (fetchMore) { response = await this.esClient().search(query); const numOfHits = response?.hits.hits.length; @@ -860,27 +860,30 @@ export class TelemetryReceiver implements ITelemetryReceiver { } fetchMore = numOfHits > 0 && numOfHits < 1_000; - } catch (e) { - tlog(this.logger, e); - fetchMore = false; - } - - if (response == null) { - await this.closePointInTime(pitId); - return; - } + if (response == null) { + await this.closePointInTime(pitId); + return; + } - const alerts: TelemetryEvent[] = response.hits.hits.flatMap((h) => - h._source != null ? ([h._source] as TelemetryEvent[]) : [] - ); + const alerts: TelemetryEvent[] = response.hits.hits.flatMap((h) => + h._source != null ? ([h._source] as TelemetryEvent[]) : [] + ); - if (response?.pit_id != null) { - pitId = response?.pit_id; - } + if (response?.pit_id != null) { + pitId = response?.pit_id; + } - tlog(this.logger, `Prebuilt rule alerts to return: ${alerts.length}`); + tlog(this.logger, `Prebuilt rule alerts to return: ${alerts.length}`); - yield alerts; + yield alerts; + } + } catch (e) { + // to keep backward compatibility with the previous implementation, silent return + // once we start using `paginate` this error should be managed downstream + tlog(this.logger, e); + return; + } finally { + await this.closePointInTime(pitId); } } From 45a4ec5b3ceedf07412be92a2237385be3562ccd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Wed, 20 Mar 2024 14:10:06 +0100 Subject: [PATCH 17/19] Enrich task metrics doc --- .../server/lib/telemetry/async_sender.ts | 22 +++++++++++++---- .../task_based/all_types.ts | 4 ++-- .../task_based/detection_rules.ts | 24 +++++++++---------- .../task_based/security_lists.ts | 14 +++++------ ...remove_time_fields_from_telemetry_stats.ts | 5 ++-- 5 files changed, 41 insertions(+), 28 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/async_sender.ts b/x-pack/plugins/security_solution/server/lib/telemetry/async_sender.ts index 1edd3cb598521..b87ce9b4af963 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/async_sender.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/async_sender.ts @@ -268,15 +268,27 @@ export class AsyncTelemetryEventsSender implements IAsyncTelemetryEventsSender { private enrich(event: Event): Event { const clusterInfo = this.telemetryReceiver?.getClusterInfo(); - // TODO(szaffarano): does it make sense to include cluster info in task metric payloads? - if (typeof event.payload === 'object' && event.channel !== TelemetryChannel.TASK_METRICS) { + // TODO(szaffarano): generalize the enrichment at channel level to not hardcode the logic here + if (typeof event.payload === 'object') { + let additional = {}; + + if (event.channel !== TelemetryChannel.TASK_METRICS) { + additional = { + cluster_name: clusterInfo?.cluster_name, + cluster_uuid: clusterInfo?.cluster_uuid, + }; + } else { + additional = { + cluster_uuid: clusterInfo?.cluster_uuid, + }; + } + event.payload = { ...event.payload, - // TODO(szaffarano): include license info if available - cluster_uuid: clusterInfo?.cluster_uuid, - cluster_name: clusterInfo?.cluster_name, + ...additional, }; } + return event; } diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/task_based/all_types.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/task_based/all_types.ts index adaed08158794..8ec7a4eeb56cd 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/task_based/all_types.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/task_based/all_types.ts @@ -7,7 +7,7 @@ import expect from '@kbn/expect'; -import { getSecurityTelemetryStats, removeTimeFieldsFromTelemetryStats } from '../../../utils'; +import { getSecurityTelemetryStats, removeExtraFieldsFromTelemetryStats } from '../../../utils'; import { createAlertsIndex, deleteAllRules, @@ -46,7 +46,7 @@ export default ({ getService }: FtrProviderContext) => { it('@skipInQA should only have task metric values when no rules are running', async () => { await retry.try(async () => { const stats = await getSecurityTelemetryStats(supertest, log); - removeTimeFieldsFromTelemetryStats(stats); + removeExtraFieldsFromTelemetryStats(stats); expect(stats).to.eql({ detection_rules: [ [ diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/task_based/detection_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/task_based/detection_rules.ts index 48f8be9a3d2c5..abfe92a4cec04 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/task_based/detection_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/task_based/detection_rules.ts @@ -16,7 +16,7 @@ import { getSecurityTelemetryStats, createExceptionList, createExceptionListItem, - removeTimeFieldsFromTelemetryStats, + removeExtraFieldsFromTelemetryStats, } from '../../../utils'; import { createRule, @@ -99,7 +99,7 @@ export default ({ getService }: FtrProviderContext) => { // Get the stats and ensure they're empty await retry.try(async () => { const stats = await getSecurityTelemetryStats(supertest, log); - removeTimeFieldsFromTelemetryStats(stats); + removeExtraFieldsFromTelemetryStats(stats); expect(stats.detection_rules).to.eql([ [ { @@ -155,7 +155,7 @@ export default ({ getService }: FtrProviderContext) => { // Get the stats and ensure they're empty await retry.try(async () => { const stats = await getSecurityTelemetryStats(supertest, log); - removeTimeFieldsFromTelemetryStats(stats); + removeExtraFieldsFromTelemetryStats(stats); expect(stats.detection_rules).to.eql([ [ { @@ -211,7 +211,7 @@ export default ({ getService }: FtrProviderContext) => { // Get the stats and ensure they're empty await retry.try(async () => { const stats = await getSecurityTelemetryStats(supertest, log); - removeTimeFieldsFromTelemetryStats(stats); + removeExtraFieldsFromTelemetryStats(stats); expect(stats.detection_rules).to.eql([ [ { @@ -267,7 +267,7 @@ export default ({ getService }: FtrProviderContext) => { // Get the stats and ensure they're empty await retry.try(async () => { const stats = await getSecurityTelemetryStats(supertest, log); - removeTimeFieldsFromTelemetryStats(stats); + removeExtraFieldsFromTelemetryStats(stats); expect(stats.detection_rules).to.eql([ [ { @@ -323,7 +323,7 @@ export default ({ getService }: FtrProviderContext) => { // Get the stats and ensure they're empty await retry.try(async () => { const stats = await getSecurityTelemetryStats(supertest, log); - removeTimeFieldsFromTelemetryStats(stats); + removeExtraFieldsFromTelemetryStats(stats); expect(stats.detection_rules).to.eql([ [ { @@ -451,7 +451,7 @@ export default ({ getService }: FtrProviderContext) => { await retry.try(async () => { const stats = await getSecurityTelemetryStats(supertest, log); - removeTimeFieldsFromTelemetryStats(stats); + removeExtraFieldsFromTelemetryStats(stats); const detectionRules = stats.detection_rules .flat() .map((obj: any) => (obj.passed != null ? obj : obj.detection_rule)); @@ -528,7 +528,7 @@ export default ({ getService }: FtrProviderContext) => { await retry.try(async () => { const stats = await getSecurityTelemetryStats(supertest, log); - removeTimeFieldsFromTelemetryStats(stats); + removeExtraFieldsFromTelemetryStats(stats); const detectionRules = stats.detection_rules .flat() .map((obj: any) => (obj.passed != null ? obj : obj.detection_rule)); @@ -605,7 +605,7 @@ export default ({ getService }: FtrProviderContext) => { await retry.try(async () => { const stats = await getSecurityTelemetryStats(supertest, log); - removeTimeFieldsFromTelemetryStats(stats); + removeExtraFieldsFromTelemetryStats(stats); const detectionRules = stats.detection_rules .flat() .map((obj: any) => (obj.passed != null ? obj : obj.detection_rule)); @@ -682,7 +682,7 @@ export default ({ getService }: FtrProviderContext) => { await retry.try(async () => { const stats = await getSecurityTelemetryStats(supertest, log); - removeTimeFieldsFromTelemetryStats(stats); + removeExtraFieldsFromTelemetryStats(stats); const detectionRules = stats.detection_rules .flat() .map((obj: any) => (obj.passed != null ? obj : obj.detection_rule)); @@ -759,7 +759,7 @@ export default ({ getService }: FtrProviderContext) => { await retry.try(async () => { const stats = await getSecurityTelemetryStats(supertest, log); - removeTimeFieldsFromTelemetryStats(stats); + removeExtraFieldsFromTelemetryStats(stats); const detectionRules = stats.detection_rules .flat() .map((obj: any) => (obj.passed != null ? obj : obj.detection_rule)); @@ -860,7 +860,7 @@ export default ({ getService }: FtrProviderContext) => { await retry.try(async () => { const stats = await getSecurityTelemetryStats(supertest, log); - removeTimeFieldsFromTelemetryStats(stats); + removeExtraFieldsFromTelemetryStats(stats); const detectionRules = stats.detection_rules .flat() .map((obj: any) => (obj.passed != null ? obj : obj.detection_rule)) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/task_based/security_lists.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/task_based/security_lists.ts index 088016d868caa..ac7a99899a336 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/task_based/security_lists.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/task_based/security_lists.ts @@ -15,7 +15,7 @@ import { getSecurityTelemetryStats, createExceptionListItem, createExceptionList, - removeTimeFieldsFromTelemetryStats, + removeExtraFieldsFromTelemetryStats, } from '../../../utils'; import { createAlertsIndex, @@ -76,7 +76,7 @@ export default ({ getService }: FtrProviderContext) => { await retry.try(async () => { const stats = await getSecurityTelemetryStats(supertest, log); - removeTimeFieldsFromTelemetryStats(stats); + removeExtraFieldsFromTelemetryStats(stats); const trustedApplication = stats.security_lists .flat() .map((obj: any) => (obj.passed != null ? obj : obj.trusted_application)); @@ -146,7 +146,7 @@ export default ({ getService }: FtrProviderContext) => { await retry.try(async () => { const stats = await getSecurityTelemetryStats(supertest, log); - removeTimeFieldsFromTelemetryStats(stats); + removeExtraFieldsFromTelemetryStats(stats); const trustedApplication = stats.security_lists .flat() .map((obj: any) => (obj.passed != null ? obj : obj.trusted_application)) @@ -222,7 +222,7 @@ export default ({ getService }: FtrProviderContext) => { await retry.try(async () => { const stats = await getSecurityTelemetryStats(supertest, log); - removeTimeFieldsFromTelemetryStats(stats); + removeExtraFieldsFromTelemetryStats(stats); const securityLists = stats.security_lists .flat() .map((obj: any) => (obj.passed != null ? obj : obj.endpoint_exception)); @@ -288,7 +288,7 @@ export default ({ getService }: FtrProviderContext) => { await retry.try(async () => { const stats = await getSecurityTelemetryStats(supertest, log); - removeTimeFieldsFromTelemetryStats(stats); + removeExtraFieldsFromTelemetryStats(stats); const securityLists = stats.security_lists .flat() .map((obj: any) => (obj.passed != null ? obj : obj.endpoint_exception)) @@ -368,7 +368,7 @@ export default ({ getService }: FtrProviderContext) => { await retry.try(async () => { const stats = await getSecurityTelemetryStats(supertest, log); - removeTimeFieldsFromTelemetryStats(stats); + removeExtraFieldsFromTelemetryStats(stats); const endPointEventFilter = stats.security_lists .flat() .map((obj: any) => (obj.passed != null ? obj : obj.endpoint_event_filter)); @@ -435,7 +435,7 @@ export default ({ getService }: FtrProviderContext) => { await retry.try(async () => { const stats = await getSecurityTelemetryStats(supertest, log); - removeTimeFieldsFromTelemetryStats(stats); + removeExtraFieldsFromTelemetryStats(stats); const endPointEventFilter = stats.security_lists .flat() .map((obj: any) => (obj.passed != null ? obj : obj.endpoint_event_filter)) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/telemetry/remove_time_fields_from_telemetry_stats.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/telemetry/remove_time_fields_from_telemetry_stats.ts index 7c0931b4b5ae9..912649862a24f 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/telemetry/remove_time_fields_from_telemetry_stats.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/telemetry/remove_time_fields_from_telemetry_stats.ts @@ -7,13 +7,14 @@ import { unset } from 'lodash'; -export const removeTimeFieldsFromTelemetryStats = (stats: any) => { +export const removeExtraFieldsFromTelemetryStats = (stats: any) => { Object.entries(stats).forEach(([, value]: [unknown, any]) => { value.forEach((entry: any, i: number) => { - entry.forEach((e: any, j: number) => { + entry.forEach((_e: any, j: number) => { unset(value, `[${i}][${j}].time_executed_in_ms`); unset(value, `[${i}][${j}].start_time`); unset(value, `[${i}][${j}].end_time`); + unset(value, `[${i}][${j}].cluster_uuid`); }); }); }); From 90cccfcd9af751128501577cabf39867b271abd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Tue, 2 Apr 2024 13:09:43 +0200 Subject: [PATCH 18/19] Fix import --- .../security_solution/server/lib/telemetry/tasks/endpoint.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts index 6bcc2e68826a9..8bd9aaecf7319 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts @@ -32,8 +32,8 @@ import { isPackagePolicyList, newTelemetryLogger, safeValue, - type TelemetryLogger, } from '../helpers'; +import type { TelemetryLogger } from '../telemetry_logger'; import type { PolicyData } from '../../../../common/endpoint/types'; import { TELEMETRY_CHANNEL_ENDPOINT_META } from '../constants'; From 409f670c20ebd600a3010eacb33b496cfefc484f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Tue, 2 Apr 2024 14:45:30 +0200 Subject: [PATCH 19/19] Fix type --- .../server/lib/telemetry/tasks/configuration.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/configuration.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/configuration.ts index 88602de627829..701f42a89f4c0 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/configuration.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/configuration.ts @@ -100,7 +100,7 @@ export function createTelemetryConfigurationTaskConfig() { } if (configArtifact.pagination_config) { - log('Updating pagination configuration'); + log.l('Updating pagination configuration'); telemetryConfiguration.pagination_config = configArtifact.pagination_config; _receiver.setMaxPageSizeBytes(configArtifact.pagination_config.max_page_size_bytes); _receiver.setNumDocsToSample(configArtifact.pagination_config.num_docs_to_sample);