From d8ca854a7d3b627b46886d1c66fc127630adf35b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B4mulo=20Farias?= Date: Fri, 22 May 2026 17:34:53 +0200 Subject: [PATCH] [Entity Store] Change defaults to conservative values (#270617) ## Summary Update entity store defaults to a conservative number: - maxLogsPerPage: 50k - maxLogsPerWindow: 100k Meaning, we will process at most 50k logs in each query of entity store, and in a single window, if it observes more than 100k logs (two pages) we don't process those logs. This can be configured via update api to better align with customers' setup and needs. --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> (cherry picked from commit b9560c557118b03d3fa340140df992abc81307e8) # Conflicts: # x-pack/solutions/security/plugins/entity_store/server/domain/asset_manager/asset_manager_client.test.ts --- oas_docs/output/kibana.serverless.yaml | 4 ++-- oas_docs/output/kibana.yaml | 4 ++-- .../logs_extraction/logs_extraction_client.test.ts | 12 ++++-------- .../domain/saved_objects/global_state/constants.ts | 4 ++-- 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/oas_docs/output/kibana.serverless.yaml b/oas_docs/output/kibana.serverless.yaml index 1598f69b9678e..4aa81f7ec52aa 100644 --- a/oas_docs/output/kibana.serverless.yaml +++ b/oas_docs/output/kibana.serverless.yaml @@ -70384,12 +70384,12 @@ paths: pattern: '[smdh]$' type: string maxLogsPerPage: - default: 40000 + default: 50000 maximum: 9007199254740991 minimum: 1 type: integer maxLogsPerWindow: - default: 500000 + default: 100000 maximum: 9007199254740991 minimum: 0 type: integer diff --git a/oas_docs/output/kibana.yaml b/oas_docs/output/kibana.yaml index 3ef3956b17181..d224d83f0d03d 100644 --- a/oas_docs/output/kibana.yaml +++ b/oas_docs/output/kibana.yaml @@ -74377,12 +74377,12 @@ paths: pattern: '[smdh]$' type: string maxLogsPerPage: - default: 40000 + default: 50000 maximum: 9007199254740991 minimum: 1 type: integer maxLogsPerWindow: - default: 500000 + default: 100000 maximum: 9007199254740991 minimum: 0 type: integer diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/logs_extraction/logs_extraction_client.test.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/logs_extraction/logs_extraction_client.test.ts index bfc1c0877fbb7..795922b04a1f8 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/logs_extraction/logs_extraction_client.test.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/logs_extraction/logs_extraction_client.test.ts @@ -112,6 +112,7 @@ function createMockGlobalStateClient( lookbackPeriod: string; delay: string; maxTimeWindowSize: string; + maxLogsPerWindow: number; excludedIndexPatterns: string[]; additionalIndexPatterns: string[]; }> @@ -125,6 +126,9 @@ function createMockGlobalStateClient( // Default to a very large cap so existing tests run as a single sub-window. The dedicated // sub-window cap describe block overrides this to exercise capping behavior. maxTimeWindowSize: logExtractionOverrides?.maxTimeWindowSize ?? '999d', + // Default to 0 (disabled) so volume-cap logic doesn't interfere with unrelated tests. + // The dedicated volume-cap describe block overrides this via setupVolCapTest. + maxLogsPerWindow: logExtractionOverrides?.maxLogsPerWindow ?? 0, }); const state = { logsExtraction } as EntityStoreGlobalState; return { @@ -975,9 +979,6 @@ describe('LogsExtractionClient', () => { ['2025-01-15T11:00:01.000Z', 'hash2', '2025-01-15T11:00:01.000Z', 'entity2'], ], }; - // effectiveMaxLogsPerPage = min(40000, maxLogsPerWindow=1) = 1. - // Probe LIMIT 1 → total_logs = 1 → sliceLogCount = 1. - // totalLogs = 1 >= maxLogsPerWindow=1 → cap fires. setupVolCapTest({ maxLogsPerWindow: 1, maxLogsPerWindowCapBehavior: 'defer' }); mockExtractSuccessSequence(mainExtractionResponse, 1); mockIngestEntities.mockResolvedValue(undefined); @@ -1014,9 +1015,6 @@ describe('LogsExtractionClient', () => { ['2025-01-15T11:00:01.000Z', 'hash2', '2025-01-15T11:00:01.000Z', 'entity2'], ], }; - // effectiveMaxLogsPerPage = min(40000, maxLogsPerWindow=1) = 1. - // Probe LIMIT 1 → total_logs = 1 → sliceLogCount = 1. - // totalLogs = 1 >= maxLogsPerWindow=1 → cap fires. setupVolCapTest({ maxLogsPerWindow: 1, maxLogsPerWindowCapBehavior: 'drop' }); mockExtractSuccessSequence(mainExtractionResponse, 1); mockIngestEntities.mockResolvedValue(undefined); @@ -1108,7 +1106,6 @@ describe('LogsExtractionClient', () => { expect(result.success).toBe(true); if (!result.success) return; expect(result.logsCapApplied).toBe(true); - // effectiveMaxLogsPerPage = min(40000, maxLogsPerWindow=1) = 1 → sliceLogCount = 1 expect(result.logsProcessed).toBe(1); // defer: lastSearchTimestamp is where the loop stopped, NOT the window end expect(result.lastSearchTimestamp).toBe(lastPageTimestamp); @@ -1142,7 +1139,6 @@ describe('LogsExtractionClient', () => { expect(result.success).toBe(true); if (!result.success) return; expect(result.logsCapApplied).toBe(true); - // effectiveMaxLogsPerPage = min(40000, maxLogsPerWindow=1) = 1 → sliceLogCount = 1 expect(result.logsProcessed).toBe(1); // drop: lastSearchTimestamp is advanced to the window end expect(result.lastSearchTimestamp).toBe(toDateISO); diff --git a/x-pack/solutions/security/plugins/entity_store/server/domain/saved_objects/global_state/constants.ts b/x-pack/solutions/security/plugins/entity_store/server/domain/saved_objects/global_state/constants.ts index a9d28f7e1121b..f097a6de17947 100644 --- a/x-pack/solutions/security/plugins/entity_store/server/domain/saved_objects/global_state/constants.ts +++ b/x-pack/solutions/security/plugins/entity_store/server/domain/saved_objects/global_state/constants.ts @@ -15,11 +15,11 @@ export const LOG_EXTRACTION_FREQUENCY_DEFAULT = '1m'; // Max amount of entities to extract in one ESQL query export const LOG_EXTRACTION_DOCS_LIMIT_DEFAULT = 10000; // Max raw log documents per logs to be processed in a query (inside elastic search) -export const LOG_EXTRACTION_MAX_LOGS_PER_PAGE_DEFAULT = 40000; +export const LOG_EXTRACTION_MAX_LOGS_PER_PAGE_DEFAULT = 50_000; export const LOG_EXTRACTION_TIMEOUT_DEFAULT = '59s'; export const LOG_EXTRACTION_MAX_TIME_WINDOW_SIZE_DEFAULT = '15m'; // Max total raw log documents to process per task run; 0 = no cap -export const LOG_EXTRACTION_MAX_LOGS_PER_WINDOW_DEFAULT = 500_000; +export const LOG_EXTRACTION_MAX_LOGS_PER_WINDOW_DEFAULT = 100_000; export const LOG_EXTRACTION_CAP_BEHAVIOR_DEFAULT = 'drop' as const; export type LogExtractionConfig = z.infer;