Skip to content

[9.4] [Entity Store] Change defaults to conservative values (#270617) #270915

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
romulets merged 1 commit into from
May 25, 2026
Merged

[9.4] [Entity Store] Change defaults to conservative values (#270617) #270915

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions oas_docs/output/kibana.serverless.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70384,12 +70384,12 @@ paths:
pattern: '[smdh]$'
type: string
maxLogsPerPage:
default: 40000
default: 50000
maximum: 9007199254740991
minimum: 1
type: integer
maxLogsPerWindow:
default: 500000
default: 100000
maximum: 9007199254740991
minimum: 0
type: integer
Expand Down
4 changes: 2 additions & 2 deletions oas_docs/output/kibana.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74377,12 +74377,12 @@ paths:
pattern: '[smdh]$'
type: string
maxLogsPerPage:
default: 40000
default: 50000
maximum: 9007199254740991
minimum: 1
type: integer
maxLogsPerWindow:
default: 500000
default: 100000
maximum: 9007199254740991
minimum: 0
type: integer
Expand Down
12 changes: 4 additions & 8 deletions ...ecurity/plugins/entity_store/server/domain/logs_extraction/logs_extraction_client.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ function createMockGlobalStateClient(
lookbackPeriod: string;
delay: string;
maxTimeWindowSize: string;
maxLogsPerWindow: number;
excludedIndexPatterns: string[];
additionalIndexPatterns: string[];
}>
Expand All @@ -125,6 +126,9 @@ function createMockGlobalStateClient(
// Default to a very large cap so existing tests run as a single sub-window. The dedicated
// sub-window cap describe block overrides this to exercise capping behavior.
maxTimeWindowSize: logExtractionOverrides?.maxTimeWindowSize ?? '999d',
// Default to 0 (disabled) so volume-cap logic doesn't interfere with unrelated tests.
// The dedicated volume-cap describe block overrides this via setupVolCapTest.
maxLogsPerWindow: logExtractionOverrides?.maxLogsPerWindow ?? 0,
});
const state = { logsExtraction } as EntityStoreGlobalState;
return {
Expand Down Expand Up @@ -975,9 +979,6 @@ describe('LogsExtractionClient', () => {
['2025-01-15T11:00:01.000Z', 'hash2', '2025-01-15T11:00:01.000Z', 'entity2'],
],
};
// effectiveMaxLogsPerPage = min(40000, maxLogsPerWindow=1) = 1.
// Probe LIMIT 1 → total_logs = 1 → sliceLogCount = 1.
// totalLogs = 1 >= maxLogsPerWindow=1 → cap fires.
setupVolCapTest({ maxLogsPerWindow: 1, maxLogsPerWindowCapBehavior: 'defer' });
mockExtractSuccessSequence(mainExtractionResponse, 1);
mockIngestEntities.mockResolvedValue(undefined);
Expand Down Expand Up @@ -1014,9 +1015,6 @@ describe('LogsExtractionClient', () => {
['2025-01-15T11:00:01.000Z', 'hash2', '2025-01-15T11:00:01.000Z', 'entity2'],
],
};
// effectiveMaxLogsPerPage = min(40000, maxLogsPerWindow=1) = 1.
// Probe LIMIT 1 → total_logs = 1 → sliceLogCount = 1.
// totalLogs = 1 >= maxLogsPerWindow=1 → cap fires.
setupVolCapTest({ maxLogsPerWindow: 1, maxLogsPerWindowCapBehavior: 'drop' });
mockExtractSuccessSequence(mainExtractionResponse, 1);
mockIngestEntities.mockResolvedValue(undefined);
Expand Down Expand Up @@ -1108,7 +1106,6 @@ describe('LogsExtractionClient', () => {
expect(result.success).toBe(true);
if (!result.success) return;
expect(result.logsCapApplied).toBe(true);
// effectiveMaxLogsPerPage = min(40000, maxLogsPerWindow=1) = 1 → sliceLogCount = 1
expect(result.logsProcessed).toBe(1);
// defer: lastSearchTimestamp is where the loop stopped, NOT the window end
expect(result.lastSearchTimestamp).toBe(lastPageTimestamp);
Expand Down Expand Up @@ -1142,7 +1139,6 @@ describe('LogsExtractionClient', () => {
expect(result.success).toBe(true);
if (!result.success) return;
expect(result.logsCapApplied).toBe(true);
// effectiveMaxLogsPerPage = min(40000, maxLogsPerWindow=1) = 1 → sliceLogCount = 1
expect(result.logsProcessed).toBe(1);
// drop: lastSearchTimestamp is advanced to the window end
expect(result.lastSearchTimestamp).toBe(toDateISO);
Expand Down
4 changes: 2 additions & 2 deletions ...tions/security/plugins/entity_store/server/domain/saved_objects/global_state/constants.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,11 @@ export const LOG_EXTRACTION_FREQUENCY_DEFAULT = '1m';
// Max amount of entities to extract in one ESQL query
export const LOG_EXTRACTION_DOCS_LIMIT_DEFAULT = 10000;
// Max raw log documents per logs to be processed in a query (inside elastic search)
export const LOG_EXTRACTION_MAX_LOGS_PER_PAGE_DEFAULT = 40000;
export const LOG_EXTRACTION_MAX_LOGS_PER_PAGE_DEFAULT = 50_000;
export const LOG_EXTRACTION_TIMEOUT_DEFAULT = '59s';
export const LOG_EXTRACTION_MAX_TIME_WINDOW_SIZE_DEFAULT = '15m';
// Max total raw log documents to process per task run; 0 = no cap
export const LOG_EXTRACTION_MAX_LOGS_PER_WINDOW_DEFAULT = 500_000;
export const LOG_EXTRACTION_MAX_LOGS_PER_WINDOW_DEFAULT = 100_000;
export const LOG_EXTRACTION_CAP_BEHAVIOR_DEFAULT = 'drop' as const;

export type LogExtractionConfig = z.infer<typeof LogExtractionConfig>;
Expand Down
Loading