diff --git a/oas_docs/output/kibana.serverless.yaml b/oas_docs/output/kibana.serverless.yaml index 143f999aca563..5cfe7044b8e62 100644 --- a/oas_docs/output/kibana.serverless.yaml +++ b/oas_docs/output/kibana.serverless.yaml @@ -48398,6 +48398,25 @@ paths: items: type: string type: array + filters: + items: + type: object + properties: + entity_types: + items: + enum: + - host + - user + - service + type: string + type: array + filter: + description: KQL filter string + type: string + required: + - entity_types + - filter + type: array range: type: object properties: diff --git a/oas_docs/output/kibana.yaml b/oas_docs/output/kibana.yaml index 5ae758014a5a2..1da958a3c5f5b 100644 --- a/oas_docs/output/kibana.yaml +++ b/oas_docs/output/kibana.yaml @@ -53755,6 +53755,25 @@ paths: items: type: string type: array + filters: + items: + type: object + properties: + entity_types: + items: + enum: + - host + - user + - service + type: string + type: array + filter: + description: KQL filter string + type: string + required: + - entity_types + - filter + type: array range: type: object properties: diff --git a/packages/kbn-check-saved-objects-cli/current_fields.json b/packages/kbn-check-saved-objects-cli/current_fields.json index f2d22c63a5051..2b7febd9c9bb7 100644 --- a/packages/kbn-check-saved-objects-cli/current_fields.json +++ b/packages/kbn-check-saved-objects-cli/current_fields.json @@ -948,6 +948,9 @@ "enabled", "excludeAlertStatuses", "filter", + "filters", + "filters.entity_types", + "filters.filter", "identifierType", "interval", "pageSize", diff --git a/packages/kbn-check-saved-objects-cli/current_mappings.json b/packages/kbn-check-saved-objects-cli/current_mappings.json index 26daeae23217e..7c418452c2121 100644 --- a/packages/kbn-check-saved-objects-cli/current_mappings.json +++ b/packages/kbn-check-saved-objects-cli/current_mappings.json @@ -3154,6 +3154,17 @@ "dynamic": false, "properties": {} }, + "filters": { + "properties": { + "entity_types": { + "type": "keyword" + }, + "filter": { + "type": "text" + } + }, + "type": "nested" + }, "identifierType": { "type": "keyword" }, diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index cfd316f96957c..767863fc7a333 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -159,7 +159,7 @@ describe('checking migration metadata changes on all registered SO types', () => "privmon-api-key": "c06b1614786ce7271087378b47d465c956ab1537", "product-doc-install-status": "f94e3e5ad2cc933df918f2cd159044c626e01011", "query": "1966ccce8e9853018111fb8a1dee500228731d9e", - "risk-engine-configuration": "ad7bf1d048a5dad258c2dd8823265adb4debf9a6", + "risk-engine-configuration": "f5ca37ab60d0bb0756869c9a4171146afbdd67a7", "rules-settings": "53f94e5ce61f5e75d55ab8adbc1fb3d0937d2e0b", "sample-data-telemetry": "c38daf1a49ed24f2a4fb091e6e1e833fccf19935", "search": "d81feb3845eb84c00dabfd3e89dc798c854c07dd", @@ -997,8 +997,9 @@ describe('checking migration metadata changes on all registered SO types', () => "query|7.16.0: b1a3b62b35f9e5c5adef5983e5c83a0a174ac679", "======================================================", "risk-engine-configuration|global: 0ca55e55c439cebd5bad7ecec17d81a6264f4ea4", - "risk-engine-configuration|mappings: eeef5029f25635e3c973fb1047f6ef6d73ac7b9a", + "risk-engine-configuration|mappings: 20b5659d79a49b6d1850003dc9bb35a76d836b6f", "risk-engine-configuration|schemas: da39a3ee5e6b4b0d3255bfef95601890afd80709", + "risk-engine-configuration|10.4.0: cb95557ad3ff0c0983274d3acf6cbde458424652", "risk-engine-configuration|10.3.0: c859b4f9fe7e1e4c97ba3adc8715004f2b06c5e0", "risk-engine-configuration|10.2.0: 9db1cd4b80df6c7b6198d7021623c8302b58a764", "risk-engine-configuration|10.1.0: 6502b46de13e13ff2ce499d3a8188955f89b6e0d", @@ -1345,7 +1346,7 @@ describe('checking migration metadata changes on all registered SO types', () => "privmon-api-key": "10.0.0", "product-doc-install-status": "10.1.0", "query": "10.2.0", - "risk-engine-configuration": "10.3.0", + "risk-engine-configuration": "10.4.0", "rules-settings": "10.1.0", "sample-data-telemetry": "10.0.0", "search": "10.9.0", @@ -1492,7 +1493,7 @@ describe('checking migration metadata changes on all registered SO types', () => "privmon-api-key": "0.0.0", "product-doc-install-status": "10.1.0", "query": "10.2.0", - "risk-engine-configuration": "10.3.0", + "risk-engine-configuration": "10.4.0", "rules-settings": "10.1.0", "sample-data-telemetry": "0.0.0", "search": "10.9.0", diff --git a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/risk_engine/engine_configure_saved_object_route.gen.ts b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/risk_engine/engine_configure_saved_object_route.gen.ts index 58953b4ea4aa3..8ee3606a245e1 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/risk_engine/engine_configure_saved_object_route.gen.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/risk_engine/engine_configure_saved_object_route.gen.ts @@ -42,6 +42,17 @@ export const ConfigureRiskEngineSavedObjectRequestBody = z.object({ .optional(), exclude_alert_tags: z.array(z.string()).optional(), enable_reset_to_zero: z.boolean().optional(), + filters: z + .array( + z.object({ + entity_types: z.array(z.enum(['host', 'user', 'service'])), + /** + * KQL filter string + */ + filter: z.string(), + }) + ) + .optional(), }); export type ConfigureRiskEngineSavedObjectRequestBodyInput = z.input< typeof ConfigureRiskEngineSavedObjectRequestBody diff --git a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/risk_engine/engine_configure_saved_object_route.schema.yaml b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/risk_engine/engine_configure_saved_object_route.schema.yaml index 6f8f367fcfc72..bf84e7974efa1 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/risk_engine/engine_configure_saved_object_route.schema.yaml +++ b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/risk_engine/engine_configure_saved_object_route.schema.yaml @@ -35,6 +35,22 @@ paths: type: string enable_reset_to_zero: type: boolean + filters: + type: array + items: + type: object + properties: + entity_types: + type: array + items: + type: string + enum: [host, user, service] + filter: + type: string + description: KQL filter string + required: + - entity_types + - filter responses: "200": description: Successful response diff --git a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/risk_engine/engine_settings_route.gen.ts b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/risk_engine/engine_settings_route.gen.ts index be357e1cae0d4..93909e7b4d0d6 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/risk_engine/engine_settings_route.gen.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/risk_engine/engine_settings_route.gen.ts @@ -29,4 +29,15 @@ export const ReadRiskEngineSettingsResponse = z.object({ * Whether to enable resetting risk scores to zero when there are no alerts in the selected date range */ enableResetToZero: z.boolean().optional(), + filters: z + .array( + z.object({ + entity_types: z.array(z.enum(['host', 'user', 'service'])), + /** + * KQL filter string + */ + filter: z.string(), + }) + ) + .optional(), }); diff --git a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/risk_engine/engine_settings_route.schema.yaml b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/risk_engine/engine_settings_route.schema.yaml index 39606a817c8b0..887ca1d65b2bd 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/risk_engine/engine_settings_route.schema.yaml +++ b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/risk_engine/engine_settings_route.schema.yaml @@ -27,3 +27,19 @@ paths: enableResetToZero: type: boolean description: Whether to enable resetting risk scores to zero when there are no alerts in the selected date range + filters: + type: array + items: + type: object + properties: + entity_types: + type: array + items: + type: string + enum: [host, user, service] + filter: + type: string + description: KQL filter string + required: + - entity_types + - filter diff --git a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/risk_engine/preview_route.gen.ts b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/risk_engine/preview_route.gen.ts index 00d58b32c580d..c1963a6afc38b 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/risk_engine/preview_route.gen.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/risk_engine/preview_route.gen.ts @@ -63,6 +63,23 @@ export const RiskScoresPreviewRequest = z.object({ * A list of alert tags to exclude from the risk score calculation. If unspecified, all alert tags are included. */ exclude_alert_tags: z.array(z.string()).optional(), + /** + * Custom KQL filters to exclude from risk scoring queries, allowing more targeted risk analysis by filtering out specific alerts. + */ + filters: z + .array( + z.object({ + /** + * The entity types this filter applies to + */ + entity_types: z.array(z.enum(['host', 'user', 'service'])), + /** + * KQL filter expression to exclude (alerts matching this filter will be excluded from risk score calculation) + */ + filter: z.string(), + }) + ) + .optional(), }); export type RiskScoresPreviewResponse = z.infer; diff --git a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/risk_engine/preview_route.schema.yaml b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/risk_engine/preview_route.schema.yaml index a1c0f51c8a0c5..9b18b09c0ad02 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/risk_engine/preview_route.schema.yaml +++ b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/risk_engine/preview_route.schema.yaml @@ -68,6 +68,24 @@ components: type: array items: type: string + filters: + description: Custom KQL filters to exclude from risk scoring queries, allowing more targeted risk analysis by filtering out specific alerts. + type: array + items: + type: object + properties: + entity_types: + type: array + items: + type: string + enum: [host, user, service] + description: The entity types this filter applies to + filter: + type: string + description: KQL filter expression to exclude (alerts matching this filter will be excluded from risk score calculation) + required: + - entity_types + - filter RiskScoresPreviewResponse: diff --git a/x-pack/solutions/security/plugins/security_solution/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml b/x-pack/solutions/security/plugins/security_solution/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml index d31100793e561..e1bc5282b988e 100644 --- a/x-pack/solutions/security/plugins/security_solution/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml +++ b/x-pack/solutions/security/plugins/security_solution/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml @@ -1294,6 +1294,25 @@ paths: items: type: string type: array + filters: + items: + type: object + properties: + entity_types: + items: + enum: + - host + - user + - service + type: string + type: array + filter: + description: KQL filter string + type: string + required: + - entity_types + - filter + type: array range: type: object properties: diff --git a/x-pack/solutions/security/plugins/security_solution/docs/openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml b/x-pack/solutions/security/plugins/security_solution/docs/openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml index ea30a0e33cbf0..969a7a730a229 100644 --- a/x-pack/solutions/security/plugins/security_solution/docs/openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml +++ b/x-pack/solutions/security/plugins/security_solution/docs/openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml @@ -1294,6 +1294,25 @@ paths: items: type: string type: array + filters: + items: + type: object + properties: + entity_types: + items: + enum: + - host + - user + - service + type: string + type: array + filter: + description: KQL filter string + type: string + required: + - entity_types + - filter + type: array range: type: object properties: diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/routes/configure_saved_object.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/routes/configure_saved_object.ts index 6146056fcc8da..64a5fa04263c2 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/routes/configure_saved_object.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/routes/configure_saved_object.ts @@ -94,6 +94,7 @@ export const riskEngineConfigureSavedObjectRoute = ( range: request.body.range, excludeAlertTags: request.body.exclude_alert_tags, enableResetToZero: request.body.enable_reset_to_zero, + filters: request.body.filters, }); return response.ok({ body: { risk_engine_saved_object_configured: true } }); } catch (e) { diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/routes/settings.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/routes/settings.ts index d06215be7f7f7..8f69d0cbf9f83 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/routes/settings.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/routes/settings.ts @@ -59,6 +59,10 @@ export const riskEngineSettingsRoute = (router: EntityAnalyticsRoutesDeps['route Array.isArray(result?.excludeAlertStatuses) && !result.excludeAlertStatuses.includes('closed'), enableResetToZero: result.enableResetToZero, + filters: (result.filters || []).map((f) => ({ + entity_types: f.entity_types as Array<'host' | 'user' | 'service'>, + filter: f.filter, + })), }, }); } catch (e) { diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/saved_object/risk_engine_configuration_type.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/saved_object/risk_engine_configuration_type.test.ts new file mode 100644 index 0000000000000..341f009194ca2 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/saved_object/risk_engine_configuration_type.test.ts @@ -0,0 +1,131 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + SavedObjectsModelVersion, + SavedObjectModelTransformationContext, +} from '@kbn/core-saved-objects-server'; +import { riskEngineConfigurationType } from './risk_engine_configuration_type'; + +describe('riskEngineConfigurationType', () => { + it('should have the correct model versions', () => { + expect(riskEngineConfigurationType.modelVersions).toHaveProperty('1'); + expect(riskEngineConfigurationType.modelVersions).toHaveProperty('2'); + expect(riskEngineConfigurationType.modelVersions).toHaveProperty('3'); + expect(riskEngineConfigurationType.modelVersions).toHaveProperty('4'); + }); + + it('should have filters field in mappings', () => { + expect(riskEngineConfigurationType.mappings.properties).toHaveProperty('filters'); + expect(riskEngineConfigurationType.mappings.properties.filters).toEqual({ + type: 'nested', + properties: { + entity_types: { + type: 'keyword', + }, + filter: { + type: 'text', + }, + }, + }); + }); + + describe('version 4 migration', () => { + it('should add filters field to existing configurations', () => { + const version4 = ( + riskEngineConfigurationType.modelVersions as Record + )?.['4']; + expect(version4).toBeDefined(); + + const mockDocument = { + id: 'test-id', + type: 'risk-engine-configuration', + attributes: { + dataViewId: 'test-dataview', + enabled: true, + filter: {}, + identifierType: 'host', + interval: '1h', + pageSize: 1000, + range: { start: 'now-30d', end: 'now' }, + _meta: { mappingsVersion: 2 }, + }, + references: [], + migrationVersion: {}, + coreMigrationVersion: '8.0.0', + typeMigrationVersion: '8.0.0', + updated_at: '2023-01-01T00:00:00.000Z', + version: '1', + namespaces: ['default'], + originId: 'test-origin', + }; + + const result = + version4?.changes[1]?.type === 'data_backfill' + ? version4.changes[1].backfillFn( + mockDocument, + {} as SavedObjectModelTransformationContext + ) + : null; + + expect(result).toEqual({ + attributes: { + ...mockDocument.attributes, + filters: [], + }, + }); + }); + + it('should preserve existing filters if they exist', () => { + const version4 = ( + riskEngineConfigurationType.modelVersions as Record + )?.['4']; + expect(version4).toBeDefined(); + + const existingFilters = [{ entity_types: ['host'], filter: 'agent.type: filebeat' }]; + + const mockDocument = { + id: 'test-id', + type: 'risk-engine-configuration', + attributes: { + dataViewId: 'test-dataview', + enabled: true, + filter: {}, + identifierType: 'host', + interval: '1h', + pageSize: 1000, + range: { start: 'now-30d', end: 'now' }, + filters: existingFilters, + _meta: { mappingsVersion: 2 }, + }, + references: [], + migrationVersion: {}, + coreMigrationVersion: '8.0.0', + typeMigrationVersion: '8.0.0', + updated_at: '2023-01-01T00:00:00.000Z', + version: '1', + namespaces: ['default'], + originId: 'test-origin', + }; + + const result = + version4?.changes[1]?.type === 'data_backfill' + ? version4.changes[1].backfillFn( + mockDocument, + {} as SavedObjectModelTransformationContext + ) + : null; + + expect(result).toEqual({ + attributes: { + ...mockDocument.attributes, + filters: existingFilters, + }, + }); + }); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/saved_object/risk_engine_configuration_type.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/saved_object/risk_engine_configuration_type.ts index 161f2a040ab05..73ad5b27d8941 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/saved_object/risk_engine_configuration_type.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/saved_object/risk_engine_configuration_type.ts @@ -51,6 +51,17 @@ export const riskEngineConfigurationTypeMappings: SavedObjectsType['mappings'] = excludeAlertStatuses: { type: 'keyword', }, + filters: { + type: 'nested', + properties: { + entity_types: { + type: 'keyword', + }, + filter: { + type: 'text', + }, + }, + }, }, }; @@ -109,6 +120,34 @@ const version3: SavedObjectsModelVersion = { ], }; +const version4: SavedObjectsModelVersion = { + changes: [ + { + type: 'mappings_addition', + addedMappings: { + filters: { + type: 'nested', + properties: { + entity_types: { type: 'keyword' }, + filter: { type: 'text' }, + }, + }, + }, + }, + { + type: 'data_backfill', + backfillFn: (document) => { + return { + attributes: { + ...document.attributes, + filters: document.attributes.filters || [], + }, + }; + }, + }, + ], +}; + export const riskEngineConfigurationType: SavedObjectsType = { name: riskEngineConfigurationTypeName, indexPattern: SECURITY_SOLUTION_SAVED_OBJECT_INDEX, @@ -119,5 +158,6 @@ export const riskEngineConfigurationType: SavedObjectsType = { 1: version1, 2: version2, 3: version3, + 4: version4, }, }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/utils/saved_object_configuration.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/utils/saved_object_configuration.test.ts index 1c27e1ab9c7d6..b994963e2c343 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/utils/saved_object_configuration.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/utils/saved_object_configuration.test.ts @@ -13,7 +13,7 @@ describe('#getDefaultRiskEngineConfiguration', () => { const namespace = 'default'; const config = getDefaultRiskEngineConfiguration({ namespace }); - expect(config._meta.mappingsVersion).toEqual(4); + expect(config._meta.mappingsVersion).toEqual(5); expect(riskScoreFieldMap).toMatchInlineSnapshot(` Object { "@timestamp": Object { diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/utils/saved_object_configuration.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/utils/saved_object_configuration.ts index b7cccce74fd56..0a652d6883a82 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/utils/saved_object_configuration.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/utils/saved_object_configuration.ts @@ -34,7 +34,7 @@ export const getDefaultRiskEngineConfiguration = ({ excludeAlertStatuses: ['closed'], _meta: { // Upgrade this property when changing mappings - mappingsVersion: 4, + mappingsVersion: 5, }, }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/__snapshots__/calculate_esql_risk_scores.test.ts.snap b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/__snapshots__/calculate_esql_risk_scores.test.ts.snap index 4e5230753a9f8..16a38a56670bb 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/__snapshots__/calculate_esql_risk_scores.test.ts.snap +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/__snapshots__/calculate_esql_risk_scores.test.ts.snap @@ -10,10 +10,10 @@ exports[`Calculate risk scores with ESQL ESQL query matches snapshot 1`] = ` kibana.alert.uuid as alert_id, event.kind as category, @timestamp as time - | EVAL input = CONCAT(\\"\\"\\" {\\"score\\": \\"\\"\\"\\", risk_score::keyword, \\"\\"\\"\\", \\"time\\": \\"\\"\\"\\", time::keyword, \\"\\"\\"\\", \\"index\\": \\"\\"\\"\\", _index, \\"\\"\\"\\", \\"rule_name\\": \\"\\"\\"\\", rule_name, \\"\\"\\"\\", \\"category\\": \\"\\"\\"\\", category, \\"\\"\\"\\", \\"id\\": \\"\\"\\"\\", alert_id, \\"\\"\\"\\" } \\"\\"\\") + | EVAL input = CONCAT(\\"\\"\\" {\\"risk_score\\": \\"\\"\\"\\", risk_score::keyword, \\"\\"\\"\\", \\"time\\": \\"\\"\\"\\", time::keyword, \\"\\"\\"\\", \\"index\\": \\"\\"\\"\\", _index, \\"\\"\\"\\", \\"rule_name\\": \\"\\"\\"\\", rule_name, \\"\\"\\"\\", \\"category\\": \\"\\"\\"\\", category, \\"\\"\\"\\", \\"id\\": \\"\\"\\"\\", alert_id, \\"\\"\\"\\" } \\"\\"\\") | STATS alert_count = count(risk_score), - scores = 1 * MV_PSERIES_WEIGHTED_SUM(TOP(risk_score, 10000, \\"desc\\"), 1.5), + scores = MV_PSERIES_WEIGHTED_SUM(TOP(risk_score, 10000, \\"desc\\"), 1.5), risk_inputs = TOP(input, 10, \\"desc\\") BY host.name | SORT scores DESC, host.name ASC diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_and_persist_risk_scores.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_and_persist_risk_scores.test.ts index b2ce82155229e..74c46162b0920 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_and_persist_risk_scores.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_and_persist_risk_scores.test.ts @@ -78,7 +78,7 @@ describe('calculateAndPersistRiskScores', () => { const results = await calculate(); const entities = { - host: ['hostname'], + host: [], user: [], service: [], generic: [], @@ -126,7 +126,7 @@ describe('calculateAndPersistRiskScores', () => { const results = await calculate(); const entities = { - host: ['hostname'], + host: [], user: [], service: [], generic: [], diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_and_persist_risk_scores.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_and_persist_risk_scores.ts index ff130f91011a6..93fbdea098364 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_and_persist_risk_scores.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_and_persist_risk_scores.ts @@ -8,19 +8,18 @@ import type { ElasticsearchClient, Logger } from '@kbn/core/server'; import type { ExperimentalFeatures } from '../../../../common'; -import type { - EntityType, - RiskScoresCalculationResponse, -} from '../../../../common/api/entity_analytics'; +import type { EntityType } from '../../../../common/search_strategy'; import type { RiskScoreDataClient } from './risk_score_data_client'; import type { AssetCriticalityService } from '../asset_criticality/asset_criticality_service'; import { calculateRiskScores } from './calculate_risk_scores'; import type { CalculateAndPersistScoresParams } from '../types'; import { calculateScoresWithESQL } from './calculate_esql_risk_scores'; +import type { RiskScoresCalculationResponse } from '../../../../common/api/entity_analytics'; export type CalculationResults = RiskScoresCalculationResponse & { entities: Record; }; + export const calculateAndPersistRiskScores = async ( params: CalculateAndPersistScoresParams & { assetCriticalityService: AssetCriticalityService; @@ -40,10 +39,23 @@ export const calculateAndPersistRiskScores = async ( const calculate = params.experimentalFeatures.disableESQLRiskScoring ? calculateRiskScores : calculateScoresWithESQL; - const { after_keys: afterKeys, scores, entities } = await calculate(rest); + const { after_keys: afterKeys, scores } = await calculate(rest); + + // Extract entity IDs from scores for reset-to-zero functionality + const entities: Record = { + host: scores.host?.map((score: { id_value: string }) => score.id_value) || [], + user: scores.user?.map((score: { id_value: string }) => score.id_value) || [], + service: scores.service?.map((score: { id_value: string }) => score.id_value) || [], + generic: scores.generic?.map((score: { id_value: string }) => score.id_value) || [], + }; if (!scores.host?.length && !scores.user?.length && !scores.service?.length) { - return { after_keys: {}, errors: [], scores_written: 0, entities }; + return { + after_keys: {}, + errors: [], + scores_written: 0, + entities, + }; } try { @@ -56,7 +68,12 @@ export const calculateAndPersistRiskScores = async ( const { errors, docs_written: scoresWritten } = await writer.bulk({ ...scores, refresh }); - const result = { after_keys: afterKeys, errors, scores_written: scoresWritten, entities }; + const result = { + after_keys: afterKeys, + errors, + scores_written: scoresWritten, + entities, + }; return returnScores ? { ...result, scores } : result; }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_esql_risk_scores.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_esql_risk_scores.test.ts index 5eeeb2dd77b7b..e25ebce03b61d 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_esql_risk_scores.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_esql_risk_scores.test.ts @@ -14,14 +14,7 @@ import { RIEMANN_ZETA_S_VALUE, RIEMANN_ZETA_VALUE } from './constants'; describe('Calculate risk scores with ESQL', () => { describe('ESQL query', () => { it('matches snapshot', () => { - const q = getESQL({ - entityType: EntityType.host, - bounds: { lower: 'abel', upper: 'zuzanna' }, - sampleSize: 10000, - pageSize: 3500, - index: '.alerts-security.alerts-default', - weight: 1, - }); + const q = getESQL(EntityType.host, { lower: 'abel', upper: 'zuzanna' }, 10000, 3500); expect(q).toMatchSnapshot(); }); }); @@ -29,11 +22,11 @@ describe('Calculate risk scores with ESQL', () => { describe('buildRiskScoreBucket', () => { it('parses esql results into RiskScoreBucket', () => { const inputs = [ - '{ "score": "50", "time": "2021-08-23T18:00:05.000Z", "rule_name": "Test rule 5", "id": "test_id_5" }', - '{ "score": "40", "time": "2021-08-22T18:00:04.000Z", "rule_name": "Test rule 4", "id": "test_id_4" }', - '{ "score": "30", "time": "2021-08-21T18:00:03.000Z", "rule_name": "Test rule 3", "id": "test_id_3" }', - '{ "score": "20", "time": "2021-08-20T18:00:02.000Z", "rule_name": "Test rule 2", "id": "test_id_2" }', - '{ "score": "10", "time": "2021-08-19T18:00:01.000Z", "rule_name": "Test rule 1", "id": "test_id_1" }', + '{ "risk_score": "50", "time": "2021-08-23T18:00:05.000Z", "rule_name": "Test rule 5", "id": "test_id_5" }', + '{ "risk_score": "40", "time": "2021-08-22T18:00:04.000Z", "rule_name": "Test rule 4", "id": "test_id_4" }', + '{ "risk_score": "30", "time": "2021-08-21T18:00:03.000Z", "rule_name": "Test rule 3", "id": "test_id_3" }', + '{ "risk_score": "20", "time": "2021-08-20T18:00:02.000Z", "rule_name": "Test rule 2", "id": "test_id_2" }', + '{ "risk_score": "10", "time": "2021-08-19T18:00:01.000Z", "rule_name": "Test rule 1", "id": "test_id_1" }', ]; const alertCount = 10; const riskScore = 100; @@ -56,8 +49,8 @@ describe('Calculate risk scores with ESQL', () => { score: riskScore, normalized_score: riskScore / RIEMANN_ZETA_VALUE, notes: [], - category_1_score: riskScore, - category_1_count: 10, + category_1_score: riskScore, // Don't normalize here - will be normalized in calculate_risk_scores.ts + category_1_count: alertCount, risk_inputs: [ { index: '.alerts-security.alerts-default', diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_esql_risk_scores.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_esql_risk_scores.ts index 3313eb4f302b0..7566d8587e2cd 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_esql_risk_scores.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_esql_risk_scores.ts @@ -8,6 +8,7 @@ import { isEmpty } from 'lodash'; import type { FieldValue, QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; import type { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { fromKueryExpression, toElasticsearchQuery } from '@kbn/es-query'; import { ALERT_RISK_SCORE, ALERT_WORKFLOW_STATUS, @@ -28,20 +29,13 @@ import type { import { withSecuritySpan } from '../../../utils/with_security_span'; import type { AssetCriticalityService } from '../asset_criticality/asset_criticality_service'; -import type { - CalculateResults, - CalculateScoresParams, - RiskScoreBucket, - RiskScoreCompositeBuckets, -} from '../types'; +import type { RiskScoresPreviewResponse } from '../../../../common/api/entity_analytics'; +import type { CalculateScoresParams, RiskScoreBucket, RiskScoreCompositeBuckets } from '../types'; import { RIEMANN_ZETA_S_VALUE, RIEMANN_ZETA_VALUE } from './constants'; -import { - filterFromRange, - getGlobalWeightForIdentifierType, - processScores, -} from './calculate_risk_scores'; +import { filterFromRange, processScores } from './calculate_risk_scores'; + type ESQLResults = Array< - [EntityType, { scores: EntityRiskScoreRecord[]; afterKey: EntityAfterKey }, string[]] + [EntityType, { scores: EntityRiskScoreRecord[]; afterKey: EntityAfterKey }] >; export const calculateScoresWithESQL = async ( @@ -50,98 +44,161 @@ export const calculateScoresWithESQL = async ( esClient: ElasticsearchClient; logger: Logger; experimentalFeatures: ExperimentalFeatures; - } & CalculateScoresParams -): Promise => + } & CalculateScoresParams & { + filters?: Array<{ entity_types: string[]; filter: string }>; + } +): Promise => withSecuritySpan('calculateRiskScores', async () => { - const { - afterKeys, - alertSampleSizePerShard, - assetCriticalityService, - esClient, - identifierType, - index, - logger, - pageSize, - weights, - } = params; + const { identifierType, logger, esClient } = params; const now = new Date().toISOString(); - const filter = getFilters(params); - const identifierTypes: EntityType[] = identifierType ? [identifierType] : getEntityAnalyticsEntityTypes(); - const compositeQuery = getCompositeQuery(identifierTypes, filter, params); + // Create separate queries for each entity type with entity-specific filters + const entityQueries = identifierTypes.map((entityType) => { + const filter = getFilters(params, entityType); + return { + entityType, + query: getCompositeQuery([entityType], filter, params), + }; + }); + logger.trace( - `STEP ONE: Executing ESQL Risk Score composite query:\n${JSON.stringify(compositeQuery)}` + `STEP ONE: Executing ESQL Risk Score queries for entity types: ${identifierTypes.join(', ')}` ); - const response = await esClient - .search(compositeQuery) - .catch((e) => { - logger.error(`Error executing composite query: ${e.message}`); - }); - if (!response?.aggregations) { - return { - after_keys: {}, - scores: { - host: [], - user: [], - service: [], - }, - entities: { - user: [], - service: [], - host: [], - generic: [], - }, - }; + // Execute queries for each entity type + const responses = await Promise.all( + entityQueries.map(async ({ entityType, query }) => { + logger.trace( + `Executing ESQL Risk Score query for ${entityType}:\n${JSON.stringify(query)}` + ); + + let error: unknown = null; + const response = await esClient + .search(query) + .catch((e) => { + logger.error(`Error executing composite query for ${entityType}: ${e.message}`); + error = e; + return null; + }); + + return { + entityType, + response, + query, + error, + }; + }) + ); + + // Combine results from all entity queries + const combinedAggregations: Partial = {}; + responses.forEach(({ entityType, response }) => { + if ( + response?.aggregations && + (response.aggregations as unknown as Record)[entityType] + ) { + (combinedAggregations as Record)[entityType] = ( + response.aggregations as unknown as Record + )[entityType]; + } + }); + + // Check if all queries that had errors failed due to index_not_found_exception + const errorsPresent = responses.filter(({ error }) => error).length; + const indexNotFoundErrors = responses.filter(({ error }) => { + if (!error) return false; + const errorMessage = error instanceof Error ? error.message : String(error); + return ( + errorMessage.includes('index_not_found_exception') || + errorMessage.includes('no such index') || + errorMessage.includes('NoShardAvailableActionException') + ); + }).length; + + // If we have no aggregations, return empty scores if: + // 1. All queries that had errors were index-not-found errors + // 2. OR there were no errors at all (valid index pattern with no data) + const shouldReturnEmptyScores = + errorsPresent === 0 || (errorsPresent > 0 && errorsPresent === indexNotFoundErrors); + + if (Object.keys(combinedAggregations).length === 0) { + if (shouldReturnEmptyScores) { + return { + after_keys: {}, + scores: { + host: [], + user: [], + service: [], + }, + }; + } + // Log the actual errors for debugging + responses.forEach(({ entityType, error }) => { + if (error) { + logger.error( + `Query failed for ${entityType}: ${ + error instanceof Error ? error.message : String(error) + }` + ); + } + }); + // Otherwise, throw an error as before + throw new Error('No aggregations in any composite response'); } - const promises = toEntries(response.aggregations).map>( - ([entityType, { buckets, after_key: afterKey }]) => { - const entities = buckets.map(({ key }) => key[EntityTypeToIdentifierField[entityType]]); + const promises = toEntries(combinedAggregations as Record).map( + async ([entityType, aggregationData]: [string, unknown]) => { + const { buckets, after_key: afterKey } = aggregationData as { + buckets: Array<{ key: Record }>; + after_key?: Record; + }; + const entities = buckets.map( + ({ key }) => key[(EntityTypeToIdentifierField as Record)[entityType]] + ); if (entities.length === 0) { return Promise.resolve([ entityType as EntityType, - { afterKey: afterKey as EntityAfterKey, scores: [] }, - entities, + { afterKey: afterKey || {}, scores: [] }, ] satisfies ESQLResults[number]); } const bounds = { - lower: afterKeys[entityType]?.[EntityTypeToIdentifierField[entityType]], - upper: afterKey?.[EntityTypeToIdentifierField[entityType]], + lower: (params.afterKeys as Record>)[entityType]?.[ + (EntityTypeToIdentifierField as Record)[entityType] + ], + upper: afterKey?.[(EntityTypeToIdentifierField as Record)[entityType]], }; - const weight = getGlobalWeightForIdentifierType(entityType as EntityType, weights) || 1; - - const query = getESQL({ - entityType: entityType as EntityType, + const query = getESQL( + entityType as EntityType, bounds, - sampleSize: alertSampleSizePerShard || 10000, - pageSize, - index, - weight, - }); + params.alertSampleSizePerShard || 10000, + params.pageSize, + params.index + ); + + const entityFilter = getFilters(params, entityType as EntityType); return esClient.esql .query({ query, - filter: { bool: { filter } }, + filter: { bool: { filter: entityFilter } }, }) - .then((rs) => - rs.values.map(buildRiskScoreBucket(entityType as EntityType, index, weight)) - ) + .then((rs) => rs.values.map(buildRiskScoreBucket(entityType as EntityType, params.index))) .then((riskScoreBuckets) => { return processScores({ - assetCriticalityService, + assetCriticalityService: params.assetCriticalityService, buckets: riskScoreBuckets, - identifierField: EntityTypeToIdentifierField[entityType], + identifierField: (EntityTypeToIdentifierField as Record)[entityType], logger, now, + identifierType: entityType as EntityType, + weights: params.weights, }); }) .then((scores: EntityRiskScoreRecord[]): ESQLResults[number] => { @@ -151,7 +208,6 @@ export const calculateScoresWithESQL = async ( scores, afterKey: afterKey as EntityAfterKey, }, - entities, ]; }) @@ -162,33 +218,33 @@ export const calculateScoresWithESQL = async ( logger.error(`Query: ${query}`); return [ entityType as EntityType, - { afterKey: afterKey as EntityAfterKey, scores: [] }, - entities, - ]; + { afterKey: afterKey || {}, scores: [] }, + ] satisfies ESQLResults[number]; }); } ); const esqlResults = await Promise.all(promises); - const results: CalculateResults = esqlResults.reduce<{ - after_keys: Record; - scores: Record; - entities: Record; - }>( - (res, [entityType, { afterKey, scores }, entities]) => { - res.after_keys[entityType] = afterKey; + const results: RiskScoresPreviewResponse = esqlResults.reduce( + (res, [entityType, { afterKey, scores }]) => { + res.after_keys[entityType] = afterKey || {}; res.scores[entityType] = scores; - res.entities[entityType] = entities; return res; }, - { after_keys: {}, scores: {}, entities: { user: [], service: [], host: [], generic: [] } } + { after_keys: {}, scores: {} } ); return results; }); -const getFilters = (options: CalculateScoresParams) => { - const { excludeAlertStatuses = [], excludeAlertTags = [], range, filter: userFilter } = options; +const getFilters = (options: CalculateScoresParams, entityType?: EntityType) => { + const { + excludeAlertStatuses = [], + excludeAlertTags = [], + range, + filter: userFilter, + filters: customFilters, + } = options; const filters = [filterFromRange(range), { exists: { field: ALERT_RISK_SCORE } }]; if (excludeAlertStatuses.length > 0) { filters.push({ @@ -204,6 +260,25 @@ const getFilters = (options: CalculateScoresParams) => { }); } + // Apply entity-specific custom filters (EXCLUSIVE - exclude matching alerts) + if (customFilters && customFilters.length > 0 && entityType) { + customFilters + .filter((customFilter) => customFilter.entity_types.includes(entityType)) + .forEach((customFilter) => { + try { + const kqlQuery = fromKueryExpression(customFilter.filter); + const esQuery = toElasticsearchQuery(kqlQuery); + if (esQuery) { + filters.push({ + bool: { must_not: esQuery }, + }); + } + } catch (error) { + // Silently ignore invalid KQL filters to prevent query failures + } + }); + } + return filters; }; @@ -212,12 +287,11 @@ export const getCompositeQuery = ( filter: QueryDslQueryContainer[], params: CalculateScoresParams ) => { - const { index, pageSize, runtimeMappings, afterKeys } = params; return { size: 0, - index, + index: params.index, ignore_unavailable: true, - runtime_mappings: runtimeMappings, + runtime_mappings: params.runtimeMappings, query: { function_score: { query: { @@ -241,9 +315,9 @@ export const getCompositeQuery = ( ...aggs, [entityType]: { composite: { - size: pageSize, + size: params.pageSize, sources: [{ [idField]: { terms: { field: idField } } }], - after: afterKeys[entityType], + after: params.afterKeys[entityType], }, }, }; @@ -251,36 +325,26 @@ export const getCompositeQuery = ( }; }; -export interface GetESQLParams { - bounds: { +export const getESQL = ( + entityType: EntityType, + afterKeys: { lower?: string; upper?: string; - }; - entityType: EntityType; - index: string; - pageSize: number; - sampleSize: number; - weight: number; -} - -export const getESQL = ({ - entityType, - bounds, - sampleSize, - pageSize, - index, - weight, -}: GetESQLParams) => { + }, + sampleSize: number, + pageSize: number, + index: string = '.alerts-security.alerts-default' +) => { const identifierField = EntityTypeToIdentifierField[entityType]; - const lower = bounds.lower ? `${identifierField} > ${bounds.lower}` : undefined; - const upper = bounds.upper ? `${identifierField} <= ${bounds.upper}` : undefined; + const lower = afterKeys.lower ? `${identifierField} > ${afterKeys.lower}` : undefined; + const upper = afterKeys.upper ? `${identifierField} <= ${afterKeys.upper}` : undefined; if (!lower && !upper) { throw new Error('Either lower or upper after key must be provided for pagination'); } const rangeClause = [lower, upper].filter(Boolean).join(' and '); - const query = /* ESQL */ ` + const query = /* SQL */ ` FROM ${index} METADATA _index | WHERE kibana.alert.risk_score IS NOT NULL AND KQL("${rangeClause}") | RENAME kibana.alert.risk_score as risk_score, @@ -289,10 +353,10 @@ export const getESQL = ({ kibana.alert.uuid as alert_id, event.kind as category, @timestamp as time - | EVAL input = CONCAT(""" {"score": """", risk_score::keyword, """", "time": """", time::keyword, """", "index": """", _index, """", "rule_name": """", rule_name, """\", "category": """", category, """\", "id": \"""", alert_id, """\" } """) + | EVAL input = CONCAT(""" {"risk_score": """", risk_score::keyword, """", "time": """", time::keyword, """", "index": """", _index, """", "rule_name": """", rule_name, """\", "category": """", category, """\", "id": \"""", alert_id, """\" } """) | STATS alert_count = count(risk_score), - scores = ${weight} * MV_PSERIES_WEIGHTED_SUM(TOP(risk_score, ${sampleSize}, "desc"), ${RIEMANN_ZETA_S_VALUE}), + scores = MV_PSERIES_WEIGHTED_SUM(TOP(risk_score, ${sampleSize}, "desc"), ${RIEMANN_ZETA_S_VALUE}), risk_inputs = TOP(input, 10, "desc") BY ${identifierField} | SORT scores DESC, ${identifierField} ASC @@ -303,7 +367,7 @@ export const getESQL = ({ }; export const buildRiskScoreBucket = - (entityType: EntityType, index: string, weight: number = 1) => + (entityType: EntityType, index: string) => (row: FieldValue[]): RiskScoreBucket => { const [count, score, _inputs, entity] = row as [ number, @@ -314,10 +378,11 @@ export const buildRiskScoreBucket = const inputs = (Array.isArray(_inputs) ? _inputs : [_inputs]).map((input, i) => { const parsedRiskInputData = JSON.parse(input); - const value = parseFloat(parsedRiskInputData.score); + const value = parseFloat(parsedRiskInputData.risk_score); const currentScore = value / Math.pow(i + 1, RIEMANN_ZETA_S_VALUE); + const { risk_score: _, ...otherFields } = parsedRiskInputData; return { - ...parsedRiskInputData, + ...otherFields, score: value, contribution: currentScore / RIEMANN_ZETA_VALUE, index, @@ -334,7 +399,7 @@ export const buildRiskScoreBucket = score, normalized_score: score / RIEMANN_ZETA_VALUE, // normalize value to be between 0-100 notes: [], - category_1_score: score / weight, // category score before global weight applied and normalization + category_1_score: score, // Don't normalize here - will be normalized in calculate_risk_scores.ts category_1_count: count, risk_inputs: inputs, }, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_risk_scores.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_risk_scores.test.ts index 076113c16b07a..0fe0c853ab881 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_risk_scores.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_risk_scores.test.ts @@ -5,17 +5,200 @@ * 2.0. */ +import { buildFiltersForEntityType, calculateRiskScores } from './calculate_risk_scores'; +import type { EntityType } from '../../../../common/entity_analytics/types'; import type { ElasticsearchClient, Logger } from '@kbn/core/server'; import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks'; import { assetCriticalityServiceMock } from '../asset_criticality/asset_criticality_service.mock'; -import { calculateRiskScores } from './calculate_risk_scores'; -import { calculateRiskScoresMock } from './calculate_risk_scores.mock'; - -import { ALERT_WORKFLOW_STATUS } from '@kbn/rule-registry-plugin/common/technical_rule_data_field_names'; -import { EntityType } from '../../../../common/search_strategy'; import { allowedExperimentalValues } from '../../../../common'; +describe('buildFiltersForEntityType', () => { + const mockUserFilter = { term: { 'user.name': 'test-user' } }; + const mockExcludeAlertStatuses = ['closed']; + const mockExcludeAlertTags = ['test-tag']; + + it('should build basic filters without custom filters', () => { + const filters = buildFiltersForEntityType( + 'host' as EntityType, + mockUserFilter, + [], + mockExcludeAlertStatuses, + mockExcludeAlertTags + ); + + expect(filters).toHaveLength(4); + expect(filters[0]).toEqual({ exists: { field: 'kibana.alert.risk_score' } }); + expect(filters[1]).toEqual(mockUserFilter); + expect(filters[2]).toEqual({ + bool: { must_not: { terms: { 'kibana.alert.workflow_status': mockExcludeAlertStatuses } } }, + }); + expect(filters[3]).toEqual({ + bool: { must_not: { terms: { 'kibana.alert.workflow_tags': mockExcludeAlertTags } } }, + }); + }); + + it('should apply entity-specific custom filters (exclusive)', () => { + const customFilters = [ + { entity_types: ['host'], filter: 'agent.type: filebeat' }, + { entity_types: ['user'], filter: 'user.name: ubuntu' }, + ]; + + const hostFilters = buildFiltersForEntityType( + 'host' as EntityType, + mockUserFilter, + customFilters, + mockExcludeAlertStatuses, + mockExcludeAlertTags + ); + + const userFilters = buildFiltersForEntityType( + 'user' as EntityType, + mockUserFilter, + customFilters, + mockExcludeAlertStatuses, + mockExcludeAlertTags + ); + + // Host filters should exclude the host-specific filter (must_not) + expect(hostFilters).toHaveLength(5); + expect(hostFilters[4]).toEqual( + expect.objectContaining({ + bool: expect.objectContaining({ + must_not: expect.objectContaining({ + bool: expect.objectContaining({ + should: expect.arrayContaining([ + expect.objectContaining({ + match: expect.objectContaining({ + 'agent.type': 'filebeat', + }), + }), + ]), + }), + }), + }), + }) + ); + + // User filters should exclude the user-specific filter (must_not) + expect(userFilters).toHaveLength(5); + expect(userFilters[4]).toEqual( + expect.objectContaining({ + bool: expect.objectContaining({ + must_not: expect.objectContaining({ + bool: expect.objectContaining({ + should: expect.arrayContaining([ + expect.objectContaining({ + match: expect.objectContaining({ + 'user.name': 'ubuntu', + }), + }), + ]), + }), + }), + }), + }) + ); + }); + + it('should apply multiple exclusive filters for the same entity type', () => { + const customFilters = [ + { entity_types: ['host'], filter: 'agent.type: filebeat' }, + { entity_types: ['host'], filter: 'host.os.name: linux' }, + ]; + + const filters = buildFiltersForEntityType( + 'host' as EntityType, + mockUserFilter, + customFilters, + mockExcludeAlertStatuses, + mockExcludeAlertTags + ); + + expect(filters).toHaveLength(6); // 4 base filters + 2 custom filters + }); + + it('should handle empty custom filters array', () => { + const filters = buildFiltersForEntityType( + 'host' as EntityType, + mockUserFilter, + [], + mockExcludeAlertStatuses, + mockExcludeAlertTags + ); + + expect(filters).toHaveLength(4); + }); + + it('should handle invalid KQL filters gracefully', () => { + const customFilters = [{ entity_types: ['host'], filter: 'invalid kql syntax {' }]; + + const filters = buildFiltersForEntityType( + 'host' as EntityType, + mockUserFilter, + customFilters, + mockExcludeAlertStatuses, + mockExcludeAlertTags + ); + + // Should still return the base filters even if custom filter is invalid + // Invalid KQL filters are silently ignored to prevent query failures + expect(filters).toHaveLength(4); // Base filters + exclude filters, invalid custom filter is ignored + }); + + it('should handle empty user filter', () => { + const filters = buildFiltersForEntityType( + 'host' as EntityType, + {}, + [], + mockExcludeAlertStatuses, + mockExcludeAlertTags + ); + + expect(filters).toHaveLength(3); // No user filter added + expect(filters[0]).toEqual({ exists: { field: 'kibana.alert.risk_score' } }); + }); + + it('should handle empty exclude arrays', () => { + const filters = buildFiltersForEntityType('host' as EntityType, mockUserFilter, [], [], []); + + expect(filters).toHaveLength(2); // Only base filters + user filter + expect(filters[0]).toEqual({ exists: { field: 'kibana.alert.risk_score' } }); + expect(filters[1]).toEqual(mockUserFilter); + }); + + it('should handle service entity type', () => { + const customFilters = [{ entity_types: ['service'], filter: 'service.name: nginx' }]; + + const filters = buildFiltersForEntityType( + 'service' as EntityType, + mockUserFilter, + customFilters, + mockExcludeAlertStatuses, + mockExcludeAlertTags + ); + + expect(filters).toHaveLength(5); + expect(filters[4]).toEqual( + expect.objectContaining({ + bool: expect.objectContaining({ + must_not: expect.objectContaining({ + bool: expect.objectContaining({ + should: expect.arrayContaining([ + expect.objectContaining({ + match: expect.objectContaining({ + 'service.name': 'nginx', + }), + }), + ]), + }), + }), + }), + }) + ); + }); +}); + describe('calculateRiskScores()', () => { let params: Parameters[0]; let esClient: ElasticsearchClient; @@ -60,232 +243,5 @@ describe('calculateRiskScores()', () => { (esClient.search as jest.Mock).mock.calls[0][0].query.function_score.query.bool.filter ).toEqual(expect.not.arrayContaining([{}])); }); - - it('drops an empty array filter if specified by the caller', async () => { - params.filter = []; - await calculateRiskScores(params); - - expect( - (esClient.search as jest.Mock).mock.calls[0][0].query.function_score.query.bool.filter - ).toEqual(expect.not.arrayContaining([[]])); - }); - - describe('identifierType', () => { - it('creates aggs for both host and user by default', async () => { - await calculateRiskScores(params); - expect(esClient.search).toHaveBeenCalledWith( - expect.objectContaining({ - aggs: expect.objectContaining({ host: expect.anything(), user: expect.anything() }), - }) - ); - }); - - it('creates an aggregation per specified identifierType', async () => { - params = { ...params, identifierType: EntityType.host }; - await calculateRiskScores(params); - const [[call]] = (esClient.search as jest.Mock).mock.calls; - expect(call).toEqual( - expect.objectContaining({ aggs: expect.objectContaining({ host: expect.anything() }) }) - ); - expect(call.aggs).toHaveProperty('host'); - expect(call.aggs).not.toHaveProperty('user'); - }); - }); - - describe('after_keys', () => { - it('applies a single after_key to the correct aggregation', async () => { - params = { ...params, afterKeys: { host: { 'host.name': 'foo' } } }; - await calculateRiskScores(params); - const [[call]] = (esClient.search as jest.Mock).mock.calls; - expect(call).toEqual( - expect.objectContaining({ - aggs: expect.objectContaining({ - host: expect.objectContaining({ - composite: expect.objectContaining({ after: { 'host.name': 'foo' } }), - }), - }), - }) - ); - }); - - it('applies multiple after_keys to the correct aggregations', async () => { - params = { - ...params, - afterKeys: { - host: { 'host.name': 'foo' }, - user: { 'user.name': 'bar' }, - }, - }; - await calculateRiskScores(params); - const [[call]] = (esClient.search as jest.Mock).mock.calls; - - expect(call).toEqual( - expect.objectContaining({ - aggs: expect.objectContaining({ - host: expect.objectContaining({ - composite: expect.objectContaining({ after: { 'host.name': 'foo' } }), - }), - user: expect.objectContaining({ - composite: expect.objectContaining({ after: { 'user.name': 'bar' } }), - }), - }), - }) - ); - }); - - it('uses an undefined after_key by default', async () => { - await calculateRiskScores(params); - const [[call]] = (esClient.search as jest.Mock).mock.calls; - - expect(call).toEqual( - expect.objectContaining({ - aggs: expect.objectContaining({ - host: expect.objectContaining({ - composite: expect.objectContaining({ after: undefined }), - }), - user: expect.objectContaining({ - composite: expect.objectContaining({ after: undefined }), - }), - }), - }) - ); - }); - }); - - describe('excludeAlertStatuses', () => { - it('should not add the filter when excludeAlertStatuses is empty', async () => { - params = { ...params, excludeAlertStatuses: [] }; - await calculateRiskScores(params); - expect( - (esClient.search as jest.Mock).mock.calls[0][0].query.function_score.query.bool.filter - ).toEqual( - expect.not.arrayContaining([ - { - bool: { - must_not: { terms: { [ALERT_WORKFLOW_STATUS]: params.excludeAlertStatuses } }, - }, - }, - ]) - ); - }); - - it('should add the filter when excludeAlertStatuses is not empty', async () => { - // eslint-disable-next-line @typescript-eslint/no-unused-expressions - esClient.search as jest.Mock; - params = { ...params, excludeAlertStatuses: ['closed'] }; - await calculateRiskScores(params); - expect( - (esClient.search as jest.Mock).mock.calls[0][0].query.function_score.query.bool.filter - ).toEqual( - expect.arrayContaining([ - { - bool: { - must_not: { terms: { [ALERT_WORKFLOW_STATUS]: params.excludeAlertStatuses } }, - }, - }, - ]) - ); - }); - }); - }); - - describe('outputs', () => { - beforeEach(() => { - // stub out a reasonable response - (esClient.search as jest.Mock).mockResolvedValueOnce({ - aggregations: calculateRiskScoresMock.buildAggregationResponse(), - }); - }); - - it('returns a flattened list of risk scores', async () => { - const response = await calculateRiskScores(params); - expect(response).toHaveProperty('scores'); - expect(response.scores.host).toHaveLength(2); - expect(response.scores.user).toHaveLength(2); - expect(response.scores.service).toHaveLength(2); - }); - - it('returns scores in the expected format', async () => { - const { - scores: { host: hostScores }, - } = await calculateRiskScores(params); - const [score] = hostScores ?? []; - expect(score).toEqual( - expect.objectContaining({ - '@timestamp': expect.any(String), - id_field: expect.any(String), - id_value: expect.any(String), - calculated_level: 'Low', - calculated_score: expect.any(Number), - calculated_score_norm: expect.any(Number), - category_1_score: expect.any(Number), - category_1_count: expect.any(Number), - notes: expect.any(Array), - }) - ); - }); - - it('returns risk inputs in the expected format', async () => { - const { - scores: { user: userScores }, - } = await calculateRiskScores(params); - const [score] = userScores ?? []; - expect(score).toEqual( - expect.objectContaining({ - inputs: expect.arrayContaining([ - expect.objectContaining({ - id: expect.any(String), - index: expect.any(String), - category: expect.any(String), - description: expect.any(String), - risk_score: expect.any(Number), - timestamp: expect.any(String), - }), - ]), - }) - ); - }); - }); - - describe('error conditions', () => { - it('raises an error if elasticsearch client rejects', async () => { - (esClient.search as jest.Mock).mockRejectedValueOnce({ - aggregations: calculateRiskScoresMock.buildAggregationResponse(), - }); - - await expect(() => calculateRiskScores(params)).rejects.toEqual({ - aggregations: calculateRiskScoresMock.buildAggregationResponse(), - }); - }); - - describe('when the asset criticality service throws an error', () => { - beforeEach(() => { - (esClient.search as jest.Mock).mockResolvedValueOnce({ - aggregations: calculateRiskScoresMock.buildAggregationResponse(), - }); - ( - params.assetCriticalityService.getCriticalitiesByIdentifiers as jest.Mock - ).mockRejectedValueOnce(new Error('foo')); - }); - - it('logs the error but proceeds if asset criticality service throws', async () => { - await expect(calculateRiskScores(params)).resolves.toEqual( - expect.objectContaining({ - scores: expect.objectContaining({ - host: expect.arrayContaining([ - expect.objectContaining({ - calculated_level: expect.any(String), - id_field: expect.any(String), - id_value: expect.any(String), - }), - ]), - }), - }) - ); - expect(logger.warn).toHaveBeenCalledWith( - 'Error retrieving criticality: Error: foo. Scoring will proceed without criticality information.' - ); - }); - }); }); }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_risk_scores.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_risk_scores.ts index b1dc7c57de21f..640d750ce4f9f 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_risk_scores.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_risk_scores.ts @@ -16,14 +16,19 @@ import { ALERT_WORKFLOW_STATUS, ALERT_WORKFLOW_TAGS, } from '@kbn/rule-registry-plugin/common/technical_rule_data_field_names'; +import { toElasticsearchQuery, fromKueryExpression } from '@kbn/es-query'; import { getEntityAnalyticsEntityTypes } from '../../../../common/entity_analytics/utils'; import type { EntityType } from '../../../../common/search_strategy'; import type { ExperimentalFeatures } from '../../../../common'; -import type { AssetCriticalityRecord } from '../../../../common/api/entity_analytics'; +import type { + AssetCriticalityRecord, + RiskScoresPreviewResponse, +} from '../../../../common/api/entity_analytics'; import type { AfterKeys, EntityRiskScoreRecord, RiskScoreWeights, + RiskScoreWeight, } from '../../../../common/api/entity_analytics/common'; import { getRiskLevel, @@ -35,14 +40,12 @@ import type { AssetCriticalityService } from '../asset_criticality/asset_critica import { applyCriticalityToScore, getCriticalityModifier } from '../asset_criticality/helpers'; import { getAfterKeyForIdentifierType, getFieldForIdentifier } from './helpers'; import type { - CalculateResults, CalculateRiskScoreAggregations, CalculateScoresParams, RiskScoreBucket, } from '../types'; import { RIEMANN_ZETA_VALUE, RIEMANN_ZETA_S_VALUE } from './constants'; import { getPainlessScripts, type PainlessScripts } from './painless'; -import { EntityTypeToIdentifierField } from '../../../../common/entity_analytics/types'; const max10DecimalPlaces = (num: number) => Math.round(num * 1e10) / 1e10; @@ -52,22 +55,32 @@ const formatForResponse = ({ now, identifierField, includeNewFields, + globalWeight, }: { bucket: RiskScoreBucket; criticality?: AssetCriticalityRecord; now: string; identifierField: string; includeNewFields: boolean; + globalWeight?: number; }): EntityRiskScoreRecord => { const riskDetails = bucket.top_inputs.risk_details; + // Apply global weight to the score if provided + const weightedScore = + globalWeight !== undefined ? riskDetails.value.score * globalWeight : riskDetails.value.score; + const weightedNormalizedScore = + globalWeight !== undefined + ? riskDetails.value.normalized_score * globalWeight + : riskDetails.value.normalized_score; + const criticalityModifier = getCriticalityModifier(criticality?.criticality_level); const normalizedScoreWithCriticality = applyCriticalityToScore({ - score: riskDetails.value.normalized_score, + score: weightedNormalizedScore, modifier: criticalityModifier, }); const calculatedLevel = getRiskLevel(normalizedScoreWithCriticality); - const categoryTwoScore = normalizedScoreWithCriticality - riskDetails.value.normalized_score; + const categoryTwoScore = normalizedScoreWithCriticality - weightedNormalizedScore; const categoryTwoCount = criticalityModifier ? 1 : 0; const newFields = { @@ -82,7 +95,7 @@ const formatForResponse = ({ id_field: identifierField, id_value: bucket.key[identifierField], calculated_level: calculatedLevel, - calculated_score: max10DecimalPlaces(riskDetails.value.score), + calculated_score: max10DecimalPlaces(weightedScore), calculated_score_norm: max10DecimalPlaces(normalizedScoreWithCriticality), category_1_score: max10DecimalPlaces(riskDetails.value.category_1_score / RIEMANN_ZETA_VALUE), // normalize value to be between 0-100 category_1_count: riskDetails.value.category_1_count, @@ -104,6 +117,55 @@ export const filterFromRange = (range: CalculateScoresParams['range']): QueryDsl range: { '@timestamp': { lt: range.end, gte: range.start } }, }); +/** + * Builds filters for a specific entity type, including entity-specific custom filters + */ +export const buildFiltersForEntityType = ( + entityType: EntityType, + userFilter: QueryDslQueryContainer, + customFilters: Array<{ entity_types: string[]; filter: string }> = [], + excludeAlertStatuses: string[] = [], + excludeAlertTags: string[] = [] +): QueryDslQueryContainer[] => { + const filters: QueryDslQueryContainer[] = [{ exists: { field: ALERT_RISK_SCORE } }]; + + // Add existing user filter (backward compatibility) + if (!isEmpty(userFilter)) { + filters.push(userFilter); + } + + // Add alert status exclusions + if (excludeAlertStatuses.length > 0) { + filters.push({ + bool: { must_not: { terms: { [ALERT_WORKFLOW_STATUS]: excludeAlertStatuses } } }, + }); + } + + // Add alert tag exclusions + if (excludeAlertTags.length > 0) { + filters.push({ + bool: { must_not: { terms: { [ALERT_WORKFLOW_TAGS]: excludeAlertTags } } }, + }); + } + + // Add entity-specific custom filters (EXCLUSIVE - exclude matching alerts) + customFilters + .filter((f) => f.entity_types.includes(entityType)) + .forEach((f) => { + try { + const esQuery = toElasticsearchQuery(fromKueryExpression(f.filter)); + filters.push({ + bool: { must_not: esQuery }, + }); + } catch (error) { + // Log warning but don't fail the entire query + // Note: Invalid KQL filters are silently ignored to prevent query failures + } + }); + + return filters; +}; + const buildIdentifierTypeAggregation = ({ afterKeys, identifierType, @@ -168,12 +230,16 @@ export const processScores = async ({ identifierField, logger, now, + identifierType, + weights, }: { assetCriticalityService: AssetCriticalityService; buckets: RiskScoreBucket[]; identifierField: string; logger: Logger; now: string; + identifierType?: EntityType; + weights?: RiskScoreWeights; }): Promise => { if (buckets.length === 0) { return []; @@ -193,12 +259,23 @@ export const processScores = async ({ ); } + const globalWeight = identifierType + ? getGlobalWeightForIdentifierType(identifierType, weights) + : undefined; + return buckets.map((bucket) => { const criticality = criticalities.find( (c) => c.id_field === identifierField && c.id_value === bucket.key[identifierField] ); - return formatForResponse({ bucket, criticality, identifierField, now, includeNewFields: true }); + return formatForResponse({ + bucket, + criticality, + identifierField, + now, + includeNewFields: true, + globalWeight, + }); }); }; @@ -206,7 +283,9 @@ export const getGlobalWeightForIdentifierType = ( identifierType: EntityType, weights?: RiskScoreWeights ): number | undefined => - weights?.find((weight) => weight.type === RiskWeightTypes.global)?.[identifierType]; + weights?.find((weight: RiskScoreWeight) => weight.type === RiskWeightTypes.global)?.[ + identifierType + ]; export const calculateRiskScores = async ({ afterKeys: userAfterKeys, @@ -225,115 +304,128 @@ export const calculateRiskScores = async ({ excludeAlertStatuses = [], experimentalFeatures, excludeAlertTags = [], + filters: customFilters = [], }: { assetCriticalityService: AssetCriticalityService; esClient: ElasticsearchClient; logger: Logger; experimentalFeatures: ExperimentalFeatures; -} & CalculateScoresParams): Promise => +} & CalculateScoresParams & { + filters?: Array<{ entity_types: string[]; filter: string }>; + }): Promise => withSecuritySpan('calculateRiskScores', async () => { const now = new Date().toISOString(); const scriptedMetricPainless = await getPainlessScripts(); - const filter = [filterFromRange(range), { exists: { field: ALERT_RISK_SCORE } }]; - if (excludeAlertStatuses.length > 0) { - filter.push({ - bool: { must_not: { terms: { [ALERT_WORKFLOW_STATUS]: excludeAlertStatuses } } }, - }); - } - if (!isEmpty(userFilter)) { - filter.push(userFilter as QueryDslQueryContainer); - } - if (excludeAlertTags.length > 0) { - filter.push({ - bool: { must_not: { terms: { [ALERT_WORKFLOW_TAGS]: excludeAlertTags } } }, - }); - } const identifierTypes: EntityType[] = identifierType ? [identifierType] : getEntityAnalyticsEntityTypes(); - const request = { - size: 0, - _source: false, - index, - ignore_unavailable: true, - runtime_mappings: runtimeMappings, - query: { - function_score: { + // Build base filters that apply to all entity types + const baseFilters = [filterFromRange(range), { exists: { field: ALERT_RISK_SCORE } }]; + + // Create separate queries for each entity type with entity-specific filters + const entityQueries = identifierTypes.map((_identifierType) => { + // Build entity-specific filters + const entityFilters = buildFiltersForEntityType( + _identifierType, + userFilter as QueryDslQueryContainer, + customFilters, + excludeAlertStatuses, + excludeAlertTags + ); + + // Combine base filters with entity-specific filters + const allFilters = [...baseFilters, ...entityFilters]; + + return { + entityType: _identifierType, + request: { + size: 0, + _source: false, + index, + ignore_unavailable: true, + runtime_mappings: runtimeMappings, query: { - bool: { - filter, - should: [ - { - match_all: {}, // This forces ES to calculate score + function_score: { + query: { + bool: { + filter: allFilters, + should: [ + { + match_all: {}, // This forces ES to calculate score + }, + ], }, - ], + }, + field_value_factor: { + field: ALERT_RISK_SCORE, // sort by risk score + }, }, }, - field_value_factor: { - field: ALERT_RISK_SCORE, // sort by risk score + aggs: { + [_identifierType]: buildIdentifierTypeAggregation({ + afterKeys: userAfterKeys, + identifierType: _identifierType, + pageSize, + weights, + alertSampleSizePerShard, + scriptedMetricPainless, + }), }, }, - }, - aggs: identifierTypes.reduce((aggs, _identifierType) => { - aggs[_identifierType] = buildIdentifierTypeAggregation({ - afterKeys: userAfterKeys, - identifierType: _identifierType, - pageSize, - weights, - alertSampleSizePerShard, - scriptedMetricPainless, - }); - return aggs; - }, {} as Record), - }; - - if (debug) { - logger.info(`Executing Risk Score query:\n${JSON.stringify(request)}`); - } + }; + }); - const response = await esClient.search(request); + // Execute queries for each entity type + const responses = await Promise.all( + entityQueries.map(async ({ entityType, request: entityRequest }) => { + if (debug) { + logger.info( + `Executing Risk Score query for ${entityType}:\n${JSON.stringify(entityRequest)}` + ); + } - if (debug) { - logger.info(`Received Risk Score response:\n${JSON.stringify(response)}`); - } + const response = await esClient.search( + entityRequest + ); - if (!response.aggregations) { - return { - ...(debug ? { request, response } : {}), - after_keys: {}, - scores: { - host: [], - user: [], - service: [], - }, - entities: { user: [], host: [], service: [], generic: [] }, - }; - } + if (debug) { + logger.info( + `Received Risk Score response for ${entityType}:\n${JSON.stringify(response)}` + ); + } - const hosts = - response.aggregations?.host?.buckets.map( - ({ key }) => key[EntityTypeToIdentifierField.host] - ) || []; + return { + entityType, + response, + request: entityRequest, + }; + }) + ); - const users = - response.aggregations?.user?.buckets.map( - ({ key }) => key[EntityTypeToIdentifierField.user] - ) || []; + // Combine results from all entity queries + const combinedAggregations: Partial = {}; + const combinedAfterKeys: Partial = {}; - const services = - response.aggregations?.service?.buckets.map( - ({ key }) => key[EntityTypeToIdentifierField.service] - ) || []; + responses.forEach(({ entityType, response }) => { + if (response.aggregations && (response.aggregations as Record)[entityType]) { + (combinedAggregations as Record)[entityType] = ( + response.aggregations as Record + )[entityType]; + (combinedAfterKeys as Record)[entityType] = ( + response.aggregations as Record }> + )[entityType]?.after_key; + } + }); - const userBuckets = response.aggregations.user?.buckets ?? []; - const hostBuckets = response.aggregations.host?.buckets ?? []; - const serviceBuckets = response.aggregations.service?.buckets ?? []; + const userBuckets = combinedAggregations.user?.buckets ?? []; + const hostBuckets = combinedAggregations.host?.buckets ?? []; + const serviceBuckets = combinedAggregations.service?.buckets ?? []; const afterKeys = { - host: response.aggregations.host?.after_key, - user: response.aggregations.user?.after_key, - service: experimentalFeatures ? response.aggregations.service?.after_key : undefined, + host: combinedAfterKeys.host, + user: combinedAfterKeys.user, + service: experimentalFeatures ? combinedAfterKeys.service : undefined, }; const hostScores = await processScores({ @@ -359,18 +451,23 @@ export const calculateRiskScores = async ({ }); return { - ...(debug ? { request, response } : {}), + ...(debug + ? { + debug: { + request: JSON.stringify( + responses.map(({ entityType, request }) => ({ entityType, request })) + ), + response: JSON.stringify( + responses.map(({ entityType, response }) => ({ entityType, response })) + ), + }, + } + : {}), after_keys: afterKeys, scores: { host: hostScores, user: userScores, service: serviceScores, }, - entities: { - user: users, - host: hosts, - service: services, - generic: [], - }, }; }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/risk_score_service.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/risk_score_service.ts index 57932b7437146..675c44ceae068 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/risk_score_service.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/risk_score_service.ts @@ -83,9 +83,7 @@ export const riskScoreServiceFactory = ({ esClient, logger, experimentalFeatures, - }).catch((err) => { - logger.error(`Error calculating risk scores: ${err}`); - throw err; + filters: params.filters || [], }); }, calculateAndPersistScores: (params) => diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/routes/preview.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/routes/preview.test.ts index 2eb6272177427..ad302bf6bbf05 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/routes/preview.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/routes/preview.test.ts @@ -288,5 +288,44 @@ describe('POST risk_engine/preview route', () => { ); }); }); + + describe('filters', () => { + it('respects the provided filters', async () => { + const filters = [ + { entity_types: ['host'], filter: 'agent.type: filebeat' }, + { entity_types: ['user'], filter: 'user.name: ubuntu' }, + ]; + const request = buildRequest({ filters }); + + const response = await server.inject(request, requestContextMock.convertContext(context)); + + expect(response.status).toEqual(200); + expect(mockRiskScoreService.calculateScores).toHaveBeenCalledWith( + expect.objectContaining({ filters }) + ); + }); + + it('handles empty filters array', async () => { + const request = buildRequest({ filters: [] }); + + const response = await server.inject(request, requestContextMock.convertContext(context)); + + expect(response.status).toEqual(200); + expect(mockRiskScoreService.calculateScores).toHaveBeenCalledWith( + expect.objectContaining({ filters: [] }) + ); + }); + + it('handles undefined filters', async () => { + const request = buildRequest(); + + const response = await server.inject(request, requestContextMock.convertContext(context)); + + expect(response.status).toEqual(200); + expect(mockRiskScoreService.calculateScores).toHaveBeenCalledWith( + expect.objectContaining({ filters: [] }) + ); + }); + }); }); }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/routes/preview.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/routes/preview.ts index 9485a736810c6..8825636e97bc9 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/routes/preview.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/routes/preview.ts @@ -68,6 +68,7 @@ export const riskScorePreviewRoute = ( weights, exclude_alert_statuses: excludedStatuses, exclude_alert_tags: excludedTags, + filters: customFilters, } = request.body; const entityAnalyticsConfig = await riskScoreService.getConfigurationWithDefaults( @@ -88,6 +89,7 @@ export const riskScorePreviewRoute = ( const pageSize = userPageSize ?? DEFAULT_RISK_SCORE_PAGE_SIZE; const excludeAlertStatuses = excludedStatuses || ['closed']; const excludeAlertTags = excludedTags || []; + const filters = customFilters || []; const result = await riskScoreService.calculateScores({ afterKeys, @@ -102,6 +104,7 @@ export const riskScorePreviewRoute = ( alertSampleSizePerShard, excludeAlertStatuses, excludeAlertTags, + filters, }); securityContext.getAuditLogger()?.log({ @@ -116,6 +119,20 @@ export const riskScorePreviewRoute = ( return response.ok({ body: result }); } catch (e) { + // If the error is related to a non-existent index, return empty scores instead of an error + if (e instanceof Error && e.message && e.message.includes('index_not_found_exception')) { + return response.ok({ + body: { + after_keys: {}, + scores: { + host: [], + user: [], + service: [], + }, + }, + }); + } + const error = transformError(e); return siemResponse.error({ diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/types.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/types.ts index 33481b3e5c664..0923f56a61e52 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/types.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/types.ts @@ -85,6 +85,10 @@ export interface RiskEngineConfiguration { excludeAlertStatuses?: string[]; excludeAlertTags?: string[]; enableResetToZero: boolean; + filters?: Array<{ + entity_types: string[]; + filter: string; + }>; } export interface CalculateScoresParams { @@ -100,6 +104,7 @@ export interface CalculateScoresParams { alertSampleSizePerShard?: number; excludeAlertStatuses?: string[]; excludeAlertTags?: string[]; + filters?: Array<{ entity_types: string[]; filter: string }>; } export interface CalculateAndPersistScoresParams { diff --git a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/init_and_status_apis.ts b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/init_and_status_apis.ts index 3d32c4a168c9a..9822f7692c4cc 100644 --- a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/init_and_status_apis.ts +++ b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/init_and_status_apis.ts @@ -647,7 +647,7 @@ export default ({ getService }: FtrProviderContext) => { start: 'now-30d', }, _meta: { - mappingsVersion: 4, + mappingsVersion: 5, }, enableResetToZero: true, });