diff --git a/x-pack/solutions/security/plugins/security_solution/common/entity_analytics/lead_generation/constants.ts b/x-pack/solutions/security/plugins/security_solution/common/entity_analytics/lead_generation/constants.ts index 0c8a363616674..8cf8b707ebc9e 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/entity_analytics/lead_generation/constants.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/entity_analytics/lead_generation/constants.ts @@ -8,7 +8,12 @@ export const LEAD_GENERATION_URL = '/internal/entity_analytics/leads' as const; export const GENERATE_LEADS_URL = `${LEAD_GENERATION_URL}/generate` as const; export const GET_LEADS_URL = LEAD_GENERATION_URL as string; +export const GET_LEAD_BY_ID_URL = `${LEAD_GENERATION_URL}/{id}` as const; export const LEAD_GENERATION_STATUS_URL = `${LEAD_GENERATION_URL}/status` as const; +export const DISMISS_LEAD_URL = `${LEAD_GENERATION_URL}/{id}/_dismiss` as const; +export const BULK_UPDATE_LEADS_URL = `${LEAD_GENERATION_URL}/bulk_update` as const; +export const ENABLE_LEAD_GENERATION_URL = `${LEAD_GENERATION_URL}/enable` as const; +export const DISABLE_LEAD_GENERATION_URL = `${LEAD_GENERATION_URL}/disable` as const; export type LeadGenerationMode = 'adhoc' | 'scheduled'; diff --git a/x-pack/solutions/security/plugins/security_solution/common/entity_analytics/lead_generation/types.ts b/x-pack/solutions/security/plugins/security_solution/common/entity_analytics/lead_generation/types.ts index f06160a8df4ef..0e2fd5b89d600 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/entity_analytics/lead_generation/types.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/entity_analytics/lead_generation/types.ts @@ -49,8 +49,6 @@ export const leadEntitySchema = z.object({ name: z.string(), }); -export type LeadEntity = z.infer; - // --------------------------------------------------------------------------- // Lead // --------------------------------------------------------------------------- @@ -74,52 +72,31 @@ export const leadSchema = z.object({ export type Lead = z.infer; -// --------------------------------------------------------------------------- -// Engine configuration -// --------------------------------------------------------------------------- - -export const leadGenerationEngineConfigSchema = z.object({ - minObservations: z.number().int().min(0).default(1), - maxLeads: z.number().int().min(1).default(10), -}); - -export type LeadGenerationEngineConfig = z.infer; - // --------------------------------------------------------------------------- // API request / response schemas // --------------------------------------------------------------------------- export const generateLeadsRequestSchema = z.object({ maxLeads: z.number().int().min(1).max(50).optional(), - connectorId: z.string().optional(), }); -export type GenerateLeadsRequest = z.infer; - export const findLeadsRequestSchema = z.object({ - page: z.number().int().min(1).optional().default(1), - perPage: z.number().int().min(1).max(100).optional().default(20), + page: z.coerce.number().int().min(1).optional().default(1), + perPage: z.coerce.number().int().min(1).max(100).optional().default(20), sortField: z.enum(['priority', 'timestamp']).optional().default('priority'), sortOrder: z.enum(['asc', 'desc']).optional().default('desc'), status: LeadStatusEnum.optional(), }); -export type FindLeadsRequest = z.infer; - -export const findLeadsResponseSchema = z.object({ - leads: z.array(leadSchema), - total: z.number(), - page: z.number(), - perPage: z.number(), +export const getLeadByIdRequestSchema = z.object({ + id: z.string().min(1), }); -export type FindLeadsResponse = z.infer; - -export const leadGenerationStatusSchema = z.object({ - isEnabled: z.boolean(), - indexExists: z.boolean(), - totalLeads: z.number(), - lastRun: z.string().datetime().nullable(), +export const dismissLeadRequestSchema = z.object({ + id: z.string().min(1), }); -export type LeadGenerationStatus = z.infer; +export const bulkUpdateLeadsRequestSchema = z.object({ + ids: z.array(z.string().min(1)).min(1).max(100), + status: LeadStatusEnum, +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/engine/lead_generation_engine.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/engine/lead_generation_engine.ts index 6f7ef70692e6b..ced5a50116ebd 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/engine/lead_generation_engine.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/engine/lead_generation_engine.ts @@ -15,7 +15,7 @@ import type { Observation, ObservationModule, } from '../types'; -import { DEFAULT_ENGINE_CONFIG, computeStaleness } from '../types'; +import { DEFAULT_ENGINE_CONFIG } from '../types'; import { entityToKey } from '../observation_modules/utils'; import { llmSynthesizeLeadContent } from './llm_synthesize'; @@ -236,7 +236,7 @@ const groupIntoLeads = async ( priority: maxPriority, chatRecommendations: recommendations, timestamp: now.toISOString(), - staleness: computeStaleness(now, now), + staleness: 'fresh', observations: allObservations, }); } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/engine/llm_synthesize.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/engine/llm_synthesize.ts index 2a52aedfba018..092db47ae4b62 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/engine/llm_synthesize.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/engine/llm_synthesize.ts @@ -31,7 +31,7 @@ Respond ONLY with a valid JSON object (no markdown fences, no extra text) matchi {{ "title": "string - MAXIMUM 4 WORDS. A short threat label, not a sentence. Good: 'Anomalous behavior', 'Credential harvesting', 'Lateral movement detected', 'Privilege escalation'. Bad: 'Suspected Multi-Tactic Attack Targeting DevOps User with Container Escape'", "description": "string - a narrative paragraph (plain text, NO markdown, NO bold/italic markers) connecting the evidence, referencing specific data points (scores, alert counts, escalation deltas), explaining why this matters and what the attacker may be doing. Do NOT use asterisks or markdown formatting.", - "tags": ["string array - 3 to 6 tags. Use human-readable technique or rule names, NOT numeric IDs. Good: 'Container Escape Attempt', 'Remote Service Execution', 'Credential Access via Brute Force'. Bad: 'T1075', 'T1078'. Also include short contextual tags like 'Privilege Escalation', 'Lateral Movement'."], + "tags": ["string array - 3 to 6 tags. Use human-readable technique or rule names, NOT numeric IDs. Only use rule names that appear explicitly in the observation data below; do not invent or guess rule names. Good: 'Container Escape Attempt', 'Remote Service Execution', 'Credential Access via Brute Force'. Bad: 'T1075', 'T1078'. Also include short contextual tags like 'Privilege Escalation', 'Lateral Movement'."], "recommendations": ["string - a single chat message an analyst can paste into an AI chat assistant to start investigating. It must be a direct request or question the analyst would type. Example: 'Show me the critical/high severity alerts for user \\"jsmith\\" from the last 7 days, grouped by detection rule name, and correlate with the risk score trend over the last 30 days'. Do NOT write generic advice like 'Isolate the account' or 'Review logs'. Write an actual chat prompt."] }} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/entity_conversion.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/entity_conversion.ts index 70ceab71781dc..134d0881bf0ba 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/entity_conversion.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/entity_conversion.ts @@ -5,21 +5,54 @@ * 2.0. */ +import type { EntityStoreCRUDClient } from '@kbn/entity-store/server'; import type { Entity } from '../../../../common/api/entity_analytics/entity_store/entities/common.gen'; import type { LeadEntity } from './types'; +const ENTITY_PAGE_SIZE = 1000; + /** * Convert an Entity Store V2 record into a LeadEntity, extracting the * convenience `type` and `name` fields from the nested `entity` object. * Falls back to `entity.id` (EUID) when `entity.name` is absent. + * + * Accepts `Record` so it works with Entity types from + * both the security_solution and entity_store plugins (structurally + * equivalent but separate Zod-generated types). */ -export const entityRecordToLeadEntity = (record: Entity): LeadEntity => { - const entityField = (record as Record).entity as - | { name?: string; type?: string; id?: string } +export const entityRecordToLeadEntity = (record: Record): LeadEntity => { + const entityField = record.entity as + | { name?: string; type?: string; id?: string; EngineMetadata?: { Type?: string } } | undefined; return { - record, - type: entityField?.type ?? 'unknown', + record: record as Entity, + type: entityField?.EngineMetadata?.Type ?? entityField?.type ?? 'unknown', name: entityField?.name ?? entityField?.id ?? 'unknown', }; }; + +/** + * Paginate through all entities in the V2 unified index via + * `CRUDClient.listEntities()`, accumulating results across pages. + */ +export const fetchAllLeadEntities = async ( + crudClient: EntityStoreCRUDClient +): Promise => { + const allEntities: LeadEntity[] = []; + let searchAfter: Array | undefined; + + do { + const { entities, nextSearchAfter } = await crudClient.listEntities({ + size: ENTITY_PAGE_SIZE, + searchAfter, + }); + + for (const entity of entities) { + allEntities.push(entityRecordToLeadEntity(entity)); + } + + searchAfter = nextSearchAfter; + } while (searchAfter !== undefined); + + return allEntities; +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/index.ts index 8b0f523191d48..551e54c721c16 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/index.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/index.ts @@ -6,6 +6,11 @@ */ export { createLeadIndexService, type LeadIndexService } from './indices'; +export { + createLeadDataClient, + type LeadDataClient, + type LeadDataClientDeps, +} from './lead_data_client'; export { ObservationModuleRegistry, type ObservationEntity } from './observation_modules'; export { createLeadGenerationEngine } from './engine'; export { @@ -13,7 +18,7 @@ export { createTemporalStateModule, createBehavioralAnalysisModule, } from './observation_modules'; -export { entityRecordToLeadEntity } from './entity_conversion'; +export { entityRecordToLeadEntity, fetchAllLeadEntities } from './entity_conversion'; export { createLeadGenerationService } from './services/lead_generation_service'; export type { Lead, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/lead_data_client.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/lead_data_client.test.ts new file mode 100644 index 0000000000000..0d764ae79cad7 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/lead_data_client.test.ts @@ -0,0 +1,435 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks'; +import { createLeadDataClient } from './lead_data_client'; +import type { LeadDataClient } from './lead_data_client'; +import { getLeadsIndexName } from '../../../../common/entity_analytics/lead_generation/constants'; +import type { Lead } from '../../../../common/entity_analytics/lead_generation/types'; + +const makeTestLead = (overrides: Partial = {}): Lead => ({ + id: 'lead-1', + title: 'Test Lead', + byline: 'Entity X has suspicious activity', + description: 'Detailed investigation guide', + entities: [{ type: 'user', name: 'admin' }], + tags: ['brute_force'], + priority: 8, + chatRecommendations: ['What alerts exist?', 'Check risk score history'], + timestamp: new Date().toISOString(), + staleness: 'fresh', + status: 'active', + observations: [ + { + entityId: 'user:admin', + moduleId: 'risk_analysis', + type: 'high_risk_score', + score: 85, + severity: 'high', + confidence: 0.9, + description: 'Risk score is 85', + metadata: { scoreNorm: 85 }, + }, + ], + executionUuid: '550e8400-e29b-41d4-a716-446655440000', + sourceType: 'adhoc', + ...overrides, +}); + +describe('LeadDataClient', () => { + const spaceId = 'default'; + const adhocIndex = getLeadsIndexName(spaceId, 'adhoc'); + const scheduledIndex = getLeadsIndexName(spaceId, 'scheduled'); + const allIndices = `${adhocIndex},${scheduledIndex}`; + + let esClient: ReturnType; + let logger: ReturnType; + let client: LeadDataClient; + + beforeEach(() => { + jest.clearAllMocks(); + esClient = elasticsearchServiceMock.createElasticsearchClient(); + logger = loggingSystemMock.createLogger(); + client = createLeadDataClient({ esClient, logger, spaceId }); + }); + + describe('createLeads', () => { + it('bulk indexes leads with snake_case fields and cleans up stale docs', async () => { + esClient.bulk.mockResolvedValueOnce({ errors: false, items: [], took: 1 }); + esClient.deleteByQuery.mockResolvedValueOnce({ + deleted: 0, + failures: [], + timed_out: false, + took: 1, + total: 0, + }); + + const lead = makeTestLead(); + await client.createLeads({ + leads: [lead], + executionId: 'exec-1', + sourceType: 'adhoc', + }); + + expect(esClient.bulk).toHaveBeenCalledTimes(1); + const [bulkCall] = esClient.bulk.mock.calls; + const body = bulkCall[0].body as unknown[]; + + // Verify the index action targets the adhoc index + expect(body[0]).toEqual({ index: { _index: adhocIndex, _id: lead.id } }); + + // Verify snake_case fields in the document + const doc = body[1] as Record; + expect(doc.chat_recommendations).toEqual(lead.chatRecommendations); + expect(doc.execution_uuid).toBe('exec-1'); + expect(doc.source_type).toBe('adhoc'); + expect(doc.observations).toEqual([ + expect.objectContaining({ + entity_id: 'user:admin', + module_id: 'risk_analysis', + }), + ]); + + // camelCase fields should NOT be present + expect(doc).not.toHaveProperty('chatRecommendations'); + expect(doc).not.toHaveProperty('executionUuid'); + expect(doc).not.toHaveProperty('sourceType'); + + // Verify stale cleanup uses snake_case execution_uuid + expect(esClient.deleteByQuery).toHaveBeenCalledWith( + expect.objectContaining({ + index: adhocIndex, + query: { bool: { must_not: [{ term: { 'execution_uuid.keyword': 'exec-1' } }] } }, + }) + ); + }); + + it('uses the scheduled index when sourceType is scheduled', async () => { + esClient.bulk.mockResolvedValueOnce({ errors: false, items: [], took: 1 }); + esClient.deleteByQuery.mockResolvedValueOnce({ + deleted: 0, + failures: [], + timed_out: false, + took: 1, + total: 0, + }); + + await client.createLeads({ + leads: [makeTestLead()], + executionId: 'exec-2', + sourceType: 'scheduled', + }); + + const [bulkCall] = esClient.bulk.mock.calls; + const body = bulkCall[0].body as unknown[]; + expect((body[0] as Record).index).toEqual( + expect.objectContaining({ _index: scheduledIndex }) + ); + }); + + it('skips bulk indexing when leads array is empty but still cleans up stale docs', async () => { + esClient.deleteByQuery.mockResolvedValueOnce({ + deleted: 2, + failures: [], + timed_out: false, + took: 1, + total: 2, + }); + + await client.createLeads({ + leads: [], + executionId: 'exec-3', + sourceType: 'adhoc', + }); + + expect(esClient.bulk).not.toHaveBeenCalled(); + expect(esClient.deleteByQuery).toHaveBeenCalledTimes(1); + }); + + it('logs a warning and does not throw on persistence failure', async () => { + esClient.bulk.mockRejectedValueOnce(new Error('ES unavailable')); + + await expect( + client.createLeads({ + leads: [makeTestLead()], + executionId: 'exec-4', + sourceType: 'adhoc', + }) + ).resolves.toBeUndefined(); + + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('Failed to persist leads')); + }); + }); + + describe('findLeads', () => { + it('queries both indices with pagination and transforms response to camelCase', async () => { + const esDoc = { + id: 'lead-1', + title: 'Test Lead', + byline: 'Entity X', + description: 'Details', + entities: [{ type: 'user', name: 'admin' }], + tags: ['brute_force'], + priority: 8, + chat_recommendations: ['Question 1'], + timestamp: new Date().toISOString(), + staleness: 'fresh', + status: 'active', + observations: [ + { + entity_id: 'user:admin', + module_id: 'risk_analysis', + type: 'high_risk_score', + score: 85, + severity: 'high', + confidence: 0.9, + description: 'Risk score 85', + metadata: {}, + }, + ], + execution_uuid: 'exec-uuid', + source_type: 'adhoc', + }; + + esClient.search.mockResolvedValueOnce({ + hits: { + total: { value: 1, relation: 'eq' }, + hits: [{ _source: esDoc, _id: 'lead-1', _index: adhocIndex }], + }, + } as never); + + const result = await client.findLeads({ page: 1, perPage: 10 }); + + expect(esClient.search).toHaveBeenCalledWith( + expect.objectContaining({ + index: allIndices, + size: 10, + from: 0, + track_total_hits: true, + }) + ); + + expect(result.total).toBe(1); + expect(result.page).toBe(1); + expect(result.perPage).toBe(10); + expect(result.leads).toHaveLength(1); + + // Verify camelCase transformation + const lead = result.leads[0]; + expect(lead.chatRecommendations).toEqual(['Question 1']); + expect(lead.executionUuid).toBe('exec-uuid'); + expect(lead.sourceType).toBe('adhoc'); + expect(lead.observations[0].entityId).toBe('user:admin'); + expect(lead.observations[0].moduleId).toBe('risk_analysis'); + }); + + it('applies status filter when provided', async () => { + esClient.search.mockResolvedValueOnce({ + hits: { total: { value: 0, relation: 'eq' }, hits: [] }, + } as never); + + await client.findLeads({ status: 'dismissed' }); + + const searchCall = esClient.search.mock.calls[0]; + expect(searchCall).toBeDefined(); + expect((searchCall[0] as Record).query).toEqual({ + bool: { filter: [{ term: { 'status.keyword': 'dismissed' } }] }, + }); + }); + + it('returns empty results when indices are unavailable', async () => { + esClient.search.mockRejectedValueOnce(new Error('index_not_found_exception')); + + const result = await client.findLeads({}); + expect(result).toEqual({ leads: [], total: 0, page: 1, perPage: 20 }); + }); + }); + + describe('getLeadById', () => { + it('returns the lead when found', async () => { + esClient.search.mockResolvedValueOnce({ + hits: { + total: { value: 1, relation: 'eq' }, + hits: [ + { + _id: 'lead-1', + _index: adhocIndex, + _source: { + id: 'lead-1', + title: 'Found Lead', + byline: '', + description: '', + entities: [], + tags: [], + priority: 5, + chat_recommendations: [], + timestamp: new Date().toISOString(), + staleness: 'fresh', + status: 'active', + observations: [], + execution_uuid: 'e-1', + source_type: 'adhoc', + }, + }, + ], + }, + } as never); + + const lead = await client.getLeadById('lead-1'); + + expect(lead).not.toBeNull(); + expect(lead!.id).toBe('lead-1'); + expect(lead!.title).toBe('Found Lead'); + }); + + it('returns null when lead is not found', async () => { + esClient.search.mockResolvedValueOnce({ + hits: { total: { value: 0, relation: 'eq' }, hits: [] }, + } as never); + + const lead = await client.getLeadById('nonexistent'); + expect(lead).toBeNull(); + }); + + it('returns null on search error', async () => { + esClient.search.mockRejectedValueOnce(new Error('index_not_found')); + + const lead = await client.getLeadById('lead-1'); + expect(lead).toBeNull(); + }); + }); + + describe('dismissLead', () => { + it('sets status to dismissed via updateByQuery', async () => { + esClient.updateByQuery.mockResolvedValueOnce({ + updated: 1, + failures: [], + timed_out: false, + took: 1, + total: 1, + }); + + const result = await client.dismissLead('lead-1'); + expect(result).toBe(true); + + expect(esClient.updateByQuery).toHaveBeenCalledWith( + expect.objectContaining({ + index: allIndices, + query: { term: { 'id.keyword': 'lead-1' } }, + }) + ); + }); + + it('returns false when no document matched', async () => { + esClient.updateByQuery.mockResolvedValueOnce({ + updated: 0, + failures: [], + timed_out: false, + took: 1, + total: 0, + }); + + const result = await client.dismissLead('nonexistent'); + expect(result).toBe(false); + }); + }); + + describe('bulkUpdateLeads', () => { + it('updates multiple leads by ids', async () => { + esClient.updateByQuery.mockResolvedValueOnce({ + updated: 3, + failures: [], + timed_out: false, + took: 1, + total: 3, + }); + + const count = await client.bulkUpdateLeads(['a', 'b', 'c'], { status: 'dismissed' }); + expect(count).toBe(3); + + const [call] = esClient.updateByQuery.mock.calls; + expect(call[0].query).toEqual({ terms: { 'id.keyword': ['a', 'b', 'c'] } }); + expect(call[0].script).toEqual( + expect.objectContaining({ + params: { status: 'dismissed' }, + }) + ); + }); + + it('returns 0 for an empty ids array', async () => { + const count = await client.bulkUpdateLeads([], { status: 'active' }); + expect(count).toBe(0); + expect(esClient.updateByQuery).not.toHaveBeenCalled(); + }); + + it('throws on error so the route can surface it', async () => { + esClient.updateByQuery.mockRejectedValueOnce(new Error('cluster error')); + + await expect(client.bulkUpdateLeads(['a'], { status: 'dismissed' })).rejects.toThrow( + 'cluster error' + ); + }); + }); + + describe('getStatus', () => { + it('returns status with total count and last run timestamp', async () => { + esClient.search.mockResolvedValueOnce({ + hits: { + total: { value: 42, relation: 'eq' }, + hits: [ + { + _id: 'lead-1', + _index: adhocIndex, + _source: { timestamp: '2026-03-10T00:00:00.000Z' }, + }, + ], + }, + } as never); + + const status = await client.getStatus(); + expect(status).toEqual({ + isEnabled: false, + indexExists: true, + totalLeads: 42, + lastRun: '2026-03-10T00:00:00.000Z', + }); + }); + + it('returns defaults when indices do not exist', async () => { + esClient.search.mockRejectedValueOnce(new Error('index_not_found')); + + const status = await client.getStatus(); + expect(status).toEqual({ + isEnabled: false, + indexExists: false, + totalLeads: 0, + lastRun: null, + }); + }); + }); + + describe('deleteAllLeads', () => { + it('deletes all docs from both indices', async () => { + esClient.deleteByQuery.mockResolvedValueOnce({ + deleted: 10, + failures: [], + timed_out: false, + took: 1, + total: 10, + }); + + await client.deleteAllLeads(); + + expect(esClient.deleteByQuery).toHaveBeenCalledWith( + expect.objectContaining({ + index: allIndices, + query: { match_all: {} }, + }) + ); + }); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/lead_data_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/lead_data_client.ts new file mode 100644 index 0000000000000..17e5c41a8303e --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/lead_data_client.ts @@ -0,0 +1,452 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient, Logger } from '@kbn/core/server'; +import type { estypes } from '@elastic/elasticsearch'; + +import { + getLeadsIndexName, + type LeadGenerationMode, +} from '../../../../common/entity_analytics/lead_generation/constants'; +import type { + Lead, + LeadStatus, + LeadStaleness, +} from '../../../../common/entity_analytics/lead_generation/types'; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export interface LeadDataClientDeps { + readonly esClient: ElasticsearchClient; + readonly logger: Logger; + readonly spaceId: string; +} + +export interface CreateLeadsParams { + readonly leads: readonly Lead[]; + readonly executionId: string; + readonly sourceType: LeadGenerationMode; +} + +export interface FindLeadsParams { + readonly page?: number; + readonly perPage?: number; + readonly sortField?: 'priority' | 'timestamp'; + readonly sortOrder?: 'asc' | 'desc'; + readonly status?: LeadStatus; +} + +export interface FindLeadsResult { + readonly leads: Lead[]; + readonly total: number; + readonly page: number; + readonly perPage: number; +} + +export interface LeadDataClient { + createLeads(params: CreateLeadsParams): Promise; + findLeads(params: FindLeadsParams): Promise; + getLeadById(id: string): Promise; + updateLead(id: string, updates: Partial>): Promise; + dismissLead(id: string): Promise; + bulkUpdateLeads(ids: readonly string[], updates: { status: LeadStatus }): Promise; + getStatus(): Promise<{ + isEnabled: boolean; + indexExists: boolean; + totalLeads: number; + lastRun: string | null; + }>; + deleteAllLeads(): Promise; +} + +// --------------------------------------------------------------------------- +// Staleness computation (timestamp-based, computed at read time) +// --------------------------------------------------------------------------- + +const STALENESS_THRESHOLDS = { + fresh: 24 * 60 * 60 * 1000, + stale: 72 * 60 * 60 * 1000, +}; + +const computeStaleness = (timestamp: string): LeadStaleness => { + const ageMs = Date.now() - new Date(timestamp).getTime(); + if (ageMs <= STALENESS_THRESHOLDS.fresh) return 'fresh'; + if (ageMs <= STALENESS_THRESHOLDS.stale) return 'stale'; + return 'expired'; +}; + +// --------------------------------------------------------------------------- +// camelCase ↔ snake_case transform layer +// +// Index mappings (source of truth) use snake_case. API and in-memory types +// use camelCase. This transform bridges the two. +// --------------------------------------------------------------------------- + +interface EsObservationDoc { + entity_id: string; + module_id: string; + type: string; + score: number; + severity: string; + confidence: number; + description: string; + metadata: Record; +} + +interface EsLeadDoc { + id: string; + title: string; + byline: string; + description: string; + entities: Array<{ type: string; name: string }>; + tags: string[]; + priority: number; + chat_recommendations: string[]; + timestamp: string; + staleness: string; + status: string; + observations: EsObservationDoc[]; + execution_uuid: string; + source_type: string; +} + +const leadToEsDoc = ( + lead: Lead, + executionId: string, + sourceType: LeadGenerationMode +): EsLeadDoc => ({ + id: lead.id, + title: lead.title, + byline: lead.byline, + description: lead.description, + entities: lead.entities.map(({ type, name }) => ({ type, name })), + tags: lead.tags, + priority: lead.priority, + chat_recommendations: lead.chatRecommendations, + timestamp: lead.timestamp, + staleness: lead.staleness, + status: lead.status ?? 'active', + observations: lead.observations.map((obs) => ({ + entity_id: obs.entityId, + module_id: obs.moduleId, + type: obs.type, + score: obs.score, + severity: obs.severity, + confidence: obs.confidence, + description: obs.description, + metadata: obs.metadata, + })), + execution_uuid: executionId, + source_type: sourceType, +}); + +const esDocToLead = (doc: Record): Lead => { + const observations = (doc.observations as EsObservationDoc[] | undefined) ?? []; + const timestamp = (doc.timestamp as string) ?? new Date().toISOString(); + + return { + id: doc.id as string, + title: doc.title as string, + byline: (doc.byline as string) ?? '', + description: (doc.description as string) ?? '', + entities: (doc.entities as Array<{ type: string; name: string }>) ?? [], + tags: (doc.tags as string[]) ?? [], + priority: (doc.priority as number) ?? 1, + chatRecommendations: (doc.chat_recommendations as string[]) ?? [], + timestamp, + staleness: computeStaleness(timestamp), + status: (doc.status as LeadStatus) ?? 'active', + observations: observations.map((obs) => ({ + entityId: obs.entity_id, + moduleId: obs.module_id, + type: obs.type, + score: obs.score, + severity: obs.severity as Lead['observations'][number]['severity'], + confidence: obs.confidence, + description: obs.description, + metadata: obs.metadata ?? {}, + })), + executionUuid: (doc.execution_uuid as string) ?? '', + sourceType: (doc.source_type as Lead['sourceType']) ?? 'adhoc', + }; +}; + +// --------------------------------------------------------------------------- +// Factory +// --------------------------------------------------------------------------- + +export const createLeadDataClient = ({ + esClient, + logger, + spaceId, +}: LeadDataClientDeps): LeadDataClient => { + const adhocIndex = getLeadsIndexName(spaceId, 'adhoc'); + const scheduledIndex = getLeadsIndexName(spaceId, 'scheduled'); + const allIndices = `${adhocIndex},${scheduledIndex}`; + + // ----------------------------------------------------------------------- + // createLeads — bulk index + gap-free stale cleanup + // ----------------------------------------------------------------------- + const createLeads = async ({ + leads, + executionId, + sourceType, + }: CreateLeadsParams): Promise => { + const indexName = getLeadsIndexName(spaceId, sourceType); + + try { + if (leads.length > 0) { + const bulkBody = leads.flatMap((lead) => [ + { index: { _index: indexName, _id: lead.id } }, + leadToEsDoc(lead, executionId, sourceType), + ]); + const bulkResp = await esClient.bulk({ body: bulkBody }); + + if (bulkResp.errors) { + const failedItems = bulkResp.items.filter((item) => item.index?.error); + const failedIds = failedItems.map((item) => item.index?._id); + logger.error( + `[LeadGeneration] Bulk indexing had ${failedItems.length}/${leads.length} failures ` + + `(executionId=${executionId}, index=${indexName}): ${JSON.stringify(failedIds)}` + ); + return; + } + + logger.debug(`[LeadGeneration] Persisted ${leads.length} leads to "${indexName}"`); + } + + await esClient.deleteByQuery({ + index: indexName, + query: { + bool: { must_not: [{ term: { 'execution_uuid.keyword': executionId } }] }, + }, + refresh: true, + conflicts: 'proceed', + slices: 'auto', + ignore_unavailable: true, + }); + } catch (e) { + logger.warn(`[LeadGeneration] Failed to persist leads to "${indexName}": ${e}`); + } + }; + + // ----------------------------------------------------------------------- + // findLeads — paginated search across both indices + // ----------------------------------------------------------------------- + const findLeads = async ({ + page = 1, + perPage = 20, + sortField = 'priority', + sortOrder = 'desc', + status, + }: FindLeadsParams): Promise => { + const from = (page - 1) * perPage; + const filters: estypes.QueryDslQueryContainer[] = []; + if (status) { + filters.push({ term: { 'status.keyword': status } }); + } + const query: estypes.QueryDslQueryContainer = + filters.length > 0 ? { bool: { filter: filters } } : { match_all: {} }; + + try { + const resp = await esClient.search({ + index: allIndices, + size: perPage, + from, + track_total_hits: true, + sort: [ + { [sortField]: { order: sortOrder as estypes.SortOrder } }, + { timestamp: { order: 'desc' as estypes.SortOrder } }, + ], + query, + ignore_unavailable: true, + }); + + const total = + typeof resp.hits.total === 'number' ? resp.hits.total : resp.hits.total?.value ?? 0; + + const leads = resp.hits.hits + .map((hit) => hit._source) + .filter((doc): doc is Record => doc != null) + .map(esDocToLead); + + return { leads, total, page, perPage }; + } catch (e) { + const isIndexNotFound = + (e as { meta?: { body?: { error?: { type?: string } } } })?.meta?.body?.error?.type === + 'index_not_found_exception'; + const errorMessage = e instanceof Error ? e.message : String(e); + if (isIndexNotFound) { + logger.debug(`[LeadGeneration] Leads indices not available yet: ${errorMessage}`); + } else { + logger.error(`[LeadGeneration] Unable to find leads due to error: ${errorMessage}`); + } + return { leads: [], total: 0, page, perPage }; + } + }; + + // ----------------------------------------------------------------------- + // getLeadById — search by doc id across both indices + // ----------------------------------------------------------------------- + const getLeadById = async (id: string): Promise => { + try { + const resp = await esClient.search({ + index: allIndices, + size: 1, + query: { term: { 'id.keyword': id } }, + ignore_unavailable: true, + }); + + const hit = resp.hits.hits[0]; + if (!hit?._source) return null; + return esDocToLead(hit._source as Record); + } catch (e) { + logger.debug(`[LeadGeneration] Error fetching lead ${id}: ${e}`); + return null; + } + }; + + // ----------------------------------------------------------------------- + // updateLead — partial update by doc id + // ----------------------------------------------------------------------- + const updateLead = async ( + id: string, + updates: Partial> + ): Promise => { + try { + const scriptParts: string[] = []; + const params: Record = {}; + let paramIdx = 0; + for (const [key, val] of Object.entries(updates)) { + const paramName = `p${paramIdx++}`; + scriptParts.push(`ctx._source['${key}'] = params.${paramName}`); + params[paramName] = val; + } + + const resp = await esClient.updateByQuery({ + index: allIndices, + query: { term: { 'id.keyword': id } }, + script: { + source: scriptParts.join('; '), + lang: 'painless', + params, + }, + refresh: true, + conflicts: 'proceed', + ignore_unavailable: true, + }); + return (resp.updated ?? 0) > 0; + } catch (e) { + logger.error(`[LeadGeneration] Error updating lead ${id}: ${e}`); + return false; + } + }; + + // ----------------------------------------------------------------------- + // dismissLead — set status to 'dismissed' + // ----------------------------------------------------------------------- + const dismissLead = async (id: string): Promise => { + return updateLead(id, { status: 'dismissed' }); + }; + + // ----------------------------------------------------------------------- + // bulkUpdateLeads — bulk status change via updateByQuery + // ----------------------------------------------------------------------- + const bulkUpdateLeads = async ( + ids: readonly string[], + updates: { status: LeadStatus } + ): Promise => { + if (ids.length === 0) return 0; + + const resp = await esClient.updateByQuery({ + index: allIndices, + query: { terms: { 'id.keyword': [...ids] } }, + script: { + source: `ctx._source['status'] = params.status`, + lang: 'painless', + params: { status: updates.status }, + }, + refresh: true, + conflicts: 'proceed', + slices: 'auto', + ignore_unavailable: true, + }); + return resp.updated ?? 0; + }; + + // ----------------------------------------------------------------------- + // getStatus — engine status (cheap count query) + // ----------------------------------------------------------------------- + const getStatus = async (): Promise<{ + isEnabled: boolean; + indexExists: boolean; + totalLeads: number; + lastRun: string | null; + }> => { + let indexExists = false; + let totalLeads = 0; + let lastRun: string | null = null; + + try { + const resp = await esClient.search({ + index: allIndices, + size: 1, + sort: [{ timestamp: { order: 'desc' } }], + _source: ['timestamp'], + track_total_hits: true, + request_cache: true, + ignore_unavailable: true, + }); + + indexExists = true; + totalLeads = + typeof resp.hits.total === 'number' ? resp.hits.total : resp.hits.total?.value ?? 0; + + const latestHit = resp.hits.hits[0]; + if (latestHit?._source) { + lastRun = (latestHit._source as Record).timestamp as string; + } + } catch (e) { + logger.debug(`[LeadGeneration] Status check — indices not available: ${e}`); + } + + // TODO: Wire to Task Manager (#15955) to report actual isEnabled state + return { isEnabled: false, indexExists, totalLeads, lastRun }; + }; + + // ----------------------------------------------------------------------- + // deleteAllLeads — used by disable route for cleanup + // ----------------------------------------------------------------------- + const deleteAllLeads = async (): Promise => { + try { + await esClient.deleteByQuery({ + index: allIndices, + query: { match_all: {} }, + refresh: true, + conflicts: 'proceed', + slices: 'auto', + ignore_unavailable: true, + }); + logger.info(`[LeadGeneration] Deleted all leads from space "${spaceId}"`); + } catch (e) { + logger.warn(`[LeadGeneration] Failed to delete all leads: ${e}`); + } + }; + + return { + createLeads, + findLeads, + getLeadById, + updateLead, + dismissLead, + bulkUpdateLeads, + getStatus, + deleteAllLeads, + }; +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/observation_modules/behavioral_analysis_module.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/observation_modules/behavioral_analysis_module.test.ts index 9321ad1da55f9..edc30dfe68f32 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/observation_modules/behavioral_analysis_module.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/observation_modules/behavioral_analysis_module.test.ts @@ -7,7 +7,7 @@ import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; import { loggingSystemMock } from '@kbn/core/server/mocks'; -import { createBehavioralAnalysisModule } from './alert_analysis_module'; +import { createBehavioralAnalysisModule } from './behavioral_analysis_module'; import type { LeadEntity } from '../types'; const createEntity = (type: string, name: string, email?: string): LeadEntity => ({ diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/observation_modules/alert_analysis_module/config.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/observation_modules/behavioral_analysis_module/config.ts similarity index 100% rename from x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/observation_modules/alert_analysis_module/config.ts rename to x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/observation_modules/behavioral_analysis_module/config.ts diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/observation_modules/alert_analysis_module/data_access.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/observation_modules/behavioral_analysis_module/data_access.ts similarity index 100% rename from x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/observation_modules/alert_analysis_module/data_access.ts rename to x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/observation_modules/behavioral_analysis_module/data_access.ts diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/observation_modules/alert_analysis_module/index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/observation_modules/behavioral_analysis_module/index.ts similarity index 74% rename from x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/observation_modules/alert_analysis_module/index.ts rename to x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/observation_modules/behavioral_analysis_module/index.ts index 3ebb6e7e44fae..e39c1d21bc76b 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/observation_modules/alert_analysis_module/index.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/observation_modules/behavioral_analysis_module/index.ts @@ -5,4 +5,4 @@ * 2.0. */ -export { createBehavioralAnalysisModule, createAlertAnalysisModule } from './module'; +export { createBehavioralAnalysisModule } from './module'; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/observation_modules/alert_analysis_module/module.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/observation_modules/behavioral_analysis_module/module.ts similarity index 92% rename from x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/observation_modules/alert_analysis_module/module.ts rename to x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/observation_modules/behavioral_analysis_module/module.ts index f638eb171a894..4f7d8005f28e1 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/observation_modules/alert_analysis_module/module.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/observation_modules/behavioral_analysis_module/module.ts @@ -48,6 +48,3 @@ export const createBehavioralAnalysisModule = ({ return observations; }, }); - -/** @deprecated Use createBehavioralAnalysisModule. */ -export const createAlertAnalysisModule = createBehavioralAnalysisModule; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/observation_modules/alert_analysis_module/observations.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/observation_modules/behavioral_analysis_module/observations.ts similarity index 100% rename from x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/observation_modules/alert_analysis_module/observations.ts rename to x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/observation_modules/behavioral_analysis_module/observations.ts diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/observation_modules/index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/observation_modules/index.ts index a328f3857254c..e35a514a1a9ae 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/observation_modules/index.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/observation_modules/index.ts @@ -9,5 +9,5 @@ export type { ObservationModule, ObservationModuleConfig, ObservationEntity } fr export { ObservationModuleRegistry } from './observation_module_registry'; export { createRiskScoreModule } from './risk_score_module'; export { createTemporalStateModule } from './temporal_state_module'; -export { createBehavioralAnalysisModule, createAlertAnalysisModule } from './alert_analysis_module'; +export { createBehavioralAnalysisModule } from './behavioral_analysis_module'; export { entityToKey, extractIsPrivileged, groupEntitiesByType, makeObservation } from './utils'; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/observation_modules/temporal_state_module.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/observation_modules/temporal_state_module.ts index a6685bfdfeddf..03415413de308 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/observation_modules/temporal_state_module.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/observation_modules/temporal_state_module.ts @@ -110,9 +110,8 @@ const fetchPrivilegeEscalations = async ( if (hit) { const entityField = hit._source?.entity as Record | undefined; const attrs = entityField?.attributes as { privileged?: boolean } | undefined; - const wasPrivileged = attrs?.privileged === true; - if (!wasPrivileged) { + if (attrs !== undefined && attrs.privileged === false) { escalated.add(`${entityType}:${bucket.key}`); } } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/bulk_update_leads.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/bulk_update_leads.test.ts new file mode 100644 index 0000000000000..64e8cab94ac3f --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/bulk_update_leads.test.ts @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggingSystemMock } from '@kbn/core/server/mocks'; +import { bulkUpdateLeadsRoute } from './bulk_update_leads'; +import { BULK_UPDATE_LEADS_URL } from '../../../../../common/entity_analytics/lead_generation/constants'; +import { + serverMock, + requestContextMock, + requestMock, +} from '../../../detection_engine/routes/__mocks__'; + +const mockBulkUpdateLeads = jest.fn(); +jest.mock('../lead_data_client', () => ({ + createLeadDataClient: () => ({ bulkUpdateLeads: mockBulkUpdateLeads }), +})); + +describe('bulkUpdateLeadsRoute', () => { + let server: ReturnType; + let context: ReturnType; + const logger = loggingSystemMock.createLogger(); + + beforeEach(() => { + jest.clearAllMocks(); + server = serverMock.create(); + const { clients } = requestContextMock.createTools(); + context = requestContextMock.convertContext(requestContextMock.create({ ...clients })); + bulkUpdateLeadsRoute(server.router, logger); + }); + + it('returns 200 with updated count', async () => { + mockBulkUpdateLeads.mockResolvedValueOnce(3); + + const request = requestMock.create({ + method: 'post', + path: BULK_UPDATE_LEADS_URL, + body: { ids: ['a', 'b', 'c'], status: 'dismissed' }, + }); + + const response = await server.inject(request, context); + expect(response.status).toEqual(200); + expect(response.body).toEqual({ updated: 3 }); + }); + + it('passes ids and status through to data client', async () => { + mockBulkUpdateLeads.mockResolvedValueOnce(2); + + const request = requestMock.create({ + method: 'post', + path: BULK_UPDATE_LEADS_URL, + body: { ids: ['x', 'y'], status: 'active' }, + }); + + await server.inject(request, context); + expect(mockBulkUpdateLeads).toHaveBeenCalledWith(['x', 'y'], { status: 'active' }); + }); + + it('returns 500 on unexpected error', async () => { + mockBulkUpdateLeads.mockRejectedValueOnce(new Error('cluster down')); + + const request = requestMock.create({ + method: 'post', + path: BULK_UPDATE_LEADS_URL, + body: { ids: ['a'], status: 'dismissed' }, + }); + + const response = await server.inject(request, context); + expect(response.status).toEqual(500); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/bulk_update_leads.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/bulk_update_leads.ts new file mode 100644 index 0000000000000..d2be836baf1ab --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/bulk_update_leads.ts @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, Logger } from '@kbn/core/server'; +import { buildSiemResponse } from '@kbn/lists-plugin/server/routes/utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; +import { buildRouteValidationWithZod } from '@kbn/zod-helpers/v4'; + +import { BULK_UPDATE_LEADS_URL } from '../../../../../common/entity_analytics/lead_generation/constants'; +import { bulkUpdateLeadsRequestSchema } from '../../../../../common/entity_analytics/lead_generation/types'; +import { API_VERSIONS } from '../../../../../common/entity_analytics/constants'; +import { APP_ID } from '../../../../../common'; +import type { EntityAnalyticsRoutesDeps } from '../../types'; +import { createLeadDataClient } from '../lead_data_client'; +import { withMinimumLicense } from '../../utils/with_minimum_license'; + +export const bulkUpdateLeadsRoute = ( + router: EntityAnalyticsRoutesDeps['router'], + logger: Logger +) => { + router.versioned + .post({ + access: 'internal', + path: BULK_UPDATE_LEADS_URL, + security: { + authz: { + requiredPrivileges: ['securitySolution', `${APP_ID}-entity-analytics`], + }, + }, + }) + .addVersion( + { + version: API_VERSIONS.internal.v1, + validate: { + request: { + body: buildRouteValidationWithZod(bulkUpdateLeadsRequestSchema), + }, + }, + }, + + withMinimumLicense(async (context, request, response): Promise => { + const siemResponse = buildSiemResponse(response); + + try { + const { getSpaceId } = await context.securitySolution; + const spaceId = getSpaceId(); + const esClient = (await context.core).elasticsearch.client.asCurrentUser; + + const { ids, status } = request.body; + const leadDataClient = createLeadDataClient({ esClient, logger, spaceId }); + const updated = await leadDataClient.bulkUpdateLeads(ids, { status }); + + return response.ok({ body: { updated } }); + } catch (e) { + logger.error(`[LeadGeneration] Error bulk-updating leads: ${e}`); + const error = transformError(e); + return siemResponse.error({ + statusCode: error.statusCode, + body: error.message, + }); + } + }) + ); +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/disable_lead_generation.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/disable_lead_generation.test.ts new file mode 100644 index 0000000000000..d21bbf4aa67ae --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/disable_lead_generation.test.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggingSystemMock } from '@kbn/core/server/mocks'; +import { disableLeadGenerationRoute } from './disable_lead_generation'; +import { DISABLE_LEAD_GENERATION_URL } from '../../../../../common/entity_analytics/lead_generation/constants'; +import { + serverMock, + requestContextMock, + requestMock, +} from '../../../detection_engine/routes/__mocks__'; + +describe('disableLeadGenerationRoute', () => { + let server: ReturnType; + let context: ReturnType; + const logger = loggingSystemMock.createLogger(); + + beforeEach(() => { + jest.clearAllMocks(); + server = serverMock.create(); + const { clients } = requestContextMock.createTools(); + context = requestContextMock.convertContext(requestContextMock.create({ ...clients })); + disableLeadGenerationRoute(server.router, logger); + }); + + it('returns 200 with success (placeholder)', async () => { + const request = requestMock.create({ + method: 'post', + path: DISABLE_LEAD_GENERATION_URL, + body: {}, + }); + + const response = await server.inject(request, context); + expect(response.status).toEqual(200); + expect(response.body).toEqual({ success: true }); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/disable_lead_generation.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/disable_lead_generation.ts new file mode 100644 index 0000000000000..62d202bbc11f4 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/disable_lead_generation.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, Logger } from '@kbn/core/server'; +import { buildSiemResponse } from '@kbn/lists-plugin/server/routes/utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; + +import { DISABLE_LEAD_GENERATION_URL } from '../../../../../common/entity_analytics/lead_generation/constants'; +import { API_VERSIONS } from '../../../../../common/entity_analytics/constants'; +import { APP_ID } from '../../../../../common'; +import type { EntityAnalyticsRoutesDeps } from '../../types'; +import { withMinimumLicense } from '../../utils/with_minimum_license'; + +export const disableLeadGenerationRoute = ( + router: EntityAnalyticsRoutesDeps['router'], + logger: Logger +) => { + router.versioned + .post({ + access: 'internal', + path: DISABLE_LEAD_GENERATION_URL, + security: { + authz: { + requiredPrivileges: ['securitySolution', `${APP_ID}-entity-analytics`], + }, + }, + }) + .addVersion( + { + version: API_VERSIONS.internal.v1, + validate: {}, + }, + + withMinimumLicense(async (_context, _request, response): Promise => { + const siemResponse = buildSiemResponse(response); + + try { + // TODO: Wire to Task Manager (#15955) — remove the recurring task + logger.info('[LeadGeneration] Disable requested — Task Manager wiring pending (#15955)'); + + return response.ok({ body: { success: true } }); + } catch (e) { + logger.error(`[LeadGeneration] Error disabling lead generation: ${e}`); + const error = transformError(e); + return siemResponse.error({ + statusCode: error.statusCode, + body: error.message, + }); + } + }) + ); +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/dismiss_lead.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/dismiss_lead.test.ts new file mode 100644 index 0000000000000..b9ec00b4137d5 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/dismiss_lead.test.ts @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggingSystemMock } from '@kbn/core/server/mocks'; +import { dismissLeadRoute } from './dismiss_lead'; +import { DISMISS_LEAD_URL } from '../../../../../common/entity_analytics/lead_generation/constants'; +import { + serverMock, + requestContextMock, + requestMock, +} from '../../../detection_engine/routes/__mocks__'; + +const mockDismissLead = jest.fn(); +jest.mock('../lead_data_client', () => ({ + createLeadDataClient: () => ({ dismissLead: mockDismissLead }), +})); + +describe('dismissLeadRoute', () => { + let server: ReturnType; + let context: ReturnType; + const logger = loggingSystemMock.createLogger(); + + beforeEach(() => { + jest.clearAllMocks(); + server = serverMock.create(); + const { clients } = requestContextMock.createTools(); + context = requestContextMock.convertContext(requestContextMock.create({ ...clients })); + dismissLeadRoute(server.router, logger); + }); + + it('returns 200 with success when lead is dismissed', async () => { + mockDismissLead.mockResolvedValueOnce(true); + + const request = requestMock.create({ + method: 'post', + path: DISMISS_LEAD_URL, + params: { id: 'lead-1' }, + }); + + const response = await server.inject(request, context); + expect(response.status).toEqual(200); + expect(response.body).toEqual({ success: true }); + }); + + it('returns 404 when lead is not found', async () => { + mockDismissLead.mockResolvedValueOnce(false); + + const request = requestMock.create({ + method: 'post', + path: DISMISS_LEAD_URL, + params: { id: 'nonexistent' }, + }); + + const response = await server.inject(request, context); + expect(response.status).toEqual(404); + }); + + it('returns 500 on unexpected error', async () => { + mockDismissLead.mockRejectedValueOnce(new Error('unexpected')); + + const request = requestMock.create({ + method: 'post', + path: DISMISS_LEAD_URL, + params: { id: 'lead-1' }, + }); + + const response = await server.inject(request, context); + expect(response.status).toEqual(500); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/dismiss_lead.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/dismiss_lead.ts new file mode 100644 index 0000000000000..c3a082a5972e8 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/dismiss_lead.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, Logger } from '@kbn/core/server'; +import { buildSiemResponse } from '@kbn/lists-plugin/server/routes/utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; +import { buildRouteValidationWithZod } from '@kbn/zod-helpers/v4'; + +import { DISMISS_LEAD_URL } from '../../../../../common/entity_analytics/lead_generation/constants'; +import { dismissLeadRequestSchema } from '../../../../../common/entity_analytics/lead_generation/types'; +import { API_VERSIONS } from '../../../../../common/entity_analytics/constants'; +import { APP_ID } from '../../../../../common'; +import type { EntityAnalyticsRoutesDeps } from '../../types'; +import { createLeadDataClient } from '../lead_data_client'; +import { withMinimumLicense } from '../../utils/with_minimum_license'; + +export const dismissLeadRoute = (router: EntityAnalyticsRoutesDeps['router'], logger: Logger) => { + router.versioned + .post({ + access: 'internal', + path: DISMISS_LEAD_URL, + security: { + authz: { + requiredPrivileges: ['securitySolution', `${APP_ID}-entity-analytics`], + }, + }, + }) + .addVersion( + { + version: API_VERSIONS.internal.v1, + validate: { + request: { + params: buildRouteValidationWithZod(dismissLeadRequestSchema), + }, + }, + }, + + withMinimumLicense(async (context, request, response): Promise => { + const siemResponse = buildSiemResponse(response); + + try { + const { getSpaceId } = await context.securitySolution; + const spaceId = getSpaceId(); + const esClient = (await context.core).elasticsearch.client.asCurrentUser; + + const leadDataClient = createLeadDataClient({ esClient, logger, spaceId }); + const dismissed = await leadDataClient.dismissLead(request.params.id); + + if (!dismissed) { + return siemResponse.error({ + statusCode: 404, + body: `Lead ${request.params.id} not found`, + }); + } + + return response.ok({ body: { success: true } }); + } catch (e) { + logger.error(`[LeadGeneration] Error dismissing lead: ${e}`); + const error = transformError(e); + return siemResponse.error({ + statusCode: error.statusCode, + body: error.message, + }); + } + }) + ); +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/enable_lead_generation.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/enable_lead_generation.test.ts new file mode 100644 index 0000000000000..4fcab65e33ba1 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/enable_lead_generation.test.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggingSystemMock } from '@kbn/core/server/mocks'; +import { enableLeadGenerationRoute } from './enable_lead_generation'; +import { ENABLE_LEAD_GENERATION_URL } from '../../../../../common/entity_analytics/lead_generation/constants'; +import { + serverMock, + requestContextMock, + requestMock, +} from '../../../detection_engine/routes/__mocks__'; + +describe('enableLeadGenerationRoute', () => { + let server: ReturnType; + let context: ReturnType; + const logger = loggingSystemMock.createLogger(); + + beforeEach(() => { + jest.clearAllMocks(); + server = serverMock.create(); + const { clients } = requestContextMock.createTools(); + context = requestContextMock.convertContext(requestContextMock.create({ ...clients })); + enableLeadGenerationRoute(server.router, logger); + }); + + it('returns 200 with success (placeholder)', async () => { + const request = requestMock.create({ + method: 'post', + path: ENABLE_LEAD_GENERATION_URL, + body: {}, + }); + + const response = await server.inject(request, context); + expect(response.status).toEqual(200); + expect(response.body).toEqual({ success: true }); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/enable_lead_generation.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/enable_lead_generation.ts new file mode 100644 index 0000000000000..8b618fb6bde12 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/enable_lead_generation.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, Logger } from '@kbn/core/server'; +import { buildSiemResponse } from '@kbn/lists-plugin/server/routes/utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; + +import { ENABLE_LEAD_GENERATION_URL } from '../../../../../common/entity_analytics/lead_generation/constants'; +import { API_VERSIONS } from '../../../../../common/entity_analytics/constants'; +import { APP_ID } from '../../../../../common'; +import type { EntityAnalyticsRoutesDeps } from '../../types'; +import { withMinimumLicense } from '../../utils/with_minimum_license'; + +export const enableLeadGenerationRoute = ( + router: EntityAnalyticsRoutesDeps['router'], + logger: Logger +) => { + router.versioned + .post({ + access: 'internal', + path: ENABLE_LEAD_GENERATION_URL, + security: { + authz: { + requiredPrivileges: ['securitySolution', `${APP_ID}-entity-analytics`], + }, + }, + }) + .addVersion( + { + version: API_VERSIONS.internal.v1, + validate: {}, + }, + + withMinimumLicense(async (_context, _request, response): Promise => { + const siemResponse = buildSiemResponse(response); + + try { + // TODO: Wire to Task Manager (#15955) — schedule the 24h recurring task + logger.info('[LeadGeneration] Enable requested — Task Manager wiring pending (#15955)'); + + return response.ok({ body: { success: true } }); + } catch (e) { + logger.error(`[LeadGeneration] Error enabling lead generation: ${e}`); + const error = transformError(e); + return siemResponse.error({ + statusCode: error.statusCode, + body: error.message, + }); + } + }) + ); +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/generate_leads.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/generate_leads.test.ts index b6591510eeccc..e2762231edb63 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/generate_leads.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/generate_leads.test.ts @@ -12,7 +12,8 @@ import { formatLeadForResponse, type FormattedLead, } from '../services/lead_generation_service'; -import { entityRecordToLeadEntity } from '../entity_conversion'; +import { entityRecordToLeadEntity, fetchAllLeadEntities } from '../entity_conversion'; +import type { EntityStoreCRUDClient } from '@kbn/entity-store/server'; import type { Lead } from '../types'; const createMockLead = (overrides: Partial = {}): Lead => ({ @@ -137,7 +138,7 @@ describe('lead generation helpers', () => { expect(esClient.deleteByQuery).toHaveBeenCalledTimes(1); expect(esClient.deleteByQuery).toHaveBeenCalledWith( expect.objectContaining({ - query: { bool: { must_not: [{ term: { executionId: 'exec-1' } }] } }, + query: { bool: { must_not: [{ term: { 'execution_uuid.keyword': 'exec-1' } }] } }, refresh: true, conflicts: 'proceed', ignore_unavailable: true, @@ -215,21 +216,36 @@ describe('lead generation helpers', () => { }); describe('entityRecordToLeadEntity', () => { - it('prefers entity.type and uses entity.name', () => { + it('prefers EngineMetadata.Type over entity.type', () => { const record = { entity: { id: 'euid-1', name: 'alice', type: 'user', + EngineMetadata: { Type: 'host' }, }, } as never; const result = entityRecordToLeadEntity(record); - expect(result.type).toBe('user'); + expect(result.type).toBe('host'); expect(result.name).toBe('alice'); expect(result.record).toBe(record); }); + it('falls back to entity.type when EngineMetadata.Type is absent', () => { + const record = { + entity: { + id: 'euid-1', + name: 'alice', + type: 'user', + }, + } as never; + const result = entityRecordToLeadEntity(record); + + expect(result.type).toBe('user'); + expect(result.name).toBe('alice'); + }); + it('falls back to entity.id for name when entity.name is missing', () => { const record = { entity: { id: 'euid-host-1', type: 'host' }, @@ -248,4 +264,74 @@ describe('lead generation helpers', () => { expect(result.name).toBe('unknown'); }); }); + + describe('fetchAllLeadEntities', () => { + const makeEntity = (name: string) => + ({ entity: { name, type: 'user', id: `euid-${name}` } } as never); + + const createMockCRUDClient = ( + pages: Array<{ + entities: ReturnType[]; + nextSearchAfter?: Array; + }> + ): EntityStoreCRUDClient => { + let callIdx = 0; + return { + listEntities: jest.fn().mockImplementation(() => { + const page = pages[callIdx] ?? { entities: [], nextSearchAfter: undefined }; + callIdx++; + return Promise.resolve(page); + }), + } as unknown as EntityStoreCRUDClient; + }; + + it('returns empty when no entities exist', async () => { + const client = createMockCRUDClient([{ entities: [] }]); + const result = await fetchAllLeadEntities(client); + + expect(result).toEqual([]); + expect(client.listEntities).toHaveBeenCalledTimes(1); + }); + + it('fetches a single page of entities', async () => { + const client = createMockCRUDClient([{ entities: [makeEntity('alice'), makeEntity('bob')] }]); + const result = await fetchAllLeadEntities(client); + + expect(result).toHaveLength(2); + expect(result[0].name).toBe('alice'); + expect(result[1].name).toBe('bob'); + }); + + it('paginates through multiple pages using searchAfter', async () => { + const client = createMockCRUDClient([ + { entities: [makeEntity('alice')], nextSearchAfter: ['cursor-1'] }, + { entities: [makeEntity('bob')], nextSearchAfter: ['cursor-2'] }, + { entities: [makeEntity('carol')] }, + ]); + const result = await fetchAllLeadEntities(client); + + expect(result).toHaveLength(3); + expect(result.map((e) => e.name)).toEqual(['alice', 'bob', 'carol']); + expect(client.listEntities).toHaveBeenCalledTimes(3); + expect(client.listEntities).toHaveBeenNthCalledWith(2, { + size: 1000, + searchAfter: ['cursor-1'], + }); + expect(client.listEntities).toHaveBeenNthCalledWith(3, { + size: 1000, + searchAfter: ['cursor-2'], + }); + }); + + it('converts entities to LeadEntity format', async () => { + const client = createMockCRUDClient([{ entities: [makeEntity('alice')] }]); + const result = await fetchAllLeadEntities(client); + + expect(result[0]).toEqual({ + record: expect.any(Object), + type: 'user', + name: 'alice', + }); + }); + }); }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/generate_leads.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/generate_leads.ts index abdaa6d156eb6..c529f94d8298c 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/generate_leads.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/generate_leads.ts @@ -5,7 +5,8 @@ * 2.0. */ -import type { IKibanaResponse, Logger } from '@kbn/core/server'; +import { v4 as uuidv4 } from 'uuid'; +import type { IKibanaResponse, Logger, StartServicesAccessor } from '@kbn/core/server'; import { buildSiemResponse } from '@kbn/lists-plugin/server/routes/utils'; import { transformError } from '@kbn/securitysolution-es-utils'; import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; @@ -15,13 +16,20 @@ import { GENERATE_LEADS_URL } from '../../../../../common/entity_analytics/lead_ import { API_VERSIONS } from '../../../../../common/entity_analytics/constants'; import { APP_ID } from '../../../../../common'; import type { EntityAnalyticsRoutesDeps } from '../../types'; +import type { StartPlugins } from '../../../../plugin'; import { createLeadGenerationService } from '../services/lead_generation_service'; +import { fetchAllLeadEntities } from '../entity_conversion'; +import { withMinimumLicense } from '../../utils/with_minimum_license'; const GenerateLeadsRequestBody = z.object({ mode: z.enum(['adhoc', 'scheduled']).optional().default('adhoc'), }); -export const generateLeadsRoute = (router: EntityAnalyticsRoutesDeps['router'], logger: Logger) => { +export const generateLeadsRoute = ( + router: EntityAnalyticsRoutesDeps['router'], + logger: Logger, + getStartServices: StartServicesAccessor +) => { router.versioned .post({ access: 'internal', @@ -42,30 +50,45 @@ export const generateLeadsRoute = (router: EntityAnalyticsRoutesDeps['router'], }, }, - async (context, request, response): Promise => { + withMinimumLicense(async (context, request, response): Promise => { const siemResponse = buildSiemResponse(response); try { const securitySolution = await context.securitySolution; const spaceId = securitySolution.getSpaceId(); const esClient = (await context.core).elasticsearch.client.asCurrentUser; + const executionUuid = uuidv4(); + + const [, startPlugins] = await getStartServices(); + const crudClient = startPlugins.entityStore.createCRUDClient(esClient, spaceId); const service = createLeadGenerationService({ esClient, logger, spaceId, - entityStoreDataClient: securitySolution.getEntityStoreDataClient(), + fetchEntities: () => fetchAllLeadEntities(crudClient), riskScoreDataClient: securitySolution.getRiskScoreDataClient(), }); - const result = await service.generate(request.body.mode ?? 'adhoc'); + void (async () => { + try { + await service.generate(request.body.mode ?? 'adhoc', executionUuid); + logger.info( + `[LeadGeneration] Background generation completed (executionUuid=${executionUuid})` + ); + } catch (pipelineError) { + logger.error( + `[LeadGeneration] Background generation failed (executionUuid=${executionUuid}): ${pipelineError}` + ); + } + })(); - return response.ok({ body: result }); + return response.ok({ body: { executionUuid } }); } catch (e) { - logger.error(`[LeadGeneration] Error generating leads: ${e}`); + logger.error(`[LeadGeneration] Error initiating lead generation: ${e}`); const error = transformError(e); return siemResponse.error({ statusCode: error.statusCode, body: error.message }); } - } + }) ); }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/get_lead_by_id.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/get_lead_by_id.test.ts new file mode 100644 index 0000000000000..7658f1db1dfb9 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/get_lead_by_id.test.ts @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggingSystemMock } from '@kbn/core/server/mocks'; +import { getLeadByIdRoute } from './get_lead_by_id'; +import { GET_LEAD_BY_ID_URL } from '../../../../../common/entity_analytics/lead_generation/constants'; +import { + serverMock, + requestContextMock, + requestMock, +} from '../../../detection_engine/routes/__mocks__'; + +const mockGetLeadById = jest.fn(); +jest.mock('../lead_data_client', () => ({ + createLeadDataClient: () => ({ getLeadById: mockGetLeadById }), +})); + +describe('getLeadByIdRoute', () => { + let server: ReturnType; + let context: ReturnType; + const logger = loggingSystemMock.createLogger(); + + beforeEach(() => { + jest.clearAllMocks(); + server = serverMock.create(); + const { clients } = requestContextMock.createTools(); + context = requestContextMock.convertContext(requestContextMock.create({ ...clients })); + getLeadByIdRoute(server.router, logger); + }); + + it('returns 200 with the lead when found', async () => { + const lead = { id: 'lead-1', title: 'Found Lead', priority: 8 }; + mockGetLeadById.mockResolvedValueOnce(lead); + + const request = requestMock.create({ + method: 'get', + path: GET_LEAD_BY_ID_URL, + params: { id: 'lead-1' }, + }); + + const response = await server.inject(request, context); + expect(response.status).toEqual(200); + expect(response.body.id).toBe('lead-1'); + }); + + it('returns 404 when lead is not found', async () => { + mockGetLeadById.mockResolvedValueOnce(null); + + const request = requestMock.create({ + method: 'get', + path: GET_LEAD_BY_ID_URL, + params: { id: 'nonexistent' }, + }); + + const response = await server.inject(request, context); + expect(response.status).toEqual(404); + expect(response.body.message).toContain('nonexistent'); + }); + + it('returns 500 on unexpected error', async () => { + mockGetLeadById.mockRejectedValueOnce(new Error('unexpected')); + + const request = requestMock.create({ + method: 'get', + path: GET_LEAD_BY_ID_URL, + params: { id: 'lead-1' }, + }); + + const response = await server.inject(request, context); + expect(response.status).toEqual(500); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/get_lead_by_id.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/get_lead_by_id.ts new file mode 100644 index 0000000000000..a6d1cf1476f2e --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/get_lead_by_id.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, Logger } from '@kbn/core/server'; +import { buildSiemResponse } from '@kbn/lists-plugin/server/routes/utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; +import { buildRouteValidationWithZod } from '@kbn/zod-helpers/v4'; + +import { GET_LEAD_BY_ID_URL } from '../../../../../common/entity_analytics/lead_generation/constants'; +import { getLeadByIdRequestSchema } from '../../../../../common/entity_analytics/lead_generation/types'; +import { API_VERSIONS } from '../../../../../common/entity_analytics/constants'; +import { APP_ID } from '../../../../../common'; +import type { EntityAnalyticsRoutesDeps } from '../../types'; +import { createLeadDataClient } from '../lead_data_client'; +import { withMinimumLicense } from '../../utils/with_minimum_license'; + +export const getLeadByIdRoute = (router: EntityAnalyticsRoutesDeps['router'], logger: Logger) => { + router.versioned + .get({ + access: 'internal', + path: GET_LEAD_BY_ID_URL, + security: { + authz: { + requiredPrivileges: ['securitySolution', `${APP_ID}-entity-analytics`], + }, + }, + }) + .addVersion( + { + version: API_VERSIONS.internal.v1, + validate: { + request: { + params: buildRouteValidationWithZod(getLeadByIdRequestSchema), + }, + }, + }, + + withMinimumLicense(async (context, request, response): Promise => { + const siemResponse = buildSiemResponse(response); + + try { + const { getSpaceId } = await context.securitySolution; + const spaceId = getSpaceId(); + const esClient = (await context.core).elasticsearch.client.asCurrentUser; + + const leadDataClient = createLeadDataClient({ esClient, logger, spaceId }); + const lead = await leadDataClient.getLeadById(request.params.id); + + if (!lead) { + return siemResponse.error({ + statusCode: 404, + body: `Lead ${request.params.id} not found`, + }); + } + + return response.ok({ body: lead }); + } catch (e) { + logger.error(`[LeadGeneration] Error fetching lead by id: ${e}`); + const error = transformError(e); + return siemResponse.error({ + statusCode: error.statusCode, + body: error.message, + }); + } + }) + ); +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/get_lead_generation_status.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/get_lead_generation_status.test.ts new file mode 100644 index 0000000000000..0d8332fbba467 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/get_lead_generation_status.test.ts @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggingSystemMock } from '@kbn/core/server/mocks'; +import { getLeadGenerationStatusRoute } from './get_lead_generation_status'; +import { LEAD_GENERATION_STATUS_URL } from '../../../../../common/entity_analytics/lead_generation/constants'; +import { + serverMock, + requestContextMock, + requestMock, +} from '../../../detection_engine/routes/__mocks__'; + +const mockGetStatus = jest.fn(); +jest.mock('../lead_data_client', () => ({ + createLeadDataClient: () => ({ getStatus: mockGetStatus }), +})); + +describe('getLeadGenerationStatusRoute', () => { + let server: ReturnType; + let context: ReturnType; + const logger = loggingSystemMock.createLogger(); + + beforeEach(() => { + jest.clearAllMocks(); + server = serverMock.create(); + const { clients } = requestContextMock.createTools(); + context = requestContextMock.convertContext(requestContextMock.create({ ...clients })); + getLeadGenerationStatusRoute(server.router, logger); + }); + + it('returns 200 with engine status', async () => { + mockGetStatus.mockResolvedValueOnce({ + isEnabled: false, + indexExists: true, + totalLeads: 42, + lastRun: '2026-03-10T00:00:00.000Z', + }); + + const request = requestMock.create({ + method: 'get', + path: LEAD_GENERATION_STATUS_URL, + }); + + const response = await server.inject(request, context); + expect(response.status).toEqual(200); + expect(response.body).toEqual({ + isEnabled: false, + indexExists: true, + totalLeads: 42, + lastRun: '2026-03-10T00:00:00.000Z', + }); + }); + + it('returns 500 on error', async () => { + mockGetStatus.mockRejectedValueOnce(new Error('status check failed')); + + const request = requestMock.create({ + method: 'get', + path: LEAD_GENERATION_STATUS_URL, + }); + + const response = await server.inject(request, context); + expect(response.status).toEqual(500); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/get_lead_generation_status.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/get_lead_generation_status.ts new file mode 100644 index 0000000000000..e5beb3dff4604 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/get_lead_generation_status.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, Logger } from '@kbn/core/server'; +import { buildSiemResponse } from '@kbn/lists-plugin/server/routes/utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; + +import { LEAD_GENERATION_STATUS_URL } from '../../../../../common/entity_analytics/lead_generation/constants'; +import { API_VERSIONS } from '../../../../../common/entity_analytics/constants'; +import { APP_ID } from '../../../../../common'; +import type { EntityAnalyticsRoutesDeps } from '../../types'; +import { createLeadDataClient } from '../lead_data_client'; +import { withMinimumLicense } from '../../utils/with_minimum_license'; + +export const getLeadGenerationStatusRoute = ( + router: EntityAnalyticsRoutesDeps['router'], + logger: Logger +) => { + router.versioned + .get({ + access: 'internal', + path: LEAD_GENERATION_STATUS_URL, + security: { + authz: { + requiredPrivileges: ['securitySolution', `${APP_ID}-entity-analytics`], + }, + }, + }) + .addVersion( + { + version: API_VERSIONS.internal.v1, + validate: {}, + }, + + withMinimumLicense(async (context, _request, response): Promise => { + const siemResponse = buildSiemResponse(response); + + try { + const { getSpaceId } = await context.securitySolution; + const spaceId = getSpaceId(); + const esClient = (await context.core).elasticsearch.client.asCurrentUser; + + const leadDataClient = createLeadDataClient({ esClient, logger, spaceId }); + const status = await leadDataClient.getStatus(); + + return response.ok({ body: status }); + } catch (e) { + logger.error(`[LeadGeneration] Error fetching status: ${e}`); + const error = transformError(e); + return siemResponse.error({ + statusCode: error.statusCode, + body: error.message, + }); + } + }) + ); +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/get_leads.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/get_leads.test.ts new file mode 100644 index 0000000000000..360743b1d0157 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/get_leads.test.ts @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggingSystemMock } from '@kbn/core/server/mocks'; +import { getLeadsRoute } from './get_leads'; +import { GET_LEADS_URL } from '../../../../../common/entity_analytics/lead_generation/constants'; +import { + serverMock, + requestContextMock, + requestMock, +} from '../../../detection_engine/routes/__mocks__'; + +const mockFindLeads = jest.fn(); +jest.mock('../lead_data_client', () => ({ + createLeadDataClient: () => ({ findLeads: mockFindLeads }), +})); + +describe('getLeadsRoute', () => { + let server: ReturnType; + let context: ReturnType; + const logger = loggingSystemMock.createLogger(); + + beforeEach(() => { + jest.clearAllMocks(); + server = serverMock.create(); + const { clients } = requestContextMock.createTools(); + context = requestContextMock.convertContext(requestContextMock.create({ ...clients })); + getLeadsRoute(server.router, logger); + }); + + it('returns 200 with paginated leads', async () => { + mockFindLeads.mockResolvedValueOnce({ + leads: [{ id: 'lead-1', title: 'Test' }], + total: 1, + page: 1, + perPage: 20, + }); + + const request = requestMock.create({ + method: 'get', + path: GET_LEADS_URL, + query: { page: '1', perPage: '20' }, + }); + + const response = await server.inject(request, context); + expect(response.status).toEqual(200); + expect(response.body.total).toBe(1); + expect(response.body.leads).toHaveLength(1); + }); + + it('passes query params through to findLeads', async () => { + mockFindLeads.mockResolvedValueOnce({ + leads: [], + total: 0, + page: 2, + perPage: 10, + }); + + const request = requestMock.create({ + method: 'get', + path: GET_LEADS_URL, + query: { + page: '2', + perPage: '10', + sortField: 'timestamp', + sortOrder: 'asc', + status: 'dismissed', + }, + }); + + await server.inject(request, context); + expect(mockFindLeads).toHaveBeenCalledWith( + expect.objectContaining({ + page: 2, + perPage: 10, + sortField: 'timestamp', + sortOrder: 'asc', + status: 'dismissed', + }) + ); + }); + + it('applies default query params when not provided', async () => { + mockFindLeads.mockResolvedValueOnce({ + leads: [], + total: 0, + page: 1, + perPage: 20, + }); + + const request = requestMock.create({ + method: 'get', + path: GET_LEADS_URL, + query: {}, + }); + + await server.inject(request, context); + expect(mockFindLeads).toHaveBeenCalledWith( + expect.objectContaining({ + page: 1, + perPage: 20, + sortField: 'priority', + sortOrder: 'desc', + }) + ); + }); + + it('returns 500 when data client throws', async () => { + mockFindLeads.mockRejectedValueOnce(new Error('ES failure')); + + const request = requestMock.create({ + method: 'get', + path: GET_LEADS_URL, + query: {}, + }); + + const response = await server.inject(request, context); + expect(response.status).toEqual(500); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/get_leads.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/get_leads.ts index 8fc61843a0631..ace2714b4b485 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/get_leads.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/get_leads.ts @@ -8,24 +8,15 @@ import type { IKibanaResponse, Logger } from '@kbn/core/server'; import { buildSiemResponse } from '@kbn/lists-plugin/server/routes/utils'; import { transformError } from '@kbn/securitysolution-es-utils'; -import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; -import { z } from '@kbn/zod'; +import { buildRouteValidationWithZod } from '@kbn/zod-helpers/v4'; -import { - GET_LEADS_URL, - getLeadsIndexName, -} from '../../../../../common/entity_analytics/lead_generation/constants'; +import { GET_LEADS_URL } from '../../../../../common/entity_analytics/lead_generation/constants'; +import { findLeadsRequestSchema } from '../../../../../common/entity_analytics/lead_generation/types'; import { API_VERSIONS } from '../../../../../common/entity_analytics/constants'; import { APP_ID } from '../../../../../common'; import type { EntityAnalyticsRoutesDeps } from '../../types'; -import { computeStaleness } from '../types'; - -const MAX_PAGE_SIZE = 200; - -const GetLeadsQueryParams = z.object({ - size: z.coerce.number().int().min(1).max(MAX_PAGE_SIZE).optional().default(50), - from: z.coerce.number().int().min(0).optional().default(0), -}); +import { createLeadDataClient } from '../lead_data_client'; +import { withMinimumLicense } from '../../utils/with_minimum_license'; export const getLeadsRoute = (router: EntityAnalyticsRoutesDeps['router'], logger: Logger) => { router.versioned @@ -43,12 +34,12 @@ export const getLeadsRoute = (router: EntityAnalyticsRoutesDeps['router'], logge version: API_VERSIONS.internal.v1, validate: { request: { - query: buildRouteValidationWithZod(GetLeadsQueryParams), + query: buildRouteValidationWithZod(findLeadsRequestSchema), }, }, }, - async (context, request, response): Promise => { + withMinimumLicense(async (context, request, response): Promise => { const siemResponse = buildSiemResponse(response); try { @@ -56,49 +47,10 @@ export const getLeadsRoute = (router: EntityAnalyticsRoutesDeps['router'], logge const spaceId = getSpaceId(); const esClient = (await context.core).elasticsearch.client.asCurrentUser; - const { size, from } = request.query; - const adhocIndex = getLeadsIndexName(spaceId, 'adhoc'); - const scheduledIndex = getLeadsIndexName(spaceId, 'scheduled'); - - let leads: unknown[] = []; - let total = 0; - - try { - const resp = await esClient.search({ - index: `${adhocIndex},${scheduledIndex}`, - size, - from, - track_total_hits: true, - sort: [{ priority: { order: 'desc' } }, { timestamp: { order: 'desc' } }], - query: { match_all: {} }, - ignore_unavailable: true, - }); + const leadDataClient = createLeadDataClient({ esClient, logger, spaceId }); + const result = await leadDataClient.findLeads(request.query); - total = - typeof resp.hits.total === 'number' ? resp.hits.total : resp.hits.total?.value ?? 0; - - leads = resp.hits.hits - .map((hit) => hit._source) - .filter((doc): doc is Record => doc != null) - .map((doc) => ({ - ...doc, - staleness: - typeof doc.timestamp === 'string' - ? computeStaleness(new Date(doc.timestamp), new Date()) - : doc.staleness, - })); - } catch (searchError) { - logger.debug(`[LeadGeneration] Leads indices not available yet: ${searchError}`); - } - - return response.ok({ - body: { - leads, - total, - size, - from, - }, - }); + return response.ok({ body: result }); } catch (e) { logger.error(`[LeadGeneration] Error reading leads: ${e}`); const error = transformError(e); @@ -107,6 +59,6 @@ export const getLeadsRoute = (router: EntityAnalyticsRoutesDeps['router'], logge body: error.message, }); } - } + }) ); }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/register_lead_generation_routes.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/register_lead_generation_routes.ts index 633e8e0d7c8ab..101909dca781f 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/register_lead_generation_routes.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/routes/register_lead_generation_routes.ts @@ -8,8 +8,24 @@ import type { EntityAnalyticsRoutesDeps } from '../../types'; import { generateLeadsRoute } from './generate_leads'; import { getLeadsRoute } from './get_leads'; +import { getLeadByIdRoute } from './get_lead_by_id'; +import { getLeadGenerationStatusRoute } from './get_lead_generation_status'; +import { dismissLeadRoute } from './dismiss_lead'; +import { bulkUpdateLeadsRoute } from './bulk_update_leads'; +import { enableLeadGenerationRoute } from './enable_lead_generation'; +import { disableLeadGenerationRoute } from './disable_lead_generation'; -export const registerLeadGenerationRoutes = ({ router, logger }: EntityAnalyticsRoutesDeps) => { - generateLeadsRoute(router, logger); +export const registerLeadGenerationRoutes = ({ + router, + logger, + getStartServices, +}: EntityAnalyticsRoutesDeps) => { + generateLeadsRoute(router, logger, getStartServices); getLeadsRoute(router, logger); + getLeadByIdRoute(router, logger); + getLeadGenerationStatusRoute(router, logger); + dismissLeadRoute(router, logger); + bulkUpdateLeadsRoute(router, logger); + enableLeadGenerationRoute(router, logger); + disableLeadGenerationRoute(router, logger); }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/services/lead_generation_service.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/services/lead_generation_service.ts index aef2758d058ab..be1193812e0a0 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/services/lead_generation_service.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/services/lead_generation_service.ts @@ -5,9 +5,7 @@ * 2.0. */ -import { v4 as uuidv4 } from 'uuid'; import type { ElasticsearchClient, Logger } from '@kbn/core/server'; -import type { EntityStoreDataClient } from '../../entity_store/entity_store_data_client'; import type { RiskScoreDataClient } from '../../risk_score/risk_score_data_client'; import type { LeadGenerationMode } from '../../../../../common/entity_analytics/lead_generation/constants'; import { getLeadsIndexName } from '../../../../../common/entity_analytics/lead_generation/constants'; @@ -15,43 +13,14 @@ import { getAlertsIndex } from '../../../../../common/entity_analytics/utils'; import { createLeadGenerationEngine } from '../engine/lead_generation_engine'; import { createRiskScoreModule } from '../observation_modules/risk_score_module'; import { createTemporalStateModule } from '../observation_modules/temporal_state_module'; -import { createBehavioralAnalysisModule } from '../observation_modules/alert_analysis_module'; -import { entityRecordToLeadEntity } from '../entity_conversion'; -import type { Lead } from '../types'; - -const ENTITY_SOURCE_FIELDS = [ - '@timestamp', - 'entity.name', - 'entity.type', - 'entity.EngineMetadata.Type', - 'entity.id', - 'entity.risk', - 'entity.attributes', - 'entity.behaviors', - 'entity.lifecycle', - 'entity.relationships', - 'user.name', - 'user.id', - 'user.email', - 'user.full_name', - 'user.roles', - 'user.domain', - 'host.name', - 'host.hostname', - 'host.id', - 'host.ip', - 'host.os.name', - 'host.type', - 'host.domain', - 'host.architecture', - 'asset.criticality', -]; +import { createBehavioralAnalysisModule } from '../observation_modules/behavioral_analysis_module'; +import type { Lead, LeadEntity } from '../types'; interface LeadGenerationServiceDeps { readonly esClient: ElasticsearchClient; readonly logger: Logger; readonly spaceId: string; - readonly entityStoreDataClient: EntityStoreDataClient; + readonly fetchEntities: () => Promise; readonly riskScoreDataClient: RiskScoreDataClient; } @@ -64,28 +33,24 @@ export const createLeadGenerationService = ({ esClient, logger, spaceId, - entityStoreDataClient, + fetchEntities, riskScoreDataClient, }: LeadGenerationServiceDeps) => ({ - async generate(mode: LeadGenerationMode): Promise { + async generate(mode: LeadGenerationMode, executionId: string): Promise { const routeStart = Date.now(); const fetchStart = Date.now(); - const entityRecords = await entityStoreDataClient.fetchAllUnifiedLatestEntities({ - sourceFields: ENTITY_SOURCE_FIELDS, - }); + const leadEntities = await fetchEntities(); logger.debug( `[LeadGeneration] Entity fetch: ${Date.now() - fetchStart}ms (${ - entityRecords.length - } records)` + leadEntities.length + } entities)` ); - if (entityRecords.length === 0) { + if (leadEntities.length === 0) { return { leads: [], total: 0 }; } - const leadEntities = entityRecords.map(entityRecordToLeadEntity); - const engine = createLeadGenerationEngine({ logger }); engine.registerModule(createRiskScoreModule({ riskScoreDataClient, logger })); engine.registerModule(createTemporalStateModule({ esClient, logger, spaceId })); @@ -103,7 +68,6 @@ export const createLeadGenerationService = ({ `[LeadGeneration] Engine pipeline: ${Date.now() - generateStart}ms (${leads.length} leads)` ); - const executionId = uuidv4(); const formattedLeads = leads.map((lead) => formatLeadForResponse(lead, executionId)); const persistStart = Date.now(); @@ -148,10 +112,36 @@ export const formatLeadForResponse = (lead: Lead, executionId: string) => ({ export type FormattedLead = ReturnType; +const toSnakeCaseDoc = (lead: FormattedLead, sourceType: LeadGenerationMode) => ({ + id: lead.id, + title: lead.title, + byline: lead.byline, + description: lead.description, + entities: lead.entities, + tags: lead.tags, + priority: lead.priority, + chat_recommendations: lead.chatRecommendations, + timestamp: lead.timestamp, + staleness: lead.staleness, + status: 'active', + observations: lead.observations.map((obs) => ({ + entity_id: obs.entityId, + module_id: obs.moduleId, + type: obs.type, + score: obs.score, + severity: obs.severity, + confidence: obs.confidence, + description: obs.description, + metadata: obs.metadata, + })), + execution_uuid: lead.executionId, + source_type: sourceType, +}); + /** * Gap-free replace pattern: * 1. Bulk upsert new leads (visible immediately). - * 2. Delete any docs whose executionId differs (stale from previous runs). + * 2. Delete any docs whose execution_uuid differs (stale from previous runs). */ export const persistLeads = async ( esClient: ElasticsearchClient, @@ -167,7 +157,7 @@ export const persistLeads = async ( if (leads.length > 0) { const bulkBody = leads.flatMap((lead) => [ { index: { _index: indexName, _id: lead.id } }, - lead, + toSnakeCaseDoc(lead, mode), ]); await esClient.bulk({ body: bulkBody, refresh: 'wait_for' }); pLogger.debug(`[LeadGeneration] Persisted ${leads.length} leads to "${indexName}"`); @@ -175,7 +165,7 @@ export const persistLeads = async ( await esClient.deleteByQuery({ index: indexName, - query: { bool: { must_not: [{ term: { executionId } }] } }, + query: { bool: { must_not: [{ term: { 'execution_uuid.keyword': executionId } }] } }, refresh: true, conflicts: 'proceed', ignore_unavailable: true, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/types.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/types.ts index 44fcf05bbd9f6..9f1f5a650f09a 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/types.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/lead_generation/types.ts @@ -111,9 +111,18 @@ export interface LeadGenerationEngineConfig { readonly minObservations: number; /** Maximum number of leads to return */ readonly maxLeads: number; + /** Bonus multiplier applied when multiple observations come from the same module (0.0–1.0) */ + readonly corroborationBonus: number; + /** Bonus multiplier applied when observations come from multiple modules (0.0–1.0) */ + readonly diversityBonus: number; + /** Raw score ceiling used when normalizing to the 1–10 priority scale */ + readonly normalizationCeiling: number; } export const DEFAULT_ENGINE_CONFIG: LeadGenerationEngineConfig = { minObservations: 1, maxLeads: 10, + corroborationBonus: 0.15, + diversityBonus: 0.1, + normalizationCeiling: 100, };