From facb4977438de9045ab5c74aff7b5d7a4f2968b7 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Wed, 3 Jun 2020 17:13:48 -0400 Subject: [PATCH 01/14] Adding alerts route --- .../siem/common/endpoint/schema/resolver.ts | 14 ++++ x-pack/plugins/siem/common/endpoint/types.ts | 10 +++ .../server/endpoint/routes/resolver/alerts.ts | 37 ++++++++++ .../routes/resolver/queries/alerts.ts | 73 +++++++++++++++++++ .../server/endpoint/routes/resolver/tree.ts | 5 +- .../endpoint/routes/resolver/utils/fetch.ts | 33 ++++++++- .../endpoint/routes/resolver/utils/node.ts | 20 +++++ .../endpoint/routes/resolver/utils/tree.ts | 17 +++++ 8 files changed, 207 insertions(+), 2 deletions(-) create mode 100644 x-pack/plugins/siem/server/endpoint/routes/resolver/alerts.ts create mode 100644 x-pack/plugins/siem/server/endpoint/routes/resolver/queries/alerts.ts diff --git a/x-pack/plugins/siem/common/endpoint/schema/resolver.ts b/x-pack/plugins/siem/common/endpoint/schema/resolver.ts index 8d60a532aa67c..f5c3fd519c9c5 100644 --- a/x-pack/plugins/siem/common/endpoint/schema/resolver.ts +++ b/x-pack/plugins/siem/common/endpoint/schema/resolver.ts @@ -16,7 +16,9 @@ export const validateTree = { generations: schema.number({ defaultValue: 3, min: 0, max: 3 }), ancestors: schema.number({ defaultValue: 3, min: 0, max: 5 }), events: schema.number({ defaultValue: 100, min: 0, max: 1000 }), + alerts: schema.number({ defaultValue: 100, min: 0, max: 1000 }), afterEvent: schema.maybe(schema.string()), + afterAlert: schema.maybe(schema.string()), afterChild: schema.maybe(schema.string()), legacyEndpointID: schema.maybe(schema.string()), }), @@ -34,6 +36,18 @@ export const validateEvents = { }), }; +/** + * Used to validate GET requests for alerts for a specific process. + */ +export const validateAlerts = { + params: schema.object({ id: schema.string() }), + query: schema.object({ + alerts: schema.number({ defaultValue: 100, min: 1, max: 1000 }), + afterAlert: schema.maybe(schema.string()), + legacyEndpointID: schema.maybe(schema.string()), + }), +}; + /** * Used to validate GET requests for the ancestors of a process event. */ diff --git a/x-pack/plugins/siem/common/endpoint/types.ts b/x-pack/plugins/siem/common/endpoint/types.ts index 45b5cf2526e12..65e1d0203dfce 100644 --- a/x-pack/plugins/siem/common/endpoint/types.ts +++ b/x-pack/plugins/siem/common/endpoint/types.ts @@ -91,6 +91,7 @@ export interface ResolverTree { entityID: string; children: ResolverChildren; relatedEvents: Omit; + relatedAlerts: Omit; ancestry: ResolverAncestry; lifecycle: ResolverEvent[]; stats: ResolverNodeStats; @@ -132,6 +133,15 @@ export interface ResolverRelatedEvents { nextEvent: string | null; } +/** + * Response structure for the alerts route. + */ +export interface ResolverAlerts { + entityID: string; + alerts: ResolverEvent[]; + nextAlert: string | null; +} + /** * Returned by the server via /api/endpoint/metadata */ diff --git a/x-pack/plugins/siem/server/endpoint/routes/resolver/alerts.ts b/x-pack/plugins/siem/server/endpoint/routes/resolver/alerts.ts new file mode 100644 index 0000000000000..04171dd8137e5 --- /dev/null +++ b/x-pack/plugins/siem/server/endpoint/routes/resolver/alerts.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { TypeOf } from '@kbn/config-schema'; +import { RequestHandler, Logger } from 'kibana/server'; +import { validateAlerts } from '../../../../common/endpoint/schema/resolver'; +import { Fetcher } from './utils/fetch'; +import { EndpointAppContext } from '../../types'; + +export function handleAlerts( + log: Logger, + endpointAppContext: EndpointAppContext +): RequestHandler, TypeOf> { + return async (context, req, res) => { + const { + params: { id }, + query: { alerts, afterAlert, legacyEndpointID: endpointID }, + } = req; + try { + const indexRetriever = endpointAppContext.service.getIndexPatternRetriever(); + const client = context.core.elasticsearch.legacy.client; + const indexPattern = await indexRetriever.getEventIndexPattern(context); + + const fetcher = new Fetcher(client, id, indexPattern, endpointID); + + return res.ok({ + body: await fetcher.alerts(alerts, afterAlert), + }); + } catch (err) { + log.warn(err); + return res.internalError({ body: err }); + } + }; +} diff --git a/x-pack/plugins/siem/server/endpoint/routes/resolver/queries/alerts.ts b/x-pack/plugins/siem/server/endpoint/routes/resolver/queries/alerts.ts new file mode 100644 index 0000000000000..1d316eca40ed3 --- /dev/null +++ b/x-pack/plugins/siem/server/endpoint/routes/resolver/queries/alerts.ts @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { SearchResponse } from 'elasticsearch'; +import { ResolverEvent } from '../../../../../common/endpoint/types'; +import { ResolverQuery } from './base'; +import { PaginationBuilder, PaginatedResults } from '../utils/pagination'; +import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/public'; + +/** + * Builds a query for retrieving alerts for a node. + */ +export class AlertsQuery extends ResolverQuery { + constructor( + private readonly pagination: PaginationBuilder, + indexPattern: string, + endpointID?: string + ) { + super(indexPattern, endpointID); + } + + protected legacyQuery(endpointID: string, uniquePIDs: string[]): JsonObject { + return { + query: { + bool: { + filter: [ + { + terms: { 'endgame.unique_pid': uniquePIDs }, + }, + { + term: { 'agent.id': endpointID }, + }, + { + term: { 'event.kind': 'alert' }, + }, + ], + }, + }, + ...this.pagination.buildQueryFields( + uniquePIDs.length, + 'endgame.serial_event_id', + 'endgame.unique_pid' + ), + }; + } + + protected query(entityIDs: string[]): JsonObject { + return { + query: { + bool: { + filter: [ + { + terms: { 'process.entity_id': entityIDs }, + }, + { + term: { 'event.kind': 'alert' }, + }, + ], + }, + }, + ...this.pagination.buildQueryFields(entityIDs.length, 'event.id', 'process.entity_id'), + }; + } + + formatResponse(response: SearchResponse): PaginatedResults { + return { + results: ResolverQuery.getResults(response), + totals: PaginationBuilder.getTotals(response.aggregations), + }; + } +} diff --git a/x-pack/plugins/siem/server/endpoint/routes/resolver/tree.ts b/x-pack/plugins/siem/server/endpoint/routes/resolver/tree.ts index d750fb256a4a0..be7f75dfd52fa 100644 --- a/x-pack/plugins/siem/server/endpoint/routes/resolver/tree.ts +++ b/x-pack/plugins/siem/server/endpoint/routes/resolver/tree.ts @@ -23,6 +23,8 @@ export function handleTree( generations, ancestors, events, + alerts, + afterAlert, afterEvent, afterChild, legacyEndpointID: endpointID, @@ -35,10 +37,11 @@ export function handleTree( const fetcher = new Fetcher(client, id, indexPattern, endpointID); - const [childrenNodes, ancestry, relatedEvents] = await Promise.all([ + const [childrenNodes, ancestry, relatedEvents, alertsResp] = await Promise.all([ fetcher.children(children, generations, afterChild), fetcher.ancestors(ancestors), fetcher.events(events, afterEvent), + fetcher.alerts(alerts, afterAlert), ]); const tree = new Tree(id, { ancestry, children: childrenNodes, relatedEvents }); diff --git a/x-pack/plugins/siem/server/endpoint/routes/resolver/utils/fetch.ts b/x-pack/plugins/siem/server/endpoint/routes/resolver/utils/fetch.ts index 4b14c555d49b7..f488ed9c8ac53 100644 --- a/x-pack/plugins/siem/server/endpoint/routes/resolver/utils/fetch.ts +++ b/x-pack/plugins/siem/server/endpoint/routes/resolver/utils/fetch.ts @@ -9,6 +9,7 @@ import { ResolverChildren, ResolverRelatedEvents, ResolverAncestry, + ResolverAlerts, } from '../../../../../common/endpoint/types'; import { entityId, parentEntityId } from '../../../../../common/endpoint/models/event'; import { PaginationBuilder } from './pagination'; @@ -17,8 +18,9 @@ import { LifecycleQuery } from '../queries/lifecycle'; import { ChildrenQuery } from '../queries/children'; import { EventsQuery } from '../queries/events'; import { StatsQuery } from '../queries/stats'; -import { createAncestry, createRelatedEvents, createLifecycle } from './node'; +import { createAncestry, createRelatedEvents, createLifecycle, createAlerts } from './node'; import { ChildrenNodesHelper } from './children_helper'; +import { AlertsQuery } from '../queries/alerts'; /** * Handles retrieving nodes of a resolver tree. @@ -80,6 +82,16 @@ export class Fetcher { return this.doEvents(limit, after); } + /** + * Retrieves the alerts for the origin node. + * + * @param limit the upper bound number of alerts to return + * @param after a cursor to use as the starting point for retrieving alerts + */ + public async alerts(limit: number, after?: string): Promise { + return this.doAlerts(limit, after); + } + /** * Enriches a resolver tree with statistics for how many related events and alerts exist for each node in the tree. * @@ -138,6 +150,25 @@ export class Fetcher { ); } + private async doAlerts(limit: number, after?: string) { + const query = new AlertsQuery( + PaginationBuilder.createBuilder(limit, after), + this.indexPattern, + this.endpointID + ); + + const { totals, results } = await query.search(this.client, this.id); + if (results.length === 0) { + // return an empty set of results + return createAlerts(this.id); + } + if (!totals[this.id]) { + throw new Error(`Could not find the totals for related events entity_id: ${this.id}`); + } + + return createAlerts(this.id, results, PaginationBuilder.buildCursor(totals[this.id], results)); + } + private async doChildren( cache: ChildrenNodesHelper, ids: string[], diff --git a/x-pack/plugins/siem/server/endpoint/routes/resolver/utils/node.ts b/x-pack/plugins/siem/server/endpoint/routes/resolver/utils/node.ts index ae078b5368a96..4768051a35cfc 100644 --- a/x-pack/plugins/siem/server/endpoint/routes/resolver/utils/node.ts +++ b/x-pack/plugins/siem/server/endpoint/routes/resolver/utils/node.ts @@ -11,6 +11,7 @@ import { ResolverRelatedEvents, ResolverTree, ChildNode, + ResolverAlerts, } from '../../../../../common/endpoint/types'; /** @@ -28,6 +29,21 @@ export function createRelatedEvents( return { entityID, events, nextEvent }; } +/** + * Creates an alert object that the alerts handler would return + * + * @param entityID the entity_id for these related events + * @param alerts array of alerts + * @param nextAlert the cursor to retrieve the next alert + */ +export function createAlerts( + entityID: string, + alerts: ResolverEvent[] = [], + nextAlert: string | null = null +): ResolverAlerts { + return { entityID, alerts, nextAlert }; +} + /** * Creates a child node that would be used in the child handler response * @@ -74,6 +90,10 @@ export function createTree(entityID: string): ResolverTree { events: [], nextEvent: null, }, + relatedAlerts: { + alerts: [], + nextAlert: null, + }, lifecycle: [], ancestry: { ancestors: [], diff --git a/x-pack/plugins/siem/server/endpoint/routes/resolver/utils/tree.ts b/x-pack/plugins/siem/server/endpoint/routes/resolver/utils/tree.ts index 048964068324b..73382c72359f0 100644 --- a/x-pack/plugins/siem/server/endpoint/routes/resolver/utils/tree.ts +++ b/x-pack/plugins/siem/server/endpoint/routes/resolver/utils/tree.ts @@ -12,6 +12,7 @@ import { ResolverAncestry, ResolverTree, ResolverChildren, + ResolverAlerts, } from '../../../../../common/endpoint/types'; import { createTree } from './node'; @@ -25,6 +26,7 @@ export interface Options { relatedEvents?: ResolverRelatedEvents; ancestry?: ResolverAncestry; children?: ResolverChildren; + alerts?: ResolverAlerts; } /** @@ -74,6 +76,7 @@ export class Tree { this.addRelatedEvents(options.relatedEvents); this.addAncestors(options.ancestry); this.addChildren(options.children); + this.addAlerts(options.alerts); } /** @@ -108,6 +111,20 @@ export class Tree { this.tree.relatedEvents.nextEvent = relatedEventsInfo.nextEvent; } + /** + * Add alerts for the tree's origin node. Alerts cannot be added for other nodes. + * + * @param alertInfo is the alerts and pagination information to add to the tree. + */ + private addAlerts(alertInfo: ResolverAlerts | undefined) { + if (!alertInfo) { + return; + } + + this.tree.relatedAlerts.alerts = alertInfo.alerts; + this.tree.relatedAlerts.nextAlert = alertInfo.nextAlert; + } + /** * Add ancestors to the tree. * From 2e3374318406513f4dfe622dfe5b187681141271 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Fri, 5 Jun 2020 14:22:45 -0400 Subject: [PATCH 02/14] Adding related alerts generator changes, tests, and script updates --- .../common/endpoint/generate_data.test.ts | 78 ++++++++++---- .../common/endpoint/generate_data.ts | 101 +++++++++++++----- .../common/endpoint/types.ts | 4 +- .../scripts/endpoint/README.md | 6 +- .../scripts/endpoint/resolver_generator.ts | 9 +- .../server/endpoint/routes/resolver.ts | 11 ++ .../server/endpoint/routes/resolver/tree.ts | 9 +- .../endpoint/routes/resolver/utils/fetch.ts | 14 ++- .../endpoint/routes/resolver/utils/node.ts | 6 +- .../endpoint/routes/resolver/utils/tree.ts | 8 +- .../api_integration/apis/endpoint/resolver.ts | 63 ++++++++++- 11 files changed, 243 insertions(+), 66 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts index 6c8c5e3f51808..ba4f2251564e8 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts @@ -100,11 +100,40 @@ describe('data generator', () => { expect(processEvent.process.name).not.toBeNull(); }); + describe('creates an origin alert when no related alerts are requested', () => { + let tree: Tree; + beforeEach(() => { + tree = generator.generateTree({ + alwaysGenMaxChildrenPerNode: true, + ancestors: 3, + children: 3, + generations: 3, + percentTerminated: 100, + percentWithRelated: 100, + relatedEvents: 0, + relatedAlerts: 0, + }); + }); + + it('creates an alert for the origin node but no other nodes', () => { + for (const node of tree.ancestry.values()) { + expect(node.relatedAlerts.length).toEqual(0); + } + + for (const node of tree.children.values()) { + expect(node.relatedAlerts.length).toEqual(0); + } + + expect(tree.origin.relatedAlerts.length).toEqual(1); + }); + }); + describe('creates a resolver tree structure', () => { let tree: Tree; const ancestors = 3; const childrenPerNode = 3; const generations = 3; + const relatedAlerts = 4; beforeEach(() => { tree = generator.generateTree({ alwaysGenMaxChildrenPerNode: true, @@ -118,14 +147,16 @@ describe('data generator', () => { { category: RelatedEventCategory.File, count: 2 }, { category: RelatedEventCategory.Network, count: 1 }, ], + relatedAlerts, }); }); const eventInNode = (event: Event, node: TreeNode) => { const inLifecycle = node.lifecycle.includes(event); const inRelated = node.relatedEvents.includes(event); + const inRelatedAlerts = node.relatedAlerts.includes(event); - return (inRelated || inLifecycle) && event.process.entity_id === node.id; + return (inRelated || inRelatedAlerts || inLifecycle) && event.process.entity_id === node.id; }; it('has the right related events for each node', () => { @@ -158,6 +189,18 @@ describe('data generator', () => { checkRelatedEvents(tree.origin); }); + it('has the right number of related alerts for each node', () => { + for (const node of tree.ancestry.values()) { + expect(node.relatedAlerts.length).toEqual(relatedAlerts); + } + + for (const node of tree.children.values()) { + expect(node.relatedAlerts.length).toEqual(relatedAlerts); + } + + expect(tree.origin.relatedAlerts.length).toEqual(relatedAlerts); + }); + it('has the right number of ancestors', () => { expect(tree.ancestry.size).toEqual(ancestors); }); @@ -187,33 +230,28 @@ describe('data generator', () => { expect(tree.allEvents.length).toBeGreaterThan(0); tree.allEvents.forEach((event) => { - if (event.event.kind === 'alert') { - expect(event).toEqual(tree.alertEvent); - } else { - const ancestor = tree.ancestry.get(event.process.entity_id); - if (ancestor) { - expect(eventInNode(event, ancestor)).toBeTruthy(); - return; - } - - const children = tree.children.get(event.process.entity_id); - if (children) { - expect(eventInNode(event, children)).toBeTruthy(); - return; - } + const ancestor = tree.ancestry.get(event.process.entity_id); + if (ancestor) { + expect(eventInNode(event, ancestor)).toBeTruthy(); + return; + } - expect(eventInNode(event, tree.origin)).toBeTruthy(); + const children = tree.children.get(event.process.entity_id); + if (children) { + expect(eventInNode(event, children)).toBeTruthy(); + return; } + + expect(eventInNode(event, tree.origin)).toBeTruthy(); }); }); const nodeEventCount = (node: TreeNode) => { - return node.lifecycle.length + node.relatedEvents.length; + return node.lifecycle.length + node.relatedEvents.length + node.relatedAlerts.length; }; it('has the correct number of total events', () => { - // starts at 1 because the alert is in the allEvents array - let total = 1; + let total = 0; for (const node of tree.ancestry.values()) { total += nodeEventCount(node); } @@ -232,7 +270,7 @@ describe('data generator', () => { let events: Event[]; beforeEach(() => { - events = generator.createAlertEventAncestry(3, 0, 0, 0); + events = generator.createAlertEventAncestry(3, 0, 0, 0, 0); }); it('with n-1 process events', () => { diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts index b17a5aa28ac6a..d17432ebfe7e7 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts @@ -212,6 +212,7 @@ export interface TreeNode { id: string; lifecycle: Event[]; relatedEvents: Event[]; + relatedAlerts: Event[]; } /** @@ -227,7 +228,6 @@ export interface Tree { */ ancestry: Map; origin: TreeNode; - alertEvent: Event; /** * All events from children, ancestry, origin, and the alert in a single array */ @@ -241,7 +241,8 @@ export interface TreeOptions { ancestors?: number; generations?: number; children?: number; - relatedEvents?: RelatedEventInfo[]; + relatedEvents?: RelatedEventInfo[] | number; + relatedAlerts?: number; percentWithRelated?: number; percentTerminated?: number; alwaysGenMaxChildrenPerNode?: boolean; @@ -465,30 +466,33 @@ export class EndpointDocGenerator { // and add the event to the right array. let node = nodeMap.get(nodeId); if (!node) { - node = { id: nodeId, lifecycle: [], relatedEvents: [] }; + node = { id: nodeId, lifecycle: [], relatedEvents: [], relatedAlerts: [] }; } // place the event in the right array depending on its category - if (event.event.category === 'process') { - node.lifecycle.push(event); - } else { - node.relatedEvents.push(event); + if (event.event.kind === 'event') { + if (event.event.category === 'process') { + node.lifecycle.push(event); + } else { + node.relatedEvents.push(event); + } + } else if (event.event.kind === 'alert') { + node.relatedAlerts.push(event); } + return nodeMap.set(nodeId, node); }; const ancestry = this.createAlertEventAncestry( options.ancestors, options.relatedEvents, + options.relatedAlerts, options.percentWithRelated, options.percentTerminated ); - // create a mapping of entity_id -> lifecycle and related events - // slice gets everything but the last item which is an alert - const ancestryNodes: Map = ancestry - .slice(0, -1) - .reduce(addEventToMap, new Map()); + // create a mapping of entity_id -> {lifecycle, related events, and related alerts} + const ancestryNodes: Map = ancestry.reduce(addEventToMap, new Map()); const alert = ancestry[ancestry.length - 1]; const origin = ancestryNodes.get(alert.process.entity_id); @@ -505,6 +509,7 @@ export class EndpointDocGenerator { options.generations, options.children, options.relatedEvents, + options.relatedAlerts, options.percentWithRelated, options.percentTerminated, options.alwaysGenMaxChildrenPerNode @@ -516,7 +521,6 @@ export class EndpointDocGenerator { return { children: childrenNodes, ancestry: ancestryNodes, - alertEvent: alert, allEvents: [...ancestry, ...children], origin, }; @@ -530,6 +534,7 @@ export class EndpointDocGenerator { * @param childGenerations - number of child generations to create relative to the alert * @param maxChildrenPerNode - maximum number of children for any given node in the tree * @param relatedEventsPerNode - number of related events (file, registry, etc) to create for each process event in the tree + * @param relatedAlertsPerNode - number of alerts to generate for each node, if this is 0 an alert will still be generated for the origin node * @param percentNodesWithRelated - percent of nodes which should have related events * @param percentTerminated - percent of nodes which will have process termination events * @param alwaysGenMaxChildrenPerNode - flag to always return the max children per node instead of it being a random number of children @@ -540,6 +545,7 @@ export class EndpointDocGenerator { childGenerations?: number, maxChildrenPerNode?: number, relatedEventsPerNode?: number, + relatedAlertsPerNode?: number, percentNodesWithRelated?: number, percentTerminated?: number, alwaysGenMaxChildrenPerNode?: boolean @@ -550,6 +556,7 @@ export class EndpointDocGenerator { childGenerations, maxChildrenPerNode, relatedEventsPerNode, + relatedAlertsPerNode, percentNodesWithRelated, percentTerminated, alwaysGenMaxChildrenPerNode @@ -566,6 +573,7 @@ export class EndpointDocGenerator { * @param maxChildrenPerNode - maximum number of children for any given node in the tree * @param relatedEventsPerNode - can be an array of RelatedEventInfo objects describing the related events that should be generated for each process node * or a number which defines the number of related events and will default to random categories + * @param relatedAlertsPerNode - number of alerts to generate for each node, if this is 0 an alert will still be generated for the origin node * @param percentNodesWithRelated - percent of nodes which should have related events * @param percentTerminated - percent of nodes which will have process termination events * @param alwaysGenMaxChildrenPerNode - flag to always return the max children per node instead of it being a random number of children @@ -575,6 +583,7 @@ export class EndpointDocGenerator { childGenerations?: number, maxChildrenPerNode?: number, relatedEventsPerNode?: RelatedEventInfo[] | number, + relatedAlertsPerNode?: number, percentNodesWithRelated?: number, percentTerminated?: number, alwaysGenMaxChildrenPerNode?: boolean @@ -594,6 +603,7 @@ export class EndpointDocGenerator { childGenerations, maxChildrenPerNode, relatedEventsPerNode, + relatedAlertsPerNode, percentNodesWithRelated, percentTerminated, alwaysGenMaxChildrenPerNode @@ -605,12 +615,14 @@ export class EndpointDocGenerator { * @param alertAncestors - number of ancestor generations to create * @param relatedEventsPerNode - can be an array of RelatedEventInfo objects describing the related events that should be generated for each process node * or a number which defines the number of related events and will default to random categories - * @param pctWithRelated - percent of ancestors that will have related events + * @param relatedAlertsPerNode - number of alerts to generate for each node, if this is 0 an alert will still be generated for the origin node + * @param pctWithRelated - percent of ancestors that will have related events and alerts * @param pctWithTerminated - percent of ancestors that will have termination events */ public createAlertEventAncestry( alertAncestors = 3, relatedEventsPerNode: RelatedEventInfo[] | number = 5, + relatedAlertsPerNode: number = 3, pctWithRelated = 30, pctWithTerminated = 100 ): Event[] { @@ -621,16 +633,32 @@ export class EndpointDocGenerator { let ancestor = root; let timestamp = root['@timestamp'] + 1000; - // generate related alerts for root - const processDuration: number = 6 * 3600; - if (this.randomN(100) < pctWithRelated) { + const addRelatedAlerts = ( + node: Event, + alertsPerNode: number, + secBeforeAlert: number, + eventList: Event[] + ) => { + for (const relatedAlert of this.relatedAlertsGenerator(node, alertsPerNode, secBeforeAlert)) { + eventList.push(relatedAlert); + } + }; + + const addRelatedEvents = (node: Event, secBeforeEvent: number, eventList: Event[]) => { for (const relatedEvent of this.relatedEventsGenerator( - ancestor, + node, relatedEventsPerNode, - processDuration + secBeforeEvent )) { - events.push(relatedEvent); + eventList.push(relatedEvent); } + }; + + // generate related alerts for rootW + const processDuration: number = 6 * 3600; + if (this.randomN(100) < pctWithRelated) { + addRelatedEvents(ancestor, processDuration, events); + addRelatedAlerts(ancestor, relatedAlertsPerNode, processDuration, events); } // generate the termination event for the root @@ -670,13 +698,15 @@ export class EndpointDocGenerator { // generate related alerts for ancestor if (this.randomN(100) < pctWithRelated) { - for (const relatedEvent of this.relatedEventsGenerator( - ancestor, - relatedEventsPerNode, - processDuration - )) { - events.push(relatedEvent); + addRelatedEvents(ancestor, processDuration, events); + let numAlertsPerNode = relatedAlertsPerNode; + // if this is the last ancestor, create one less related alert so that we have a uniform amount of related alerts + // for each node. The last alert at the end of this function should always be created even if the related alerts + // amount is 0 + if (i === alertAncestors - 1) { + numAlertsPerNode -= 1; } + addRelatedAlerts(ancestor, numAlertsPerNode, processDuration, events); } } events.push( @@ -701,6 +731,7 @@ export class EndpointDocGenerator { generations = 2, maxChildrenPerNode = 2, relatedEventsPerNode: RelatedEventInfo[] | number = 3, + relatedAlertsPerNode: number = 3, percentNodesWithRelated = 100, percentChildrenTerminated = 100, alwaysGenMaxChildrenPerNode = false @@ -759,6 +790,7 @@ export class EndpointDocGenerator { } if (this.randomN(100) < percentNodesWithRelated) { yield* this.relatedEventsGenerator(child, relatedEventsPerNode, processDuration); + yield* this.relatedAlertsGenerator(child, relatedAlertsPerNode, processDuration); } } } @@ -803,6 +835,23 @@ export class EndpointDocGenerator { } } + /** + * Creates related alerts for a process event + * @param node - process event to relate alerts to by entityID + * @param relatedAlerts - number which defines the number of related alerts to create + * @param alertCreationTime - maximum number of seconds after process event that related alert timestamp can be + */ + public *relatedAlertsGenerator( + node: Event, + relatedAlerts: number = 3, + alertCreationTime: number = 6 * 3600 + ) { + for (let i = 0; i < relatedAlerts; i++) { + const ts = node['@timestamp'] + this.randomN(alertCreationTime) * 1000; + yield this.generateAlert(ts, node.process.entity_id, node.process.parent?.entity_id); + } + } + /** * Generates an Ingest `datasource` that includes the Endpoint Policy data */ diff --git a/x-pack/plugins/security_solution/common/endpoint/types.ts b/x-pack/plugins/security_solution/common/endpoint/types.ts index 48ec564ba21ac..8b1f4f73f21c3 100644 --- a/x-pack/plugins/security_solution/common/endpoint/types.ts +++ b/x-pack/plugins/security_solution/common/endpoint/types.ts @@ -107,7 +107,7 @@ export interface ResolverTree { entityID: string; children: ResolverChildren; relatedEvents: Omit; - relatedAlerts: Omit; + relatedAlerts: Omit; ancestry: ResolverAncestry; lifecycle: ResolverEvent[]; stats: ResolverNodeStats; @@ -152,7 +152,7 @@ export interface ResolverRelatedEvents { /** * Response structure for the alerts route. */ -export interface ResolverAlerts { +export interface ResolverRelatedAlerts { entityID: string; alerts: ResolverEvent[]; nextAlert: string | null; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/README.md b/x-pack/plugins/security_solution/scripts/endpoint/README.md index e97bd9be61d3b..79ca768a3c135 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/README.md +++ b/x-pack/plugins/security_solution/scripts/endpoint/README.md @@ -33,7 +33,7 @@ Options: --eventIndex, --ei index to store events in [string] [default: "events-endpoint-1"] --metadataIndex, --mi index to store host metadata in - [string] [default: "metrics-endpoint-default-1"] + [string] [default: "metrics-endpoint.metadata-default-1"] --policyIndex, --pi index to store host policy in [string] [default: "metrics-endpoint.policy-default-1"] --auth elasticsearch username and password, separated @@ -46,8 +46,10 @@ Options: [number] [default: 3] --relatedEvents, --related number of related events to create for each process event [number] [default: 5] + --relatedAlerts, --relAlerts number of related alerts to create for each + process event [number] [default: 5] --percentWithRelated, --pr percent of process events to add related events - to [number] [default: 30] + and related alerts to [number] [default: 30] --percentTerminated, --pt percent of process events to add termination event for [number] [default: 30] --maxChildrenPerNode, --maxCh always generate the max number of children per diff --git a/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator.ts b/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator.ts index 63c1507718137..e5e8c6e34bb10 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator.ts @@ -107,9 +107,15 @@ async function main() { type: 'number', default: 5, }, + relatedAlerts: { + alias: 'relAlerts', + describe: 'number of related alerts to create for each process event', + type: 'number', + default: 5, + }, percentWithRelated: { alias: 'pr', - describe: 'percent of process events to add related events to', + describe: 'percent of process events to add related events and related alerts to', type: 'number', default: 30, }, @@ -248,6 +254,7 @@ async function main() { argv.generations, argv.children, argv.relatedEvents, + argv.relatedAlerts, argv.percentWithRelated, argv.percentTerminated, argv.maxChildrenPerNode diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver.ts index 9a4f55770f934..9b45a1a6c5354 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver.ts @@ -11,11 +11,13 @@ import { validateEvents, validateChildren, validateAncestry, + validateAlerts, } from '../../../common/endpoint/schema/resolver'; import { handleEvents } from './resolver/events'; import { handleChildren } from './resolver/children'; import { handleAncestry } from './resolver/ancestry'; import { handleTree } from './resolver/tree'; +import { handleAlerts } from './resolver/alerts'; export function registerResolverRoutes(router: IRouter, endpointAppContext: EndpointAppContext) { const log = endpointAppContext.logFactory.get('resolver'); @@ -29,6 +31,15 @@ export function registerResolverRoutes(router: IRouter, endpointAppContext: Endp handleEvents(log, endpointAppContext) ); + router.get( + { + path: '/api/endpoint/resolver/{id}/alerts', + validate: validateAlerts, + options: { authRequired: true }, + }, + handleAlerts(log, endpointAppContext) + ); + router.get( { path: '/api/endpoint/resolver/{id}/children', diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree.ts index be7f75dfd52fa..baad56c74b8a8 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree.ts @@ -37,14 +37,19 @@ export function handleTree( const fetcher = new Fetcher(client, id, indexPattern, endpointID); - const [childrenNodes, ancestry, relatedEvents, alertsResp] = await Promise.all([ + const [childrenNodes, ancestry, relatedEvents, relatedAlerts] = await Promise.all([ fetcher.children(children, generations, afterChild), fetcher.ancestors(ancestors), fetcher.events(events, afterEvent), fetcher.alerts(alerts, afterAlert), ]); - const tree = new Tree(id, { ancestry, children: childrenNodes, relatedEvents }); + const tree = new Tree(id, { + ancestry, + children: childrenNodes, + relatedEvents, + relatedAlerts, + }); const enrichedTree = await fetcher.stats(tree); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts index 4aef30933a325..da6d8e2cbde81 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts @@ -9,7 +9,7 @@ import { ResolverChildren, ResolverRelatedEvents, ResolverAncestry, - ResolverAlerts, + ResolverRelatedAlerts, } from '../../../../../common/endpoint/types'; import { entityId, parentEntityId } from '../../../../../common/endpoint/models/event'; import { PaginationBuilder } from './pagination'; @@ -18,7 +18,7 @@ import { LifecycleQuery } from '../queries/lifecycle'; import { ChildrenQuery } from '../queries/children'; import { EventsQuery } from '../queries/events'; import { StatsQuery } from '../queries/stats'; -import { createAncestry, createRelatedEvents, createLifecycle, createAlerts } from './node'; +import { createAncestry, createRelatedEvents, createLifecycle, createRelatedAlerts } from './node'; import { ChildrenNodesHelper } from './children_helper'; import { AlertsQuery } from '../queries/alerts'; @@ -88,7 +88,7 @@ export class Fetcher { * @param limit the upper bound number of alerts to return * @param after a cursor to use as the starting point for retrieving alerts */ - public async alerts(limit: number, after?: string): Promise { + public async alerts(limit: number, after?: string): Promise { return this.doAlerts(limit, after); } @@ -160,13 +160,17 @@ export class Fetcher { const { totals, results } = await query.search(this.client, this.id); if (results.length === 0) { // return an empty set of results - return createAlerts(this.id); + return createRelatedAlerts(this.id); } if (!totals[this.id]) { throw new Error(`Could not find the totals for related events entity_id: ${this.id}`); } - return createAlerts(this.id, results, PaginationBuilder.buildCursor(totals[this.id], results)); + return createRelatedAlerts( + this.id, + results, + PaginationBuilder.buildCursor(totals[this.id], results) + ); } private async doChildren( diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/node.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/node.ts index f4b63c43313e7..58aa9efc1fc56 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/node.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/node.ts @@ -11,7 +11,7 @@ import { ResolverRelatedEvents, ResolverTree, ChildNode, - ResolverAlerts, + ResolverRelatedAlerts, } from '../../../../../common/endpoint/types'; /** @@ -36,11 +36,11 @@ export function createRelatedEvents( * @param alerts array of alerts * @param nextAlert the cursor to retrieve the next alert */ -export function createAlerts( +export function createRelatedAlerts( entityID: string, alerts: ResolverEvent[] = [], nextAlert: string | null = null -): ResolverAlerts { +): ResolverRelatedAlerts { return { entityID, alerts, nextAlert }; } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/tree.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/tree.ts index 73382c72359f0..9e47f4eb94485 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/tree.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/tree.ts @@ -12,7 +12,7 @@ import { ResolverAncestry, ResolverTree, ResolverChildren, - ResolverAlerts, + ResolverRelatedAlerts, } from '../../../../../common/endpoint/types'; import { createTree } from './node'; @@ -26,7 +26,7 @@ export interface Options { relatedEvents?: ResolverRelatedEvents; ancestry?: ResolverAncestry; children?: ResolverChildren; - alerts?: ResolverAlerts; + relatedAlerts?: ResolverRelatedAlerts; } /** @@ -76,7 +76,7 @@ export class Tree { this.addRelatedEvents(options.relatedEvents); this.addAncestors(options.ancestry); this.addChildren(options.children); - this.addAlerts(options.alerts); + this.addRelatedAlerts(options.relatedAlerts); } /** @@ -116,7 +116,7 @@ export class Tree { * * @param alertInfo is the alerts and pagination information to add to the tree. */ - private addAlerts(alertInfo: ResolverAlerts | undefined) { + private addRelatedAlerts(alertInfo: ResolverRelatedAlerts | undefined) { if (!alertInfo) { return; } diff --git a/x-pack/test/api_integration/apis/endpoint/resolver.ts b/x-pack/test/api_integration/apis/endpoint/resolver.ts index 43f42f700a4c8..462eeaab8f092 100644 --- a/x-pack/test/api_integration/apis/endpoint/resolver.ts +++ b/x-pack/test/api_integration/apis/endpoint/resolver.ts @@ -15,6 +15,7 @@ import { ResolverTree, LegacyEndpointEvent, ResolverNodeStats, + ResolverRelatedAlerts, } from '../../../../plugins/security_solution/common/endpoint/types'; import { parentEntityId } from '../../../../plugins/security_solution/common/endpoint/models/event'; import { FtrProviderContext } from '../../ftr_provider_context'; @@ -199,6 +200,7 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC const treeOptions: Options = { ancestors: 5, relatedEvents: relatedEventsToGen, + relatedAlerts: 4, children: 3, generations: 2, percentTerminated: 100, @@ -219,6 +221,62 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC await esArchiver.unload('endpoint/resolver/api_feature'); }); + describe('related alerts route', () => { + describe('endpoint events', () => { + it('should not find any alerts', async () => { + const { body }: { body: ResolverRelatedAlerts } = await supertest + .get(`/api/endpoint/resolver/5555/alerts`) + .expect(200); + expect(body.nextAlert).to.eql(null); + expect(body.alerts).to.be.empty(); + }); + + it('should return details for the root node', async () => { + const { body }: { body: ResolverRelatedAlerts } = await supertest + .get(`/api/endpoint/resolver/${tree.origin.id}/alerts`) + .expect(200); + expect(body.alerts.length).to.eql(4); + compareArrays(tree.origin.relatedAlerts, body.alerts, true); + expect(body.nextAlert).to.eql(null); + }); + + it('should return paginated results for the root node', async () => { + let { body }: { body: ResolverRelatedAlerts } = await supertest + .get(`/api/endpoint/resolver/${tree.origin.id}/alerts?alerts=2`) + .expect(200); + expect(body.alerts.length).to.eql(2); + compareArrays(tree.origin.relatedAlerts, body.alerts); + expect(body.nextAlert).not.to.eql(null); + + ({ body } = await supertest + .get( + `/api/endpoint/resolver/${tree.origin.id}/alerts?alerts=2&afterAlert=${body.nextAlert}` + ) + .expect(200)); + expect(body.alerts.length).to.eql(2); + compareArrays(tree.origin.relatedAlerts, body.alerts); + expect(body.nextAlert).to.not.eql(null); + + ({ body } = await supertest + .get( + `/api/endpoint/resolver/${tree.origin.id}/alerts?alerts=2&afterAlert=${body.nextAlert}` + ) + .expect(200)); + expect(body.alerts).to.be.empty(); + expect(body.nextAlert).to.eql(null); + }); + + it('should return the first page of information when the cursor is invalid', async () => { + const { body }: { body: ResolverRelatedAlerts } = await supertest + .get(`/api/endpoint/resolver/${tree.origin.id}/alerts?afterAlert=blah`) + .expect(200); + expect(body.alerts.length).to.eql(4); + compareArrays(tree.origin.relatedAlerts, body.alerts, true); + expect(body.nextAlert).to.eql(null); + }); + }); + }); + describe('related events route', () => { describe('legacy events', () => { const endpointID = '5a0c957f-b8e7-4538-965e-57e8bb86ad3a'; @@ -604,7 +662,7 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC it('returns a tree', async () => { const { body }: { body: ResolverTree } = await supertest .get( - `/api/endpoint/resolver/${tree.origin.id}?children=100&generations=3&ancestors=5&events=4` + `/api/endpoint/resolver/${tree.origin.id}?children=100&generations=3&ancestors=5&events=4&alerts=4` ) .expect(200); @@ -620,6 +678,9 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC expect(body.relatedEvents.nextEvent).to.equal(null); compareArrays(tree.origin.relatedEvents, body.relatedEvents.events, true); + expect(body.relatedAlerts.nextAlert).to.equal(null); + compareArrays(tree.origin.relatedAlerts, body.relatedAlerts.alerts, true); + compareArrays(tree.origin.lifecycle, body.lifecycle, true); verifyStats(body.stats, relatedEventsToGen); }); From 5031536e4ec11bd338867b1f4e5bab0ae6b8db33 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Mon, 8 Jun 2020 15:31:02 -0400 Subject: [PATCH 03/14] Fixing missed parameter --- .../endpoint/routes/resolver/utils/children_helper.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.test.ts index 11f3dd69b3f95..51c9cef08a466 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.test.ts @@ -34,7 +34,7 @@ describe('Children helper', () => { const root = generator.generateEvent(); it('builds the children response structure', () => { - const children = Array.from(generator.descendantsTreeGenerator(root, 3, 3, 0, 0, 100, true)); + const children = Array.from(generator.descendantsTreeGenerator(root, 3, 3, 0, 0, 0, 100, true)); // because we requested the generator to always return the max children, there will always be at least 2 parents const parents = findParents(children); From b804ddb038f7ef338f8734ec0c022835bed2c525 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Tue, 9 Jun 2020 13:49:22 -0400 Subject: [PATCH 04/14] Aligning the AlertEvent and ResolverEvent definition --- .../common/endpoint/generate_data.ts | 5 +++++ .../security_solution/common/endpoint/types.ts | 16 +++++++--------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts index d17432ebfe7e7..f45d4709c5fb6 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts @@ -177,6 +177,7 @@ interface HostInfo { agent: { version: string; id: string; + type: string; }; host: Host; endpoint: { @@ -280,6 +281,7 @@ export class EndpointDocGenerator { agent: { version: this.randomVersion(), id: this.seededUUIDv4(), + type: 'endpoint', }, elastic: { agent: { @@ -327,6 +329,9 @@ export class EndpointDocGenerator { return { ...this.commonInfo, '@timestamp': ts, + ecs: { + version: '1.4.0', + }, event: { action: this.randomChoice(FILE_OPERATIONS), kind: 'alert', diff --git a/x-pack/plugins/security_solution/common/endpoint/types.ts b/x-pack/plugins/security_solution/common/endpoint/types.ts index 8b1f4f73f21c3..124579e611a92 100644 --- a/x-pack/plugins/security_solution/common/endpoint/types.ts +++ b/x-pack/plugins/security_solution/common/endpoint/types.ts @@ -246,11 +246,15 @@ interface DllFields { /** * Describes an Alert Event. */ -export type AlertEvent = Immutable<{ +export interface AlertEvent { '@timestamp': number; agent: { id: string; version: string; + type: string; + }; + ecs: { + version: string; }; event: { id: string; @@ -327,7 +331,7 @@ export type AlertEvent = Immutable<{ }; host: Host; dll?: DllFields[]; -}>; +} /** * The status of the host @@ -425,13 +429,7 @@ export interface EndpointEvent { id: string; kind: string; }; - host: { - id: string; - hostname: string; - ip: string[]; - mac: string[]; - os: HostOS; - }; + host: Host; process: { entity_id: string; name: string; From d63287bcccc1ea4f4cf5a4287fc2199323b08600 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Wed, 10 Jun 2020 15:45:41 -0400 Subject: [PATCH 05/14] Fixing type errors --- .../security_solution/common/endpoint_alerts/types.ts | 7 +++---- .../public/endpoint_alerts/view/index.tsx | 3 ++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint_alerts/types.ts b/x-pack/plugins/security_solution/common/endpoint_alerts/types.ts index 3fbde79414aa0..d37051faeb740 100644 --- a/x-pack/plugins/security_solution/common/endpoint_alerts/types.ts +++ b/x-pack/plugins/security_solution/common/endpoint_alerts/types.ts @@ -15,7 +15,6 @@ import { AlertEvent, KbnConfigSchemaInputTypeOf, AppLocation, - Immutable, } from '../endpoint/types'; /** @@ -119,7 +118,7 @@ export type AlertListData = AlertResultList; export interface AlertListState { /** Array of alert items. */ - readonly alerts: Immutable; + readonly alerts: AlertData[]; /** The total number of alerts on the page. */ readonly total: number; @@ -131,10 +130,10 @@ export interface AlertListState { readonly pageIndex: number; /** Current location object from React Router history. */ - readonly location?: Immutable; + readonly location?: AppLocation; /** Specific Alert data to be shown in the details view */ - readonly alertDetails?: Immutable; + readonly alertDetails?: AlertDetails; /** Search bar state including indexPatterns */ readonly searchBar: AlertsSearchBarState; diff --git a/x-pack/plugins/security_solution/public/endpoint_alerts/view/index.tsx b/x-pack/plugins/security_solution/public/endpoint_alerts/view/index.tsx index b39ea678596a4..6ad0d5ec94219 100644 --- a/x-pack/plugins/security_solution/public/endpoint_alerts/view/index.tsx +++ b/x-pack/plugins/security_solution/public/endpoint_alerts/view/index.tsx @@ -32,6 +32,7 @@ import { useAlertListSelector } from './hooks/use_alerts_selector'; import { AlertDetailsOverview } from './details'; import { FormattedDate } from './formatted_date'; import { AlertIndexSearchBar } from './index_search_bar'; +import { Immutable } from '../../../common/endpoint/types'; export const AlertIndex = memo(() => { const history = useHistory(); @@ -145,7 +146,7 @@ export const AlertIndex = memo(() => { history.push(urlFromQueryParams(paramsWithoutFlyoutDetails)); }, [history, queryParams]); - const timestampForRows: Map = useMemo(() => { + const timestampForRows: Map, number> = useMemo(() => { return new Map( alertListData.map((alertData) => { return [alertData, alertData['@timestamp']]; From 7155add664afb4e4ca2a034549fb2d0028ca60f1 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Wed, 10 Jun 2020 16:31:46 -0400 Subject: [PATCH 06/14] Fixing import error --- .../server/endpoint/routes/resolver/queries/alerts.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/alerts.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/alerts.ts index 1d316eca40ed3..013bc4302de2e 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/alerts.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/alerts.ts @@ -7,7 +7,7 @@ import { SearchResponse } from 'elasticsearch'; import { ResolverEvent } from '../../../../../common/endpoint/types'; import { ResolverQuery } from './base'; import { PaginationBuilder, PaginatedResults } from '../utils/pagination'; -import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/public'; +import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/common'; /** * Builds a query for retrieving alerts for a node. From e6e674991e81330f09b56bd85f569bb65f129d67 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Mon, 15 Jun 2020 12:36:36 -0400 Subject: [PATCH 07/14] Adding ancestry functionality in generator --- .../common/endpoint/generate_data.ts | 31 ++++++++++++++++--- .../common/endpoint/types.ts | 6 ++++ 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts index 7944d7d365ed8..38a50dc2e1edb 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts @@ -26,6 +26,7 @@ interface EventOptions { eventType?: string; eventCategory?: string | string[]; processName?: string; + ancestry?: string[]; } const Windows: HostOS[] = [ @@ -341,7 +342,8 @@ export class EndpointDocGenerator { public generateAlert( ts = new Date().getTime(), entityID = this.randomString(10), - parentEntityID?: string + parentEntityID?: string, + ancestryArray: string[] = [] ): AlertEvent { return { ...this.commonInfo, @@ -416,6 +418,7 @@ export class EndpointDocGenerator { sha1: 'fake sha1', sha256: 'fake sha256', }, + Ext: { ancestry: ancestryArray }, }, dll: [ { @@ -469,6 +472,7 @@ export class EndpointDocGenerator { entity_id: options.entityID ? options.entityID : this.randomString(10), parent: options.parentEntityID ? { entity_id: options.parentEntityID } : undefined, name: options.processName ? options.processName : randomProcessName(), + Ext: { ancestry: options.ancestry?.slice() || [] }, }, }; } @@ -676,7 +680,7 @@ export class EndpointDocGenerator { } }; - // generate related alerts for rootW + // generate related alerts for root const processDuration: number = 6 * 3600; if (this.randomN(100) < pctWithRelated) { addRelatedEvents(ancestor, processDuration, events); @@ -701,6 +705,8 @@ export class EndpointDocGenerator { ancestor = this.generateEvent({ timestamp, parentEntityID: ancestor.process.entity_id, + // add the parent to the ancestry array + ancestry: [ancestor.process.entity_id, ...ancestor.process.Ext.ancestry], }); events.push(ancestor); timestamp = timestamp + 1000; @@ -714,6 +720,7 @@ export class EndpointDocGenerator { parentEntityID: ancestor.process.parent?.entity_id, eventCategory: 'process', eventType: 'end', + ancestry: ancestor.process.Ext.ancestry, }) ); } @@ -732,7 +739,12 @@ export class EndpointDocGenerator { } } events.push( - this.generateAlert(timestamp, ancestor.process.entity_id, ancestor.process.parent?.entity_id) + this.generateAlert( + timestamp, + ancestor.process.entity_id, + ancestor.process.parent?.entity_id, + ancestor.process.Ext.ancestry + ) ); return events; } @@ -787,6 +799,10 @@ export class EndpointDocGenerator { const child = this.generateEvent({ timestamp, parentEntityID: currentState.event.process.entity_id, + ancestry: [ + currentState.event.process.entity_id, + ...currentState.event.process.Ext.ancestry, + ], }); maxChildren = this.randomN(maxChildrenPerNode + 1); @@ -808,6 +824,7 @@ export class EndpointDocGenerator { parentEntityID: child.process.parent?.entity_id, eventCategory: 'process', eventType: 'end', + ancestry: child.process.Ext.ancestry, }); } if (this.randomN(100) < percentNodesWithRelated) { @@ -852,6 +869,7 @@ export class EndpointDocGenerator { parentEntityID: node.process.parent?.entity_id, eventCategory: eventInfo.category, eventType: eventInfo.creationType, + ancestry: node.process.Ext.ancestry, }); } } @@ -870,7 +888,12 @@ export class EndpointDocGenerator { ) { for (let i = 0; i < relatedAlerts; i++) { const ts = node['@timestamp'] + this.randomN(alertCreationTime) * 1000; - yield this.generateAlert(ts, node.process.entity_id, node.process.parent?.entity_id); + yield this.generateAlert( + ts, + node.process.entity_id, + node.process.parent?.entity_id, + node.process.Ext.ancestry + ); } } diff --git a/x-pack/plugins/security_solution/common/endpoint/types.ts b/x-pack/plugins/security_solution/common/endpoint/types.ts index 0341b7593caf0..b75081d3ff1ab 100644 --- a/x-pack/plugins/security_solution/common/endpoint/types.ts +++ b/x-pack/plugins/security_solution/common/endpoint/types.ts @@ -313,6 +313,9 @@ export interface AlertEvent { thread?: ThreadFields[]; uptime: number; user: string; + Ext: { + ancestry: string[]; + }; }; file: { owner: string; @@ -445,6 +448,9 @@ export interface EndpointEvent { entity_id: string; name?: string; }; + Ext: { + ancestry: string[]; + }; }; } From 1050a5653d2eed81114ce61b05d05a39084e75a4 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Mon, 15 Jun 2020 15:18:48 -0400 Subject: [PATCH 08/14] Creating some tests for ancestry field --- .../security_solution/common/endpoint/generate_data.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts index ba4f2251564e8..79ce31a024b04 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts @@ -159,6 +159,10 @@ describe('data generator', () => { return (inRelated || inRelatedAlerts || inLifecycle) && event.process.entity_id === node.id; }; + it('has ancestry array defined', () => { + expect(tree.origin.lifecycle[0].process.Ext.ancestry.length).toBe(3); + }); + it('has the right related events for each node', () => { const checkRelatedEvents = (node: TreeNode) => { expect(node.relatedEvents.length).toEqual(4); From 8f46beaa0e4150fc8a28d9261cbf84916e85a6ee Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Mon, 15 Jun 2020 20:11:47 -0400 Subject: [PATCH 09/14] Making progress on the ancestry --- .../common/endpoint/generate_data.test.ts | 34 ++++-- .../common/endpoint/generate_data.ts | 16 ++- .../common/endpoint/models/event.ts | 7 ++ .../endpoint/routes/resolver/utils/fetch.ts | 115 ++++++++++++------ .../api_integration/apis/endpoint/resolver.ts | 13 +- 5 files changed, 129 insertions(+), 56 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts index 79ce31a024b04..074ee9b23bbd4 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts @@ -10,6 +10,7 @@ import { TreeNode, RelatedEventCategory, ECSCategory, + ANCESTRY_LIMIT, } from './generate_data'; interface Node { @@ -116,6 +117,7 @@ describe('data generator', () => { }); it('creates an alert for the origin node but no other nodes', () => { + tree.ancestry.delete(tree.origin.id); for (const node of tree.ancestry.values()) { expect(node.relatedAlerts.length).toEqual(0); } @@ -159,8 +161,28 @@ describe('data generator', () => { return (inRelated || inRelatedAlerts || inLifecycle) && event.process.entity_id === node.id; }; + const verifyAncestry = (event: Event, genTree: Tree) => { + if (event.process.Ext.ancestry.length > 0) { + expect(event.process.parent?.entity_id).toBe(event.process.Ext.ancestry[0]); + } + for (let i = 0; i < event.process.Ext.ancestry.length; i++) { + const ancestor = event.process.Ext.ancestry[i]; + const parent = genTree.children.get(ancestor) || genTree.ancestry.get(ancestor); + expect(ancestor).toBe(parent.lifecycle[0].process.entity_id); + + // the next ancestor should be the grandparent + if (i + 1 < event.process.Ext.ancestry.length) { + const grandparent = event.process.Ext.ancestry[i + 1]; + expect(grandparent).toBe(parent.lifecycle[0].process.parent?.entity_id); + } + } + }; + it('has ancestry array defined', () => { - expect(tree.origin.lifecycle[0].process.Ext.ancestry.length).toBe(3); + expect(tree.origin.lifecycle[0].process.Ext.ancestry.length).toBe(ANCESTRY_LIMIT); + for (const event of tree.allEvents) { + verifyAncestry(event, tree); + } }); it('has the right related events for each node', () => { @@ -189,8 +211,6 @@ describe('data generator', () => { for (const node of tree.children.values()) { checkRelatedEvents(node); } - - checkRelatedEvents(tree.origin); }); it('has the right number of related alerts for each node', () => { @@ -206,7 +226,8 @@ describe('data generator', () => { }); it('has the right number of ancestors', () => { - expect(tree.ancestry.size).toEqual(ancestors); + // +1 for the origin node + expect(tree.ancestry.size).toEqual(ancestors + 1); }); it('has the right number of total children', () => { @@ -243,10 +264,7 @@ describe('data generator', () => { const children = tree.children.get(event.process.entity_id); if (children) { expect(eventInNode(event, children)).toBeTruthy(); - return; } - - expect(eventInNode(event, tree.origin)).toBeTruthy(); }); }); @@ -264,8 +282,6 @@ describe('data generator', () => { total += nodeEventCount(node); } - total += nodeEventCount(tree.origin); - expect(tree.allEvents.length).toEqual(total); }); }); diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts index 38a50dc2e1edb..7ddf411e14a3b 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts @@ -18,6 +18,7 @@ import { import { factory as policyFactory } from './models/policy_config'; export type Event = AlertEvent | EndpointEvent; +export const ANCESTRY_LIMIT: number = 2; interface EventOptions { timestamp?: number; @@ -239,6 +240,7 @@ export interface Tree { * Map of entity_id to node */ ancestry: Map; + // TODO add the origin to the ancestry array to make test verification easier origin: TreeNode; /** * All events from children, ancestry, origin, and the alert in a single array @@ -343,7 +345,8 @@ export class EndpointDocGenerator { ts = new Date().getTime(), entityID = this.randomString(10), parentEntityID?: string, - ancestryArray: string[] = [] + ancestryArray: string[] = [], + ancestryLimit: number = 2 ): AlertEvent { return { ...this.commonInfo, @@ -418,7 +421,9 @@ export class EndpointDocGenerator { sha1: 'fake sha1', sha256: 'fake sha256', }, - Ext: { ancestry: ancestryArray }, + // simulate a finite ancestry array size, the endpoint limits the ancestry array to 20 entries we'll use + // 2 so that the backend can handle that case + Ext: { ancestry: ancestryArray.slice(0, ANCESTRY_LIMIT) }, }, dll: [ { @@ -472,7 +477,9 @@ export class EndpointDocGenerator { entity_id: options.entityID ? options.entityID : this.randomString(10), parent: options.parentEntityID ? { entity_id: options.parentEntityID } : undefined, name: options.processName ? options.processName : randomProcessName(), - Ext: { ancestry: options.ancestry?.slice() || [] }, + // simulate a finite ancestry array size, the endpoint limits the ancestry array to 20 entries we'll use + // 2 so that the backend can handle that case + Ext: { ancestry: options.ancestry?.slice(0, ANCESTRY_LIMIT) || [] }, }, }; } @@ -526,9 +533,6 @@ export class EndpointDocGenerator { throw Error(`could not find origin while building tree: ${alert.process.entity_id}`); } - // remove the origin node from the ancestry array - ancestryNodes.delete(alert.process.entity_id); - const children = Array.from( this.descendantsTreeGenerator( alert, diff --git a/x-pack/plugins/security_solution/common/endpoint/models/event.ts b/x-pack/plugins/security_solution/common/endpoint/models/event.ts index 9eea6bf320db8..0d6293ad1eb00 100644 --- a/x-pack/plugins/security_solution/common/endpoint/models/event.ts +++ b/x-pack/plugins/security_solution/common/endpoint/models/event.ts @@ -52,3 +52,10 @@ export function parentEntityId(event: ResolverEvent): string | undefined { } return event.process.parent?.entity_id; } + +export function ancestryArray(event: ResolverEvent): string[] { + if (isLegacyEvent(event)) { + return []; + } + return event.process.Ext.ancestry; +} diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts index da6d8e2cbde81..4c910d5e77362 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts @@ -10,8 +10,10 @@ import { ResolverRelatedEvents, ResolverAncestry, ResolverRelatedAlerts, + LifecycleNode, + ResolverEvent, } from '../../../../../common/endpoint/types'; -import { entityId, parentEntityId } from '../../../../../common/endpoint/models/event'; +import { entityId, ancestryArray } from '../../../../../common/endpoint/models/event'; import { PaginationBuilder } from './pagination'; import { Tree } from './tree'; import { LifecycleQuery } from '../queries/lifecycle'; @@ -48,9 +50,19 @@ export class Fetcher { * @param limit upper limit of ancestors to retrieve */ public async ancestors(limit: number): Promise { - const root = createAncestry(); - await this.doAncestors(this.id, limit + 1, root); - return root; + const ancestryInfo = createAncestry(); + const originNode = await this.getNode(this.id); + if (originNode) { + ancestryInfo.ancestors.push(originNode); + await this.doAncestors( + // limit the ancestors we're looking for to the number of levels + // the array could be up to length 20 but that could change + ancestryArray(originNode.lifecycle[0]).slice(0, limit), + limit, + ancestryInfo + ); + } + return ancestryInfo; } /** @@ -89,7 +101,26 @@ export class Fetcher { * @param after a cursor to use as the starting point for retrieving alerts */ public async alerts(limit: number, after?: string): Promise { - return this.doAlerts(limit, after); + const query = new AlertsQuery( + PaginationBuilder.createBuilder(limit, after), + this.indexPattern, + this.endpointID + ); + + const { totals, results } = await query.search(this.client, this.id); + if (results.length === 0) { + // return an empty set of results + return createRelatedAlerts(this.id); + } + if (!totals[this.id]) { + throw new Error(`Could not find the totals for related events entity_id: ${this.id}`); + } + + return createRelatedAlerts( + this.id, + results, + PaginationBuilder.buildCursor(totals[this.id], results) + ); } /** @@ -102,56 +133,64 @@ export class Fetcher { return tree; } + private async getNode(entityID: string): Promise { + const query = new LifecycleQuery(this.indexPattern, this.endpointID); + const results = await query.search(this.client, entityID); + if (results.length === 0) { + return; + } + + return createLifecycle(entityID, results); + } + private async doAncestors( - curNodeID: string, + ancestors: string[], levels: number, ancestorInfo: ResolverAncestry ): Promise { - if (levels === 0) { - ancestorInfo.nextAncestor = curNodeID; + if (levels <= 0) { + if (ancestors.length !== 0) { + ancestorInfo.nextAncestor = ancestors[0]; + } return; } const query = new LifecycleQuery(this.indexPattern, this.endpointID); - const results = await query.search(this.client, curNodeID); + const results = await query.search(this.client, ancestors); if (results.length === 0) { return; } - ancestorInfo.ancestors.push(createLifecycle(curNodeID, results)); - const next = parentEntityId(results[0]); - if (next === undefined) { - return; - } - await this.doAncestors(next, levels - 1, ancestorInfo); - } + // bucket the start and end events together for a single node + const ancestryNodes = results.reduce( + (nodes: Map, ancestorEvent: ResolverEvent) => { + const nodeId = entityId(ancestorEvent); + let node = nodes.get(nodeId); + if (!node) { + node = createLifecycle(nodeId, []); + } - private async doEvents(limit: number, after?: string) { - const query = new EventsQuery( - PaginationBuilder.createBuilder(limit, after), - this.indexPattern, - this.endpointID + node.lifecycle.push(ancestorEvent); + return nodes.set(nodeId, node); + }, + new Map() ); - const { totals, results } = await query.search(this.client, this.id); - if (results.length === 0) { - // return an empty set of results - return createRelatedEvents(this.id); - } - if (!totals[this.id]) { - throw new Error(`Could not find the totals for related events entity_id: ${this.id}`); - } + // the order of this array is going to be weird, it will look like this + // [furthest grandparent...closer grandparent, next recursive call furthest grandparent...closer grandparent] + ancestorInfo.ancestors.push(...ancestryNodes.values()); - return createRelatedEvents( - this.id, - results, - PaginationBuilder.buildCursor(totals[this.id], results) - ); + const levelsLeft = levels - ancestryNodes.size; + // the results come back in ascending order on timestamp so the first entry in the + // results should be the further ancestor (most distant grandparent) + const next = ancestryArray(results[0]).slice(0, levelsLeft); + // the ancestry array currently only holds up to 20 values but we can't rely on that so keep recursing + await this.doAncestors(next, levelsLeft, ancestorInfo); } - private async doAlerts(limit: number, after?: string) { - const query = new AlertsQuery( + private async doEvents(limit: number, after?: string) { + const query = new EventsQuery( PaginationBuilder.createBuilder(limit, after), this.indexPattern, this.endpointID @@ -160,13 +199,13 @@ export class Fetcher { const { totals, results } = await query.search(this.client, this.id); if (results.length === 0) { // return an empty set of results - return createRelatedAlerts(this.id); + return createRelatedEvents(this.id); } if (!totals[this.id]) { throw new Error(`Could not find the totals for related events entity_id: ${this.id}`); } - return createRelatedAlerts( + return createRelatedEvents( this.id, results, PaginationBuilder.buildCursor(totals[this.id], results) diff --git a/x-pack/test/api_integration/apis/endpoint/resolver.ts b/x-pack/test/api_integration/apis/endpoint/resolver.ts index 2bb52473508b3..fda4965e95904 100644 --- a/x-pack/test/api_integration/apis/endpoint/resolver.ts +++ b/x-pack/test/api_integration/apis/endpoint/resolver.ts @@ -79,14 +79,17 @@ const verifyAncestry = (ancestors: LifecycleNode[], tree: Tree, verifyLastParent expect(Object.keys(groupedAncestors).length).to.eql(ancestors.length); // make sure there aren't any nodes with the same parent entity_id expect(Object.keys(groupedAncestorsParent).length).to.eql(ancestors.length); - ancestors.forEach((node) => { + + const endAncestor = verifyLastParent ? ancestors.length : ancestors.length - 1; + for (let i = 0; i < endAncestor; i++) { + const node = ancestors[i]; const parentID = parentEntityId(node.lifecycle[0]); // the last node generated will have `undefined` as the parent entity_id - if (parentID !== undefined && verifyLastParent) { + if (parentID !== undefined) { expect(groupedAncestors[parentID]).to.be.ok(); } expectLifecycleNodeInMap(node, tree.ancestry); - }); + } }; /** @@ -457,6 +460,10 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC const ancestryInfo = getRootAndAncestry(body); verifyAncestryFromOrigin(ancestryInfo.root, ancestryInfo.ancestry, tree, false); expect(body.nextAncestor).to.eql( + // TODO this won't be valid anymore since the order of the array will be wrong. We'll need to go through + // each ensure that they represent a direct lineage from the origin up the ancestry + // to make sure it isn't returning the wrong set of ancestry + // it should be the parent entity id on the last element of the ancestry array parentEntityId(ancestryInfo.ancestry[ancestryInfo.ancestry.length - 1].lifecycle[0]) ); From 0060c24106eccd18b95b7611b2bd6833234d7e9b Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Mon, 15 Jun 2020 21:15:46 -0400 Subject: [PATCH 10/14] Fixing the ancestry verification --- .../api_integration/apis/endpoint/resolver.ts | 71 ++++++++++--------- 1 file changed, 38 insertions(+), 33 deletions(-) diff --git a/x-pack/test/api_integration/apis/endpoint/resolver.ts b/x-pack/test/api_integration/apis/endpoint/resolver.ts index fda4965e95904..6ca545bf9601b 100644 --- a/x-pack/test/api_integration/apis/endpoint/resolver.ts +++ b/x-pack/test/api_integration/apis/endpoint/resolver.ts @@ -41,25 +41,6 @@ const expectLifecycleNodeInMap = (node: LifecycleNode, nodeMap: Map { - compareArrays(tree.origin.lifecycle, origin.lifecycle, true); - verifyAncestry(ancestors, tree, verifyLastParent); -}; - /** * Verify that all the ancestor nodes are valid and optionally have parents. * @@ -80,15 +61,34 @@ const verifyAncestry = (ancestors: LifecycleNode[], tree: Tree, verifyLastParent // make sure there aren't any nodes with the same parent entity_id expect(Object.keys(groupedAncestorsParent).length).to.eql(ancestors.length); - const endAncestor = verifyLastParent ? ancestors.length : ancestors.length - 1; - for (let i = 0; i < endAncestor; i++) { - const node = ancestors[i]; + // make sure each of the ancestors' lifecycle events are in the generated tree + for (const node of ancestors) { + expectLifecycleNodeInMap(node, tree.ancestry); + } + + // start at the origin which is always the first element of the array and make sure we have a connection + // using parent id between each of the nodes + let foundParents = 0; + let node = ancestors[0]; + for (let i = 0; i < ancestors.length; i++) { const parentID = parentEntityId(node.lifecycle[0]); - // the last node generated will have `undefined` as the parent entity_id if (parentID !== undefined) { - expect(groupedAncestors[parentID]).to.be.ok(); + // the grouped nodes should only have a single entry since each entity is unique + node = groupedAncestors[parentID][0]; + if (node) { + foundParents++; + } else { + break; + } } - expectLifecycleNodeInMap(node, tree.ancestry); + } + + if (verifyLastParent) { + expect(foundParents).to.eql(ancestors.length); + } else { + // if we only retrieved a portion of all the ancestors then the most distant grandparent's parent will not necessarily + // be in the results + expect(foundParents).to.eql(ancestors.length - 1); } }; @@ -428,9 +428,12 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC }); describe('endpoint events', () => { - const getRootAndAncestry = (ancestry: ResolverAncestry) => { - return { root: ancestry.ancestors[0], ancestry: ancestry.ancestors.slice(1) }; - }; + it('should return the origin node at the front of the array', async () => { + const { body }: { body: ResolverAncestry } = await supertest + .get(`/api/endpoint/resolver/${tree.origin.id}/ancestry?ancestors=9`) + .expect(200); + expect(body.ancestors[0].entityID).to.eql(tree.origin.id); + }); it('should return details for the root node', async () => { const { body }: { body: ResolverAncestry } = await supertest @@ -438,8 +441,8 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC .expect(200); // the tree we generated had 5 ancestors + 1 origin node expect(body.ancestors.length).to.eql(6); - const ancestryInfo = getRootAndAncestry(body); - verifyAncestryFromOrigin(ancestryInfo.root, ancestryInfo.ancestry, tree, true); + expect(body.ancestors[0].entityID).to.eql(tree.origin.id); + verifyAncestry(body.ancestors, tree, true); expect(body.nextAncestor).to.eql(null); }); @@ -457,15 +460,17 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC .expect(200); // it should have 2 ancestors + 1 origin expect(body.ancestors.length).to.eql(3); - const ancestryInfo = getRootAndAncestry(body); - verifyAncestryFromOrigin(ancestryInfo.root, ancestryInfo.ancestry, tree, false); + verifyAncestry(body.ancestors, tree, true); + // TODO fix this test + expect(false).to.be.ok(); expect(body.nextAncestor).to.eql( // TODO this won't be valid anymore since the order of the array will be wrong. We'll need to go through // each ensure that they represent a direct lineage from the origin up the ancestry // to make sure it isn't returning the wrong set of ancestry // it should be the parent entity id on the last element of the ancestry array - parentEntityId(ancestryInfo.ancestry[ancestryInfo.ancestry.length - 1].lifecycle[0]) + // parentEntityId(ancestryInfo.ancestry[ancestryInfo.ancestry.length - 1].lifecycle[0]) + {} ); }); From db147d8b7bbbd09cf1b73e1ba7d2259a018314b9 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Tue, 16 Jun 2020 13:20:04 -0400 Subject: [PATCH 11/14] Fixing existing tests --- .../common/endpoint/generate_data.test.ts | 4 +- .../endpoint/routes/resolver/utils/fetch.ts | 14 ++++-- .../api_integration/apis/endpoint/resolver.ts | 48 ++++++++++++------- 3 files changed, 42 insertions(+), 24 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts index 074ee9b23bbd4..2ec5d54d9aecf 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts @@ -168,12 +168,12 @@ describe('data generator', () => { for (let i = 0; i < event.process.Ext.ancestry.length; i++) { const ancestor = event.process.Ext.ancestry[i]; const parent = genTree.children.get(ancestor) || genTree.ancestry.get(ancestor); - expect(ancestor).toBe(parent.lifecycle[0].process.entity_id); + expect(ancestor).toBe(parent?.lifecycle[0].process.entity_id); // the next ancestor should be the grandparent if (i + 1 < event.process.Ext.ancestry.length) { const grandparent = event.process.Ext.ancestry[i + 1]; - expect(grandparent).toBe(parent.lifecycle[0].process.parent?.entity_id); + expect(grandparent).toBe(parent?.lifecycle[0].process.parent?.entity_id); } } }; diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts index 4c910d5e77362..fce8e4d934382 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts @@ -13,7 +13,11 @@ import { LifecycleNode, ResolverEvent, } from '../../../../../common/endpoint/types'; -import { entityId, ancestryArray } from '../../../../../common/endpoint/models/event'; +import { + entityId, + ancestryArray, + parentEntityId, +} from '../../../../../common/endpoint/models/event'; import { PaginationBuilder } from './pagination'; import { Tree } from './tree'; import { LifecycleQuery } from '../queries/lifecycle'; @@ -54,6 +58,8 @@ export class Fetcher { const originNode = await this.getNode(this.id); if (originNode) { ancestryInfo.ancestors.push(originNode); + // If the request is only for the origin node then set next to its parent + ancestryInfo.nextAncestor = parentEntityId(originNode.lifecycle[0]) || null; await this.doAncestors( // limit the ancestors we're looking for to the number of levels // the array could be up to length 20 but that could change @@ -149,9 +155,6 @@ export class Fetcher { ancestorInfo: ResolverAncestry ): Promise { if (levels <= 0) { - if (ancestors.length !== 0) { - ancestorInfo.nextAncestor = ancestors[0]; - } return; } @@ -159,6 +162,7 @@ export class Fetcher { const results = await query.search(this.client, ancestors); if (results.length === 0) { + ancestorInfo.nextAncestor = null; return; } @@ -180,7 +184,7 @@ export class Fetcher { // the order of this array is going to be weird, it will look like this // [furthest grandparent...closer grandparent, next recursive call furthest grandparent...closer grandparent] ancestorInfo.ancestors.push(...ancestryNodes.values()); - + ancestorInfo.nextAncestor = parentEntityId(results[0]) || null; const levelsLeft = levels - ancestryNodes.size; // the results come back in ascending order on timestamp so the first entry in the // results should be the further ancestor (most distant grandparent) diff --git a/x-pack/test/api_integration/apis/endpoint/resolver.ts b/x-pack/test/api_integration/apis/endpoint/resolver.ts index 6ca545bf9601b..b702914fc16c5 100644 --- a/x-pack/test/api_integration/apis/endpoint/resolver.ts +++ b/x-pack/test/api_integration/apis/endpoint/resolver.ts @@ -73,14 +73,14 @@ const verifyAncestry = (ancestors: LifecycleNode[], tree: Tree, verifyLastParent for (let i = 0; i < ancestors.length; i++) { const parentID = parentEntityId(node.lifecycle[0]); if (parentID !== undefined) { - // the grouped nodes should only have a single entry since each entity is unique - node = groupedAncestors[parentID][0]; - if (node) { - foundParents++; - } else { + const nextNode = groupedAncestors[parentID]; + if (!nextNode) { break; } + // the grouped nodes should only have a single entry since each entity is unique + node = nextNode[0]; } + foundParents++; } if (verifyLastParent) { @@ -92,6 +92,29 @@ const verifyAncestry = (ancestors: LifecycleNode[], tree: Tree, verifyLastParent } }; +/** + * Retrieves the most distant ancestor in the given array. + * + * @param ancestors an array of ancestor nodes + */ +const retrieveDistantAncestor = (ancestors: LifecycleNode[]) => { + // group the ancestors by their entity_id mapped to a lifecycle node + const groupedAncestors = _.groupBy(ancestors, (ancestor) => ancestor.entityID); + let node = ancestors[0]; + for (let i = 0; i < ancestors.length; i++) { + const parentID = parentEntityId(node.lifecycle[0]); + if (parentID !== undefined) { + const nextNode = groupedAncestors[parentID]; + if (nextNode) { + node = nextNode[0]; + } else { + return node; + } + } + } + return node; +}; + /** * Verify that the children nodes are correct * @@ -460,18 +483,9 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC .expect(200); // it should have 2 ancestors + 1 origin expect(body.ancestors.length).to.eql(3); - verifyAncestry(body.ancestors, tree, true); - // TODO fix this test - expect(false).to.be.ok(); - expect(body.nextAncestor).to.eql( - // TODO this won't be valid anymore since the order of the array will be wrong. We'll need to go through - // each ensure that they represent a direct lineage from the origin up the ancestry - // to make sure it isn't returning the wrong set of ancestry - - // it should be the parent entity id on the last element of the ancestry array - // parentEntityId(ancestryInfo.ancestry[ancestryInfo.ancestry.length - 1].lifecycle[0]) - {} - ); + verifyAncestry(body.ancestors, tree, false); + const distantGrandparent = retrieveDistantAncestor(body.ancestors); + expect(body.nextAncestor).to.eql(parentEntityId(distantGrandparent.lifecycle[0])); }); it('should handle multiple ancestor requests', async () => { From 882b2f92bbe1e1170d652744ebf740b2782c6e16 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Tue, 16 Jun 2020 17:21:23 -0400 Subject: [PATCH 12/14] Removing unused code and fixing test --- .../common/endpoint/generate_data.test.ts | 2 +- .../endpoint/routes/resolver/utils/fetch.ts | 23 ------------------- 2 files changed, 1 insertion(+), 24 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts index 2ec5d54d9aecf..9e03d8fb569fc 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts @@ -114,10 +114,10 @@ describe('data generator', () => { relatedEvents: 0, relatedAlerts: 0, }); + tree.ancestry.delete(tree.origin.id); }); it('creates an alert for the origin node but no other nodes', () => { - tree.ancestry.delete(tree.origin.id); for (const node of tree.ancestry.values()) { expect(node.relatedAlerts.length).toEqual(0); } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts index fcf10c0102347..fce8e4d934382 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts @@ -216,29 +216,6 @@ export class Fetcher { ); } - private async doAlerts(limit: number, after?: string) { - const query = new AlertsQuery( - PaginationBuilder.createBuilder(limit, after), - this.indexPattern, - this.endpointID - ); - - const { totals, results } = await query.search(this.client, this.id); - if (results.length === 0) { - // return an empty set of results - return createRelatedAlerts(this.id); - } - if (!totals[this.id]) { - throw new Error(`Could not find the totals for related events entity_id: ${this.id}`); - } - - return createRelatedAlerts( - this.id, - results, - PaginationBuilder.buildCursor(totals[this.id], results) - ); - } - private async doChildren( cache: ChildrenNodesHelper, ids: string[], From f8e898f976aef40dd0d4404fb8b7b4ea697655c9 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Tue, 16 Jun 2020 17:40:48 -0400 Subject: [PATCH 13/14] Adding more comments --- .../common/endpoint/generate_data.ts | 13 ++++++++++--- .../security_solution/common/endpoint/types.ts | 10 ++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts index 7ddf411e14a3b..d65298ad87c0c 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts @@ -18,6 +18,15 @@ import { import { factory as policyFactory } from './models/policy_config'; export type Event = AlertEvent | EndpointEvent; +/** + * This value indicates the limit for the size of the ancestry array. The endpoint currently saves up to 20 values + * in its messages. To simulate a limit on the array size I'm using 2 here so that we can't rely on there being a large + * number like 20. The ancestry array contains entity_ids for the ancestors of a particular process. + * + * The array has a special format. The entity_ids towards the beginning of the array are closer ancestors and the + * values towards the end of the array are more distant ancestors (grandparents). Therefore + * ancestry_array[0] == process.parent.entity_id and ancestry_array[1] == process.parent.parent.entity_id + */ export const ANCESTRY_LIMIT: number = 2; interface EventOptions { @@ -240,7 +249,6 @@ export interface Tree { * Map of entity_id to node */ ancestry: Map; - // TODO add the origin to the ancestry array to make test verification easier origin: TreeNode; /** * All events from children, ancestry, origin, and the alert in a single array @@ -345,8 +353,7 @@ export class EndpointDocGenerator { ts = new Date().getTime(), entityID = this.randomString(10), parentEntityID?: string, - ancestryArray: string[] = [], - ancestryLimit: number = 2 + ancestryArray: string[] = [] ): AlertEvent { return { ...this.commonInfo, diff --git a/x-pack/plugins/security_solution/common/endpoint/types.ts b/x-pack/plugins/security_solution/common/endpoint/types.ts index b75081d3ff1ab..e8616ecf1ff74 100644 --- a/x-pack/plugins/security_solution/common/endpoint/types.ts +++ b/x-pack/plugins/security_solution/common/endpoint/types.ts @@ -313,6 +313,11 @@ export interface AlertEvent { thread?: ThreadFields[]; uptime: number; user: string; + /* + * The array has a special format. The entity_ids towards the beginning of the array are closer ancestors and the + * values towards the end of the array are more distant ancestors (grandparents). Therefore + * ancestry_array[0] == process.parent.entity_id and ancestry_array[1] == process.parent.parent.entity_id + */ Ext: { ancestry: string[]; }; @@ -448,6 +453,11 @@ export interface EndpointEvent { entity_id: string; name?: string; }; + /* + * The array has a special format. The entity_ids towards the beginning of the array are closer ancestors and the + * values towards the end of the array are more distant ancestors (grandparents). Therefore + * ancestry_array[0] == process.parent.entity_id and ancestry_array[1] == process.parent.parent.entity_id + */ Ext: { ancestry: string[]; }; From e253fed7e89ba9c8e8f1b7a7436da7ee64bca108 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Thu, 18 Jun 2020 13:36:30 -0400 Subject: [PATCH 14/14] Fixing endgame queries --- .../common/endpoint/models/event.ts | 4 ++-- .../endpoint/routes/resolver/utils/fetch.ts | 18 ++++++++++++++++-- .../api_integration/apis/endpoint/resolver.ts | 1 + 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/models/event.ts b/x-pack/plugins/security_solution/common/endpoint/models/event.ts index 0d6293ad1eb00..fdaef41dbf118 100644 --- a/x-pack/plugins/security_solution/common/endpoint/models/event.ts +++ b/x-pack/plugins/security_solution/common/endpoint/models/event.ts @@ -53,9 +53,9 @@ export function parentEntityId(event: ResolverEvent): string | undefined { return event.process.parent?.entity_id; } -export function ancestryArray(event: ResolverEvent): string[] { +export function ancestryArray(event: ResolverEvent): string[] | undefined { if (isLegacyEvent(event)) { - return []; + return undefined; } return event.process.Ext.ancestry; } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts index fce8e4d934382..8d7c3d7b73158 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts @@ -63,7 +63,7 @@ export class Fetcher { await this.doAncestors( // limit the ancestors we're looking for to the number of levels // the array could be up to length 20 but that could change - ancestryArray(originNode.lifecycle[0]).slice(0, limit), + Fetcher.getAncestryAsArray(originNode.lifecycle[0]).slice(0, limit), limit, ancestryInfo ); @@ -149,6 +149,20 @@ export class Fetcher { return createLifecycle(entityID, results); } + private static getAncestryAsArray(event: ResolverEvent): string[] { + const ancestors = ancestryArray(event); + if (ancestors) { + return ancestors; + } + + const parentID = parentEntityId(event); + if (parentID) { + return [parentID]; + } + + return []; + } + private async doAncestors( ancestors: string[], levels: number, @@ -188,7 +202,7 @@ export class Fetcher { const levelsLeft = levels - ancestryNodes.size; // the results come back in ascending order on timestamp so the first entry in the // results should be the further ancestor (most distant grandparent) - const next = ancestryArray(results[0]).slice(0, levelsLeft); + const next = Fetcher.getAncestryAsArray(results[0]).slice(0, levelsLeft); // the ancestry array currently only holds up to 20 values but we can't rely on that so keep recursing await this.doAncestors(next, levelsLeft, ancestorInfo); } diff --git a/x-pack/test/api_integration/apis/endpoint/resolver.ts b/x-pack/test/api_integration/apis/endpoint/resolver.ts index b702914fc16c5..c9356de07f711 100644 --- a/x-pack/test/api_integration/apis/endpoint/resolver.ts +++ b/x-pack/test/api_integration/apis/endpoint/resolver.ts @@ -424,6 +424,7 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC ) .expect(200); expect(body.ancestors[0].lifecycle.length).to.eql(2); + expect(body.ancestors.length).to.eql(2); expect(body.nextAncestor).to.eql(null); });