diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts index d7b653916970f..4516007580edc 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts @@ -10,6 +10,7 @@ import { TreeNode, RelatedEventCategory, ECSCategory, + ANCESTRY_LIMIT, } from './generate_data'; interface Node { @@ -113,6 +114,7 @@ describe('data generator', () => { relatedEvents: 0, relatedAlerts: 0, }); + tree.ancestry.delete(tree.origin.id); }); it('creates an alert for the origin node but no other nodes', () => { @@ -159,6 +161,30 @@ describe('data generator', () => { return (inRelated || inRelatedAlerts || inLifecycle) && event.process.entity_id === node.id; }; + const verifyAncestry = (event: Event, genTree: Tree) => { + if (event.process.Ext.ancestry.length > 0) { + expect(event.process.parent?.entity_id).toBe(event.process.Ext.ancestry[0]); + } + for (let i = 0; i < event.process.Ext.ancestry.length; i++) { + const ancestor = event.process.Ext.ancestry[i]; + const parent = genTree.children.get(ancestor) || genTree.ancestry.get(ancestor); + expect(ancestor).toBe(parent?.lifecycle[0].process.entity_id); + + // the next ancestor should be the grandparent + if (i + 1 < event.process.Ext.ancestry.length) { + const grandparent = event.process.Ext.ancestry[i + 1]; + expect(grandparent).toBe(parent?.lifecycle[0].process.parent?.entity_id); + } + } + }; + + it('has ancestry array defined', () => { + expect(tree.origin.lifecycle[0].process.Ext.ancestry.length).toBe(ANCESTRY_LIMIT); + for (const event of tree.allEvents) { + verifyAncestry(event, tree); + } + }); + it('has the right related events for each node', () => { const checkRelatedEvents = (node: TreeNode) => { expect(node.relatedEvents.length).toEqual(4); @@ -185,8 +211,6 @@ describe('data generator', () => { for (const node of tree.children.values()) { checkRelatedEvents(node); } - - checkRelatedEvents(tree.origin); }); it('has the right number of related alerts for each node', () => { @@ -202,7 +226,8 @@ describe('data generator', () => { }); it('has the right number of ancestors', () => { - expect(tree.ancestry.size).toEqual(ancestors); + // +1 for the origin node + expect(tree.ancestry.size).toEqual(ancestors + 1); }); it('has the right number of total children', () => { @@ -239,10 +264,7 @@ describe('data generator', () => { const children = tree.children.get(event.process.entity_id); if (children) { expect(eventInNode(event, children)).toBeTruthy(); - return; } - - expect(eventInNode(event, tree.origin)).toBeTruthy(); }); }); @@ -260,8 +282,6 @@ describe('data generator', () => { total += nodeEventCount(node); } - total += nodeEventCount(tree.origin); - expect(tree.allEvents.length).toEqual(total); }); }); diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts index ad140e48d4c64..ea3d61564011e 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts @@ -18,6 +18,16 @@ import { import { factory as policyFactory } from './models/policy_config'; export type Event = AlertEvent | EndpointEvent; +/** + * This value indicates the limit for the size of the ancestry array. The endpoint currently saves up to 20 values + * in its messages. To simulate a limit on the array size I'm using 2 here so that we can't rely on there being a large + * number like 20. The ancestry array contains entity_ids for the ancestors of a particular process. + * + * The array has a special format. The entity_ids towards the beginning of the array are closer ancestors and the + * values towards the end of the array are more distant ancestors (grandparents). Therefore + * ancestry_array[0] == process.parent.entity_id and ancestry_array[1] == process.parent.parent.entity_id + */ +export const ANCESTRY_LIMIT: number = 2; interface EventOptions { timestamp?: number; @@ -26,6 +36,7 @@ interface EventOptions { eventType?: string; eventCategory?: string | string[]; processName?: string; + ancestry?: string[]; pid?: number; parentPid?: number; extensions?: object; @@ -352,7 +363,8 @@ export class EndpointDocGenerator { public generateAlert( ts = new Date().getTime(), entityID = this.randomString(10), - parentEntityID?: string + parentEntityID?: string, + ancestryArray: string[] = [] ): AlertEvent { return { ...this.commonInfo, @@ -412,6 +424,9 @@ export class EndpointDocGenerator { sha256: 'fake sha256', }, Ext: { + // simulate a finite ancestry array size, the endpoint limits the ancestry array to 20 entries we'll use + // 2 so that the backend can handle that case + ancestry: ancestryArray.slice(0, ANCESTRY_LIMIT), code_signature: [ { trusted: false, @@ -532,6 +547,9 @@ export class EndpointDocGenerator { } : undefined, name: processName, + // simulate a finite ancestry array size, the endpoint limits the ancestry array to 20 entries we'll use + // 2 so that the backend can handle that case + Ext: { ancestry: options.ancestry?.slice(0, ANCESTRY_LIMIT) || [] }, }, user: { domain: this.randomString(10), @@ -589,9 +607,6 @@ export class EndpointDocGenerator { throw Error(`could not find origin while building tree: ${alert.process.entity_id}`); } - // remove the origin node from the ancestry array - ancestryNodes.delete(alert.process.entity_id); - const children = Array.from( this.descendantsTreeGenerator( alert, @@ -715,7 +730,7 @@ export class EndpointDocGenerator { } }; - // generate related alerts for rootW + // generate related alerts for root const processDuration: number = 6 * 3600; if (this.randomN(100) < pctWithRelated) { addRelatedEvents(ancestor, processDuration, events); @@ -740,6 +755,8 @@ export class EndpointDocGenerator { ancestor = this.generateEvent({ timestamp, parentEntityID: ancestor.process.entity_id, + // add the parent to the ancestry array + ancestry: [ancestor.process.entity_id, ...ancestor.process.Ext.ancestry], parentPid: ancestor.process.pid, pid: this.randomN(5000), }); @@ -755,6 +772,7 @@ export class EndpointDocGenerator { parentEntityID: ancestor.process.parent?.entity_id, eventCategory: 'process', eventType: 'end', + ancestry: ancestor.process.Ext.ancestry, }) ); } @@ -773,7 +791,12 @@ export class EndpointDocGenerator { } } events.push( - this.generateAlert(timestamp, ancestor.process.entity_id, ancestor.process.parent?.entity_id) + this.generateAlert( + timestamp, + ancestor.process.entity_id, + ancestor.process.parent?.entity_id, + ancestor.process.Ext.ancestry + ) ); return events; } @@ -828,6 +851,10 @@ export class EndpointDocGenerator { const child = this.generateEvent({ timestamp, parentEntityID: currentState.event.process.entity_id, + ancestry: [ + currentState.event.process.entity_id, + ...currentState.event.process.Ext.ancestry, + ], }); maxChildren = this.randomN(maxChildrenPerNode + 1); @@ -849,6 +876,7 @@ export class EndpointDocGenerator { parentEntityID: child.process.parent?.entity_id, eventCategory: 'process', eventType: 'end', + ancestry: child.process.Ext.ancestry, }); } if (this.randomN(100) < percentNodesWithRelated) { @@ -893,6 +921,7 @@ export class EndpointDocGenerator { parentEntityID: node.process.parent?.entity_id, eventCategory: eventInfo.category, eventType: eventInfo.creationType, + ancestry: node.process.Ext.ancestry, }); } } @@ -911,7 +940,12 @@ export class EndpointDocGenerator { ) { for (let i = 0; i < relatedAlerts; i++) { const ts = node['@timestamp'] + this.randomN(alertCreationTime) * 1000; - yield this.generateAlert(ts, node.process.entity_id, node.process.parent?.entity_id); + yield this.generateAlert( + ts, + node.process.entity_id, + node.process.parent?.entity_id, + node.process.Ext.ancestry + ); } } diff --git a/x-pack/plugins/security_solution/common/endpoint/models/event.ts b/x-pack/plugins/security_solution/common/endpoint/models/event.ts index 3f07bf77abf20..98f4b4336a1c8 100644 --- a/x-pack/plugins/security_solution/common/endpoint/models/event.ts +++ b/x-pack/plugins/security_solution/common/endpoint/models/event.ts @@ -53,6 +53,13 @@ export function parentEntityId(event: ResolverEvent): string | undefined { return event.process.parent?.entity_id; } +export function ancestryArray(event: ResolverEvent): string[] | undefined { + if (isLegacyEvent(event)) { + return undefined; + } + return event.process.Ext.ancestry; +} + /** * @param event The event to get the category for */ diff --git a/x-pack/plugins/security_solution/common/endpoint/types.ts b/x-pack/plugins/security_solution/common/endpoint/types.ts index 0b93af741da22..f8cfb8f7c3bbc 100644 --- a/x-pack/plugins/security_solution/common/endpoint/types.ts +++ b/x-pack/plugins/security_solution/common/endpoint/types.ts @@ -300,6 +300,12 @@ export interface AlertEvent { thread?: ThreadFields[]; uptime: number; Ext: { + /* + * The array has a special format. The entity_ids towards the beginning of the array are closer ancestors and the + * values towards the end of the array are more distant ancestors (grandparents). Therefore + * ancestry_array[0] == process.parent.entity_id and ancestry_array[1] == process.parent.parent.entity_id + */ + ancestry: string[]; code_signature: Array<{ subject_name: string; trusted: boolean; @@ -469,6 +475,14 @@ export interface EndpointEvent { name?: string; pid?: number; }; + /* + * The array has a special format. The entity_ids towards the beginning of the array are closer ancestors and the + * values towards the end of the array are more distant ancestors (grandparents). Therefore + * ancestry_array[0] == process.parent.entity_id and ancestry_array[1] == process.parent.parent.entity_id + */ + Ext: { + ancestry: string[]; + }; }; user?: { domain?: string; diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts index da6d8e2cbde81..8d7c3d7b73158 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts @@ -10,8 +10,14 @@ import { ResolverRelatedEvents, ResolverAncestry, ResolverRelatedAlerts, + LifecycleNode, + ResolverEvent, } from '../../../../../common/endpoint/types'; -import { entityId, parentEntityId } from '../../../../../common/endpoint/models/event'; +import { + entityId, + ancestryArray, + parentEntityId, +} from '../../../../../common/endpoint/models/event'; import { PaginationBuilder } from './pagination'; import { Tree } from './tree'; import { LifecycleQuery } from '../queries/lifecycle'; @@ -48,9 +54,21 @@ export class Fetcher { * @param limit upper limit of ancestors to retrieve */ public async ancestors(limit: number): Promise { - const root = createAncestry(); - await this.doAncestors(this.id, limit + 1, root); - return root; + const ancestryInfo = createAncestry(); + const originNode = await this.getNode(this.id); + if (originNode) { + ancestryInfo.ancestors.push(originNode); + // If the request is only for the origin node then set next to its parent + ancestryInfo.nextAncestor = parentEntityId(originNode.lifecycle[0]) || null; + await this.doAncestors( + // limit the ancestors we're looking for to the number of levels + // the array could be up to length 20 but that could change + Fetcher.getAncestryAsArray(originNode.lifecycle[0]).slice(0, limit), + limit, + ancestryInfo + ); + } + return ancestryInfo; } /** @@ -89,7 +107,26 @@ export class Fetcher { * @param after a cursor to use as the starting point for retrieving alerts */ public async alerts(limit: number, after?: string): Promise { - return this.doAlerts(limit, after); + const query = new AlertsQuery( + PaginationBuilder.createBuilder(limit, after), + this.indexPattern, + this.endpointID + ); + + const { totals, results } = await query.search(this.client, this.id); + if (results.length === 0) { + // return an empty set of results + return createRelatedAlerts(this.id); + } + if (!totals[this.id]) { + throw new Error(`Could not find the totals for related events entity_id: ${this.id}`); + } + + return createRelatedAlerts( + this.id, + results, + PaginationBuilder.buildCursor(totals[this.id], results) + ); } /** @@ -102,29 +139,72 @@ export class Fetcher { return tree; } + private async getNode(entityID: string): Promise { + const query = new LifecycleQuery(this.indexPattern, this.endpointID); + const results = await query.search(this.client, entityID); + if (results.length === 0) { + return; + } + + return createLifecycle(entityID, results); + } + + private static getAncestryAsArray(event: ResolverEvent): string[] { + const ancestors = ancestryArray(event); + if (ancestors) { + return ancestors; + } + + const parentID = parentEntityId(event); + if (parentID) { + return [parentID]; + } + + return []; + } + private async doAncestors( - curNodeID: string, + ancestors: string[], levels: number, ancestorInfo: ResolverAncestry ): Promise { - if (levels === 0) { - ancestorInfo.nextAncestor = curNodeID; + if (levels <= 0) { return; } const query = new LifecycleQuery(this.indexPattern, this.endpointID); - const results = await query.search(this.client, curNodeID); + const results = await query.search(this.client, ancestors); if (results.length === 0) { + ancestorInfo.nextAncestor = null; return; } - ancestorInfo.ancestors.push(createLifecycle(curNodeID, results)); - const next = parentEntityId(results[0]); - if (next === undefined) { - return; - } - await this.doAncestors(next, levels - 1, ancestorInfo); + // bucket the start and end events together for a single node + const ancestryNodes = results.reduce( + (nodes: Map, ancestorEvent: ResolverEvent) => { + const nodeId = entityId(ancestorEvent); + let node = nodes.get(nodeId); + if (!node) { + node = createLifecycle(nodeId, []); + } + + node.lifecycle.push(ancestorEvent); + return nodes.set(nodeId, node); + }, + new Map() + ); + + // the order of this array is going to be weird, it will look like this + // [furthest grandparent...closer grandparent, next recursive call furthest grandparent...closer grandparent] + ancestorInfo.ancestors.push(...ancestryNodes.values()); + ancestorInfo.nextAncestor = parentEntityId(results[0]) || null; + const levelsLeft = levels - ancestryNodes.size; + // the results come back in ascending order on timestamp so the first entry in the + // results should be the further ancestor (most distant grandparent) + const next = Fetcher.getAncestryAsArray(results[0]).slice(0, levelsLeft); + // the ancestry array currently only holds up to 20 values but we can't rely on that so keep recursing + await this.doAncestors(next, levelsLeft, ancestorInfo); } private async doEvents(limit: number, after?: string) { @@ -150,29 +230,6 @@ export class Fetcher { ); } - private async doAlerts(limit: number, after?: string) { - const query = new AlertsQuery( - PaginationBuilder.createBuilder(limit, after), - this.indexPattern, - this.endpointID - ); - - const { totals, results } = await query.search(this.client, this.id); - if (results.length === 0) { - // return an empty set of results - return createRelatedAlerts(this.id); - } - if (!totals[this.id]) { - throw new Error(`Could not find the totals for related events entity_id: ${this.id}`); - } - - return createRelatedAlerts( - this.id, - results, - PaginationBuilder.buildCursor(totals[this.id], results) - ); - } - private async doChildren( cache: ChildrenNodesHelper, ids: string[], diff --git a/x-pack/test/api_integration/apis/endpoint/resolver.ts b/x-pack/test/api_integration/apis/endpoint/resolver.ts index 2bb52473508b3..c9356de07f711 100644 --- a/x-pack/test/api_integration/apis/endpoint/resolver.ts +++ b/x-pack/test/api_integration/apis/endpoint/resolver.ts @@ -41,25 +41,6 @@ const expectLifecycleNodeInMap = (node: LifecycleNode, nodeMap: Map { - compareArrays(tree.origin.lifecycle, origin.lifecycle, true); - verifyAncestry(ancestors, tree, verifyLastParent); -}; - /** * Verify that all the ancestor nodes are valid and optionally have parents. * @@ -79,14 +60,59 @@ const verifyAncestry = (ancestors: LifecycleNode[], tree: Tree, verifyLastParent expect(Object.keys(groupedAncestors).length).to.eql(ancestors.length); // make sure there aren't any nodes with the same parent entity_id expect(Object.keys(groupedAncestorsParent).length).to.eql(ancestors.length); - ancestors.forEach((node) => { + + // make sure each of the ancestors' lifecycle events are in the generated tree + for (const node of ancestors) { + expectLifecycleNodeInMap(node, tree.ancestry); + } + + // start at the origin which is always the first element of the array and make sure we have a connection + // using parent id between each of the nodes + let foundParents = 0; + let node = ancestors[0]; + for (let i = 0; i < ancestors.length; i++) { const parentID = parentEntityId(node.lifecycle[0]); - // the last node generated will have `undefined` as the parent entity_id - if (parentID !== undefined && verifyLastParent) { - expect(groupedAncestors[parentID]).to.be.ok(); + if (parentID !== undefined) { + const nextNode = groupedAncestors[parentID]; + if (!nextNode) { + break; + } + // the grouped nodes should only have a single entry since each entity is unique + node = nextNode[0]; } - expectLifecycleNodeInMap(node, tree.ancestry); - }); + foundParents++; + } + + if (verifyLastParent) { + expect(foundParents).to.eql(ancestors.length); + } else { + // if we only retrieved a portion of all the ancestors then the most distant grandparent's parent will not necessarily + // be in the results + expect(foundParents).to.eql(ancestors.length - 1); + } +}; + +/** + * Retrieves the most distant ancestor in the given array. + * + * @param ancestors an array of ancestor nodes + */ +const retrieveDistantAncestor = (ancestors: LifecycleNode[]) => { + // group the ancestors by their entity_id mapped to a lifecycle node + const groupedAncestors = _.groupBy(ancestors, (ancestor) => ancestor.entityID); + let node = ancestors[0]; + for (let i = 0; i < ancestors.length; i++) { + const parentID = parentEntityId(node.lifecycle[0]); + if (parentID !== undefined) { + const nextNode = groupedAncestors[parentID]; + if (nextNode) { + node = nextNode[0]; + } else { + return node; + } + } + } + return node; }; /** @@ -398,6 +424,7 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC ) .expect(200); expect(body.ancestors[0].lifecycle.length).to.eql(2); + expect(body.ancestors.length).to.eql(2); expect(body.nextAncestor).to.eql(null); }); @@ -425,9 +452,12 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC }); describe('endpoint events', () => { - const getRootAndAncestry = (ancestry: ResolverAncestry) => { - return { root: ancestry.ancestors[0], ancestry: ancestry.ancestors.slice(1) }; - }; + it('should return the origin node at the front of the array', async () => { + const { body }: { body: ResolverAncestry } = await supertest + .get(`/api/endpoint/resolver/${tree.origin.id}/ancestry?ancestors=9`) + .expect(200); + expect(body.ancestors[0].entityID).to.eql(tree.origin.id); + }); it('should return details for the root node', async () => { const { body }: { body: ResolverAncestry } = await supertest @@ -435,8 +465,8 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC .expect(200); // the tree we generated had 5 ancestors + 1 origin node expect(body.ancestors.length).to.eql(6); - const ancestryInfo = getRootAndAncestry(body); - verifyAncestryFromOrigin(ancestryInfo.root, ancestryInfo.ancestry, tree, true); + expect(body.ancestors[0].entityID).to.eql(tree.origin.id); + verifyAncestry(body.ancestors, tree, true); expect(body.nextAncestor).to.eql(null); }); @@ -454,12 +484,9 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC .expect(200); // it should have 2 ancestors + 1 origin expect(body.ancestors.length).to.eql(3); - const ancestryInfo = getRootAndAncestry(body); - verifyAncestryFromOrigin(ancestryInfo.root, ancestryInfo.ancestry, tree, false); - expect(body.nextAncestor).to.eql( - // it should be the parent entity id on the last element of the ancestry array - parentEntityId(ancestryInfo.ancestry[ancestryInfo.ancestry.length - 1].lifecycle[0]) - ); + verifyAncestry(body.ancestors, tree, false); + const distantGrandparent = retrieveDistantAncestor(body.ancestors); + expect(body.nextAncestor).to.eql(parentEntityId(distantGrandparent.lifecycle[0])); }); it('should handle multiple ancestor requests', async () => {