Skip to content

[Security Solution] Use safe type in resolver backend #76969

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jonathan-buttner merged 12 commits into from
Sep 10, 2020
Merged

[Security Solution] Use safe type in resolver backend #76969

Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
f9f3921
Moving generator to safe type version
jonathan-buttner Sep 3, 2020
3216705
Finished generator and alert
jonathan-buttner Sep 4, 2020
d2d5f4e
Gzipping again
jonathan-buttner Sep 4, 2020
304335a
Finishing type conversions for backend
jonathan-buttner Sep 8, 2020
4ef0b21
Trying to cast front end tests back to unsafe type for now
jonathan-buttner Sep 8, 2020
910fe24
Working reducer tests
jonathan-buttner Sep 8, 2020
04a588b
Fixing merge conflicts
jonathan-buttner Sep 8, 2020
c06ba93
Adding more comments and fixing alert type
jonathan-buttner Sep 8, 2020
25fa083
Restoring resolver test data
jonathan-buttner Sep 8, 2020
f2909b1
Updating snapshot with timestamp info
jonathan-buttner Sep 8, 2020
b14cb3b
Removing todo and fixing test
jonathan-buttner Sep 10, 2020
7908e51
Merge branch 'master' into use-safe-type
elasticmachine Sep 10, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
136 changes: 75 additions & 61 deletions x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,12 @@ import {
ECSCategory,
ANCESTRY_LIMIT,
} from './generate_data';
import { firstNonNullValue, values } from './models/ecs_safety_helpers';
import {
entityIDSafeVersion,
parentEntityIDSafeVersion,
timestampSafeVersion,
} from './models/event';

interface Node {
events: Event[];
Expand All @@ -30,7 +36,7 @@ describe('data generator', () => {
const event1 = generator.generateEvent();
const event2 = generator.generateEvent();

expect(event2.event.sequence).toBe(event1.event.sequence + 1);
expect(event2.event?.sequence).toBe((firstNonNullValue(event1.event?.sequence) ?? 0) + 1);
});

it('creates the same documents with same random seed', () => {
Expand Down Expand Up @@ -76,37 +82,37 @@ describe('data generator', () => {
const timestamp = new Date().getTime();
const alert = generator.generateAlert(timestamp);
expect(alert['@timestamp']).toEqual(timestamp);
expect(alert.event.action).not.toBeNull();
expect(alert.event?.action).not.toBeNull();
expect(alert.Endpoint).not.toBeNull();
expect(alert.agent).not.toBeNull();
expect(alert.host).not.toBeNull();
expect(alert.process.entity_id).not.toBeNull();
expect(alert.process?.entity_id).not.toBeNull();
});

it('creates process event documents', () => {
const timestamp = new Date().getTime();
const processEvent = generator.generateEvent({ timestamp });
expect(processEvent['@timestamp']).toEqual(timestamp);
expect(processEvent.event.category).toEqual(['process']);
expect(processEvent.event.kind).toEqual('event');
expect(processEvent.event.type).toEqual(['start']);
expect(processEvent.event?.category).toEqual(['process']);
expect(processEvent.event?.kind).toEqual('event');
expect(processEvent.event?.type).toEqual(['start']);
expect(processEvent.agent).not.toBeNull();
expect(processEvent.host).not.toBeNull();
expect(processEvent.process.entity_id).not.toBeNull();
expect(processEvent.process.name).not.toBeNull();
expect(processEvent.process?.entity_id).not.toBeNull();
expect(processEvent.process?.name).not.toBeNull();
});

it('creates other event documents', () => {
const timestamp = new Date().getTime();
const processEvent = generator.generateEvent({ timestamp, eventCategory: 'dns' });
expect(processEvent['@timestamp']).toEqual(timestamp);
expect(processEvent.event.category).toEqual('dns');
expect(processEvent.event.kind).toEqual('event');
expect(processEvent.event.type).toEqual(['start']);
expect(processEvent.event?.category).toEqual('dns');
expect(processEvent.event?.kind).toEqual('event');
expect(processEvent.event?.type).toEqual(['start']);
expect(processEvent.agent).not.toBeNull();
expect(processEvent.host).not.toBeNull();
expect(processEvent.process.entity_id).not.toBeNull();
expect(processEvent.process.name).not.toBeNull();
expect(processEvent.process?.entity_id).not.toBeNull();
expect(processEvent.process?.name).not.toBeNull();
});

describe('creates events with an empty ancestry array', () => {
Expand All @@ -128,7 +134,7 @@ describe('data generator', () => {

it('creates all events with an empty ancestry array', () => {
for (const event of tree.allEvents) {
expect(event.process.Ext!.ancestry!.length).toEqual(0);
expect(event.process?.Ext?.ancestry?.length).toEqual(0);
}
});
});
Expand Down Expand Up @@ -194,36 +200,38 @@ describe('data generator', () => {
const inRelated = node.relatedEvents.includes(event);
const inRelatedAlerts = node.relatedAlerts.includes(event);

return (inRelated || inRelatedAlerts || inLifecycle) && event.process.entity_id === node.id;
return (inRelated || inRelatedAlerts || inLifecycle) && event.process?.entity_id === node.id;
};

const verifyAncestry = (event: Event, genTree: Tree) => {
if (event.process.Ext!.ancestry!.length > 0) {
expect(event.process.parent?.entity_id).toBe(event.process.Ext!.ancestry![0]);
const ancestry = values(event.process?.Ext?.ancestry);
if (ancestry.length > 0) {
expect(event.process?.parent?.entity_id).toBe(ancestry[0]);
}
for (let i = 0; i < event.process.Ext!.ancestry!.length; i++) {
const ancestor = event.process.Ext!.ancestry![i];
for (let i = 0; i < ancestry.length; i++) {
const ancestor = ancestry[i];
const parent = genTree.children.get(ancestor) || genTree.ancestry.get(ancestor);
expect(ancestor).toBe(parent?.lifecycle[0].process.entity_id);
expect(ancestor).toBe(parent?.lifecycle[0].process?.entity_id);

// the next ancestor should be the grandparent
if (i + 1 < event.process.Ext!.ancestry!.length) {
const grandparent = event.process.Ext!.ancestry![i + 1];
expect(grandparent).toBe(parent?.lifecycle[0].process.parent?.entity_id);
if (i + 1 < ancestry.length) {
const grandparent = ancestry[i + 1];
expect(grandparent).toBe(parent?.lifecycle[0].process?.parent?.entity_id);
}
}
};

it('creates related events in ascending order', () => {
// the order should not change since it should already be in ascending order
const relatedEventsAsc = _.cloneDeep(tree.origin.relatedEvents).sort(
(event1, event2) => event1['@timestamp'] - event2['@timestamp']
(event1, event2) =>
(timestampSafeVersion(event1) ?? 0) - (timestampSafeVersion(event2) ?? 0)
);
expect(tree.origin.relatedEvents).toStrictEqual(relatedEventsAsc);
});

it('has ancestry array defined', () => {
expect(tree.origin.lifecycle[0].process.Ext!.ancestry!.length).toBe(ANCESTRY_LIMIT);
expect(values(tree.origin.lifecycle[0].process?.Ext?.ancestry).length).toBe(ANCESTRY_LIMIT);
for (const event of tree.allEvents) {
verifyAncestry(event, tree);
}
Expand Down Expand Up @@ -252,12 +260,9 @@ describe('data generator', () => {

const counts: Record<string, number> = {};
for (const event of node.relatedEvents) {
if (Array.isArray(event.event.category)) {
for (const cat of event.event.category) {
counts[cat] = counts[cat] + 1 || 1;
}
} else {
counts[event.event.category] = counts[event.event.category] + 1 || 1;
const categories = values(event.event?.category);
for (const cat of categories) {
counts[cat] = counts[cat] + 1 || 1;
}
}
expect(counts[ECSCategory.Driver]).toEqual(1);
Expand Down Expand Up @@ -316,15 +321,18 @@ describe('data generator', () => {
expect(tree.allEvents.length).toBeGreaterThan(0);

tree.allEvents.forEach((event) => {
const ancestor = tree.ancestry.get(event.process.entity_id);
if (ancestor) {
expect(eventInNode(event, ancestor)).toBeTruthy();
return;
}
const entityID = entityIDSafeVersion(event);
if (entityID) {
const ancestor = tree.ancestry.get(entityID);
if (ancestor) {
expect(eventInNode(event, ancestor)).toBeTruthy();
return;
}

const children = tree.children.get(event.process.entity_id);
if (children) {
expect(eventInNode(event, children)).toBeTruthy();
const children = tree.children.get(entityID);
if (children) {
expect(eventInNode(event, children)).toBeTruthy();
}
}
});
});
Expand All @@ -351,9 +359,8 @@ describe('data generator', () => {
let events: Event[];

const isCategoryProcess = (event: Event) => {
return (
_.isEqual(event.event.category, ['process']) || _.isEqual(event.event.category, 'process')
);
const category = values(event.event?.category);
return _.isEqual(category, ['process']);
};

beforeEach(() => {
Expand All @@ -366,12 +373,16 @@ describe('data generator', () => {

it('with n-1 process events', () => {
for (let i = events.length - 2; i > 0; ) {
const parentEntityIdOfChild = events[i].process.parent?.entity_id;
for (; --i >= -1 && (events[i].event.kind !== 'event' || !isCategoryProcess(events[i])); ) {
const parentEntityIdOfChild = parentEntityIDSafeVersion(events[i]);
for (
;
--i >= -1 && (events[i].event?.kind !== 'event' || !isCategoryProcess(events[i]));

) {
// related event - skip it
}
expect(i).toBeGreaterThanOrEqual(0);
expect(parentEntityIdOfChild).toEqual(events[i].process.entity_id);
expect(parentEntityIdOfChild).toEqual(entityIDSafeVersion(events[i]));
}
});

Expand All @@ -380,37 +391,40 @@ describe('data generator', () => {
for (
;
previousProcessEventIndex >= -1 &&
(events[previousProcessEventIndex].event.kind !== 'event' ||
(events[previousProcessEventIndex].event?.kind !== 'event' ||
!isCategoryProcess(events[previousProcessEventIndex]));
previousProcessEventIndex--
) {
// related event - skip it
}
expect(previousProcessEventIndex).toBeGreaterThanOrEqual(0);
// The alert should be last and have the same entity_id as the previous process event
expect(events[events.length - 1].process.entity_id).toEqual(
events[previousProcessEventIndex].process.entity_id
expect(events[events.length - 1].process?.entity_id).toEqual(
events[previousProcessEventIndex].process?.entity_id
);
expect(events[events.length - 1].process.parent?.entity_id).toEqual(
events[previousProcessEventIndex].process.parent?.entity_id
expect(events[events.length - 1].process?.parent?.entity_id).toEqual(
events[previousProcessEventIndex].process?.parent?.entity_id
);
expect(events[events.length - 1].event.kind).toEqual('alert');
expect(events[events.length - 1].event.category).toEqual('malware');
expect(events[events.length - 1].event?.kind).toEqual('alert');
expect(events[events.length - 1].event?.category).toEqual('malware');
});
});

function buildResolverTree(events: Event[]): Node {
// First pass we gather up all the events by entity_id
const tree: Record<string, Node> = {};
events.forEach((event) => {
if (event.process.entity_id in tree) {
tree[event.process.entity_id].events.push(event);
} else {
tree[event.process.entity_id] = {
events: [event],
children: [],
parent_entity_id: event.process.parent?.entity_id,
};
const entityID = entityIDSafeVersion(event);
if (entityID) {
if (entityID in tree) {
tree[entityID].events.push(event);
} else {
tree[entityID] = {
events: [event],
children: [],
parent_entity_id: parentEntityIDSafeVersion(event),
};
}
}
});
// Second pass add child references to each node
Expand All @@ -420,7 +434,7 @@ describe('data generator', () => {
}
}
// The root node must be first in the array or this fails
return tree[events[0].process.entity_id];
return tree[entityIDSafeVersion(events[0]) ?? ''];
Copy link
Contributor

@oatkiller oatkiller Sep 10, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think '' would be correct here. Maybe this instead?

Suggested change
return tree[entityIDSafeVersion(events[0]) ?? ''];
const entityID = entityIDSafeVersion(events[0])
if (entityID === undefined) {
// this should never happen.
throw new Error()
}
return tree[entityIDSafeVersion(events[0])];

Or this?

Suggested change
return tree[entityIDSafeVersion(events[0]) ?? ''];
return tree[entityIDSafeVersion(events[0])!];

I'm not that familiar with the code.

Copy link
Contributor Author

@jonathan-buttner jonathan-buttner Sep 10, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah good point, I'll throw an error like you suggested 👍

}

function countResolverEvents(rootNode: Node, generations: number): number {
Expand Down
Loading