From ec2922ab8fe00a4fa53b534677a046475cd4ddb6 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Wed, 2 Jun 2021 23:15:28 -0500 Subject: [PATCH 1/6] Adding boilerplate for new CTI search strategy type This is going to be a subtype of the general SecSol search strategy; the main functionality is going to be: * transformation of the incoming parameters into named equivalents * transformation of responses to include enrichment context fields (matched.*) --- .../security_solution/cti/index.ts | 10 ++++++++++ .../search_strategy/security_solution/index.ts | 2 ++ .../factory/cti/event_enrichment/index.ts | 8 ++++++++ .../factory/cti/event_enrichment/query.ts | 16 ++++++++++++++++ .../security_solution/factory/cti/index.ts | 15 +++++++++++++++ .../security_solution/factory/index.ts | 6 ++++-- 6 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 x-pack/plugins/security_solution/common/search_strategy/security_solution/cti/index.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/index.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/index.ts diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/cti/index.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/cti/index.ts new file mode 100644 index 0000000000000..c86b82fcf666e --- /dev/null +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/cti/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export enum CtiQueries { + eventEnrichment = 'eventEnrichment', +} diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts index 956b785079d8d..ea03393856087 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts @@ -66,6 +66,7 @@ import { MatrixHistogramStrategyResponse, } from './matrix_histogram'; import { TimerangeInput, SortField, PaginationInput, PaginationInputPaginated } from '../common'; +import { CtiQueries } from './cti'; export * from './hosts'; export * from './matrix_histogram'; @@ -76,6 +77,7 @@ export type FactoryQueryTypes = | HostsKpiQueries | NetworkQueries | NetworkKpiQueries + | CtiQueries | typeof MatrixHistogramQuery | typeof MatrixHistogramQueryEntities; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/index.ts new file mode 100644 index 0000000000000..66ede766e4741 --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './query'; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.ts new file mode 100644 index 0000000000000..41ab486f63898 --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { CtiQueries } from '../../../../../../common/search_strategy/security_solution/cti'; +import { SecuritySolutionFactory } from '../../types'; + +const ident = (a: unknown) => a; + +export const eventEnrichment: SecuritySolutionFactory = { + buildDsl: ident, + parse: ident, +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/index.ts new file mode 100644 index 0000000000000..5857a0417239c --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/index.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { FactoryQueryTypes } from '../../../../../common/search_strategy/security_solution'; +import { CtiQueries } from '../../../../../common/search_strategy/security_solution/cti'; +import type { SecuritySolutionFactory } from '../types'; +import { eventEnrichment } from './event_enrichment'; + +export const ctiFactoryTypes: Record> = { + [CtiQueries.eventEnrichment]: eventEnrichment, +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/index.ts index 346dd20c89441..5b54c63408d10 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/index.ts @@ -5,12 +5,13 @@ * 2.0. */ -import { FactoryQueryTypes } from '../../../../common/search_strategy/security_solution'; +import type { FactoryQueryTypes } from '../../../../common/search_strategy/security_solution'; +import type { SecuritySolutionFactory } from './types'; import { hostsFactory } from './hosts'; import { matrixHistogramFactory } from './matrix_histogram'; import { networkFactory } from './network'; -import { SecuritySolutionFactory } from './types'; +import { ctiFactoryTypes } from './cti'; export const securitySolutionFactory: Record< FactoryQueryTypes, @@ -19,4 +20,5 @@ export const securitySolutionFactory: Record< ...hostsFactory, ...matrixHistogramFactory, ...networkFactory, + ...ctiFactoryTypes, }; From 1ef6fd6190b58f1cf55e6508a2f194b70dbaa50c Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 3 Jun 2021 23:41:40 -0500 Subject: [PATCH 2/6] More boilerplate, including tests A few type errors because our functions don't actually do anything yet, nor are our request/response types fleshed out. --- .../security_solution/cti/index.ts | 11 +++++++++++ .../search_strategy/security_solution/index.ts | 10 +++++++++- .../factory/cti/event_enrichment/factory.ts | 16 ++++++++++++++++ .../factory/cti/event_enrichment/index.ts | 2 +- .../factory/cti/event_enrichment/query.test.ts | 10 ++++++++++ .../factory/cti/event_enrichment/query.ts | 10 +--------- .../cti/event_enrichment/response.test.ts | 10 ++++++++++ .../factory/cti/event_enrichment/response.ts | 8 ++++++++ 8 files changed, 66 insertions(+), 11 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/factory.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.test.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/response.test.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/response.ts diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/cti/index.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/cti/index.ts index c86b82fcf666e..9b04836c2405a 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/security_solution/cti/index.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/cti/index.ts @@ -5,6 +5,17 @@ * 2.0. */ +import { IEsSearchResponse } from 'src/plugins/data/public'; +import { Inspect } from '../../common'; +import { RequestBasicOptions } from '..'; + export enum CtiQueries { eventEnrichment = 'eventEnrichment', } + +export interface CtiEventEnrichmentRequestOptions extends RequestBasicOptions {} + +export interface CtiEventEnrichmentStrategyResponse extends IEsSearchResponse { + inspect?: Inspect; + totalCount: number; +} diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts index ea03393856087..06d4a16699b8f 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts @@ -66,7 +66,11 @@ import { MatrixHistogramStrategyResponse, } from './matrix_histogram'; import { TimerangeInput, SortField, PaginationInput, PaginationInputPaginated } from '../common'; -import { CtiQueries } from './cti'; +import { + CtiEventEnrichmentRequestOptions, + CtiEventEnrichmentStrategyResponse, + CtiQueries, +} from './cti'; export * from './hosts'; export * from './matrix_histogram'; @@ -147,6 +151,8 @@ export type StrategyResponseType = T extends HostsQ ? NetworkKpiUniquePrivateIpsStrategyResponse : T extends typeof MatrixHistogramQuery ? MatrixHistogramStrategyResponse + : T extends CtiQueries.eventEnrichment + ? CtiEventEnrichmentStrategyResponse : never; export type StrategyRequestType = T extends HostsQueries.hosts @@ -195,6 +201,8 @@ export type StrategyRequestType = T extends HostsQu ? NetworkKpiUniquePrivateIpsRequestOptions : T extends typeof MatrixHistogramQuery ? MatrixHistogramRequestOptions + : T extends CtiQueries.eventEnrichment + ? CtiEventEnrichmentRequestOptions : never; export interface DocValueFieldsInput { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/factory.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/factory.ts new file mode 100644 index 0000000000000..e5e1a14df3c1c --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/factory.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { CtiQueries } from '../../../../../../common/search_strategy/security_solution/cti'; +import { SecuritySolutionFactory } from '../../types'; +import { buildEventEnrichmentQuery } from './query'; +import { parseEventEnrichmentResponse } from './response'; + +export const eventEnrichment: SecuritySolutionFactory = { + buildDsl: buildEventEnrichmentQuery, + parse: parseEventEnrichmentResponse, +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/index.ts index 66ede766e4741..6884b7b6320cf 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/index.ts @@ -5,4 +5,4 @@ * 2.0. */ -export * from './query'; +export { eventEnrichment } from './factory'; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.test.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.test.ts new file mode 100644 index 0000000000000..b4a7542a53489 --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.test.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +describe('buildEventEnrichmentQuery', () => { + it('makes each query filter named based on its field'); +}); diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.ts index 41ab486f63898..9a193a43e3579 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.ts @@ -5,12 +5,4 @@ * 2.0. */ -import { CtiQueries } from '../../../../../../common/search_strategy/security_solution/cti'; -import { SecuritySolutionFactory } from '../../types'; - -const ident = (a: unknown) => a; - -export const eventEnrichment: SecuritySolutionFactory = { - buildDsl: ident, - parse: ident, -}; +export const buildEventEnrichmentQuery = (a: unknown) => a; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/response.test.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/response.test.ts new file mode 100644 index 0000000000000..b353b49a1b802 --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/response.test.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +describe('parseEventEnrichmentResponse', () => { + it('adds matched.* enrichment fields based on the named query'); +}); diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/response.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/response.ts new file mode 100644 index 0000000000000..1902129b3aa9a --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/response.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const parseEventEnrichmentResponse = (a: unknown) => a; From b3410e5ff0d1a7dfe70efc7291942c2421f43e88 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 4 Jun 2021 23:34:09 -0500 Subject: [PATCH 3/6] Starting to flesh out the request parsing * Defines a basic request, along with a mock * Defines helper function to generate should clauses from field values * Adds placeholder tests throughout --- .../security_solution/common/cti/constants.ts | 13 +++++ .../security_solution/cti/index.mock.ts | 23 ++++++++ .../security_solution/cti/index.ts | 4 +- .../cti/event_enrichment/helpers.test.ts | 52 +++++++++++++++++++ .../factory/cti/event_enrichment/helpers.ts | 36 +++++++++++++ .../cti/event_enrichment/query.test.ts | 12 ++++- .../factory/cti/event_enrichment/query.ts | 44 +++++++++++++++- .../cti/event_enrichment/response.test.ts | 1 + .../factory/cti/event_enrichment/response.ts | 9 +++- 9 files changed, 190 insertions(+), 4 deletions(-) create mode 100644 x-pack/plugins/security_solution/common/search_strategy/security_solution/cti/index.mock.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/helpers.test.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/helpers.ts diff --git a/x-pack/plugins/security_solution/common/cti/constants.ts b/x-pack/plugins/security_solution/common/cti/constants.ts index 3423f17e3f683..10452996eae6f 100644 --- a/x-pack/plugins/security_solution/common/cti/constants.ts +++ b/x-pack/plugins/security_solution/common/cti/constants.ts @@ -44,3 +44,16 @@ export const SORTED_THREAT_SUMMARY_FIELDS = [ INDICATOR_FIRSTSEEN, INDICATOR_LASTSEEN, ]; + +export const EVENT_ENRICHMENT_INDICATOR_FIELD_MAP = { + 'file.hash.md5': 'threatintel.indicator.file.hash.md5', + 'file.hash.sha1': 'threatintel.indicator.file.hash.sha1', + 'file.hash.sha256': 'threatintel.indicator.file.hash.sha256', + 'file.pe.imphash': 'threatintel.indicator.file.pe.imphash', + 'file.elf.telfhash': 'threatintel.indicator.file.elf.telfhash', + 'file.hash.ssdeep': 'threatintel.indicator.file.hash.ssdeep', + 'source.ip': 'threatintel.indicator.ip', + 'destination.ip': 'threatintel.indicator.ip', + 'url.full': 'threatintel.indicator.url.full', + 'registry.path': 'threatintel.indicator.registry.path', +}; diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/cti/index.mock.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/cti/index.mock.ts new file mode 100644 index 0000000000000..bcbf75aca3762 --- /dev/null +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/cti/index.mock.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { CtiEventEnrichmentRequestOptions, CtiQueries } from '.'; + +export const buildRequestOptionsMock = ( + overrides: Partial = {} +): CtiEventEnrichmentRequestOptions => ({ + defaultIndex: ['filebeat-*'], + eventFields: { + 'file.hash.md5': '1eee2bf3f56d8abed72da2bc523e7431', + 'source.ip': '127.0.0.1', + 'url.full': 'elastic.co', + }, + factoryQueryType: CtiQueries.eventEnrichment, + filterQuery: '{"bool":{"must":[],"filter":[{"match_all":{}}],"should":[],"must_not":[]}}', + timerange: { interval: '', from: '2020-09-13T09:00:43.249Z', to: '2020-09-14T09:00:43.249Z' }, + ...overrides, +}); diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/cti/index.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/cti/index.ts index 9b04836c2405a..549605ba5bff7 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/security_solution/cti/index.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/cti/index.ts @@ -13,7 +13,9 @@ export enum CtiQueries { eventEnrichment = 'eventEnrichment', } -export interface CtiEventEnrichmentRequestOptions extends RequestBasicOptions {} +export interface CtiEventEnrichmentRequestOptions extends RequestBasicOptions { + eventFields: Record; +} export interface CtiEventEnrichmentStrategyResponse extends IEsSearchResponse { inspect?: Inspect; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/helpers.test.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/helpers.test.ts new file mode 100644 index 0000000000000..69de558ad0b0e --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/helpers.test.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { buildIndicatorShouldClauses } from './helpers'; + +describe('buildIndicatorShouldClauses', () => { + it('returns an empty array given an empty fieldset', () => { + expect(buildIndicatorShouldClauses({})).toEqual([]); + }); + + it('returns an empty array given no relevant values', () => { + const eventFields = { 'url.domain': 'elastic.co' }; + expect(buildIndicatorShouldClauses(eventFields)).toEqual([]); + }); + + it('returns a clause for each relevant value', () => { + const eventFields = { 'source.ip': '127.0.0.1', 'url.full': 'elastic.co' }; + expect(buildIndicatorShouldClauses(eventFields)).toHaveLength(2); + }); + + it('excludes non-CTI fields', () => { + const eventFields = { 'source.ip': '127.0.0.1', 'url.domain': 'elastic.co' }; + expect(buildIndicatorShouldClauses(eventFields)).toHaveLength(1); + }); + + it('defines a named query where the name is the event field and the value is the event field value', () => { + const eventFields = { 'file.hash.md5': '1eee2bf3f56d8abed72da2bc523e7431' }; + + expect(buildIndicatorShouldClauses(eventFields)).toContainEqual({ + match: { + 'threatintel.indicator.file.hash.md5': { + _name: 'file.hash.md5', + query: '1eee2bf3f56d8abed72da2bc523e7431', + }, + }, + }); + }); + + it('returns valid queries for multiple valid fields', () => { + const eventFields = { 'source.ip': '127.0.0.1', 'url.full': 'elastic.co' }; + expect(buildIndicatorShouldClauses(eventFields)).toEqual( + expect.arrayContaining([ + { match: { 'threatintel.indicator.ip': { _name: 'source.ip', query: '127.0.0.1' } } }, + { match: { 'threatintel.indicator.url.full': { _name: 'url.full', query: 'elastic.co' } } }, + ]) + ); + }); +}); diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/helpers.ts new file mode 100644 index 0000000000000..8e10c0cf3f8e8 --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/helpers.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isEmpty } from 'lodash'; +import { QueryContainer } from '@elastic/elasticsearch/api/types'; + +import { EVENT_ENRICHMENT_INDICATOR_FIELD_MAP } from '../../../../../../common/cti/constants'; + +export const buildIndicatorShouldClauses = ( + eventFields: Record +): QueryContainer[] => { + const validEventFields = Object.keys(EVENT_ENRICHMENT_INDICATOR_FIELD_MAP) as Array< + keyof typeof EVENT_ENRICHMENT_INDICATOR_FIELD_MAP + >; + + return validEventFields.reduce((shoulds, eventField) => { + const eventFieldValue = eventFields[eventField]; + + if (!isEmpty(eventFieldValue)) { + shoulds.push({ + match: { + [EVENT_ENRICHMENT_INDICATOR_FIELD_MAP[eventField]]: { + query: eventFieldValue, + _name: eventField, + }, + }, + }); + } + + return shoulds; + }, []); +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.test.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.test.ts index b4a7542a53489..ec9a92467dd58 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.test.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.test.ts @@ -5,6 +5,16 @@ * 2.0. */ +import { buildRequestOptionsMock } from '../../../../../../common/search_strategy/security_solution/cti/index.mock'; +import { buildEventEnrichmentQuery } from './query'; + describe('buildEventEnrichmentQuery', () => { - it('makes each query filter named based on its field'); + it('converts each event field/value into a named filter', () => { + const options = buildRequestOptionsMock(); + const query = buildEventEnrichmentQuery(options); + expect(query.body?.query?.bool?.filter).toEqual(expect.objectContaining({ foo: 'bar' })); + }); + + it.todo('filters on indicator events'); + it.todo('includes the specified timerange'); }); diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.ts index 9a193a43e3579..e1a755d02a2db 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.ts @@ -5,4 +5,46 @@ * 2.0. */ -export const buildEventEnrichmentQuery = (a: unknown) => a; +import { isEmpty } from 'lodash'; +import { CtiQueries } from '../../../../../../common/search_strategy/security_solution/cti'; +import { createQueryFilterClauses } from '../../../../../utils/build_query'; +import { SecuritySolutionFactory } from '../../types'; +import { buildIndicatorShouldClauses } from './helpers'; + +export const buildEventEnrichmentQuery: SecuritySolutionFactory['buildDsl'] = ({ + defaultIndex, + docValueFields, + eventFields, + filterQuery, + timerange: { from, to }, +}) => { + const filter = [ + ...createQueryFilterClauses(filterQuery), + { term: { 'event.type': 'indicator' } }, + { + range: { + '@timestamp': { + gte: from, + lte: to, + format: 'strict_date_optional_time', + }, + }, + }, + ]; + + return { + allowNoIndices: true, + index: defaultIndex, + ignoreUnavailable: true, + body: { + ...(!isEmpty(docValueFields) && { docvalue_fields: docValueFields }), + query: { + bool: { + should: buildIndicatorShouldClauses(eventFields), + filter, + minimum_should_match: 1, + }, + }, + }, + }; +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/response.test.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/response.test.ts index b353b49a1b802..53afe9fd65b0e 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/response.test.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/response.test.ts @@ -7,4 +7,5 @@ describe('parseEventEnrichmentResponse', () => { it('adds matched.* enrichment fields based on the named query'); + it('includes an accurate inspect response'); }); diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/response.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/response.ts index 1902129b3aa9a..06950c6d3162a 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/response.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/response.ts @@ -5,4 +5,11 @@ * 2.0. */ -export const parseEventEnrichmentResponse = (a: unknown) => a; +import { CtiQueries } from '../../../../../../common/search_strategy/security_solution/cti'; +import { SecuritySolutionFactory } from '../../types'; + +export const parseEventEnrichmentResponse: SecuritySolutionFactory['parse'] = async ( + options, + response, + deps +) => ({}); From 146f36c8364deeb3276af8b3adf1e0b6075810c3 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Sun, 6 Jun 2021 22:41:40 -0500 Subject: [PATCH 4/6] Fleshing out unit tests around our enrichment query --- .../cti/event_enrichment/query.test.ts | 64 ++++++++++++++++++- 1 file changed, 61 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.test.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.test.ts index ec9a92467dd58..e6e73193ae714 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.test.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.test.ts @@ -12,9 +12,67 @@ describe('buildEventEnrichmentQuery', () => { it('converts each event field/value into a named filter', () => { const options = buildRequestOptionsMock(); const query = buildEventEnrichmentQuery(options); - expect(query.body?.query?.bool?.filter).toEqual(expect.objectContaining({ foo: 'bar' })); + expect(query.body?.query?.bool?.should).toEqual( + expect.arrayContaining([ + { + match: { + 'threatintel.indicator.file.hash.md5': { + _name: 'file.hash.md5', + query: '1eee2bf3f56d8abed72da2bc523e7431', + }, + }, + }, + { match: { 'threatintel.indicator.ip': { _name: 'source.ip', query: '127.0.0.1' } } }, + { match: { 'threatintel.indicator.url.full': { _name: 'url.full', query: 'elastic.co' } } }, + ]) + ); }); - it.todo('filters on indicator events'); - it.todo('includes the specified timerange'); + it('filters on indicator events', () => { + const options = buildRequestOptionsMock(); + const query = buildEventEnrichmentQuery(options); + expect(query.body?.query?.bool?.filter).toEqual( + expect.arrayContaining([{ term: { 'event.type': 'indicator' } }]) + ); + }); + + it('includes the specified timerange', () => { + const options = buildRequestOptionsMock(); + const query = buildEventEnrichmentQuery(options); + expect(query.body?.query?.bool?.filter).toEqual( + expect.arrayContaining([ + { + range: { + '@timestamp': { + format: 'strict_date_optional_time', + gte: '2020-09-13T09:00:43.249Z', + lte: '2020-09-14T09:00:43.249Z', + }, + }, + }, + ]) + ); + }); + + it('includes specified docvalue_fields', () => { + const docValueFields = [ + { field: '@timestamp', format: 'date_time' }, + { field: 'event.created', format: 'date_time' }, + { field: 'event.end', format: 'date_time' }, + ]; + const options = buildRequestOptionsMock({ docValueFields }); + const query = buildEventEnrichmentQuery(options); + expect(query.body?.docvalue_fields).toEqual(expect.arrayContaining(docValueFields)); + }); + + it('includes specified filters', () => { + const filterQuery = { + query: 'query_field: query_value', + language: 'kuery', + }; + + const options = buildRequestOptionsMock({ filterQuery }); + const query = buildEventEnrichmentQuery(options); + expect(query.body?.query?.bool?.filter).toEqual(expect.arrayContaining([filterQuery])); + }); }); From d5f420c587b4d57178b2d3b564fdca51cec77a37 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Mon, 7 Jun 2021 15:55:19 -0500 Subject: [PATCH 5/6] Fleshing out response parsing of eventEnrichment strategy --- .../common/ecs/threat/index.ts | 2 + .../security_solution/cti/index.mock.ts | 91 ++++++++++++- .../security_solution/cti/index.ts | 3 + .../cti/event_enrichment/helpers.test.ts | 122 +++++++++++++++++- .../factory/cti/event_enrichment/helpers.ts | 59 ++++++++- .../cti/event_enrichment/query.test.ts | 24 +++- .../factory/cti/event_enrichment/query.ts | 4 +- .../cti/event_enrichment/response.test.ts | 82 +++++++++++- .../factory/cti/event_enrichment/response.ts | 18 ++- 9 files changed, 386 insertions(+), 19 deletions(-) diff --git a/x-pack/plugins/security_solution/common/ecs/threat/index.ts b/x-pack/plugins/security_solution/common/ecs/threat/index.ts index 19923a82dc846..e5e7964c5d09d 100644 --- a/x-pack/plugins/security_solution/common/ecs/threat/index.ts +++ b/x-pack/plugins/security_solution/common/ecs/threat/index.ts @@ -10,6 +10,8 @@ import { EventEcs } from '../event'; interface ThreatMatchEcs { atomic?: string[]; field?: string[]; + id?: string[]; + index?: string[]; type?: string[]; } diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/cti/index.mock.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/cti/index.mock.ts index bcbf75aca3762..f3dee5a21e4c9 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/security_solution/cti/index.mock.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/cti/index.mock.ts @@ -5,9 +5,15 @@ * 2.0. */ -import { CtiEventEnrichmentRequestOptions, CtiQueries } from '.'; +import { IEsSearchResponse } from 'src/plugins/data/public'; -export const buildRequestOptionsMock = ( +import { + CtiEventEnrichmentRequestOptions, + CtiEventEnrichmentStrategyResponse, + CtiQueries, +} from '.'; + +export const buildEventEnrichmentRequestOptionsMock = ( overrides: Partial = {} ): CtiEventEnrichmentRequestOptions => ({ defaultIndex: ['filebeat-*'], @@ -21,3 +27,84 @@ export const buildRequestOptionsMock = ( timerange: { interval: '', from: '2020-09-13T09:00:43.249Z', to: '2020-09-14T09:00:43.249Z' }, ...overrides, }); + +export const buildEventEnrichmentRawResponseMock = (): IEsSearchResponse => ({ + rawResponse: { + took: 17, + timed_out: false, + _shards: { + total: 1, + successful: 1, + skipped: 0, + failed: 0, + }, + hits: { + total: { + value: 1, + relation: 'eq', + }, + max_score: 6.0637846, + hits: [ + { + _index: 'filebeat-8.0.0-2021.05.28-000001', + _id: '31408415b6d5601a92d29b86c2519658f210c194057588ae396d55cc20b3f03d', + _score: 6.0637846, + fields: { + 'event.category': ['threat'], + 'threatintel.indicator.file.type': ['html'], + 'related.hash': [ + '5529de7b60601aeb36f57824ed0e1ae8', + '15b012e6f626d0f88c2926d2bf4ca394d7b8ee07cc06d2ec05ea76bed3e8a05e', + '768:NXSFGJ/ooP6FawrB7Bo1MWnF/jRmhJImp:1SFXIqBo1Mwj2p', + ], + 'threatintel.indicator.first_seen': ['2021-05-28T18:33:29.000Z'], + 'threatintel.indicator.file.hash.tlsh': [ + 'FFB20B82F6617061C32784E2712F7A46B179B04FD1EA54A0F28CD8E9CFE4CAA1617F1C', + ], + 'service.type': ['threatintel'], + 'threatintel.indicator.file.hash.ssdeep': [ + '768:NXSFGJ/ooP6FawrB7Bo1MWnF/jRmhJImp:1SFXIqBo1Mwj2p', + ], + 'agent.type': ['filebeat'], + 'event.module': ['threatintel'], + 'threatintel.indicator.type': ['file'], + 'agent.name': ['rylastic.local'], + 'threatintel.indicator.file.hash.sha256': [ + '15b012e6f626d0f88c2926d2bf4ca394d7b8ee07cc06d2ec05ea76bed3e8a05e', + ], + 'event.kind': ['enrichment'], + 'threatintel.indicator.file.hash.md5': ['5529de7b60601aeb36f57824ed0e1ae8'], + 'fileset.name': ['abusemalware'], + 'input.type': ['httpjson'], + 'agent.hostname': ['rylastic.local'], + tags: ['threatintel-abusemalware', 'forwarded'], + 'event.ingested': ['2021-05-28T18:33:55.086Z'], + '@timestamp': ['2021-05-28T18:33:52.993Z'], + 'agent.id': ['ff93aee5-86a1-4a61-b0e6-0cdc313d01b5'], + 'ecs.version': ['1.6.0'], + 'event.reference': [ + 'https://urlhaus-api.abuse.ch/v1/download/15b012e6f626d0f88c2926d2bf4ca394d7b8ee07cc06d2ec05ea76bed3e8a05e/', + ], + 'event.type': ['indicator'], + 'event.created': ['2021-05-28T18:33:52.993Z'], + 'agent.ephemeral_id': ['d6b14f65-5bf3-430d-8315-7b5613685979'], + 'threatintel.indicator.file.size': [24738], + 'agent.version': ['8.0.0'], + 'event.dataset': ['threatintel.abusemalware'], + }, + matched_queries: ['file.hash.md5'], + }, + ], + }, + }, +}); + +export const buildEventEnrichmentResponseMock = ( + overrides: Partial = {} +): CtiEventEnrichmentStrategyResponse => ({ + ...buildEventEnrichmentRawResponseMock(), + enrichments: [], + inspect: { dsl: ['{"mocked": "json"}'] }, + totalCount: 0, + ...overrides, +}); diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/cti/index.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/cti/index.ts index 549605ba5bff7..788a44bc5b9f7 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/security_solution/cti/index.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/cti/index.ts @@ -17,7 +17,10 @@ export interface CtiEventEnrichmentRequestOptions extends RequestBasicOptions { eventFields: Record; } +export type CtiEnrichment = Record; + export interface CtiEventEnrichmentStrategyResponse extends IEsSearchResponse { + enrichments: CtiEnrichment[]; inspect?: Inspect; totalCount: number; } diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/helpers.test.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/helpers.test.ts index 69de558ad0b0e..a246b66d462ce 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/helpers.test.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/helpers.test.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { buildIndicatorShouldClauses } from './helpers'; +import { buildIndicatorEnrichments, buildIndicatorShouldClauses, getTotalCount } from './helpers'; describe('buildIndicatorShouldClauses', () => { it('returns an empty array given an empty fieldset', () => { @@ -50,3 +50,123 @@ describe('buildIndicatorShouldClauses', () => { ); }); }); + +describe('getTotalCount', () => { + it('returns 0 when total is null (not tracking)', () => { + expect(getTotalCount(null)).toEqual(0); + }); + + it('returns total when total is a number', () => { + expect(getTotalCount(5)).toEqual(5); + }); + + it('returns total.value when total is an object', () => { + expect(getTotalCount({ value: 20, relation: 'eq' })).toEqual(20); + }); +}); + +describe('buildIndicatorEnrichments', () => { + it('returns nothing if hits have no matched queries', () => { + const hits = [{ _id: '_id', _index: '_index', matched_queries: [] }]; + expect(buildIndicatorEnrichments(hits)).toEqual([]); + }); + + it("returns nothing if hits' matched queries are not valid", () => { + const hits = [{ _id: '_id', _index: '_index', matched_queries: ['invalid.field'] }]; + expect(buildIndicatorEnrichments(hits)).toEqual([]); + }); + + it('builds a single enrichment if the hit has a matched query', () => { + const hits = [ + { + _id: '_id', + _index: '_index', + matched_queries: ['file.hash.md5'], + fields: { + 'threatintel.indicator.file.hash.md5': ['indicator_value'], + }, + }, + ]; + + expect(buildIndicatorEnrichments(hits)).toEqual([ + expect.objectContaining({ + 'matched.atomic': ['indicator_value'], + 'matched.field': ['file.hash.md5'], + 'matched.id': ['_id'], + 'matched.index': ['_index'], + 'threatintel.indicator.file.hash.md5': ['indicator_value'], + }), + ]); + }); + + it('builds multiple enrichments if the hit has matched queries', () => { + const hits = [ + { + _id: '_id', + _index: '_index', + matched_queries: ['file.hash.md5', 'source.ip'], + fields: { + 'threatintel.indicator.file.hash.md5': ['indicator_value'], + 'threatintel.indicator.ip': ['127.0.0.1'], + }, + }, + ]; + + expect(buildIndicatorEnrichments(hits)).toEqual([ + expect.objectContaining({ + 'matched.atomic': ['indicator_value'], + 'matched.field': ['file.hash.md5'], + 'matched.id': ['_id'], + 'matched.index': ['_index'], + 'threatintel.indicator.file.hash.md5': ['indicator_value'], + 'threatintel.indicator.ip': ['127.0.0.1'], + }), + expect.objectContaining({ + 'matched.atomic': ['127.0.0.1'], + 'matched.field': ['source.ip'], + 'matched.id': ['_id'], + 'matched.index': ['_index'], + 'threatintel.indicator.file.hash.md5': ['indicator_value'], + 'threatintel.indicator.ip': ['127.0.0.1'], + }), + ]); + }); + + it('builds an enrichment for each hit', () => { + const hits = [ + { + _id: '_id', + _index: '_index', + matched_queries: ['file.hash.md5'], + fields: { + 'threatintel.indicator.file.hash.md5': ['indicator_value'], + }, + }, + { + _id: '_id2', + _index: '_index2', + matched_queries: ['source.ip'], + fields: { + 'threatintel.indicator.ip': ['127.0.0.1'], + }, + }, + ]; + + expect(buildIndicatorEnrichments(hits)).toEqual([ + expect.objectContaining({ + 'matched.atomic': ['indicator_value'], + 'matched.field': ['file.hash.md5'], + 'matched.id': ['_id'], + 'matched.index': ['_index'], + 'threatintel.indicator.file.hash.md5': ['indicator_value'], + }), + expect.objectContaining({ + 'matched.atomic': ['127.0.0.1'], + 'matched.field': ['source.ip'], + 'matched.id': ['_id2'], + 'matched.index': ['_index2'], + 'threatintel.indicator.ip': ['127.0.0.1'], + }), + ]); + }); +}); diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/helpers.ts index 8e10c0cf3f8e8..dbcc57689d480 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/helpers.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/helpers.ts @@ -5,18 +5,21 @@ * 2.0. */ -import { isEmpty } from 'lodash'; -import { QueryContainer } from '@elastic/elasticsearch/api/types'; +import { get, isEmpty } from 'lodash'; +import { Hit, QueryContainer, TotalHits } from '@elastic/elasticsearch/api/types'; import { EVENT_ENRICHMENT_INDICATOR_FIELD_MAP } from '../../../../../../common/cti/constants'; +import { CtiEnrichment } from '../../../../../../common/search_strategy/security_solution/cti'; + +type EventField = keyof typeof EVENT_ENRICHMENT_INDICATOR_FIELD_MAP; +const validEventFields = Object.keys(EVENT_ENRICHMENT_INDICATOR_FIELD_MAP) as EventField[]; + +const isValidEventField = (field: string): field is EventField => + validEventFields.includes(field as EventField); export const buildIndicatorShouldClauses = ( eventFields: Record ): QueryContainer[] => { - const validEventFields = Object.keys(EVENT_ENRICHMENT_INDICATOR_FIELD_MAP) as Array< - keyof typeof EVENT_ENRICHMENT_INDICATOR_FIELD_MAP - >; - return validEventFields.reduce((shoulds, eventField) => { const eventFieldValue = eventFields[eventField]; @@ -34,3 +37,47 @@ export const buildIndicatorShouldClauses = ( return shoulds; }, []); }; + +export const buildIndicatorEnrichments = (hits: Hit[]): CtiEnrichment[] => { + return hits.flatMap(({ matched_queries: matchedQueries, ...hit }) => { + return ( + matchedQueries?.reduce((enrichments, matchedQuery) => { + if (isValidEventField(matchedQuery)) { + enrichments.push({ + ...hit.fields, + ...buildIndicatorMatchedFields(hit, matchedQuery), + }); + } + + return enrichments; + }, []) ?? [] + ); + }); +}; + +const buildIndicatorMatchedFields = ( + hit: Hit, + eventField: EventField +): Record => { + const indicatorField = EVENT_ENRICHMENT_INDICATOR_FIELD_MAP[eventField]; + const atomic = get(hit.fields, indicatorField) as string[]; + + return { + 'matched.atomic': atomic, + 'matched.field': [eventField], + 'matched.id': [hit._id], + 'matched.index': [hit._index], + }; +}; + +export const getTotalCount = (total: number | TotalHits | null): number => { + if (total == null) { + return 0; + } + + if (typeof total === 'number') { + return total; + } + + return total.value; +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.test.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.test.ts index e6e73193ae714..bc96a387105c6 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.test.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.test.ts @@ -5,12 +5,12 @@ * 2.0. */ -import { buildRequestOptionsMock } from '../../../../../../common/search_strategy/security_solution/cti/index.mock'; +import { buildEventEnrichmentRequestOptionsMock } from '../../../../../../common/search_strategy/security_solution/cti/index.mock'; import { buildEventEnrichmentQuery } from './query'; describe('buildEventEnrichmentQuery', () => { it('converts each event field/value into a named filter', () => { - const options = buildRequestOptionsMock(); + const options = buildEventEnrichmentRequestOptionsMock(); const query = buildEventEnrichmentQuery(options); expect(query.body?.query?.bool?.should).toEqual( expect.arrayContaining([ @@ -29,7 +29,7 @@ describe('buildEventEnrichmentQuery', () => { }); it('filters on indicator events', () => { - const options = buildRequestOptionsMock(); + const options = buildEventEnrichmentRequestOptionsMock(); const query = buildEventEnrichmentQuery(options); expect(query.body?.query?.bool?.filter).toEqual( expect.arrayContaining([{ term: { 'event.type': 'indicator' } }]) @@ -37,7 +37,7 @@ describe('buildEventEnrichmentQuery', () => { }); it('includes the specified timerange', () => { - const options = buildRequestOptionsMock(); + const options = buildEventEnrichmentRequestOptionsMock(); const query = buildEventEnrichmentQuery(options); expect(query.body?.query?.bool?.filter).toEqual( expect.arrayContaining([ @@ -60,18 +60,30 @@ describe('buildEventEnrichmentQuery', () => { { field: 'event.created', format: 'date_time' }, { field: 'event.end', format: 'date_time' }, ]; - const options = buildRequestOptionsMock({ docValueFields }); + const options = buildEventEnrichmentRequestOptionsMock({ docValueFields }); const query = buildEventEnrichmentQuery(options); expect(query.body?.docvalue_fields).toEqual(expect.arrayContaining(docValueFields)); }); + it('requests all fields', () => { + const options = buildEventEnrichmentRequestOptionsMock(); + const query = buildEventEnrichmentQuery(options); + expect(query.body?.fields).toEqual(['*']); + }); + + it('excludes _source', () => { + const options = buildEventEnrichmentRequestOptionsMock(); + const query = buildEventEnrichmentQuery(options); + expect(query.body?._source).toEqual(false); + }); + it('includes specified filters', () => { const filterQuery = { query: 'query_field: query_value', language: 'kuery', }; - const options = buildRequestOptionsMock({ filterQuery }); + const options = buildEventEnrichmentRequestOptionsMock({ filterQuery }); const query = buildEventEnrichmentQuery(options); expect(query.body?.query?.bool?.filter).toEqual(expect.arrayContaining([filterQuery])); }); diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.ts index e1a755d02a2db..4760e6a227cd3 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.ts @@ -34,10 +34,12 @@ export const buildEventEnrichmentQuery: SecuritySolutionFactory { - it('adds matched.* enrichment fields based on the named query'); - it('includes an accurate inspect response'); + it('includes an accurate inspect response', async () => { + const options = buildEventEnrichmentRequestOptionsMock(); + const response = buildEventEnrichmentRawResponseMock(); + const parsedResponse = await parseEventEnrichmentResponse(options, response); + + const expectedInspect = expect.objectContaining({ + allowNoIndices: true, + body: { + _source: false, + fields: ['*'], + query: { + bool: { + filter: [ + { bool: { filter: [{ match_all: {} }], must: [], must_not: [], should: [] } }, + { term: { 'event.type': 'indicator' } }, + { + range: { + '@timestamp': { + format: 'strict_date_optional_time', + gte: '2020-09-13T09:00:43.249Z', + lte: '2020-09-14T09:00:43.249Z', + }, + }, + }, + ], + minimum_should_match: 1, + should: [ + { + match: { + 'threatintel.indicator.file.hash.md5': { + _name: 'file.hash.md5', + query: '1eee2bf3f56d8abed72da2bc523e7431', + }, + }, + }, + { match: { 'threatintel.indicator.ip': { _name: 'source.ip', query: '127.0.0.1' } } }, + { + match: { + 'threatintel.indicator.url.full': { _name: 'url.full', query: 'elastic.co' }, + }, + }, + ], + }, + }, + }, + ignoreUnavailable: true, + index: ['filebeat-*'], + }); + const parsedInspect = JSON.parse(parsedResponse.inspect!.dsl[0]); + expect(parsedInspect).toEqual(expectedInspect); + }); + + it('includes an accurate total count', async () => { + const options = buildEventEnrichmentRequestOptionsMock(); + const response = buildEventEnrichmentRawResponseMock(); + const parsedResponse = await parseEventEnrichmentResponse(options, response); + + expect(parsedResponse.totalCount).toEqual(1); + }); + + it('adds matched.* enrichment fields based on the named query', async () => { + const options = buildEventEnrichmentRequestOptionsMock(); + const response = buildEventEnrichmentRawResponseMock(); + const parsedResponse = await parseEventEnrichmentResponse(options, response); + + expect(parsedResponse.enrichments).toEqual([ + expect.objectContaining({ + 'matched.atomic': ['5529de7b60601aeb36f57824ed0e1ae8'], + 'matched.field': ['file.hash.md5'], + 'matched.id': ['31408415b6d5601a92d29b86c2519658f210c194057588ae396d55cc20b3f03d'], + 'matched.index': ['filebeat-8.0.0-2021.05.28-000001'], + }), + ]); + }); }); diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/response.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/response.ts index 06950c6d3162a..29a842d84558c 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/response.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/response.ts @@ -6,10 +6,26 @@ */ import { CtiQueries } from '../../../../../../common/search_strategy/security_solution/cti'; +import { inspectStringifyObject } from '../../../../../utils/build_query'; import { SecuritySolutionFactory } from '../../types'; +import { buildIndicatorEnrichments, getTotalCount } from './helpers'; +import { buildEventEnrichmentQuery } from './query'; export const parseEventEnrichmentResponse: SecuritySolutionFactory['parse'] = async ( options, response, deps -) => ({}); +) => { + const inspect = { + dsl: [inspectStringifyObject(buildEventEnrichmentQuery(options))], + }; + const totalCount = getTotalCount(response.rawResponse.hits.total); + const enrichments = buildIndicatorEnrichments(response.rawResponse.hits.hits); + + return { + ...response, + enrichments, + inspect, + totalCount, + }; +}; From 77fefd5676072168c00827f7651db334b344d54b Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 10 Jun 2021 16:30:03 -0500 Subject: [PATCH 6/6] Fix types from elasticsearch --- .../factory/cti/event_enrichment/helpers.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/helpers.ts index dbcc57689d480..e4ed05baeed77 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/helpers.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/helpers.ts @@ -6,7 +6,7 @@ */ import { get, isEmpty } from 'lodash'; -import { Hit, QueryContainer, TotalHits } from '@elastic/elasticsearch/api/types'; +import { estypes } from '@elastic/elasticsearch'; import { EVENT_ENRICHMENT_INDICATOR_FIELD_MAP } from '../../../../../../common/cti/constants'; import { CtiEnrichment } from '../../../../../../common/search_strategy/security_solution/cti'; @@ -19,8 +19,8 @@ const isValidEventField = (field: string): field is EventField => export const buildIndicatorShouldClauses = ( eventFields: Record -): QueryContainer[] => { - return validEventFields.reduce((shoulds, eventField) => { +): estypes.QueryDslQueryContainer[] => { + return validEventFields.reduce((shoulds, eventField) => { const eventFieldValue = eventFields[eventField]; if (!isEmpty(eventFieldValue)) { @@ -38,7 +38,7 @@ export const buildIndicatorShouldClauses = ( }, []); }; -export const buildIndicatorEnrichments = (hits: Hit[]): CtiEnrichment[] => { +export const buildIndicatorEnrichments = (hits: estypes.SearchHit[]): CtiEnrichment[] => { return hits.flatMap(({ matched_queries: matchedQueries, ...hit }) => { return ( matchedQueries?.reduce((enrichments, matchedQuery) => { @@ -56,7 +56,7 @@ export const buildIndicatorEnrichments = (hits: Hit[]): CtiEnrichment[] => { }; const buildIndicatorMatchedFields = ( - hit: Hit, + hit: estypes.SearchHit, eventField: EventField ): Record => { const indicatorField = EVENT_ENRICHMENT_INDICATOR_FIELD_MAP[eventField]; @@ -70,7 +70,7 @@ const buildIndicatorMatchedFields = ( }; }; -export const getTotalCount = (total: number | TotalHits | null): number => { +export const getTotalCount = (total: number | estypes.SearchTotalHits | null): number => { if (total == null) { return 0; }