Skip to content

Commit 88400cb

Browse files
FrankHassanabadFrankHassanabad
authored
Adds date time query and return fields for timestamps and overrides (#79911) (#79965)
## Summary Fixes #79865 Also fixes: * Timestamp override not being pushed down into threshold rules to use * Timestamp override not being used for lastValidDate * The return format of the date time might have been different depending on the customer mapping for both the override and the regular @timestamp so this fixes that as well. * Fixes one small type issue with fields. ### Checklist Delete any items that are not applicable to this PR. - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios
1 parent bef9cc7 commit 88400cb

File tree

10 files changed

+207
-34
lines changed

10 files changed

+207
-34
lines changed

‎x-pack/plugins/security_solution/common/detection_engine/types.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ export interface BaseHit<T> {
3535
_index: string;
3636
_id: string;
3737
_source: T;
38+
fields?: Record<string, SearchTypes[]>;
3839
}
3940

4041
export interface EqlSequence<T> {

‎x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,12 @@ describe('create_signals', () => {
2323
size: 100,
2424
ignoreUnavailable: true,
2525
body: {
26+
docvalue_fields: [
27+
{
28+
field: '@timestamp',
29+
format: 'strict_date_optional_time',
30+
},
31+
],
2632
query: {
2733
bool: {
2834
filter: [
@@ -37,6 +43,7 @@ describe('create_signals', () => {
3743
range: {
3844
'@timestamp': {
3945
gte: 'now-5m',
46+
format: 'strict_date_optional_time',
4047
},
4148
},
4249
},
@@ -51,6 +58,7 @@ describe('create_signals', () => {
5158
range: {
5259
'@timestamp': {
5360
lte: 'today',
61+
format: 'strict_date_optional_time',
5462
},
5563
},
5664
},
@@ -94,6 +102,12 @@ describe('create_signals', () => {
94102
size: 100,
95103
ignoreUnavailable: true,
96104
body: {
105+
docvalue_fields: [
106+
{
107+
field: '@timestamp',
108+
format: 'strict_date_optional_time',
109+
},
110+
],
97111
query: {
98112
bool: {
99113
filter: [
@@ -108,6 +122,7 @@ describe('create_signals', () => {
108122
range: {
109123
'@timestamp': {
110124
gte: 'now-5m',
125+
format: 'strict_date_optional_time',
111126
},
112127
},
113128
},
@@ -122,6 +137,7 @@ describe('create_signals', () => {
122137
range: {
123138
'@timestamp': {
124139
lte: 'today',
140+
format: 'strict_date_optional_time',
125141
},
126142
},
127143
},
@@ -166,6 +182,12 @@ describe('create_signals', () => {
166182
size: 100,
167183
ignoreUnavailable: true,
168184
body: {
185+
docvalue_fields: [
186+
{
187+
field: '@timestamp',
188+
format: 'strict_date_optional_time',
189+
},
190+
],
169191
query: {
170192
bool: {
171193
filter: [
@@ -180,6 +202,7 @@ describe('create_signals', () => {
180202
range: {
181203
'@timestamp': {
182204
gte: 'now-5m',
205+
format: 'strict_date_optional_time',
183206
},
184207
},
185208
},
@@ -194,6 +217,7 @@ describe('create_signals', () => {
194217
range: {
195218
'@timestamp': {
196219
lte: 'today',
220+
format: 'strict_date_optional_time',
197221
},
198222
},
199223
},
@@ -239,6 +263,12 @@ describe('create_signals', () => {
239263
size: 100,
240264
ignoreUnavailable: true,
241265
body: {
266+
docvalue_fields: [
267+
{
268+
field: '@timestamp',
269+
format: 'strict_date_optional_time',
270+
},
271+
],
242272
query: {
243273
bool: {
244274
filter: [
@@ -253,6 +283,7 @@ describe('create_signals', () => {
253283
range: {
254284
'@timestamp': {
255285
gte: 'now-5m',
286+
format: 'strict_date_optional_time',
256287
},
257288
},
258289
},
@@ -267,6 +298,7 @@ describe('create_signals', () => {
267298
range: {
268299
'@timestamp': {
269300
lte: 'today',
301+
format: 'strict_date_optional_time',
270302
},
271303
},
272304
},
@@ -311,6 +343,12 @@ describe('create_signals', () => {
311343
size: 100,
312344
ignoreUnavailable: true,
313345
body: {
346+
docvalue_fields: [
347+
{
348+
field: '@timestamp',
349+
format: 'strict_date_optional_time',
350+
},
351+
],
314352
query: {
315353
bool: {
316354
filter: [
@@ -325,6 +363,7 @@ describe('create_signals', () => {
325363
range: {
326364
'@timestamp': {
327365
gte: 'now-5m',
366+
format: 'strict_date_optional_time',
328367
},
329368
},
330369
},
@@ -339,6 +378,7 @@ describe('create_signals', () => {
339378
range: {
340379
'@timestamp': {
341380
lte: 'today',
381+
format: 'strict_date_optional_time',
342382
},
343383
},
344384
},
@@ -390,6 +430,7 @@ describe('create_signals', () => {
390430
size: 100,
391431
ignoreUnavailable: true,
392432
body: {
433+
docvalue_fields: [{ field: '@timestamp', format: 'strict_date_optional_time' }],
393434
query: {
394435
bool: {
395436
filter: [
@@ -404,6 +445,7 @@ describe('create_signals', () => {
404445
range: {
405446
'@timestamp': {
406447
gte: 'now-5m',
448+
format: 'strict_date_optional_time',
407449
},
408450
},
409451
},
@@ -418,6 +460,7 @@ describe('create_signals', () => {
418460
range: {
419461
'@timestamp': {
420462
lte: 'today',
463+
format: 'strict_date_optional_time',
421464
},
422465
},
423466
},

‎x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,25 @@ export const buildEventsSearchQuery = ({
2828
timestampOverride,
2929
}: BuildEventsSearchQuery) => {
3030
const timestamp = timestampOverride ?? '@timestamp';
31+
const docFields =
32+
timestampOverride != null
33+
? [
34+
{
35+
field: '@timestamp',
36+
format: 'strict_date_optional_time',
37+
},
38+
{
39+
field: timestampOverride,
40+
format: 'strict_date_optional_time',
41+
},
42+
]
43+
: [
44+
{
45+
field: '@timestamp',
46+
format: 'strict_date_optional_time',
47+
},
48+
];
49+
3150
const filterWithTime = [
3251
filter,
3352
{
@@ -40,6 +59,7 @@ export const buildEventsSearchQuery = ({
4059
range: {
4160
[timestamp]: {
4261
gte: from,
62+
format: 'strict_date_optional_time',
4363
},
4464
},
4565
},
@@ -54,6 +74,7 @@ export const buildEventsSearchQuery = ({
5474
range: {
5575
[timestamp]: {
5676
lte: to,
77+
format: 'strict_date_optional_time',
5778
},
5879
},
5980
},
@@ -65,12 +86,14 @@ export const buildEventsSearchQuery = ({
6586
},
6687
},
6788
];
89+
6890
const searchQuery = {
6991
allowNoIndices: true,
7092
index,
7193
size,
7294
ignoreUnavailable: true,
7395
body: {
96+
docvalue_fields: docFields,
7497
query: {
7598
bool: {
7699
filter: [

‎x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ export const buildRule = ({
102102
created_by: createdBy,
103103
updated_by: updatedBy,
104104
threat: ruleParams.threat ?? [],
105-
timestamp_override: ruleParams.timestampOverride, // TODO: Timestamp Override via timestamp_override
105+
timestamp_override: ruleParams.timestampOverride,
106106
throttle,
107107
version: ruleParams.version,
108108
created_at: createdAt,
@@ -155,7 +155,7 @@ export const buildRuleWithoutOverrides = (
155155
created_by: ruleSO.attributes.createdBy,
156156
updated_by: ruleSO.attributes.updatedBy,
157157
threat: ruleParams.threat ?? [],
158-
timestamp_override: ruleParams.timestampOverride, // TODO: Timestamp Override via timestamp_override
158+
timestamp_override: ruleParams.timestampOverride,
159159
throttle: ruleSO.attributes.throttle,
160160
version: ruleParams.version,
161161
created_at: ruleSO.attributes.createdAt,

‎x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,10 @@
66

77
import { isEmpty } from 'lodash/fp';
88

9-
import { Threshold } from '../../../../common/detection_engine/schemas/common/schemas';
9+
import {
10+
Threshold,
11+
TimestampOverrideOrUndefined,
12+
} from '../../../../common/detection_engine/schemas/common/schemas';
1013
import { singleSearchAfter } from './single_search_after';
1114

1215
import { AlertServices } from '../../../../../alerts/server';
@@ -23,6 +26,7 @@ interface FindThresholdSignalsParams {
2326
filter: unknown;
2427
threshold: Threshold;
2528
buildRuleMessage: BuildRuleMessage;
29+
timestampOverride: TimestampOverrideOrUndefined;
2630
}
2731

2832
export const findThresholdSignals = async ({
@@ -34,6 +38,7 @@ export const findThresholdSignals = async ({
3438
filter,
3539
threshold,
3640
buildRuleMessage,
41+
timestampOverride,
3742
}: FindThresholdSignalsParams): Promise<{
3843
searchResult: SignalSearchResponse;
3944
searchDuration: string;
@@ -54,7 +59,7 @@ export const findThresholdSignals = async ({
5459
return singleSearchAfter({
5560
aggregations,
5661
searchAfterSortId: undefined,
57-
timestampOverride: undefined,
62+
timestampOverride,
5863
index: inputIndexPattern,
5964
from,
6065
to,

‎x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,10 @@ export const searchAfterAndBulkCreate = async ({
9595
});
9696
toReturn = mergeReturns([
9797
toReturn,
98-
createSearchAfterReturnTypeFromResponse({ searchResult }),
98+
createSearchAfterReturnTypeFromResponse({
99+
searchResult,
100+
timestampOverride: ruleParams.timestampOverride,
101+
}),
99102
createSearchAfterReturnType({
100103
searchAfterTimes: [searchDuration],
101104
errors: searchErrors,

‎x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@ export const signalRulesAlertType = ({
9999
from,
100100
ruleId,
101101
index,
102+
eventCategoryOverride,
102103
filters,
103104
language,
104105
maxSignals,
@@ -114,6 +115,7 @@ export const signalRulesAlertType = ({
114115
threatIndex,
115116
threatMapping,
116117
threatLanguage,
118+
timestampOverride,
117119
type,
118120
exceptionsList,
119121
} = params;
@@ -313,6 +315,7 @@ export const signalRulesAlertType = ({
313315
logger,
314316
filter: esFilter,
315317
threshold,
318+
timestampOverride,
316319
buildRuleMessage,
317320
});
318321

@@ -346,7 +349,10 @@ export const signalRulesAlertType = ({
346349
});
347350
result = mergeReturns([
348351
result,
349-
createSearchAfterReturnTypeFromResponse({ searchResult: thresholdResults }),
352+
createSearchAfterReturnTypeFromResponse({
353+
searchResult: thresholdResults,
354+
timestampOverride,
355+
}),
350356
createSearchAfterReturnType({
351357
success,
352358
errors: [...errors, ...searchErrors],
@@ -464,12 +470,12 @@ export const signalRulesAlertType = ({
464470
const request = buildEqlSearchRequest(
465471
query,
466472
inputIndex,
467-
params.from,
468-
params.to,
473+
from,
474+
to,
469475
searchAfterSize,
470-
params.timestampOverride,
476+
timestampOverride,
471477
exceptionItems ?? [],
472-
params.eventCategoryOverride
478+
eventCategoryOverride
473479
);
474480
const response: EqlSignalSearchResponse = await services.callCluster(
475481
'transport.request',

0 commit comments

Comments
 (0)