Skip to content

Commit 8a0ace7

Browse files
costincostin
authored
EQL: Introduce like and regex keywords (#68791)
Built-in language support for wildcard and regex queries. Both the case sensitive and insensitive queries are allowed: field like "pattern*" // basic matching field like~ "P?tTeRn*" // case-insensitive field like ("a*", "b?", "cd" ) // multi argument field like~ ("A", "B?", "C*d") // case-insensitive multi arg field regex "[a-z]" // basic form field regex~ "[A-z]" // case-insensitive field regex ("[aA]a", "[Bb]b") // multi argument field regex~ ("aA", "bB") // case-insensitive multi arg Fix #68639
1 parent 985ae9a commit 8a0ace7

File tree

23 files changed

+977
-575
lines changed

23 files changed

+977
-575
lines changed

‎x-pack/plugin/eql/qa/common/src/main/resources/additional_test_queries.toml‎

Lines changed: 113 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -131,46 +131,6 @@ name = "numberStringConversion5"
131131
query = 'any where number(string(serial_event_id), 16) == 17'
132132
expected_event_ids = [11]
133133

134-
# [[queries]]
135-
# name = "matchWithCharacterClasses1"
136-
# expected_event_ids = [98]
137-
# notes = "regexp doesn't support character classes"
138-
# query = '''
139-
# //
140-
# // """.*?net1\s+localgroup.*?""")
141-
# process where match(command_line, """.*?net1[ ]+localgroup.*?""")
142-
# '''
143-
#
144-
# [[queries]]
145-
# name = "matchLiteAdditional"
146-
# expected_event_ids = [98]
147-
# query = '''
148-
# process where matchLite(command_line, """.*?net1.*?""")
149-
# '''
150-
#
151-
# [[queries]]
152-
# name = "matchWithCharacterClasses2"
153-
# expected_event_ids = [98]
154-
# notes = "regexp doesn't support predefined character classes (like \\s)"
155-
# query = '''
156-
# // """.*?net1\s+\w{4,15}\s+.*?"""
157-
# process where match(command_line, """.*?net1[ ]+[a-z]{4,15}[ ]+.*?""")
158-
# '''
159-
#
160-
# [[queries]]
161-
# name = "multiPatternMatch"
162-
# expected_event_ids = [50, 97, 98]
163-
# query = '''
164-
# process where match(command_line, ".*?net[1]? localgroup.*?", ".*? myappserver.py .*?")
165-
# '''
166-
#
167-
# [[queries]]
168-
# name = "matchWithSubstring"
169-
# expected_event_ids = [50, 98]
170-
# query = '''
171-
# process where match(substring(command_line, 5), ".*?net[1]? localgroup.*?", ".*? myappserver.py .*?")
172-
# '''
173-
174134
[[queries]]
175135
name = "moduloEqualsField"
176136
# Basic test for modulo function
@@ -355,3 +315,116 @@ query = '''
355315
file where file_name in~ ("winini*.exe", "lsass.e?e") and opcode == 2
356316
'''
357317
expected_event_ids = []
318+
319+
[[queries]]
320+
name = "likeWithScript"
321+
query = 'process where string(serial_event_id) like "1"'
322+
expected_event_ids = [1]
323+
324+
[[queries]]
325+
name = "likeScriptWithPattern"
326+
query = 'process where string(serial_event_id) like ("1*")'
327+
expected_event_ids = [1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
328+
329+
[[queries]]
330+
name = "likeScriptWithQuestionPattern"
331+
query = 'process where string(serial_event_id) like ("1?")'
332+
expected_event_ids = [10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
333+
334+
[[queries]]
335+
name = "likeMultipleArgs"
336+
query = '''
337+
file where file_name like ("wininit.exe", "lsass.exe") and opcode == 2
338+
'''
339+
expected_event_ids = [65, 86]
340+
341+
[[queries]]
342+
name = "likeMultipleArgsInsensitive"
343+
query = '''
344+
file where file_name like~ ("winInIT.exe", "lsASS.exe") and opcode == 2
345+
'''
346+
expected_event_ids = [65, 86]
347+
348+
[[queries]]
349+
name = "likeMultipleArgWithPattern"
350+
query = '''
351+
file where file_name like ("winini*.exe", "lsass.*") and opcode == 2
352+
'''
353+
expected_event_ids = [65, 86]
354+
355+
[[queries]]
356+
name = "likeMultipleArgWithPatternInsensitive"
357+
query = '''
358+
file where file_name like~ ("winIni*.exe", "lSaSs.*") and opcode == 2
359+
'''
360+
expected_event_ids = [65, 86]
361+
362+
[[queries]]
363+
name = "likeMultipleArgsWildcardPatternQuestionMark"
364+
query = '''
365+
file where file_name like ("winini?.exe", "lsass.e?e") and opcode == 2
366+
'''
367+
expected_event_ids = [65, 86]
368+
369+
[[queries]]
370+
name = "likeMultipleArgsWildcardPatternQuestionMarkInsensitive"
371+
query = '''
372+
file where file_name like~ ("winINI?.exE", "lSasS.E?E") and opcode == 2
373+
'''
374+
expected_event_ids = [65, 86]
375+
376+
[[queries]]
377+
name = "regexWOCharClasses"
378+
expected_event_ids = [98]
379+
notes = "regexp doesn't support character classes"
380+
query = '''
381+
//
382+
// """.*?net1\s+localgroup.*?""")
383+
process where command_line regex """.*?net1[ ]+localgroup.*?"""
384+
'''
385+
386+
[[queries]]
387+
name = "regexWOCharClasses-Insensitive"
388+
expected_event_ids = [98]
389+
notes = "regexp doesn't support character classes"
390+
query = '''
391+
process where command_line regex~ """.*?net1[ ]+localgroup.*?"""
392+
'''
393+
394+
[[queries]]
395+
name = "simpleRegex"
396+
expected_event_ids = [98]
397+
query = '''
398+
process where command_line regex """.*?net1.*?"""
399+
'''
400+
401+
[[queries]]
402+
name = "simpleRegexInsensitive"
403+
expected_event_ids = [98]
404+
query = '''
405+
process where command_line regex~ """.*?net1.*?"""
406+
'''
407+
408+
[[queries]]
409+
name = "regexWithCharacterClasses2"
410+
expected_event_ids = [98]
411+
notes = "regexp doesn't support predefined character classes (like \\s)"
412+
query = '''
413+
// """.*?net1\s+\w{4,15}\s+.*?"""
414+
process where command_line regex (""".*?net1[ ]+[a-z]{4,15}[ ]+.*?""")
415+
'''
416+
417+
[[queries]]
418+
name = "regexMultiPatternMatch"
419+
expected_event_ids = [50, 97, 98]
420+
query = '''
421+
process where command_line regex (".*?net[1]? localgroup.*?", ".*? myappserver.py .*?")
422+
'''
423+
424+
[[queries]]
425+
name = "regexWithSubstring"
426+
expected_event_ids = [50, 98]
427+
query = '''
428+
process where substring(command_line, 5) regex (".*?net[1]? localgroup.*?", ".*? myappserver.py .*?")
429+
'''
430+

‎x-pack/plugin/eql/src/main/antlr/EqlBase.g4‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -101,8 +101,8 @@ operatorExpression
101101
// https://github.com/antlr/antlr4/issues/781
102102
predicate
103103
: NOT? kind=(IN | IN_INSENSITIVE) LP expression (COMMA expression)* RP
104-
| kind=SEQ constant
105-
| kind=SEQ LP constant (COMMA constant)* RP
104+
| kind=(SEQ | LIKE | LIKE_INSENSITIVE | REGEX | REGEX_INSENSITIVE) constant
105+
| kind=(SEQ | LIKE | LIKE_INSENSITIVE | REGEX | REGEX_INSENSITIVE) LP constant (COMMA constant)* RP
106106
;
107107

108108
primaryExpression
@@ -165,11 +165,15 @@ FALSE: 'false';
165165
IN: 'in';
166166
IN_INSENSITIVE : 'in~';
167167
JOIN: 'join';
168+
LIKE: 'like';
169+
LIKE_INSENSITIVE: 'like~';
168170
MAXSPAN: 'maxspan';
169171
NOT: 'not';
170172
NULL: 'null';
171173
OF: 'of';
172174
OR: 'or';
175+
REGEX: 'regex';
176+
REGEX_INSENSITIVE: 'regex~';
173177
SEQUENCE: 'sequence';
174178
TRUE: 'true';
175179
UNTIL: 'until';

‎x-pack/plugin/eql/src/main/antlr/EqlBase.tokens‎

Lines changed: 77 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -5,79 +5,87 @@ FALSE=4
55
IN=5
66
IN_INSENSITIVE=6
77
JOIN=7
8-
MAXSPAN=8
9-
NOT=9
10-
NULL=10
11-
OF=11
12-
OR=12
13-
SEQUENCE=13
14-
TRUE=14
15-
UNTIL=15
16-
WHERE=16
17-
WITH=17
18-
SEQ=18
19-
ASGN=19
20-
EQ=20
21-
NEQ=21
22-
LT=22
23-
LTE=23
24-
GT=24
25-
GTE=25
26-
PLUS=26
27-
MINUS=27
28-
ASTERISK=28
29-
SLASH=29
30-
PERCENT=30
31-
DOT=31
32-
COMMA=32
33-
LB=33
34-
RB=34
35-
LP=35
36-
RP=36
37-
PIPE=37
38-
STRING=38
39-
INTEGER_VALUE=39
40-
DECIMAL_VALUE=40
41-
IDENTIFIER=41
42-
QUOTED_IDENTIFIER=42
43-
TILDE_IDENTIFIER=43
44-
LINE_COMMENT=44
45-
BRACKETED_COMMENT=45
46-
WS=46
8+
LIKE=8
9+
LIKE_INSENSITIVE=9
10+
MAXSPAN=10
11+
NOT=11
12+
NULL=12
13+
OF=13
14+
OR=14
15+
REGEX=15
16+
REGEX_INSENSITIVE=16
17+
SEQUENCE=17
18+
TRUE=18
19+
UNTIL=19
20+
WHERE=20
21+
WITH=21
22+
SEQ=22
23+
ASGN=23
24+
EQ=24
25+
NEQ=25
26+
LT=26
27+
LTE=27
28+
GT=28
29+
GTE=29
30+
PLUS=30
31+
MINUS=31
32+
ASTERISK=32
33+
SLASH=33
34+
PERCENT=34
35+
DOT=35
36+
COMMA=36
37+
LB=37
38+
RB=38
39+
LP=39
40+
RP=40
41+
PIPE=41
42+
STRING=42
43+
INTEGER_VALUE=43
44+
DECIMAL_VALUE=44
45+
IDENTIFIER=45
46+
QUOTED_IDENTIFIER=46
47+
TILDE_IDENTIFIER=47
48+
LINE_COMMENT=48
49+
BRACKETED_COMMENT=49
50+
WS=50
4751
'and'=1
4852
'any'=2
4953
'by'=3
5054
'false'=4
5155
'in'=5
5256
'in~'=6
5357
'join'=7
54-
'maxspan'=8
55-
'not'=9
56-
'null'=10
57-
'of'=11
58-
'or'=12
59-
'sequence'=13
60-
'true'=14
61-
'until'=15
62-
'where'=16
63-
'with'=17
64-
':'=18
65-
'='=19
66-
'=='=20
67-
'!='=21
68-
'<'=22
69-
'<='=23
70-
'>'=24
71-
'>='=25
72-
'+'=26
73-
'-'=27
74-
'*'=28
75-
'/'=29
76-
'%'=30
77-
'.'=31
78-
','=32
79-
'['=33
80-
']'=34
81-
'('=35
82-
')'=36
83-
'|'=37
58+
'like'=8
59+
'like~'=9
60+
'maxspan'=10
61+
'not'=11
62+
'null'=12
63+
'of'=13
64+
'or'=14
65+
'regex'=15
66+
'regex~'=16
67+
'sequence'=17
68+
'true'=18
69+
'until'=19
70+
'where'=20
71+
'with'=21
72+
':'=22
73+
'='=23
74+
'=='=24
75+
'!='=25
76+
'<'=26
77+
'<='=27
78+
'>'=28
79+
'>='=29
80+
'+'=30
81+
'-'=31
82+
'*'=32
83+
'/'=33
84+
'%'=34
85+
'.'=35
86+
','=36
87+
'['=37
88+
']'=38
89+
'('=39
90+
')'=40
91+
'|'=41

0 commit comments

Comments
 (0)