diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlActionTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlActionTestCase.java index 4622575c4de64..e374b1c4e60b9 100644 --- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlActionTestCase.java +++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlActionTestCase.java @@ -16,7 +16,9 @@ import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.client.eql.EqlSearchRequest; import org.elasticsearch.client.eql.EqlSearchResponse; +import org.elasticsearch.client.indices.CreateIndexRequest; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; @@ -56,6 +58,12 @@ private static void setupData(CommonEqlActionTestCase tc) throws Exception { return; } + CreateIndexRequest request = new CreateIndexRequest(testIndexName) + .mapping(Streams.readFully(CommonEqlActionTestCase.class.getResourceAsStream("/mapping-default.json")), + XContentType.JSON); + + tc.highLevelClient().indices().create(request, RequestOptions.DEFAULT); + BulkRequest bulk = new BulkRequest(); bulk.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/mapping-default.json b/x-pack/plugin/eql/qa/common/src/main/resources/mapping-default.json new file mode 100644 index 0000000000000..73a16abf380e5 --- /dev/null +++ b/x-pack/plugin/eql/qa/common/src/main/resources/mapping-default.json @@ -0,0 +1,80 @@ +{ + "properties" : { + "command_line" : { + "type" : "keyword" + }, + "event" : { + "properties" : { + "category" : { + "type" : "keyword" + } + } + }, + "md5" : { + "type" : "keyword" + }, + "parent_process_name": { + "type" : "keyword" + }, + "parent_process_path": { + "type" : "keyword" + }, + "pid" : { + "type" : "long" + }, + "ppid" : { + "type" : "long" + }, + "process_name": { + "type" : "keyword" + }, + "process_path": { + "type" : "keyword" + }, + "subtype" : { + "type" : "keyword" + }, + "@timestamp" : { + "type" : "date" + }, + "user" : { + "type" : "keyword" + }, + "user_name" : { + "type" : "keyword" + }, + "user_domain": { + "type" : "keyword" + }, + "hostname" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 + } + } + }, + "opcode" : { + "type" : "long" + }, + "file_name" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 + } + } + }, + "serial_event_id" : { + "type" : "long" + }, + "source_address" : { + "type" : "ip" + }, + "exit_code" : { + "type" : "long" + } + } +} diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml index f252030c4cc33..4889887114832 100644 --- a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml +++ b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml @@ -1057,30 +1057,6 @@ query = ''' file where between(file_path, "dev", ".json", true) == "\\testlogs\\something" ''' -[[queries]] -expected_event_ids = [75304, 75305] -query = ''' -network where cidrMatch(source_address, "10.6.48.157/8") -''' - -[[queries]] -expected_event_ids = [] -query = ''' -network where cidrMatch(source_address, "192.168.0.0/16") -''' - -[[queries]] -expected_event_ids = [75304, 75305] -query = ''' -network where cidrMatch(source_address, "192.168.0.0/16", "10.6.48.157/8") - -''' -[[queries]] -expected_event_ids = [75304, 75305] -query = ''' -network where cidrMatch(source_address, "0.0.0.0/0") -''' - [[queries]] expected_event_ids = [7, 14, 22, 29, 44] query = ''' diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java index 1c6c8792ee01b..ca16b8e44b084 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.eql.expression.function; +import org.elasticsearch.xpack.eql.expression.function.scalar.string.CIDRMatch; import org.elasticsearch.xpack.eql.expression.function.scalar.string.Between; import org.elasticsearch.xpack.eql.expression.function.scalar.string.EndsWith; import org.elasticsearch.xpack.eql.expression.function.scalar.string.Length; @@ -30,6 +31,7 @@ private static FunctionDefinition[][] functions() { // String new FunctionDefinition[] { def(Between.class, Between::new, 2, "between"), + def(CIDRMatch.class, CIDRMatch::new, "cidrmatch"), def(EndsWith.class, EndsWith::new, "endswith"), def(Length.class, Length::new, "length"), def(StartsWith.class, StartsWith::new, "startswith"), diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatch.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatch.java new file mode 100644 index 0000000000000..b7d5ea88ba3a4 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatch.java @@ -0,0 +1,114 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.expression.function.scalar.string; + +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.Expressions; +import org.elasticsearch.xpack.ql.expression.Expressions.ParamOrdinal; +import org.elasticsearch.xpack.ql.expression.function.scalar.BaseSurrogateFunction; +import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction; +import org.elasticsearch.xpack.ql.expression.predicate.logical.Or; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals; +import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.ql.type.DataTypes; +import org.elasticsearch.xpack.ql.util.CollectionUtils; + +import java.util.List; + +import static java.util.Collections.singletonList; +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isFoldable; +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isIPAndExact; +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isStringAndExact; + +/** + * EQL specific cidrMatch function + * Returns true if the source address matches any of the provided CIDR blocks. + * Refer to: https://eql.readthedocs.io/en/latest/query-guide/functions.html#cidrMatch + */ +public class CIDRMatch extends BaseSurrogateFunction { + + private final Expression field; + private final List addresses; + + public CIDRMatch(Source source, Expression field, List addresses) { + super(source, CollectionUtils.combine(singletonList(field), addresses)); + this.field = field; + this.addresses = addresses; + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, CIDRMatch::new, field, addresses); + } + + @Override + public Expression replaceChildren(List newChildren) { + if (newChildren.size() < 2) { + throw new IllegalArgumentException("expected at least [2] children but received [" + newChildren.size() + "]"); + } + return new CIDRMatch(source(), newChildren.get(0), newChildren.subList(1, newChildren.size())); + } + + @Override + public DataType dataType() { + return DataTypes.BOOLEAN; + } + + @Override + protected TypeResolution resolveType() { + if (!childrenResolved()) { + return new TypeResolution("Unresolved children"); + } + + TypeResolution resolution = isIPAndExact(field, sourceText(), Expressions.ParamOrdinal.FIRST); + if (resolution.unresolved()) { + return resolution; + } + + for (Expression addr : addresses) { + // Currently we have limited enum for ordinal numbers + // So just using default here for error messaging + resolution = isStringAndExact(addr, sourceText(), ParamOrdinal.DEFAULT); + if (resolution.unresolved()) { + return resolution; + } + } + + int index = 1; + + for (Expression addr : addresses) { + + resolution = isFoldable(addr, sourceText(), ParamOrdinal.fromIndex(index)); + if (resolution.unresolved()) { + break; + } + + resolution = isStringAndExact(addr, sourceText(), ParamOrdinal.fromIndex(index)); + if (resolution.unresolved()) { + break; + } + + index++; + } + + return resolution; + } + + @Override + public ScalarFunction makeSubstitute() { + ScalarFunction func = null; + + for (Expression address : addresses) { + final Equals eq = new Equals(source(), field, address); + func = (func == null) ? eq : new Or(source(), func, eq); + } + + return func; + } +} diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java index 4a6c87f8b27df..f8b8a875145c4 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java @@ -133,8 +133,6 @@ public void testFunctionVerificationUnknown() { error("process where serial_event_id == number('5')")); assertEquals("1:15: Unknown function [concat]", error("process where concat(serial_event_id, ':', process_name, opcode) == '5:winINIT.exe3'")); - assertEquals("1:15: Unknown function [cidrMatch]", - error("network where cidrMatch(source_address, \"192.168.0.0/16\", \"10.6.48.157/8\")")); } // Test unsupported array indexes diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java index c203026c350e2..36b53894f1adc 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java @@ -62,27 +62,35 @@ public void testBetweenWrongTypeParams() { error("process where between(process_name, \"s\", \"e\", false, 2)")); } - public void testPropertyEquationFilterUnsupported() { - QlIllegalArgumentException e = expectThrows(QlIllegalArgumentException.class, - () -> plan("process where (serial_event_id<9 and serial_event_id >= 7) or (opcode == pid)")); + public void testCIDRMatchNonIPField() { + VerificationException e = expectThrows(VerificationException.class, + () -> plan("process where cidrMatch(hostname, \"10.0.0.0/8\")")); String msg = e.getMessage(); - assertEquals("Line 1:74: Comparisons against variables are not (currently) supported; offender [pid] in [==]", msg); + assertEquals("Found 1 problem\n" + + "line 1:15: first argument of [cidrMatch(hostname, \"10.0.0.0/8\")] must be [ip], found value [hostname] type [text]", msg); } - public void testPropertyEquationInClauseFilterUnsupported() { + public void testCIDRMatchMissingValue() { + ParsingException e = expectThrows(ParsingException.class, + () -> plan("process where cidrMatch(source_address)")); + String msg = e.getMessage(); + assertEquals("line 1:16: error building [cidrmatch]: expects at least two arguments", msg); + } + + public void testCIDRMatchAgainstField() { VerificationException e = expectThrows(VerificationException.class, - () -> plan("process where opcode in (1,3) and process_name in (parent_process_name, \"SYSTEM\")")); + () -> plan("process where cidrMatch(source_address, hostname)")); String msg = e.getMessage(); - assertEquals("Found 1 problem\nline 1:35: Comparisons against variables are not (currently) supported; " + - "offender [parent_process_name] in [process_name in (parent_process_name, \"SYSTEM\")]", msg); + assertEquals("Found 1 problem\n" + + "line 1:15: second argument of [cidrMatch(source_address, hostname)] must be a constant, received [hostname]", msg); } - public void testLengthFunctionWithInexact() { + public void testCIDRMatchNonString() { VerificationException e = expectThrows(VerificationException.class, - () -> plan("process where length(plain_text) > 0")); + () -> plan("process where cidrMatch(source_address, 12345)")); String msg = e.getMessage(); - assertEquals("Found 1 problem\nline 1:15: [length(plain_text)] cannot operate on field of data type [text]: No keyword/multi-field " - + "defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg); + assertEquals("Found 1 problem\n" + + "line 1:15: argument of [cidrMatch(source_address, 12345)] must be [string], found value [12345] type [integer]", msg); } public void testEndsWithFunctionWithInexact() { @@ -93,6 +101,29 @@ public void testEndsWithFunctionWithInexact() { + "[text]: No keyword/multi-field defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg); } + public void testLengthFunctionWithInexact() { + VerificationException e = expectThrows(VerificationException.class, + () -> plan("process where length(plain_text) > 0")); + String msg = e.getMessage(); + assertEquals("Found 1 problem\nline 1:15: [length(plain_text)] cannot operate on field of data type [text]: No keyword/multi-field " + + "defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg); + } + + public void testPropertyEquationFilterUnsupported() { + QlIllegalArgumentException e = expectThrows(QlIllegalArgumentException.class, + () -> plan("process where (serial_event_id<9 and serial_event_id >= 7) or (opcode == pid)")); + String msg = e.getMessage(); + assertEquals("Line 1:74: Comparisons against variables are not (currently) supported; offender [pid] in [==]", msg); + } + + public void testPropertyEquationInClauseFilterUnsupported() { + VerificationException e = expectThrows(VerificationException.class, + () -> plan("process where opcode in (1,3) and process_name in (parent_process_name, \"SYSTEM\")")); + String msg = e.getMessage(); + assertEquals("Found 1 problem\nline 1:35: Comparisons against variables are not (currently) supported; " + + "offender [parent_process_name] in [process_name in (parent_process_name, \"SYSTEM\")]", msg); + } + public void testStartsWithFunctionWithInexact() { VerificationException e = expectThrows(VerificationException.class, () -> plan("process where startsWith(plain_text, \"foo\") == true")); diff --git a/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt b/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt index 57631c1f595f9..badf6a6cd464e 100644 --- a/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt +++ b/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt @@ -129,6 +129,27 @@ InternalEqlScriptUtils.between(InternalQlScriptUtils.docValue(doc,params.v0),par "params":{"v0":"process_name","v1":"s","v2":"e","v3":false,"v4":false,"v5":"yst"} ; +cidrMatchFunctionOne +process where cidrMatch(source_address, "10.0.0.0/8") +; +"term":{"source_address":{"value":"10.0.0.0/8" +; + +cidrMatchFunctionTwo +process where cidrMatch(source_address, "10.0.0.0/8", "192.168.0.0/16") +; +"term":{"source_address":{"value":"10.0.0.0/8" +"term":{"source_address":{"value":"192.168.0.0/16" +; + +cidrMatchFunctionThree +process where cidrMatch(source_address, "10.0.0.0/8", "192.168.0.0/16", "2001:db8::/32") +; +"term":{"source_address":{"value":"10.0.0.0/8" +"term":{"source_address":{"value":"192.168.0.0/16" +"term":{"source_address":{"value":"2001:db8::/32" +; + wildcardFunctionSingleArgument process where wildcard(process_path, "*\\red_ttp\\wininit.*") ; diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/TypeResolutions.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/TypeResolutions.java index 01307d7ef84fc..42a5bce7b9f6b 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/TypeResolutions.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/TypeResolutions.java @@ -18,6 +18,7 @@ import static org.elasticsearch.common.logging.LoggerMessageFormat.format; import static org.elasticsearch.xpack.ql.expression.Expressions.name; import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN; +import static org.elasticsearch.xpack.ql.type.DataTypes.IP; import static org.elasticsearch.xpack.ql.type.DataTypes.NULL; public final class TypeResolutions { @@ -40,6 +41,10 @@ public static TypeResolution isString(Expression e, String operationName, ParamO return isType(e, DataTypes::isString, operationName, paramOrd, "string"); } + public static TypeResolution isIP(Expression e, String operationName, ParamOrdinal paramOrd) { + return isType(e, dt -> dt == IP, operationName, paramOrd, "ip"); + } + public static TypeResolution isExact(Expression e, String message) { if (e instanceof FieldAttribute) { EsField.Exact exact = ((FieldAttribute) e).getExactInfo(); @@ -73,6 +78,15 @@ public static TypeResolution isStringAndExact(Expression e, String operationName return isExact(e, operationName, paramOrd); } + public static TypeResolution isIPAndExact(Expression e, String operationName, ParamOrdinal paramOrd) { + TypeResolution resolution = isIP(e, operationName, paramOrd); + if (resolution.unresolved()) { + return resolution; + } + + return isExact(e, operationName, paramOrd); + } + public static TypeResolution isFoldable(Expression e, String operationName, ParamOrdinal paramOrd) { if (!e.foldable()) { return new TypeResolution(format(null, "{}argument of [{}] must be a constant, received [{}]",