EQL: implement cidrMatch function (#54186)

x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlActionTestCase.java

@@ -16,7 +16,9 @@
 import org.elasticsearch.client.RestHighLevelClient;
 import org.elasticsearch.client.eql.EqlSearchRequest;
 import org.elasticsearch.client.eql.EqlSearchResponse;
+import org.elasticsearch.client.indices.CreateIndexRequest;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
@@ -56,6 +58,12 @@ private static void setupData(CommonEqlActionTestCase tc) throws Exception {
             return;
         }
 
+        CreateIndexRequest request = new CreateIndexRequest(testIndexName)
+                .mapping(Streams.readFully(CommonEqlActionTestCase.class.getResourceAsStream("/mapping-default.json")),
+                        XContentType.JSON);
+
+        tc.highLevelClient().indices().create(request, RequestOptions.DEFAULT);
+
         BulkRequest bulk = new BulkRequest();
         bulk.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
 

x-pack/plugin/eql/qa/common/src/main/resources/mapping-default.json

@@ -0,0 +1,80 @@
+{
+    "properties" : {
+        "command_line" : {
+            "type" : "keyword"
+        },
+        "event" : {
+            "properties" : {
+                "category" : {
+                    "type" : "keyword"
+                }
+            }
+        },
+        "md5" : {
+            "type" : "keyword"
+        },
+        "parent_process_name": {
+            "type" : "keyword"
+        },
+        "parent_process_path": {
+            "type" : "keyword"
+        },
+        "pid" : {
+            "type" : "long"
+        },
+        "ppid" : {
+            "type" : "long"
+        },
+        "process_name": {
+            "type" : "keyword"
+        },
+        "process_path": {
+            "type" : "keyword"
+        },
+        "subtype" : {
+            "type" : "keyword"
+        },
+        "@timestamp" : {
+            "type" : "date"
+        },
+        "user" : {
+            "type" : "keyword"
+        },
+        "user_name" : {
+            "type" : "keyword"
+        },
+        "user_domain": {
+            "type" : "keyword"
+        },
+        "hostname" : {
+            "type" : "text",
+            "fields" : {
+                "keyword" : {
+                    "type" : "keyword",
+                    "ignore_above" : 256
+                }
+            }
+        },
+        "opcode" : {
+            "type" : "long"
+        },
+        "file_name" : {
+            "type" : "text",
+            "fields" : {
+                "keyword" : {
+                    "type" : "keyword",
+                    "ignore_above" : 256
+                }
+            }
+        },
+        "serial_event_id" : {
+            "type" : "long"
+        },
+        "source_address" : {
+            "type" : "ip"
+        },
+        "exit_code" : {
+            "type" : "long"
+        }
+    }
+}

x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml

@@ -1057,30 +1057,6 @@ query = '''
 file where between(file_path, "dev", ".json", true) == "\\testlogs\\something"
 '''
 
-[[queries]]
-expected_event_ids = [75304, 75305]
-query = '''
-network where cidrMatch(source_address, "10.6.48.157/8")
-'''
-
-[[queries]]
-expected_event_ids = []
-query = '''
-network where cidrMatch(source_address, "192.168.0.0/16")
-'''
-
-[[queries]]
-expected_event_ids = [75304, 75305]
-query = '''
-network where cidrMatch(source_address, "192.168.0.0/16", "10.6.48.157/8")
-
-'''
-[[queries]]
-expected_event_ids = [75304, 75305]
-query = '''
-network where cidrMatch(source_address, "0.0.0.0/0")
-'''
-
 [[queries]]
 expected_event_ids = [7, 14, 22, 29, 44]
 query = '''

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java

@@ -6,6 +6,7 @@
 
 package org.elasticsearch.xpack.eql.expression.function;
 
+import org.elasticsearch.xpack.eql.expression.function.scalar.string.CIDRMatch;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.Between;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.EndsWith;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.Length;
@@ -30,6 +31,7 @@ private static FunctionDefinition[][] functions() {
         // String
             new FunctionDefinition[] {
                 def(Between.class, Between::new, 2, "between"),
+                def(CIDRMatch.class, CIDRMatch::new, "cidrmatch"),
                 def(EndsWith.class, EndsWith::new, "endswith"),
                 def(Length.class, Length::new, "length"),
                 def(StartsWith.class, StartsWith::new, "startswith"),

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatch.java

@@ -0,0 +1,114 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.ql.expression.Expression;
+import org.elasticsearch.xpack.ql.expression.Expressions;
+import org.elasticsearch.xpack.ql.expression.Expressions.ParamOrdinal;
+import org.elasticsearch.xpack.ql.expression.function.scalar.BaseSurrogateFunction;
+import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction;
+import org.elasticsearch.xpack.ql.expression.predicate.logical.Or;
+import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals;
+import org.elasticsearch.xpack.ql.tree.NodeInfo;
+import org.elasticsearch.xpack.ql.tree.Source;
+import org.elasticsearch.xpack.ql.type.DataType;
+import org.elasticsearch.xpack.ql.type.DataTypes;
+import org.elasticsearch.xpack.ql.util.CollectionUtils;
+
+import java.util.List;
+
+import static java.util.Collections.singletonList;
+import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isFoldable;
+import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isIPAndExact;
+import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isStringAndExact;
+
+/**
+ * EQL specific cidrMatch function
+ * Returns true if the source address matches any of the provided CIDR blocks.
+ * Refer to: https://eql.readthedocs.io/en/latest/query-guide/functions.html#cidrMatch
+ */
+public class CIDRMatch extends BaseSurrogateFunction {
+
+    private final Expression field;
+    private final List<Expression> addresses;
+
+    public CIDRMatch(Source source, Expression field, List<Expression> addresses) {
+        super(source, CollectionUtils.combine(singletonList(field), addresses));
+        this.field = field;
+        this.addresses = addresses;
+    }
+
+    @Override
+    protected NodeInfo<? extends Expression> info() {
+        return NodeInfo.create(this, CIDRMatch::new, field, addresses);
+    }
+
+    @Override
+    public Expression replaceChildren(List<Expression> newChildren) {
+        if (newChildren.size() < 2) {
+            throw new IllegalArgumentException("expected at least [2] children but received [" + newChildren.size() + "]");
+        }
+        return new CIDRMatch(source(), newChildren.get(0), newChildren.subList(1, newChildren.size()));
+    }
+
+    @Override
+    public DataType dataType() {
+        return DataTypes.BOOLEAN;
+    }
+
+    @Override
+    protected TypeResolution resolveType() {
+        if (!childrenResolved()) {
+            return new TypeResolution("Unresolved children");
+        }
+
+        TypeResolution resolution = isIPAndExact(field, sourceText(), Expressions.ParamOrdinal.FIRST);
+        if (resolution.unresolved()) {
+            return resolution;
+        }
+
+        for (Expression addr : addresses) {
+            // Currently we have limited enum for ordinal numbers
+            // So just using default here for error messaging
+            resolution = isStringAndExact(addr, sourceText(), ParamOrdinal.DEFAULT);
+            if (resolution.unresolved()) {
+                return resolution;
+            }
+        }
+
+        int index = 1;
+
+        for (Expression addr : addresses) {
+
+            resolution = isFoldable(addr, sourceText(), ParamOrdinal.fromIndex(index));
+            if (resolution.unresolved()) {
+                break;
+            }
+
+            resolution = isStringAndExact(addr, sourceText(), ParamOrdinal.fromIndex(index));
+            if (resolution.unresolved()) {
+                break;
+            }
+
+            index++;
+        }
+
+        return resolution;
+    }
+
+    @Override
+    public ScalarFunction makeSubstitute() {
+        ScalarFunction func = null;
+
+        for (Expression address : addresses) {
+            final Equals eq = new Equals(source(), field, address);
+            func = (func == null) ? eq : new Or(source(), func, eq);
+        }
+
+        return func;
+    }
+}

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java

@@ -133,8 +133,6 @@ public void testFunctionVerificationUnknown() {
                 error("process where serial_event_id == number('5')"));
         assertEquals("1:15: Unknown function [concat]",
                 error("process where concat(serial_event_id, ':', process_name, opcode) == '5:winINIT.exe3'"));
-        assertEquals("1:15: Unknown function [cidrMatch]",
-                error("network where cidrMatch(source_address, \"192.168.0.0/16\", \"10.6.48.157/8\")"));
     }
 
     // Test unsupported array indexes

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java

@@ -62,27 +62,35 @@ public void testBetweenWrongTypeParams() {
                 error("process where between(process_name, \"s\", \"e\", false, 2)"));
     }
 
-    public void testPropertyEquationFilterUnsupported() {
-        QlIllegalArgumentException e = expectThrows(QlIllegalArgumentException.class,
-                () -> plan("process where (serial_event_id<9 and serial_event_id >= 7) or (opcode == pid)"));
+    public void testCIDRMatchNonIPField() {
+        VerificationException e = expectThrows(VerificationException.class,
+                () -> plan("process where cidrMatch(hostname, \"10.0.0.0/8\")"));
         String msg = e.getMessage();
-        assertEquals("Line 1:74: Comparisons against variables are not (currently) supported; offender [pid] in [==]", msg);
+        assertEquals("Found 1 problem\n" +
+                "line 1:15: first argument of [cidrMatch(hostname, \"10.0.0.0/8\")] must be [ip], found value [hostname] type [text]", msg);
     }
 
-    public void testPropertyEquationInClauseFilterUnsupported() {
+    public void testCIDRMatchMissingValue() {
+        ParsingException e = expectThrows(ParsingException.class,
+                () -> plan("process where cidrMatch(source_address)"));
+        String msg = e.getMessage();
+        assertEquals("line 1:16: error building [cidrmatch]: expects at least two arguments", msg);
+    }
+
+    public void testCIDRMatchAgainstField() {
         VerificationException e = expectThrows(VerificationException.class,
-                () -> plan("process where opcode in (1,3) and process_name in (parent_process_name, \"SYSTEM\")"));
+                () -> plan("process where cidrMatch(source_address, hostname)"));
         String msg = e.getMessage();
-        assertEquals("Found 1 problem\nline 1:35: Comparisons against variables are not (currently) supported; " +
-                "offender [parent_process_name] in [process_name in (parent_process_name, \"SYSTEM\")]", msg);
+        assertEquals("Found 1 problem\n" +
+                "line 1:15: second argument of [cidrMatch(source_address, hostname)] must be a constant, received [hostname]", msg);
     }
 
-    public void testLengthFunctionWithInexact() {
+    public void testCIDRMatchNonString() {
         VerificationException e = expectThrows(VerificationException.class,
-                () -> plan("process where length(plain_text) > 0"));
+                () -> plan("process where cidrMatch(source_address, 12345)"));
         String msg = e.getMessage();
-        assertEquals("Found 1 problem\nline 1:15: [length(plain_text)] cannot operate on field of data type [text]: No keyword/multi-field "
-                + "defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg);
+        assertEquals("Found 1 problem\n" +
+                "line 1:15: argument of [cidrMatch(source_address, 12345)] must be [string], found value [12345] type [integer]", msg);
     }
 
     public void testEndsWithFunctionWithInexact() {
@@ -93,6 +101,29 @@ public void testEndsWithFunctionWithInexact() {
                 + "[text]: No keyword/multi-field defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg);
     }
 
+    public void testLengthFunctionWithInexact() {
+        VerificationException e = expectThrows(VerificationException.class,
+                () -> plan("process where length(plain_text) > 0"));
+        String msg = e.getMessage();
+        assertEquals("Found 1 problem\nline 1:15: [length(plain_text)] cannot operate on field of data type [text]: No keyword/multi-field "
+                + "defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg);
+    }
+
+    public void testPropertyEquationFilterUnsupported() {
+        QlIllegalArgumentException e = expectThrows(QlIllegalArgumentException.class,
+                () -> plan("process where (serial_event_id<9 and serial_event_id >= 7) or (opcode == pid)"));
+        String msg = e.getMessage();
+        assertEquals("Line 1:74: Comparisons against variables are not (currently) supported; offender [pid] in [==]", msg);
+    }
+
+    public void testPropertyEquationInClauseFilterUnsupported() {
+        VerificationException e = expectThrows(VerificationException.class,
+                () -> plan("process where opcode in (1,3) and process_name in (parent_process_name, \"SYSTEM\")"));
+        String msg = e.getMessage();
+        assertEquals("Found 1 problem\nline 1:35: Comparisons against variables are not (currently) supported; " +
+                "offender [parent_process_name] in [process_name in (parent_process_name, \"SYSTEM\")]", msg);
+    }
+
     public void testStartsWithFunctionWithInexact() {
         VerificationException e = expectThrows(VerificationException.class,
                 () -> plan("process where startsWith(plain_text, \"foo\") == true"));

x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt

@@ -129,6 +129,27 @@ InternalEqlScriptUtils.between(InternalQlScriptUtils.docValue(doc,params.v0),par
 "params":{"v0":"process_name","v1":"s","v2":"e","v3":false,"v4":false,"v5":"yst"}
 ;
 
+cidrMatchFunctionOne
+process where cidrMatch(source_address, "10.0.0.0/8")
+;
+"term":{"source_address":{"value":"10.0.0.0/8"
+;
+
+cidrMatchFunctionTwo
+process where cidrMatch(source_address, "10.0.0.0/8", "192.168.0.0/16")
+;
+"term":{"source_address":{"value":"10.0.0.0/8"
+"term":{"source_address":{"value":"192.168.0.0/16"
+;
+
+cidrMatchFunctionThree
+process where cidrMatch(source_address, "10.0.0.0/8", "192.168.0.0/16", "2001:db8::/32")
+;
+"term":{"source_address":{"value":"10.0.0.0/8"
+"term":{"source_address":{"value":"192.168.0.0/16"
+"term":{"source_address":{"value":"2001:db8::/32"
+;
+
 wildcardFunctionSingleArgument
 process where wildcard(process_path, "*\\red_ttp\\wininit.*")
 ;