From aff0389bb6da7b0423260f8df577a0b2df728017 Mon Sep 17 00:00:00 2001 From: Aleksandr Maus Date: Wed, 25 Mar 2020 09:43:13 -0400 Subject: [PATCH 1/3] EQL: implement cidrMatch function --- .../test/eql/CommonEqlActionTestCase.java | 8 ++ .../src/main/resources/mapping-default.json | 80 +++++++++++++ .../resources/test_queries_unsupported.toml | 24 ---- .../function/EqlFunctionRegistry.java | 2 + .../function/scalar/string/CIDRMatch.java | 108 ++++++++++++++++++ .../xpack/eql/optimizer/Optimizer.java | 10 ++ .../xpack/eql/analysis/VerifierTests.java | 2 - .../eql/planner/QueryFolderFailTests.java | 32 ++++++ .../src/test/resources/queryfolder_tests.txt | 17 +++ .../xpack/ql/expression/TypeResolutions.java | 14 +++ .../expression/function/FunctionRegistry.java | 23 ++++ 11 files changed, 294 insertions(+), 26 deletions(-) create mode 100644 x-pack/plugin/eql/qa/common/src/main/resources/mapping-default.json create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatch.java diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlActionTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlActionTestCase.java index a8d4cbc75defb..369e1c6d79cd7 100644 --- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlActionTestCase.java +++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlActionTestCase.java @@ -16,7 +16,9 @@ import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.client.eql.EqlSearchRequest; import org.elasticsearch.client.eql.EqlSearchResponse; +import org.elasticsearch.client.indices.CreateIndexRequest; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; @@ -55,6 +57,12 @@ private static void setupData(CommonEqlActionTestCase tc) throws Exception { return; } + CreateIndexRequest request = new CreateIndexRequest(testIndexName) + .mapping(Streams.readFully(CommonEqlActionTestCase.class.getResourceAsStream("/mapping-default.json")), + XContentType.JSON); + + tc.highLevelClient().indices().create(request, RequestOptions.DEFAULT); + BulkRequest bulk = new BulkRequest(); bulk.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/mapping-default.json b/x-pack/plugin/eql/qa/common/src/main/resources/mapping-default.json new file mode 100644 index 0000000000000..73a16abf380e5 --- /dev/null +++ b/x-pack/plugin/eql/qa/common/src/main/resources/mapping-default.json @@ -0,0 +1,80 @@ +{ + "properties" : { + "command_line" : { + "type" : "keyword" + }, + "event" : { + "properties" : { + "category" : { + "type" : "keyword" + } + } + }, + "md5" : { + "type" : "keyword" + }, + "parent_process_name": { + "type" : "keyword" + }, + "parent_process_path": { + "type" : "keyword" + }, + "pid" : { + "type" : "long" + }, + "ppid" : { + "type" : "long" + }, + "process_name": { + "type" : "keyword" + }, + "process_path": { + "type" : "keyword" + }, + "subtype" : { + "type" : "keyword" + }, + "@timestamp" : { + "type" : "date" + }, + "user" : { + "type" : "keyword" + }, + "user_name" : { + "type" : "keyword" + }, + "user_domain": { + "type" : "keyword" + }, + "hostname" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 + } + } + }, + "opcode" : { + "type" : "long" + }, + "file_name" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 + } + } + }, + "serial_event_id" : { + "type" : "long" + }, + "source_address" : { + "type" : "ip" + }, + "exit_code" : { + "type" : "long" + } + } +} diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml index f9a869f9e25f3..8d30054c957a2 100644 --- a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml +++ b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml @@ -1126,30 +1126,6 @@ query = ''' file where between(file_path, "dev", ".json", true) == "\\testlogs\\something" ''' -[[queries]] -expected_event_ids = [75304, 75305] -query = ''' -network where cidrMatch(source_address, "10.6.48.157/8") -''' - -[[queries]] -expected_event_ids = [] -query = ''' -network where cidrMatch(source_address, "192.168.0.0/16") -''' - -[[queries]] -expected_event_ids = [75304, 75305] -query = ''' -network where cidrMatch(source_address, "192.168.0.0/16", "10.6.48.157/8") - -''' -[[queries]] -expected_event_ids = [75304, 75305] -query = ''' -network where cidrMatch(source_address, "0.0.0.0/0") -''' - [[queries]] expected_event_ids = [7, 14, 22, 29, 44] query = ''' diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java index 119e12fa9f39c..92aaf6033556d 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.eql.expression.function; +import org.elasticsearch.xpack.eql.expression.function.scalar.string.CIDRMatch; import org.elasticsearch.xpack.eql.expression.function.scalar.string.Substring; import org.elasticsearch.xpack.ql.expression.function.FunctionDefinition; import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry; @@ -24,6 +25,7 @@ private static FunctionDefinition[][] functions() { // String new FunctionDefinition[] { def(Substring.class, Substring::new, "substring"), + def(CIDRMatch.class, CIDRMatch::new, "cidrmatch"), }, }; } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatch.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatch.java new file mode 100644 index 0000000000000..be020a3e99113 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatch.java @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.expression.function.scalar.string; + +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.Expressions; +import org.elasticsearch.xpack.ql.expression.Expressions.ParamOrdinal; +import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction; +import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate; +import org.elasticsearch.xpack.ql.expression.predicate.logical.Or; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals; +import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.ql.type.DataTypes; +import org.elasticsearch.xpack.ql.util.CollectionUtils; + +import java.util.List; + +import static java.util.Collections.singletonList; +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isIPAndExact; +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isStringAndExact; + +/** + * EQL specific cidrMatch function + */ +public class CIDRMatch extends ScalarFunction { + + private final Expression field; + private final List addresses; + + public CIDRMatch(Source source, Expression field, List addresses) { + super(source, CollectionUtils.combine(singletonList(field), addresses)); + this.field = field; + this.addresses = addresses; + } + + @Override + protected TypeResolution resolveType() { + if (!childrenResolved()) { + return new TypeResolution("Unresolved children"); + } + + TypeResolution resolution = isIPAndExact(field, sourceText(), Expressions.ParamOrdinal.FIRST); + if (resolution.unresolved()) { + return resolution; + } + + for (Expression addr: addresses) { + // Currently we have limited enum for ordinal numbers + // So just using default here for error messaging + resolution = isStringAndExact(addr, sourceText(), ParamOrdinal.DEFAULT); + if (resolution.unresolved()) { + return resolution; + } + } + + return resolution; + } + + @Override + public boolean foldable() { + return super.foldable() && field.foldable() && asFunction().foldable(); + } + + @Override + public Object fold() { + return asFunction().fold(); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, CIDRMatch::new, field, addresses); + } + + @Override + public ScriptTemplate asScript() { + throw new UnsupportedOperationException(); + } + + @Override + public DataType dataType() { + return DataTypes.BOOLEAN; + } + + @Override + public Expression replaceChildren(List newChildren) { + if (newChildren.size() < 2) { + throw new IllegalArgumentException("expected at least [2] children but received [" + newChildren.size() + "]"); + } + return new CIDRMatch(source(), newChildren.get(0), newChildren.subList(1, newChildren.size())); + } + + public ScalarFunction asFunction() { + ScalarFunction func = null; + + for (Expression address: addresses) { + final Equals eq = new Equals(source(), field, address); + func = (func == null) ? eq : new Or(source(), func, eq); + } + + return func; + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java index aea8042c07ec0..4e65c998e7333 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.eql.optimizer; +import org.elasticsearch.xpack.eql.expression.function.scalar.string.CIDRMatch; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.predicate.logical.Not; import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNotNull; @@ -48,6 +49,7 @@ protected Iterable.Batch> batches() { new ReplaceNullChecks(), new PropagateEquals(), new CombineBinaryComparisons(), + new ReplaceCIDRMatchFunction(), // prune/elimination new PruneFilters(), new PruneLiteralsInOrderBy() @@ -129,4 +131,12 @@ protected LogicalPlan rule(Filter filter) { }); } } + + private static class ReplaceCIDRMatchFunction extends OptimizerRule { + + @Override + protected LogicalPlan rule(Filter filter) { + return filter.transformExpressionsUp(e -> e instanceof CIDRMatch ? ((CIDRMatch) e).asFunction() : e); + } + } } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java index 1570184ced1e0..fdbd786a91348 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java @@ -153,8 +153,6 @@ public void testFunctionVerificationUnknown() { error("process where concat(serial_event_id, ':', process_name, opcode) == '5:winINIT.exe3'")); assertEquals("1:15: Unknown function [between]", error("process where between(process_name, \"s\", \"e\") == \"yst\"")); - assertEquals("1:15: Unknown function [cidrMatch]", - error("network where cidrMatch(source_address, \"192.168.0.0/16\", \"10.6.48.157/8\")")); assertEquals("1:22: Unknown function [between]", error("process where length(between(process_name, 'g', 'e')) > 0")); } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java index 471ee4c56943a..01c35abef84ac 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java @@ -6,6 +6,8 @@ package org.elasticsearch.xpack.eql.planner; +import org.elasticsearch.xpack.eql.analysis.VerificationException; +import org.elasticsearch.xpack.ql.ParsingException; import org.elasticsearch.xpack.ql.QlIllegalArgumentException; public class QueryFolderFailTests extends AbstractQueryFolderTestCase { @@ -22,4 +24,34 @@ public void testPropertyEquationInClauseFilterUnsupported() { String msg = e.getMessage(); assertEquals("Line 1:52: Comparisons against variables are not (currently) supported; offender [parent_process_name] in [==]", msg); } + + public void testCIDRMatchNonIPField() { + VerificationException e = expectThrows(VerificationException.class, + () -> plan("process where cidrMatch(hostname, \"10.0.0.0/8\")")); + String msg = e.getMessage(); + assertEquals("Found 1 problem\n" + + "line 1:15: first argument of [cidrMatch(hostname, \"10.0.0.0/8\")] must be [ip], found value [hostname] type [text]", msg); + } + + public void testCIDRMatchMissingValue() { + ParsingException e = expectThrows(ParsingException.class, + () -> plan("process where cidrMatch(source_address)")); + String msg = e.getMessage(); + assertEquals("line 1:16: error building [cidrmatch]: expects at least two arguments", msg); + } + + public void testCIDRMatchAgainstField() { + QlIllegalArgumentException e = expectThrows(QlIllegalArgumentException.class, + () -> plan("process where cidrMatch(source_address, hostname)")); + String msg = e.getMessage(); + assertEquals("Line 1:41: Comparisons against variables are not (currently) supported; offender [hostname] in [==]", msg); + } + + public void testCIDRMatchNonString() { + VerificationException e = expectThrows(VerificationException.class, + () -> plan("process where cidrMatch(source_address, 12345)")); + String msg = e.getMessage(); + assertEquals("Found 1 problem\n" + + "line 1:15: argument of [cidrMatch(source_address, 12345)] must be [string], found value [12345] type [integer]", msg); + } } diff --git a/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt b/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt index a4f73f1b19602..48169b82018ea 100644 --- a/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt +++ b/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt @@ -72,3 +72,20 @@ process where substring(file_name, -4) == '.exe' InternalEqlScriptUtils.substring(InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2),params.v3))", "params":{"v0":"file_name.keyword","v1":-4,"v2":null,"v3":".exe"} + +cidrMatchFunctionOne +process where cidrMatch(source_address, "10.0.0.0/8") +"term":{"source_address":{"value":"10.0.0.0/8" + + +cidrMatchFunctionTwo +process where cidrMatch(source_address, "10.0.0.0/8", "192.168.0.0/16") +"term":{"source_address":{"value":"10.0.0.0/8" +"term":{"source_address":{"value":"192.168.0.0/16" + + +cidrMatchFunctionThree +process where cidrMatch(source_address, "10.0.0.0/8", "192.168.0.0/16", "2001:db8::/32") +"term":{"source_address":{"value":"10.0.0.0/8" +"term":{"source_address":{"value":"192.168.0.0/16" +"term":{"source_address":{"value":"2001:db8::/32" diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/TypeResolutions.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/TypeResolutions.java index 01307d7ef84fc..42a5bce7b9f6b 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/TypeResolutions.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/TypeResolutions.java @@ -18,6 +18,7 @@ import static org.elasticsearch.common.logging.LoggerMessageFormat.format; import static org.elasticsearch.xpack.ql.expression.Expressions.name; import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN; +import static org.elasticsearch.xpack.ql.type.DataTypes.IP; import static org.elasticsearch.xpack.ql.type.DataTypes.NULL; public final class TypeResolutions { @@ -40,6 +41,10 @@ public static TypeResolution isString(Expression e, String operationName, ParamO return isType(e, DataTypes::isString, operationName, paramOrd, "string"); } + public static TypeResolution isIP(Expression e, String operationName, ParamOrdinal paramOrd) { + return isType(e, dt -> dt == IP, operationName, paramOrd, "ip"); + } + public static TypeResolution isExact(Expression e, String message) { if (e instanceof FieldAttribute) { EsField.Exact exact = ((FieldAttribute) e).getExactInfo(); @@ -73,6 +78,15 @@ public static TypeResolution isStringAndExact(Expression e, String operationName return isExact(e, operationName, paramOrd); } + public static TypeResolution isIPAndExact(Expression e, String operationName, ParamOrdinal paramOrd) { + TypeResolution resolution = isIP(e, operationName, paramOrd); + if (resolution.unresolved()) { + return resolution; + } + + return isExact(e, operationName, paramOrd); + } + public static TypeResolution isFoldable(Expression e, String operationName, ParamOrdinal paramOrd) { if (!e.foldable()) { return new TypeResolution(format(null, "{}argument of [{}] must be a constant, received [{}]", diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/FunctionRegistry.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/FunctionRegistry.java index 6b90a59289f07..f4a7c14a99921 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/FunctionRegistry.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/FunctionRegistry.java @@ -421,4 +421,27 @@ public static FunctionDefinition def(Class function, protected interface CastFunctionBuilder { T build(Source source, Expression expression, DataType dataType); } + + + @SuppressWarnings("overloads") // These are ambiguous if you aren't using ctor references but we always do + public static FunctionDefinition def(Class function, + TwoParametersVariadicBuilder ctorRef, String... names) { + FunctionBuilder builder = (source, children, distinct, cfg) -> { + boolean hasMinimumOne = OptionalArgument.class.isAssignableFrom(function); + if (hasMinimumOne && children.size() < 1) { + throw new QlIllegalArgumentException("expects at least one argument"); + } else if (!hasMinimumOne && children.size() < 2) { + throw new QlIllegalArgumentException("expects at least two arguments"); + } + if (distinct) { + throw new QlIllegalArgumentException("does not support DISTINCT yet it was specified"); + } + return ctorRef.build(source, children.get(0), children.subList(1, children.size())); + }; + return def(function, builder, false, names); + } + + protected interface TwoParametersVariadicBuilder { + T build(Source source, Expression expression, List remaining); + } } From 16ff57dbcef47a5a8003265b2d69d8b579cfc931 Mon Sep 17 00:00:00 2001 From: Aleksandr Maus Date: Sat, 4 Apr 2020 16:40:09 -0400 Subject: [PATCH 2/3] Address code review comments --- .../function/scalar/string/CIDRMatch.java | 22 ++++++++++++++++++- .../eql/planner/QueryFolderFailTests.java | 5 +++-- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatch.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatch.java index be020a3e99113..8d99e9ffea6fc 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatch.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatch.java @@ -22,11 +22,14 @@ import java.util.List; import static java.util.Collections.singletonList; +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isFoldable; import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isIPAndExact; import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isStringAndExact; /** * EQL specific cidrMatch function + * Returns true if the source address matches any of the provided CIDR blocks. + * Refer to: https://eql.readthedocs.io/en/latest/query-guide/functions.html#cidrMatch */ public class CIDRMatch extends ScalarFunction { @@ -59,12 +62,29 @@ protected TypeResolution resolveType() { } } + int index = 1; + + for (Expression addr: addresses) { + + resolution = isFoldable(addr, sourceText(), ParamOrdinal.fromIndex(index)); + if (resolution.unresolved()) { + break; + } + + resolution = isStringAndExact(addr, sourceText(), ParamOrdinal.fromIndex(index)); + if (resolution.unresolved()) { + break; + } + + index++; + } + return resolution; } @Override public boolean foldable() { - return super.foldable() && field.foldable() && asFunction().foldable(); + return field.foldable() && asFunction().foldable(); } @Override diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java index 979722951fbff..a913d5611c151 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java @@ -97,10 +97,11 @@ public void testCIDRMatchMissingValue() { } public void testCIDRMatchAgainstField() { - QlIllegalArgumentException e = expectThrows(QlIllegalArgumentException.class, + VerificationException e = expectThrows(VerificationException.class, () -> plan("process where cidrMatch(source_address, hostname)")); String msg = e.getMessage(); - assertEquals("Line 1:41: Comparisons against variables are not (currently) supported; offender [hostname] in [==]", msg); + assertEquals("Found 1 problem\n" + + "line 1:15: second argument of [cidrMatch(source_address, hostname)] must be a constant, received [hostname]", msg); } public void testCIDRMatchNonString() { From b1342e7dacb822a2f13feeabfc3a1db3232955d3 Mon Sep 17 00:00:00 2001 From: Aleksandr Maus Date: Tue, 7 Apr 2020 17:11:46 -0400 Subject: [PATCH 3/3] Convert to SurrogateFunction --- .../function/scalar/string/CIDRMatch.java | 62 +++++++------------ .../xpack/eql/optimizer/Optimizer.java | 10 --- 2 files changed, 24 insertions(+), 48 deletions(-) diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatch.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatch.java index 8d99e9ffea6fc..b7d5ea88ba3a4 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatch.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatch.java @@ -9,8 +9,8 @@ import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.Expressions; import org.elasticsearch.xpack.ql.expression.Expressions.ParamOrdinal; +import org.elasticsearch.xpack.ql.expression.function.scalar.BaseSurrogateFunction; import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction; -import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate; import org.elasticsearch.xpack.ql.expression.predicate.logical.Or; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals; import org.elasticsearch.xpack.ql.tree.NodeInfo; @@ -31,7 +31,7 @@ * Returns true if the source address matches any of the provided CIDR blocks. * Refer to: https://eql.readthedocs.io/en/latest/query-guide/functions.html#cidrMatch */ -public class CIDRMatch extends ScalarFunction { +public class CIDRMatch extends BaseSurrogateFunction { private final Expression field; private final List addresses; @@ -42,6 +42,24 @@ public CIDRMatch(Source source, Expression field, List addresses) { this.addresses = addresses; } + @Override + protected NodeInfo info() { + return NodeInfo.create(this, CIDRMatch::new, field, addresses); + } + + @Override + public Expression replaceChildren(List newChildren) { + if (newChildren.size() < 2) { + throw new IllegalArgumentException("expected at least [2] children but received [" + newChildren.size() + "]"); + } + return new CIDRMatch(source(), newChildren.get(0), newChildren.subList(1, newChildren.size())); + } + + @Override + public DataType dataType() { + return DataTypes.BOOLEAN; + } + @Override protected TypeResolution resolveType() { if (!childrenResolved()) { @@ -53,7 +71,7 @@ protected TypeResolution resolveType() { return resolution; } - for (Expression addr: addresses) { + for (Expression addr : addresses) { // Currently we have limited enum for ordinal numbers // So just using default here for error messaging resolution = isStringAndExact(addr, sourceText(), ParamOrdinal.DEFAULT); @@ -64,7 +82,7 @@ protected TypeResolution resolveType() { int index = 1; - for (Expression addr: addresses) { + for (Expression addr : addresses) { resolution = isFoldable(addr, sourceText(), ParamOrdinal.fromIndex(index)); if (resolution.unresolved()) { @@ -83,42 +101,10 @@ protected TypeResolution resolveType() { } @Override - public boolean foldable() { - return field.foldable() && asFunction().foldable(); - } - - @Override - public Object fold() { - return asFunction().fold(); - } - - @Override - protected NodeInfo info() { - return NodeInfo.create(this, CIDRMatch::new, field, addresses); - } - - @Override - public ScriptTemplate asScript() { - throw new UnsupportedOperationException(); - } - - @Override - public DataType dataType() { - return DataTypes.BOOLEAN; - } - - @Override - public Expression replaceChildren(List newChildren) { - if (newChildren.size() < 2) { - throw new IllegalArgumentException("expected at least [2] children but received [" + newChildren.size() + "]"); - } - return new CIDRMatch(source(), newChildren.get(0), newChildren.subList(1, newChildren.size())); - } - - public ScalarFunction asFunction() { + public ScalarFunction makeSubstitute() { ScalarFunction func = null; - for (Expression address: addresses) { + for (Expression address : addresses) { final Equals eq = new Equals(source(), field, address); func = (func == null) ? eq : new Or(source(), func, eq); } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java index e60fb41933da2..307e2ae2a54ca 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java @@ -6,7 +6,6 @@ package org.elasticsearch.xpack.eql.optimizer; -import org.elasticsearch.xpack.eql.expression.function.scalar.string.CIDRMatch; import org.elasticsearch.xpack.eql.util.StringUtils; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.predicate.logical.Not; @@ -53,7 +52,6 @@ protected Iterable.Batch> batches() { new ReplaceNullChecks(), new PropagateEquals(), new CombineBinaryComparisons(), - new ReplaceCIDRMatchFunction(), // prune/elimination new PruneFilters(), new PruneLiteralsInOrderBy() @@ -122,12 +120,4 @@ protected LogicalPlan rule(Filter filter) { }); } } - - private static class ReplaceCIDRMatchFunction extends OptimizerRule { - - @Override - protected LogicalPlan rule(Filter filter) { - return filter.transformExpressionsUp(e -> e instanceof CIDRMatch ? ((CIDRMatch) e).asFunction() : e); - } - } }