EQL: Add wildcard function

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java

@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.eql.expression.function;
 
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.Substring;
+import org.elasticsearch.xpack.eql.expression.function.scalar.string.Wildcard;
 import org.elasticsearch.xpack.ql.expression.function.FunctionDefinition;
 import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry;
 
@@ -17,14 +18,17 @@ public class EqlFunctionRegistry extends FunctionRegistry {
     public EqlFunctionRegistry() {
         super(functions());
     }
-    
+
     private static FunctionDefinition[][] functions() {
         return new FunctionDefinition[][] {
         // Scalar functions
         // String
             new FunctionDefinition[] {
                 def(Substring.class, Substring::new, "substring"),
             },
+            new FunctionDefinition[] {
+                def(Wildcard.class, Wildcard::new, "wildcard"),
+            },
         };
     }
 

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Wildcard.java

@@ -0,0 +1,142 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.eql.utils.StringUtils;
+import org.elasticsearch.xpack.ql.expression.Expression;
+import org.elasticsearch.xpack.ql.expression.Expressions;
+import org.elasticsearch.xpack.ql.expression.Expressions.ParamOrdinal;
+import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction;
+import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
+import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate;
+import org.elasticsearch.xpack.ql.expression.predicate.logical.Or;
+import org.elasticsearch.xpack.ql.expression.predicate.regex.Like;
+import org.elasticsearch.xpack.ql.tree.NodeInfo;
+import org.elasticsearch.xpack.ql.tree.Source;
+import org.elasticsearch.xpack.ql.type.DataType;
+import org.elasticsearch.xpack.ql.type.DataTypes;
+
+import java.util.Collections;
+import java.util.List;
+
+import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
+import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isStringAndExact;
+
+/**
+ * EQL wildcard function. Matches the form:
+ *     wildcard(field, "*wildcard*pattern*", "*wildcard*pattern*")
+ */
+
+public class Wildcard extends ScalarFunction {
+
+    private final Expression field;
+    private final List<Expression> patterns;
+
+    private static List<Expression> getArguments(Expression src, List<Expression> patterns) {
+        List<Expression> arguments = new java.util.ArrayList<>(Collections.singletonList(src));
+        arguments.addAll(patterns);
+        return arguments;
+    }
+
+    public Wildcard(Source source, Expression field, List<Expression> patterns) {
+        super(source, getArguments(field, patterns));
+        this.field = field;
+        this.patterns = patterns;
+    }
+
+    public Expression asLikes() {
+        Expression result = null;
+
+        for (Expression pattern: patterns) {
+            String wcString = pattern.fold().toString();
+            Like like = new Like(source(), field, StringUtils.toLikePattern(wcString));
+            result = result == null ? like : new Or(source(), result, like);
+        }
+
+        return result;
+    }
+
+    @Override
+    protected TypeResolution resolveType() {
+        if (!childrenResolved()) {
+            return new TypeResolution("Unresolved children");
+        }
+
+        TypeResolution lastResolution = isStringAndExact(field, sourceText(), ParamOrdinal.FIRST);
+        if (lastResolution.unresolved()) {
+            return lastResolution;
+        }
+
+        for (Expression p: patterns) {
+            lastResolution = isStringAndExact(p, sourceText(), ParamOrdinal.DEFAULT);
+            if (lastResolution.unresolved()) {
+                break;
+            }
+
+            if (p.foldable() == false) {
+                return new TypeResolution(format(null, "wildcard against variables are not (currently) supported; offender [{}] in [{}]",
+                    Expressions.name(p),
+                    sourceText()));
+            }
+        }
+
+        return lastResolution;
+    }
+
+    @Override
+    protected Pipe makePipe() {
+        Expression asLikes = asLikes();
+        if (asLikes instanceof Like) {
+            return ((Like) asLikes).makePipe();
+        } else {
+            return ((Or) asLikes).makePipe();
+        }
+    }
+
+    @Override
+    public ScriptTemplate asScript() {
+        Expression asLikes = asLikes();
+        if (asLikes instanceof Like) {
+            return ((Like) asLikes).asScript();
+        } else {
+            return ((Or) asLikes).asScript();
+        }
+    }
+
+    @Override
+    public boolean foldable() {
+        boolean foldable = field.foldable();
+        for (Expression p : patterns) {
+            foldable = foldable && p.foldable();
+        }
+        return foldable;
+    }
+
+    @Override
+    public Object fold() {
+        return asLikes().fold();
+    }
+
+    @Override
+    protected NodeInfo<? extends Expression> info() {
+        return NodeInfo.create(this, Wildcard::new, field, patterns);
+    }
+
+    @Override
+    public DataType dataType() {
+        return DataTypes.BOOLEAN;
+    }
+
+    @Override
+    public Expression replaceChildren(List<Expression> newChildren) {
+        if (newChildren.size() < 2) {
+            throw new IllegalArgumentException("expected at least [2] children but received [" + newChildren.size() + "]");
+        }
+
+        return new Wildcard(source(), newChildren.get(0), newChildren.subList(1, newChildren.size()));
+    }
+}

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java

@@ -6,6 +6,8 @@
 
 package org.elasticsearch.xpack.eql.optimizer;
 
+import org.elasticsearch.xpack.eql.expression.function.scalar.string.Wildcard;
+import org.elasticsearch.xpack.eql.utils.StringUtils;
 import org.elasticsearch.xpack.ql.expression.Expression;
 import org.elasticsearch.xpack.ql.expression.predicate.logical.Not;
 import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNotNull;
@@ -14,7 +16,6 @@
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals;
 import org.elasticsearch.xpack.ql.expression.predicate.regex.Like;
-import org.elasticsearch.xpack.ql.expression.predicate.regex.LikePattern;
 import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanLiteralsOnTheRight;
 import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanSimplification;
 import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.CombineBinaryComparisons;
@@ -48,6 +49,7 @@ protected Iterable<RuleExecutor<LogicalPlan>.Batch> batches() {
                 new ReplaceNullChecks(),
                 new PropagateEquals(),
                 new CombineBinaryComparisons(),
+                new ReplaceWildcardFunction(),
                 // prune/elimination
                 new PruneFilters(),
                 new PruneLiteralsInOrderBy()
@@ -60,6 +62,20 @@ protected Iterable<RuleExecutor<LogicalPlan>.Batch> batches() {
     }
 
 
+    private static class ReplaceWildcardFunction extends OptimizerRule<Filter> {
+
+        @Override
+        protected LogicalPlan rule(Filter filter) {
+            return filter.transformExpressionsUp(e -> {
+                if (e instanceof Wildcard) {
+                    e = ((Wildcard) e).asLikes();
+                }
+
+                return e;
+            });
+        }
+    }
+
     private static class ReplaceWildcards extends OptimizerRule<Filter> {
 
         private static boolean isWildcard(Expression expr) {
@@ -70,18 +86,6 @@ private static boolean isWildcard(Expression expr) {
             return false;
         }
 
-        private static LikePattern toLikePattern(String s) {
-            // pick a character that is guaranteed not to be in the string, because it isn't allowed to escape itself
-            char escape = 1;
-
-            // replace wildcards with % and escape special characters
-            String likeString = s.replace("%", escape + "%")
-                .replace("_", escape + "_")
-                .replace("*", "%");
-
-            return new LikePattern(likeString, escape);
-        }
-
         @Override
         protected LogicalPlan rule(Filter filter) {
             return filter.transformExpressionsUp(e -> {
@@ -91,7 +95,7 @@ protected LogicalPlan rule(Filter filter) {
 
                     if (isWildcard(cmp.right())) {
                         String wcString = cmp.right().fold().toString();
-                        Expression like = new Like(e.source(), cmp.left(), toLikePattern(wcString));
+                        Expression like = new Like(e.source(), cmp.left(), StringUtils.toLikePattern(wcString));
 
                         if (e instanceof NotEquals) {
                             like = new Not(e.source(), like);

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/utils/StringUtils.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.utils;
+
+import org.elasticsearch.xpack.ql.expression.predicate.regex.LikePattern;
+
+public class StringUtils {
+
+    public static LikePattern toLikePattern(String s) {
+        // pick a character that is guaranteed not to be in the string, because it isn't allowed to escape itself
+        char escape = 1;
+
+        // replace wildcards with % and escape special characters
+        String likeString = s.replace("%", escape + "%")
+            .replace("_", escape + "_")
+            .replace("*", "%");
+
+        return new LikePattern(likeString, escape);
+    }
+}

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/ExpressionTests.java

@@ -41,7 +41,7 @@ private Expression expr(String source) {
     }
 
 
-    public void testStrings() throws Exception {
+    public void testStrings() {
         assertEquals("hello\"world", unquoteString("'hello\"world'"));
         assertEquals("hello'world", unquoteString("\"hello'world\""));
         assertEquals("hello\nworld", unquoteString("'hello\\nworld'"));

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java

@@ -6,6 +6,8 @@
 
 package org.elasticsearch.xpack.eql.planner;
 
+import org.elasticsearch.xpack.eql.analysis.VerificationException;
+import org.elasticsearch.xpack.ql.ParsingException;
 import org.elasticsearch.xpack.ql.QlIllegalArgumentException;
 
 public class QueryFolderFailTests extends AbstractQueryFolderTestCase {
@@ -22,4 +24,19 @@ public void testPropertyEquationInClauseFilterUnsupported() {
         String msg = e.getMessage();
         assertEquals("Line 1:52: Comparisons against variables are not (currently) supported; offender [parent_process_name] in [==]", msg);
     }
+
+    public void testWildcardNotEnoughArguments() {
+        ParsingException e = expectThrows(ParsingException.class,
+            () -> plan("process where wildcard(process_name)"));
+        String msg = e.getMessage();
+        assertEquals("line 1:16: error building [wildcard]: expects at least two arguments", msg);
+    }
+
+    public void testWildcardAgainstVariable() {
+        VerificationException e = expectThrows(VerificationException.class,
+            () -> plan("process where wildcard(process_name, parent_process_name)"));
+        String msg = e.getMessage();
+        assertEquals("Found 1 problem\nline 1:15: wildcard against variables are not (currently) supported;" +
+            " offender [parent_process_name] in [wildcard(process_name, parent_process_name)]", msg);
+    }
 }

x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt

@@ -64,3 +64,22 @@ process where process_path == "*\\red_ttp\\wininit.*" and opcode in (0,1,2,3)
 "term":{"opcode":{"value":1
 "term":{"opcode":{"value":2
 "term":{"opcode":{"value":3
+
+
+wildcardFunctionSingleArgument
+process where wildcard(process_path, "*\\red_ttp\\wininit.*")
+"wildcard":{"process_path":{"wildcard":"*\\\\red_ttp\\\\wininit.*"
+
+
+wildcardFunctionTwoArguments
+process where wildcard(process_path, "*\\red_ttp\\wininit.*", "*\\abc\\*")
+"wildcard":{"process_path":{"wildcard":"*\\\\red_ttp\\\\wininit.*"
+"wildcard":{"process_path":{"wildcard":"*\\\\abc\\\\*"
+
+
+wildcardFunctionThreeArguments
+process where wildcard(process_path, "*\\red_ttp\\wininit.*", "*\\abc\\*", "*def*")
+"wildcard":{"process_path":{"wildcard":"*\\\\red_ttp\\\\wininit.*"
+"wildcard":{"process_path":{"wildcard":"*\\\\abc\\\\*"
+"wildcard":{"process_path":{"wildcard":"*def*"
+

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/FunctionRegistry.java

@@ -421,4 +421,26 @@ public static <T extends Function> FunctionDefinition def(Class<T> function,
     protected interface CastFunctionBuilder<T> {
         T build(Source source, Expression expression, DataType dataType);
     }
+
+    @SuppressWarnings("overloads")  // These are ambiguous if you aren't using ctor references but we always do
+    public static <T extends Function> FunctionDefinition def(Class<T> function,
+                                                              TwoParametersVariadicBuilder<T> ctorRef, String... names) {
+        FunctionBuilder builder = (source, children, distinct, cfg) -> {
+            boolean hasMinimumOne = OptionalArgument.class.isAssignableFrom(function);
+            if (hasMinimumOne && children.size() < 1) {
+                throw new QlIllegalArgumentException("expects at least one argument");
+            } else if (!hasMinimumOne && children.size() < 2) {
+                throw new QlIllegalArgumentException("expects at least two arguments");
+            }
+            if (distinct) {
+                throw new QlIllegalArgumentException("does not support DISTINCT yet it was specified");
+            }
+            return ctorRef.build(source, children.get(0), children.subList(1, children.size()));
+        };
+        return def(function, builder, false, names);
+    }
+
+    protected interface TwoParametersVariadicBuilder<T> {
+        T build(Source source, Expression src, List<Expression> remaining);
+    }
 }

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/logical/BinaryLogic.java

@@ -34,7 +34,7 @@ protected TypeResolution resolveInputType(Expression e, Expressions.ParamOrdinal
     }
 
     @Override
-    protected Pipe makePipe() {
+    public Pipe makePipe() {
         return new BinaryLogicPipe(source(), this, Expressions.pipe(left()), Expressions.pipe(right()), function());
     }