Add patterns and grok command

core/build.gradle

@@ -48,6 +48,7 @@ dependencies {
     api "com.fasterxml.jackson.core:jackson-core:${jackson_version}"
     api "com.fasterxml.jackson.core:jackson-databind:${jackson_version}"
     api "com.fasterxml.jackson.core:jackson-annotations:${jackson_version}"
+    implementation group: 'io.krakens', name: 'java-grok', version: '0.1.9'
     api project(':common')
 
     testImplementation('org.junit.jupiter:junit-jupiter:5.6.2')

core/src/main/java/org/opensearch/sql/analysis/Analyzer.java

@@ -36,16 +36,19 @@
 import org.opensearch.sql.ast.expression.Let;
 import org.opensearch.sql.ast.expression.Literal;
 import org.opensearch.sql.ast.expression.Map;
+import org.opensearch.sql.ast.expression.PatternsMethod;
 import org.opensearch.sql.ast.expression.UnresolvedExpression;
 import org.opensearch.sql.ast.tree.AD;
 import org.opensearch.sql.ast.tree.Aggregation;
 import org.opensearch.sql.ast.tree.Dedupe;
 import org.opensearch.sql.ast.tree.Eval;
 import org.opensearch.sql.ast.tree.Filter;
+import org.opensearch.sql.ast.tree.Grok;
 import org.opensearch.sql.ast.tree.Head;
 import org.opensearch.sql.ast.tree.Kmeans;
 import org.opensearch.sql.ast.tree.Limit;
 import org.opensearch.sql.ast.tree.Parse;
+import org.opensearch.sql.ast.tree.Patterns;
 import org.opensearch.sql.ast.tree.Project;
 import org.opensearch.sql.ast.tree.RareTopN;
 import org.opensearch.sql.ast.tree.Relation;
@@ -57,16 +60,19 @@
 import org.opensearch.sql.ast.tree.Values;
 import org.opensearch.sql.catalog.CatalogService;
 import org.opensearch.sql.data.model.ExprMissingValue;
+import org.opensearch.sql.data.model.ExprValueUtils;
 import org.opensearch.sql.data.type.ExprCoreType;
 import org.opensearch.sql.exception.SemanticCheckException;
 import org.opensearch.sql.expression.DSL;
 import org.opensearch.sql.expression.Expression;
 import org.opensearch.sql.expression.LiteralExpression;
 import org.opensearch.sql.expression.NamedExpression;
-import org.opensearch.sql.expression.ParseExpression;
 import org.opensearch.sql.expression.ReferenceExpression;
 import org.opensearch.sql.expression.aggregation.Aggregator;
 import org.opensearch.sql.expression.aggregation.NamedAggregator;
+import org.opensearch.sql.expression.parse.GrokExpression;
+import org.opensearch.sql.expression.parse.PatternsExpression;
+import org.opensearch.sql.expression.parse.RegexExpression;
 import org.opensearch.sql.planner.logical.LogicalAD;
 import org.opensearch.sql.planner.logical.LogicalAggregation;
 import org.opensearch.sql.planner.logical.LogicalDedupe;
@@ -83,7 +89,6 @@
 import org.opensearch.sql.planner.logical.LogicalSort;
 import org.opensearch.sql.planner.logical.LogicalValues;
 import org.opensearch.sql.storage.Table;
-import org.opensearch.sql.utils.ParseUtils;
 
 /**
  * Analyze the {@link UnresolvedPlan} in the {@link AnalysisContext} to construct the {@link
@@ -347,20 +352,65 @@ public LogicalPlan visitEval(Eval node, AnalysisContext context) {
   }
 
   /**
-   * Build {@link ParseExpression} to context and skip to child nodes.
+   * Build {@link GrokExpression} to context and skip to child nodes.
+   */
+  @Override
+  public LogicalPlan visitGrok(Grok node, AnalysisContext context) {
+    LogicalPlan child = node.getChild().get(0).accept(this, context);
+    Expression sourceField = expressionAnalyzer.analyze(node.getSourceField(), context);
+    String pattern = (String) node.getPattern().getValue();
+    Expression patternExpression = DSL.literal(pattern);
+
+    TypeEnvironment curEnv = context.peek();
+    GrokExpression.getNamedGroupCandidates(pattern).forEach(group -> {
+      GrokExpression expr = new GrokExpression(sourceField, patternExpression, DSL.literal(group));
+      curEnv.define(new Symbol(Namespace.FIELD_NAME, group), expr.type());
+      context.getNamedParseExpressions().add(new NamedExpression(group, expr));
+    });
+    return child;
+  }
+
+  /**
+   * Build {@link RegexExpression} to context and skip to child nodes.
    */
   @Override
   public LogicalPlan visitParse(Parse node, AnalysisContext context) {
     LogicalPlan child = node.getChild().get(0).accept(this, context);
-    Expression expression = expressionAnalyzer.analyze(node.getExpression(), context);
+    Expression sourceField = expressionAnalyzer.analyze(node.getSourceField(), context);
     String pattern = (String) node.getPattern().getValue();
     Expression patternExpression = DSL.literal(pattern);
 
     TypeEnvironment curEnv = context.peek();
-    ParseUtils.getNamedGroupCandidates(pattern).forEach(group -> {
-      curEnv.define(new Symbol(Namespace.FIELD_NAME, group), ExprCoreType.STRING);
-      context.getNamedParseExpressions().add(new NamedExpression(group,
-          new ParseExpression(expression, patternExpression, DSL.literal(group))));
+    RegexExpression.getNamedGroupCandidates(pattern).forEach(group -> {
+      RegexExpression expr =
+          new RegexExpression(sourceField, patternExpression, DSL.literal(group));
+      curEnv.define(new Symbol(Namespace.FIELD_NAME, group), expr.type());
+      context.getNamedParseExpressions().add(new NamedExpression(group, expr));
+    });
+    return child;
+  }
+
+  /**
+   * Build {@link PatternsExpression} to context and skip to child nodes.
+   */
+  @Override
+  public LogicalPlan visitPatterns(Patterns node, AnalysisContext context) {
+    LogicalPlan child = node.getChild().get(0).accept(this, context);
+    Expression sourceField = expressionAnalyzer.analyze(node.getSourceField(), context);
+    PatternsMethod patternsMethod = node.getPatternsMethod();
+    java.util.Map<String, Literal> arguments = node.getArguments();
+    Expression pattern = arguments.containsKey(PatternsExpression.PATTERN_KEY)
+        ? DSL.literal((String) arguments.get(PatternsExpression.PATTERN_KEY).getValue())
+        : DSL.literal(ExprValueUtils.stringValue(""));
+    String identifier = arguments.containsKey(PatternsExpression.NEW_FIELD_KEY)
+        ? (String) arguments.get(PatternsExpression.NEW_FIELD_KEY).getValue() : null;
+
+    TypeEnvironment curEnv = context.peek();
+    PatternsExpression.getNamedGroupCandidates(identifier).forEach(group -> {
+      PatternsExpression expr =
+          new PatternsExpression(patternsMethod, sourceField, pattern, DSL.literal(group));
+      curEnv.define(new Symbol(Namespace.FIELD_NAME, group), expr.type());
+      context.getNamedParseExpressions().add(new NamedExpression(group, expr));
     });
     return child;
   }

core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java

@@ -52,7 +52,6 @@
 import org.opensearch.sql.expression.LiteralExpression;
 import org.opensearch.sql.expression.NamedArgumentExpression;
 import org.opensearch.sql.expression.NamedExpression;
-import org.opensearch.sql.expression.ParseExpression;
 import org.opensearch.sql.expression.ReferenceExpression;
 import org.opensearch.sql.expression.aggregation.AggregationState;
 import org.opensearch.sql.expression.aggregation.Aggregator;
@@ -61,6 +60,7 @@
 import org.opensearch.sql.expression.function.BuiltinFunctionName;
 import org.opensearch.sql.expression.function.BuiltinFunctionRepository;
 import org.opensearch.sql.expression.function.FunctionName;
+import org.opensearch.sql.expression.parse.ParseExpression;
 import org.opensearch.sql.expression.span.SpanExpression;
 import org.opensearch.sql.expression.window.aggregation.AggregateWindowFunction;
 

core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java

@@ -39,10 +39,12 @@
 import org.opensearch.sql.ast.tree.Dedupe;
 import org.opensearch.sql.ast.tree.Eval;
 import org.opensearch.sql.ast.tree.Filter;
+import org.opensearch.sql.ast.tree.Grok;
 import org.opensearch.sql.ast.tree.Head;
 import org.opensearch.sql.ast.tree.Kmeans;
 import org.opensearch.sql.ast.tree.Limit;
 import org.opensearch.sql.ast.tree.Parse;
+import org.opensearch.sql.ast.tree.Patterns;
 import org.opensearch.sql.ast.tree.Project;
 import org.opensearch.sql.ast.tree.RareTopN;
 import org.opensearch.sql.ast.tree.Relation;
@@ -184,10 +186,18 @@ public T visitEval(Eval node, C context) {
     return visitChildren(node, context);
   }
 
+  public T visitGrok(Grok node, C context) {
+    return visitChildren(node, context);
+  }
+
   public T visitParse(Parse node, C context) {
     return visitChildren(node, context);
   }
 
+  public T visitPatterns(Patterns node, C context) {
+    return visitChildren(node, context);
+  }
+
   public T visitLet(Let node, C context) {
     return visitChildren(node, context);
   }

core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java

@@ -31,6 +31,7 @@
 import org.opensearch.sql.ast.expression.Map;
 import org.opensearch.sql.ast.expression.Not;
 import org.opensearch.sql.ast.expression.Or;
+import org.opensearch.sql.ast.expression.PatternsMethod;
 import org.opensearch.sql.ast.expression.QualifiedName;
 import org.opensearch.sql.ast.expression.Span;
 import org.opensearch.sql.ast.expression.SpanUnit;
@@ -44,9 +45,11 @@
 import org.opensearch.sql.ast.tree.Dedupe;
 import org.opensearch.sql.ast.tree.Eval;
 import org.opensearch.sql.ast.tree.Filter;
+import org.opensearch.sql.ast.tree.Grok;
 import org.opensearch.sql.ast.tree.Head;
 import org.opensearch.sql.ast.tree.Limit;
 import org.opensearch.sql.ast.tree.Parse;
+import org.opensearch.sql.ast.tree.Patterns;
 import org.opensearch.sql.ast.tree.Project;
 import org.opensearch.sql.ast.tree.RareTopN;
 import org.opensearch.sql.ast.tree.RareTopN.CommandType;
@@ -428,8 +431,19 @@ public static Limit limit(UnresolvedPlan input, Integer limit, Integer offset) {
     return new Limit(limit, offset).attach(input);
   }
 
-  public static Parse parse(UnresolvedPlan input, UnresolvedExpression expression,
+  public static Grok grok(UnresolvedPlan input, UnresolvedExpression sourceField,
                             Literal pattern) {
-    return new Parse(expression, pattern, input);
+    return new Grok(sourceField, pattern, input);
+  }
+
+  public static Parse parse(UnresolvedPlan input, UnresolvedExpression sourceField,
+                            Literal pattern) {
+    return new Parse(sourceField, pattern, input);
+  }
+
+  public static Patterns patterns(UnresolvedPlan input, PatternsMethod patternsMethod,
+                                  UnresolvedExpression sourceField,
+                                  java.util.Map<String, Literal> arguments) {
+    return new Patterns(patternsMethod, sourceField, arguments, input);
   }
 }

core/src/main/java/org/opensearch/sql/ast/expression/PatternsMethod.java

@@ -0,0 +1,19 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+
+package org.opensearch.sql.ast.expression;
+
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+
+@RequiredArgsConstructor
+public enum PatternsMethod {
+  REGEX("regex"),
+  PUNCT("punct");
+
+  @Getter
+  private final String name;
+}

core/src/main/java/org/opensearch/sql/ast/tree/Grok.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+
+package org.opensearch.sql.ast.tree;
+
+import com.google.common.collect.ImmutableList;
+import java.util.List;
+import lombok.AllArgsConstructor;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+import lombok.Setter;
+import lombok.ToString;
+import org.opensearch.sql.ast.AbstractNodeVisitor;
+import org.opensearch.sql.ast.expression.Literal;
+import org.opensearch.sql.ast.expression.UnresolvedExpression;
+
+/**
+ * AST node represent Grok operation.
+ */
+@Getter
+@Setter
+@ToString
+@EqualsAndHashCode(callSuper = false)
+@RequiredArgsConstructor
+@AllArgsConstructor
+public class Grok extends UnresolvedPlan {
+  /**
+   * Field.
+   */
+  private final UnresolvedExpression sourceField;
+
+  /**
+   * Pattern.
+   */
+  private final Literal pattern;
+
+  /**
+   * Child Plan.
+   */
+  private UnresolvedPlan child;
+
+  @Override
+  public Grok attach(UnresolvedPlan child) {
+    this.child = child;
+    return this;
+  }
+
+  @Override
+  public List<UnresolvedPlan> getChild() {
+    return ImmutableList.of(this.child);
+  }
+
+  @Override
+  public <T, C> T accept(AbstractNodeVisitor<T, C> nodeVisitor, C context) {
+    return nodeVisitor.visitGrok(this, context);
+  }
+}

core/src/main/java/org/opensearch/sql/ast/tree/Parse.java

@@ -19,7 +19,7 @@
 import org.opensearch.sql.ast.expression.UnresolvedExpression;
 
 /**
- * AST node represent Parse operation.
+ * AST node represent Parse with regex operation.
  */
 @Getter
 @Setter
@@ -31,7 +31,7 @@ public class Parse extends UnresolvedPlan {
   /**
    * Field.
    */
-  private final UnresolvedExpression expression;
+  private final UnresolvedExpression sourceField;
 
   /**
    * Pattern.

core/src/main/java/org/opensearch/sql/ast/tree/Patterns.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+
+package org.opensearch.sql.ast.tree;
+
+import com.google.common.collect.ImmutableList;
+import java.util.List;
+import java.util.Map;
+import lombok.AllArgsConstructor;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+import lombok.Setter;
+import lombok.ToString;
+import org.opensearch.sql.ast.AbstractNodeVisitor;
+import org.opensearch.sql.ast.expression.Literal;
+import org.opensearch.sql.ast.expression.PatternsMethod;
+import org.opensearch.sql.ast.expression.UnresolvedExpression;
+
+/**
+ * AST node represent extracting log patterns operation.
+ */
+@Getter
+@Setter
+@ToString
+@EqualsAndHashCode(callSuper = false)
+@RequiredArgsConstructor
+@AllArgsConstructor
+public class Patterns extends UnresolvedPlan {
+  private final PatternsMethod patternsMethod;
+  /**
+   * Field.
+   */
+  private final UnresolvedExpression sourceField;
+
+  private final Map<String, Literal> arguments;
+
+  /**
+   * Child Plan.
+   */
+  private UnresolvedPlan child;
+
+  @Override
+  public Patterns attach(UnresolvedPlan child) {
+    this.child = child;
+    return this;
+  }
+
+  @Override
+  public List<UnresolvedPlan> getChild() {
+    return ImmutableList.of(this.child);
+  }
+
+  @Override
+  public <T, C> T accept(AbstractNodeVisitor<T, C> nodeVisitor, C context) {
+    return nodeVisitor.visitPatterns(this, context);
+  }
+}