Resolve merge conflicts and fix tests.

Signed-off-by: Yury-Fridlyand <[email protected]>

Author: Yury-Fridlyand

common/build.gradle

@@ -36,7 +36,6 @@ dependencies {
     api group: 'com.google.guava', name: 'guava', version: '31.0.1-jre'
     api group: 'org.apache.logging.log4j', name: 'log4j-core', version:'2.17.1'
     api group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0'
-    api 'com.amazonaws:aws-encryption-sdk-java:2.4.0'
 
     testImplementation group: 'junit', name: 'junit', version: '4.13.2'
     testImplementation group: 'org.assertj', name: 'assertj-core', version: '3.9.1'

config/checkstyle/google_checks.xml

@@ -39,8 +39,9 @@
     <property name="max" value="100"/>
     <property name="ignorePattern" value="^package.*|^import.*|a href|href|http://|https://|ftp://"/>
   </module>
-
+  <module name="SuppressWarningsFilter" />
   <module name="TreeWalker">
+    <module name="SuppressWarningsHolder" />
     <module name="OuterTypeFilename"/>
     <module name="IllegalTokenText">
       <property name="tokens" value="STRING_LITERAL, CHAR_LITERAL"/>

core/src/main/java/org/opensearch/sql/analysis/Analyzer.java

@@ -63,6 +63,7 @@
 import org.opensearch.sql.ast.tree.Values;
 import org.opensearch.sql.data.model.ExprMissingValue;
 import org.opensearch.sql.data.type.ExprCoreType;
+import org.opensearch.sql.data.type.ExprType;
 import org.opensearch.sql.datasource.DataSourceService;
 import org.opensearch.sql.exception.SemanticCheckException;
 import org.opensearch.sql.expression.DSL;
@@ -152,6 +153,9 @@ public LogicalPlan visitRelation(Relation node, AnalysisContext context) {
               dataSourceSchemaIdentifierNameResolver.getIdentifierName());
     }
     table.getFieldTypes().forEach((k, v) -> curEnv.define(new Symbol(Namespace.FIELD_NAME, k), v));
+    table.getReservedFieldTypes().forEach(
+        (k, v) -> curEnv.addReservedWord(new Symbol(Namespace.FIELD_NAME, k), v)
+    );
 
     // Put index name or its alias in index namespace on type environment so qualifier
     // can be removed when analyzing qualified name. The value (expr type) here doesn't matter.
@@ -195,6 +199,9 @@ public LogicalPlan visitTableFunction(TableFunction node, AnalysisContext contex
     TypeEnvironment curEnv = context.peek();
     Table table = tableFunctionImplementation.applyArguments();
     table.getFieldTypes().forEach((k, v) -> curEnv.define(new Symbol(Namespace.FIELD_NAME, k), v));
+    table.getReservedFieldTypes().forEach(
+        (k, v) -> curEnv.addReservedWord(new Symbol(Namespace.FIELD_NAME, k), v)
+    );
     curEnv.define(new Symbol(Namespace.INDEX_NAME,
             dataSourceSchemaIdentifierNameResolver.getIdentifierName()), STRUCT);
     return new LogicalRelation(dataSourceSchemaIdentifierNameResolver.getIdentifierName(),
@@ -361,6 +368,14 @@ public LogicalPlan visitProject(Project node, AnalysisContext context) {
     List<NamedExpression> namedExpressions =
         selectExpressionAnalyzer.analyze(node.getProjectList(), context,
             new ExpressionReferenceOptimizer(expressionAnalyzer.getRepository(), child));
+
+    for (UnresolvedExpression expr : node.getProjectList()) {
+      NestedAnalyzer nestedAnalyzer = new NestedAnalyzer(
+          namedExpressions, expressionAnalyzer, child
+      );
+      child = nestedAnalyzer.analyze(expr, context);
+    }
+
     // new context
     context.push();
     TypeEnvironment newEnv = context.peek();

core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java

@@ -8,8 +8,6 @@
 
 import static org.opensearch.sql.ast.dsl.AstDSL.and;
 import static org.opensearch.sql.ast.dsl.AstDSL.compare;
-import static org.opensearch.sql.expression.function.BuiltinFunctionName.GTE;
-import static org.opensearch.sql.expression.function.BuiltinFunctionName.LTE;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
@@ -31,6 +29,7 @@
 import org.opensearch.sql.ast.expression.Case;
 import org.opensearch.sql.ast.expression.Cast;
 import org.opensearch.sql.ast.expression.Compare;
+import org.opensearch.sql.ast.expression.DataType;
 import org.opensearch.sql.ast.expression.EqualTo;
 import org.opensearch.sql.ast.expression.Field;
 import org.opensearch.sql.ast.expression.Function;
@@ -42,6 +41,7 @@
 import org.opensearch.sql.ast.expression.Or;
 import org.opensearch.sql.ast.expression.QualifiedName;
 import org.opensearch.sql.ast.expression.RelevanceFieldList;
+import org.opensearch.sql.ast.expression.ScoreFunction;
 import org.opensearch.sql.ast.expression.Span;
 import org.opensearch.sql.ast.expression.UnresolvedArgument;
 import org.opensearch.sql.ast.expression.UnresolvedAttribute;
@@ -51,6 +51,7 @@
 import org.opensearch.sql.ast.expression.Xor;
 import org.opensearch.sql.common.antlr.SyntaxCheckException;
 import org.opensearch.sql.data.model.ExprValueUtils;
+import org.opensearch.sql.data.type.ExprCoreType;
 import org.opensearch.sql.data.type.ExprType;
 import org.opensearch.sql.exception.SemanticCheckException;
 import org.opensearch.sql.expression.DSL;
@@ -67,6 +68,7 @@
 import org.opensearch.sql.expression.function.BuiltinFunctionName;
 import org.opensearch.sql.expression.function.BuiltinFunctionRepository;
 import org.opensearch.sql.expression.function.FunctionName;
+import org.opensearch.sql.expression.function.OpenSearchFunctions;
 import org.opensearch.sql.expression.parse.ParseExpression;
 import org.opensearch.sql.expression.span.SpanExpression;
 import org.opensearch.sql.expression.window.aggregation.AggregateWindowFunction;
@@ -207,6 +209,65 @@ public Expression visitHighlightFunction(HighlightFunction node, AnalysisContext
     return new HighlightExpression(expr);
   }
 
+  /**
+   * visitScoreFunction removes the score function from the AST and replaces it with the child
+   * relevance function node. If the optional boost variable is provided, the boost argument
+   * of the relevance function is combined.
+   *
+   * @param node    score function node
+   * @param context analysis context for the query
+   * @return resolved relevance function
+   */
+  public Expression visitScoreFunction(ScoreFunction node, AnalysisContext context) {
+    Literal boostArg = node.getRelevanceFieldWeight();
+    if (!boostArg.getType().equals(DataType.DOUBLE)) {
+      throw new SemanticCheckException(String.format("Expected boost type '%s' but got '%s'",
+          DataType.DOUBLE.name(), boostArg.getType().name()));
+    }
+    Double thisBoostValue = ((Double) boostArg.getValue());
+
+    // update the existing unresolved expression to add a boost argument if it doesn't exist
+    // OR multiply the existing boost argument
+    Function relevanceQueryUnresolvedExpr = (Function) node.getRelevanceQuery();
+    List<UnresolvedExpression> relevanceFuncArgs = relevanceQueryUnresolvedExpr.getFuncArgs();
+
+    boolean doesFunctionContainBoostArgument = false;
+    List<UnresolvedExpression> updatedFuncArgs = new ArrayList<>();
+    for (UnresolvedExpression expr : relevanceFuncArgs) {
+      String argumentName = ((UnresolvedArgument) expr).getArgName();
+      if (argumentName.equalsIgnoreCase("boost")) {
+        doesFunctionContainBoostArgument = true;
+        Literal boostArgLiteral = (Literal) ((UnresolvedArgument) expr).getValue();
+        Double boostValue =
+            Double.parseDouble((String) boostArgLiteral.getValue()) * thisBoostValue;
+        UnresolvedArgument newBoostArg = new UnresolvedArgument(
+                argumentName,
+                new Literal(boostValue.toString(), DataType.STRING)
+        );
+        updatedFuncArgs.add(newBoostArg);
+      } else {
+        updatedFuncArgs.add(expr);
+      }
+    }
+
+    // since nothing was found, add an argument
+    if (!doesFunctionContainBoostArgument) {
+      UnresolvedArgument newBoostArg = new UnresolvedArgument(
+              "boost", new Literal(Double.toString(thisBoostValue), DataType.STRING));
+      updatedFuncArgs.add(newBoostArg);
+    }
+
+    // create a new function expression with boost argument and resolve it
+    Function updatedRelevanceQueryUnresolvedExpr = new Function(
+            relevanceQueryUnresolvedExpr.getFuncName(),
+            updatedFuncArgs);
+    OpenSearchFunctions.OpenSearchFunction relevanceQueryExpr =
+            (OpenSearchFunctions.OpenSearchFunction) updatedRelevanceQueryUnresolvedExpr
+                    .accept(this, context);
+    relevanceQueryExpr.setScoreTracked(true);
+    return relevanceQueryExpr;
+  }
+
   @Override
   public Expression visitIn(In node, AnalysisContext context) {
     return visitIn(node.getField(), node.getValueList(), context);
@@ -297,6 +358,23 @@ public Expression visitAllFields(AllFields node, AnalysisContext context) {
   @Override
   public Expression visitQualifiedName(QualifiedName node, AnalysisContext context) {
     QualifierAnalyzer qualifierAnalyzer = new QualifierAnalyzer(context);
+
+    // check for reserved words in the identifier
+    for (String part : node.getParts()) {
+      for (TypeEnvironment typeEnv = context.peek();
+           typeEnv != null;
+           typeEnv = typeEnv.getParent()) {
+        Optional<ExprType> exprType = typeEnv.getReservedSymbolTable().lookup(
+            new Symbol(Namespace.FIELD_NAME, part));
+        if (exprType.isPresent()) {
+          return visitMetadata(
+              qualifierAnalyzer.unqualified(node),
+              (ExprCoreType) exprType.get(),
+              context
+          );
+        }
+      }
+    }
     return visitIdentifier(qualifierAnalyzer.unqualified(node), context);
   }
 
@@ -313,6 +391,19 @@ public Expression visitUnresolvedArgument(UnresolvedArgument node, AnalysisConte
     return new NamedArgumentExpression(node.getArgName(), node.getValue().accept(this, context));
   }
 
+  /**
+   * If QualifiedName is actually a reserved metadata field, return the expr type associated
+   * with the metadata field.
+   * @param ident   metadata field name
+   * @param context analysis context
+   * @return DSL reference
+   */
+  private Expression visitMetadata(String ident,
+                                   ExprCoreType exprCoreType,
+                                   AnalysisContext context) {
+    return DSL.ref(ident, exprCoreType);
+  }
+
   private Expression visitIdentifier(String ident, AnalysisContext context) {
     // ParseExpression will always override ReferenceExpression when ident conflicts
     for (NamedExpression expr : context.getNamedParseExpressions()) {
@@ -325,17 +416,6 @@ private Expression visitIdentifier(String ident, AnalysisContext context) {
     ReferenceExpression ref = DSL.ref(ident,
         typeEnv.resolve(new Symbol(Namespace.FIELD_NAME, ident)));
 
-    // Fall back to old engine too if type is not supported semantically
-    if (isTypeNotSupported(ref.type())) {
-      throw new SyntaxCheckException(String.format(
-          "Identifier [%s] of type [%s] is not supported yet", ident, ref.type()));
-    }
     return ref;
   }
-
-  // Array type is not supporte yet.
-  private boolean isTypeNotSupported(ExprType type) {
-    return "array".equalsIgnoreCase(type.typeName());
-  }
-
 }

core/src/main/java/org/opensearch/sql/analysis/ExpressionReferenceOptimizer.java

@@ -19,6 +19,7 @@
 import org.opensearch.sql.expression.conditional.cases.CaseClause;
 import org.opensearch.sql.expression.conditional.cases.WhenClause;
 import org.opensearch.sql.expression.function.BuiltinFunctionRepository;
+import org.opensearch.sql.expression.function.OpenSearchFunctions;
 import org.opensearch.sql.planner.logical.LogicalAggregation;
 import org.opensearch.sql.planner.logical.LogicalPlan;
 import org.opensearch.sql.planner.logical.LogicalPlanNodeVisitor;
@@ -70,8 +71,17 @@ public Expression visitFunction(FunctionExpression node, AnalysisContext context
       final List<Expression> args =
           node.getArguments().stream().map(expr -> expr.accept(this, context))
               .collect(Collectors.toList());
-      return (Expression) repository.compile(context.getFunctionProperties(),
-          node.getFunctionName(), args);
+      Expression optimizedFunctionExpression = (Expression) repository.compile(
+              context.getFunctionProperties(),
+              node.getFunctionName(),
+              args
+      );
+      // Propagate scoreTracked for OpenSearch functions
+      if (optimizedFunctionExpression instanceof OpenSearchFunctions.OpenSearchFunction) {
+        ((OpenSearchFunctions.OpenSearchFunction) optimizedFunctionExpression).setScoreTracked(
+                ((OpenSearchFunctions.OpenSearchFunction)node).isScoreTracked());
+      }
+      return optimizedFunctionExpression;
     }
   }
 

core/src/main/java/org/opensearch/sql/analysis/NestedAnalyzer.java

@@ -0,0 +1,111 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.analysis;
+
+import static org.opensearch.sql.data.type.ExprCoreType.STRING;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import lombok.RequiredArgsConstructor;
+import org.opensearch.sql.ast.AbstractNodeVisitor;
+import org.opensearch.sql.ast.expression.Alias;
+import org.opensearch.sql.ast.expression.Function;
+import org.opensearch.sql.ast.expression.QualifiedName;
+import org.opensearch.sql.ast.expression.UnresolvedExpression;
+import org.opensearch.sql.expression.NamedExpression;
+import org.opensearch.sql.expression.ReferenceExpression;
+import org.opensearch.sql.expression.function.BuiltinFunctionName;
+import org.opensearch.sql.planner.logical.LogicalNested;
+import org.opensearch.sql.planner.logical.LogicalPlan;
+
+/**
+ * Analyze the Nested Function in the {@link AnalysisContext} to construct the {@link
+ * LogicalPlan}.
+ */
+@RequiredArgsConstructor
+public class NestedAnalyzer extends AbstractNodeVisitor<LogicalPlan, AnalysisContext> {
+  private final List<NamedExpression> namedExpressions;
+  private final ExpressionAnalyzer expressionAnalyzer;
+  private final LogicalPlan child;
+
+  public LogicalPlan analyze(UnresolvedExpression projectItem, AnalysisContext context) {
+    LogicalPlan nested = projectItem.accept(this, context);
+    return (nested == null) ? child : nested;
+  }
+
+  @Override
+  public LogicalPlan visitAlias(Alias node, AnalysisContext context) {
+    return node.getDelegated().accept(this, context);
+  }
+
+  @Override
+  public LogicalPlan visitFunction(Function node, AnalysisContext context) {
+    if (node.getFuncName().equalsIgnoreCase(BuiltinFunctionName.NESTED.name())) {
+
+      List<UnresolvedExpression> expressions = node.getFuncArgs();
+      validateArgs(expressions);
+      ReferenceExpression nestedField =
+          (ReferenceExpression)expressionAnalyzer.analyze(expressions.get(0), context);
+      Map<String, ReferenceExpression> args;
+      if (expressions.size() == 2) {
+        args = Map.of(
+            "field", nestedField,
+            "path", (ReferenceExpression)expressionAnalyzer.analyze(expressions.get(1), context)
+        );
+      } else {
+        args = Map.of(
+            "field", (ReferenceExpression)expressionAnalyzer.analyze(expressions.get(0), context),
+            "path", generatePath(nestedField.toString())
+        );
+      }
+      if (child instanceof LogicalNested) {
+        ((LogicalNested)child).addFields(args);
+        return child;
+      } else {
+        return new LogicalNested(child, new ArrayList<>(Arrays.asList(args)), namedExpressions);
+      }
+    }
+    return null;
+  }
+
+  /**
+   * Validate each parameter used in nested function in SELECT clause. Any supplied parameter
+   * for a nested function in a SELECT statement must be a valid qualified name, and the field
+   * parameter must be nested at least one level.
+   * @param args : Arguments in nested function.
+   */
+  private void validateArgs(List<UnresolvedExpression> args) {
+    if (args.size() < 1 || args.size() > 2) {
+      throw new IllegalArgumentException(
+          "on nested object only allowed 2 parameters (field,path) or 1 parameter (field)"
+      );
+    }
+
+    for (int i = 0; i < args.size(); i++) {
+      if (!(args.get(i) instanceof QualifiedName)) {
+        throw new IllegalArgumentException(
+            String.format("Illegal nested field name: %s", args.get(i).toString())
+        );
+      }
+      if (i == 0 && ((QualifiedName)args.get(i)).getParts().size() < 2) {
+        throw new IllegalArgumentException(
+            String.format("Illegal nested field name: %s", args.get(i).toString())
+        );
+      }
+    }
+  }
+
+  /**
+   * Generate nested path dynamically. Assumes at least one level of nesting in supplied string.
+   * @param field : Nested field to generate path of.
+   * @return : Path of field derived from last level of nesting.
+   */
+  private ReferenceExpression generatePath(String field) {
+    return new ReferenceExpression(field.substring(0, field.lastIndexOf(".")), STRING);
+  }
+}