[Calcite Engine]Support metadata field

core/src/main/java/org/opensearch/sql/ast/expression/Alias.java

@@ -5,23 +5,20 @@
 
 package org.opensearch.sql.ast.expression;
 
-import lombok.AllArgsConstructor;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
-import lombok.RequiredArgsConstructor;
 import lombok.ToString;
 import org.opensearch.sql.ast.AbstractNodeVisitor;
+import org.opensearch.sql.calcite.plan.OpenSearchConstants;
 
 /**
  * Alias abstraction that associate an unnamed expression with a name. The name information
  * preserved is useful for semantic analysis and response formatting eventually. This can avoid
  * restoring the info in toString() method which is inaccurate because original info is already
  * lost.
  */
-@AllArgsConstructor
 @EqualsAndHashCode(callSuper = false)
 @Getter
-@RequiredArgsConstructor
 @ToString
 public class Alias extends UnresolvedExpression {
 
@@ -35,7 +32,36 @@ public class Alias extends UnresolvedExpression {
   private final UnresolvedExpression delegated;
 
   /** TODO. Optional field alias. This field is OpenSearch SQL-only */
-  private String alias;
+  private final String alias;
+
+  public Alias(String name, UnresolvedExpression expr) {
+    this(name, expr, false);
+  }
+
+  public Alias(String name, UnresolvedExpression expr, String alias) {
+    this(name, expr, alias, false);
+  }
+
+  public Alias(String name, UnresolvedExpression expr, boolean metaMetaFieldAllowed) {
+    this(name, expr, null, metaMetaFieldAllowed);
+  }
+
+  public Alias(String name, UnresolvedExpression expr, String alias, boolean metaMetaFieldAllowed) {
+    if (!metaMetaFieldAllowed && OpenSearchConstants.METADATAFIELD_TYPE_MAP.containsKey(name)) {
+      throw new IllegalArgumentException(
+          String.format("Cannot use metadata field [%s] as the alias.", name));
+    }
+    this.name = name;
+    this.delegated = expr;
+    this.alias = alias;
+  }
+
+  // TODO: Only for SQL. We never allow metadata field as alias but SQL view all select items as
+  //  alias. Need to remove this tricky logic after SQL fix it.
+  public static Alias newAliasAllowMetaMetaField(
+      String name, UnresolvedExpression expr, String alias) {
+    return new Alias(name, expr, alias, true);
+  }
 
   @Override
   public <T, C> T accept(AbstractNodeVisitor<T, C> nodeVisitor, C context) {

core/src/main/java/org/opensearch/sql/ast/expression/AllFields.java

@@ -5,23 +5,38 @@
 
 package org.opensearch.sql.ast.expression;
 
+import com.google.common.collect.ImmutableList;
 import java.util.Collections;
 import java.util.List;
 import lombok.EqualsAndHashCode;
+import lombok.Getter;
 import lombok.ToString;
 import org.opensearch.sql.ast.AbstractNodeVisitor;
 import org.opensearch.sql.ast.Node;
+import org.opensearch.sql.ast.tree.Project;
+import org.opensearch.sql.ast.tree.UnresolvedPlan;
 
 /** Represent the All fields which is been used in SELECT *. */
+@Getter
 @ToString
 @EqualsAndHashCode(callSuper = false)
 public class AllFields extends UnresolvedExpression {
-  public static final AllFields INSTANCE = new AllFields();
+  /** Whether exclude metadata field by force, only used by calcite engine */
+  final boolean excludeMeta;
 
-  private AllFields() {}
+  public static final AllFields INSTANCE_OF_ALL = new AllFields(false);
+  public static final AllFields INSTANCE_EXCEPT_META = new AllFields(true);
+
+  private AllFields(boolean excludeMeta) {
+    this.excludeMeta = excludeMeta;
+  }
 
   public static AllFields of() {
-    return INSTANCE;
+    return INSTANCE_OF_ALL;
+  }
+
+  public static AllFields excludeMeta() {
+    return INSTANCE_EXCEPT_META;
   }
 
   @Override
@@ -33,4 +48,12 @@ public List<? extends Node> getChild() {
   public <R, C> R accept(AbstractNodeVisitor<R, C> nodeVisitor, C context) {
     return nodeVisitor.visitAllFields(this, context);
   }
+
+  public UnresolvedPlan apply(UnresolvedPlan plan) {
+    if ((plan instanceof Project) && !((Project) plan).isExcluded()) {
+      return plan;
+    } else {
+      return new Project(ImmutableList.of(this)).attach(plan);
+    }
+  }
 }

core/src/main/java/org/opensearch/sql/ast/expression/Let.java

@@ -9,19 +9,28 @@
 import java.util.List;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
-import lombok.RequiredArgsConstructor;
 import lombok.ToString;
 import org.opensearch.sql.ast.AbstractNodeVisitor;
+import org.opensearch.sql.calcite.plan.OpenSearchConstants;
 
 /** Represent the assign operation. e.g. velocity = distance/speed. */
 @Getter
 @ToString
 @EqualsAndHashCode(callSuper = false)
-@RequiredArgsConstructor
 public class Let extends UnresolvedExpression {
   private final Field var;
   private final UnresolvedExpression expression;
 
+  public Let(Field var, UnresolvedExpression expression) {
+    String varName = var.getField().toString();
+    if (OpenSearchConstants.METADATAFIELD_TYPE_MAP.containsKey(varName)) {
+      throw new IllegalArgumentException(
+          String.format("Cannot use metadata field [%s] as the eval field.", varName));
+    }
+    this.var = var;
+    this.expression = expression;
+  }
+
   @Override
   public List<UnresolvedExpression> getChild() {
     return ImmutableList.of();

core/src/main/java/org/opensearch/sql/ast/tree/Join.java

@@ -15,11 +15,11 @@
 import lombok.RequiredArgsConstructor;
 import lombok.ToString;
 import org.opensearch.sql.ast.AbstractNodeVisitor;
+import org.opensearch.sql.ast.expression.AllFields;
 import org.opensearch.sql.ast.expression.UnresolvedExpression;
 
 @ToString
 @Getter
-@RequiredArgsConstructor
 @EqualsAndHashCode(callSuper = false)
 public class Join extends UnresolvedPlan {
   private UnresolvedPlan left;
@@ -30,9 +30,28 @@ public class Join extends UnresolvedPlan {
   private final Optional<UnresolvedExpression> joinCondition;
   private final JoinHint joinHint;
 
+  public Join(
+      UnresolvedPlan apply,
+      Optional<String> leftAlias,
+      Optional<String> rightAlias,
+      JoinType joinType,
+      Optional<UnresolvedExpression> joinCondition,
+      JoinHint joinHint) {
+    // Exclude metadata fields for join since they're meaningless for a new record
+    this.right = AllFields.excludeMeta().apply(apply);
+    this.leftAlias = leftAlias;
+    this.rightAlias = rightAlias;
+    this.joinType = joinType;
+    this.joinCondition = joinCondition;
+    this.joinHint = joinHint;
+  }
+
   @Override
   public UnresolvedPlan attach(UnresolvedPlan child) {
-    this.left = leftAlias.isEmpty() ? child : new SubqueryAlias(leftAlias.get(), child);
+    // Exclude metadata fields for join since they're meaningless for a new record
+    this.left =
+        AllFields.excludeMeta()
+            .apply(leftAlias.isEmpty() ? child : new SubqueryAlias(leftAlias.get(), child));
     return this;
   }
 

core/src/main/java/org/opensearch/sql/ast/tree/Rename.java

@@ -9,22 +9,43 @@
 import java.util.List;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
-import lombok.RequiredArgsConstructor;
 import lombok.ToString;
 import org.opensearch.sql.ast.AbstractNodeVisitor;
+import org.opensearch.sql.ast.expression.Field;
 import org.opensearch.sql.ast.expression.Map;
+import org.opensearch.sql.ast.expression.UnresolvedExpression;
+import org.opensearch.sql.calcite.plan.OpenSearchConstants;
 
 @ToString
 @EqualsAndHashCode(callSuper = false)
 @Getter
-@RequiredArgsConstructor
 public class Rename extends UnresolvedPlan {
   private final List<Map> renameList;
   private UnresolvedPlan child;
 
   public Rename(List<Map> renameList, UnresolvedPlan child) {
     this.renameList = renameList;
     this.child = child;
+    checkRename();
+  }
+
+  public Rename(List<Map> renameList) {
+    this.renameList = renameList;
+    checkRename();
+  }
+
+  private void checkRename() {
+    renameList.forEach(rename -> checkFieldName(rename.getTarget()));
+  }
+
+  private void checkFieldName(UnresolvedExpression expr) {
+    if (expr instanceof Field field) {
+      String name = field.getField().toString();
+      if (OpenSearchConstants.METADATAFIELD_TYPE_MAP.containsKey(name)) {
+        throw new IllegalArgumentException(
+            String.format("Cannot use metadata field [%s] in Rename command.", name));
+      }
+    }
   }
 
   @Override

core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java

@@ -32,7 +32,16 @@ public class CalcitePlanContext {
   public final QueryType queryType;
 
   @Getter @Setter private boolean isResolvingJoinCondition = false;
-  @Getter @Setter private boolean isResolvingExistsSubquery = false;
+  @Getter @Setter private boolean isResolvingSubquery = false;
+
+  /**
+   * The flag used to determine whether we do metadata field projection for user 1. If a project is
+   * never visited, we will do metadata field projection for user 2. Else not because user may
+   * intend to show the metadata field themselves. // TODO: use stack here if we want to do similar
+   * projection for subquery.
+   */
+  @Getter @Setter private boolean isProjectVisited = false;
+
   private final Stack<RexCorrelVariable> correlVar = new Stack<>();
 
   private CalcitePlanContext(FrameworkConfig config, QueryType queryType) {

core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java

@@ -73,6 +73,7 @@
 import org.opensearch.sql.ast.tree.TableFunction;
 import org.opensearch.sql.ast.tree.Trendline;
 import org.opensearch.sql.ast.tree.UnresolvedPlan;
+import org.opensearch.sql.calcite.plan.OpenSearchConstants;
 import org.opensearch.sql.calcite.utils.JoinAndLookupUtils;
 import org.opensearch.sql.exception.CalciteUnsupportedException;
 import org.opensearch.sql.exception.SemanticCheckException;
@@ -150,8 +151,10 @@ private boolean containsSubqueryExpression(Node expr) {
   public RelNode visitProject(Project node, CalcitePlanContext context) {
     visitChildren(node, context);
     List<RexNode> projectList;
-    if (node.getProjectList().stream().anyMatch(e -> e instanceof AllFields)) {
+    if (node.getProjectList().size() == 1
+        && node.getProjectList().getFirst() instanceof AllFields allFields) {
       tryToRemoveNestedFields(context);
+      tryToRemoveMetaFields(context, allFields.isExcludeMeta());
       return context.relBuilder.peek();
     } else {
       projectList =
@@ -162,6 +165,10 @@ public RelNode visitProject(Project node, CalcitePlanContext context) {
     if (node.isExcluded()) {
       context.relBuilder.projectExcept(projectList);
     } else {
+      // Only set when not resolving subquery and it's not projectExcept.
+      if (!context.isResolvingSubquery()) {
+        context.setProjectVisited(true);
+      }
       context.relBuilder.project(projectList);
     }
     return context.relBuilder.peek();
@@ -184,6 +191,31 @@ private void tryToRemoveNestedFields(CalcitePlanContext context) {
     }
   }
 
+  /**
+   * Try to remove metadata fields in two cases:
+   *
+   * <p>1. It's explicitly specified excluding by force, usually for join or subquery.
+   *
+   * <p>2. There is no other project ever visited in the main query
+   *
+   * @param context CalcitePlanContext
+   * @param excludeByForce whether exclude metadata fields by force
+   */
+  private static void tryToRemoveMetaFields(CalcitePlanContext context, boolean excludeByForce) {
+    if (excludeByForce || !context.isProjectVisited()) {
+      List<String> originalFields = context.relBuilder.peek().getRowType().getFieldNames();
+      List<RexNode> metaFieldsRef =
+          originalFields.stream()
+              .filter(OpenSearchConstants.METADATAFIELD_TYPE_MAP::containsKey)
+              .map(metaField -> (RexNode) context.relBuilder.field(metaField))
+              .toList();
+      // Remove metadata fields if there is and ensure there are other fields.
+      if (!metaFieldsRef.isEmpty() && metaFieldsRef.size() != originalFields.size()) {
+        context.relBuilder.projectExcept(metaFieldsRef);
+      }
+    }
+  }
+
   @Override
   public RelNode visitRename(Rename node, CalcitePlanContext context) {
     visitChildren(node, context);

core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java

@@ -398,6 +398,8 @@ public RexNode visitExistsSubquery(ExistsSubquery node, CalcitePlanContext conte
   }
 
   private RelNode resolveSubqueryPlan(UnresolvedPlan subquery, CalcitePlanContext context) {
+    boolean isNestedSubquery = context.isResolvingSubquery();
+    context.setResolvingSubquery(true);
     // clear and store the outer state
     boolean isResolvingJoinConditionOuter = context.isResolvingJoinCondition();
     if (isResolvingJoinConditionOuter) {
@@ -411,6 +413,10 @@ private RelNode resolveSubqueryPlan(UnresolvedPlan subquery, CalcitePlanContext
     if (isResolvingJoinConditionOuter) {
       context.setResolvingJoinCondition(true);
     }
+    // Only need to set isResolvingSubquery to false if it's not nested subquery.
+    if (!isNestedSubquery) {
+      context.setResolvingSubquery(false);
+    }
     return subqueryRel;
   }
 

core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java

@@ -311,7 +311,9 @@ public static ExprValue getExprValueByExprType(ExprType type, Object value) {
   public static RelDataType convertSchema(Table table) {
     List<String> fieldNameList = new ArrayList<>();
     List<RelDataType> typeList = new ArrayList<>();
-    for (Entry<String, ExprType> entry : table.getFieldTypes().entrySet()) {
+    Map<String, ExprType> fieldTypes = table.getFieldTypes();
+    fieldTypes.putAll(table.getReservedFieldTypes());
+    for (Entry<String, ExprType> entry : fieldTypes.entrySet()) {
       fieldNameList.add(entry.getKey());
       typeList.add(OpenSearchTypeFactory.convertExprTypeToRelDataType(entry.getValue()));
     }

core/src/main/java/org/opensearch/sql/storage/Table.java

@@ -43,6 +43,11 @@ default Map<String, ExprType> getReservedFieldTypes() {
     return Map.of();
   }
 
+  /** Whether include reserved fields in the schema of table */
+  default boolean includeReservedFieldTypes() {
+    return true;
+  }
+
   /**
    * Implement a {@link LogicalPlan} by {@link PhysicalPlan} in storage engine.
    *