[Calcite Engine]Support metadata field

core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java

@@ -8,6 +8,7 @@
 import org.opensearch.sql.ast.expression.AggregateFunction;
 import org.opensearch.sql.ast.expression.Alias;
 import org.opensearch.sql.ast.expression.AllFields;
+import org.opensearch.sql.ast.expression.AllFieldsExcludeMeta;
 import org.opensearch.sql.ast.expression.And;
 import org.opensearch.sql.ast.expression.Argument;
 import org.opensearch.sql.ast.expression.AttributeList;
@@ -254,6 +255,10 @@ public T visitAllFields(AllFields node, C context) {
     return visitChildren(node, context);
   }
 
+  public T visitAllFieldsExcludeMeta(AllFieldsExcludeMeta node, C context) {
+    return visitChildren(node, context);
+  }
+
   public T visitNestedAllTupleFields(NestedAllTupleFields node, C context) {
     return visitChildren(node, context);
   }

core/src/main/java/org/opensearch/sql/ast/expression/Alias.java

@@ -5,23 +5,20 @@
 
 package org.opensearch.sql.ast.expression;
 
-import lombok.AllArgsConstructor;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
-import lombok.RequiredArgsConstructor;
 import lombok.ToString;
 import org.opensearch.sql.ast.AbstractNodeVisitor;
+import org.opensearch.sql.calcite.plan.OpenSearchConstants;
 
 /**
  * Alias abstraction that associate an unnamed expression with a name. The name information
  * preserved is useful for semantic analysis and response formatting eventually. This can avoid
  * restoring the info in toString() method which is inaccurate because original info is already
  * lost.
  */
-@AllArgsConstructor
 @EqualsAndHashCode(callSuper = false)
 @Getter
-@RequiredArgsConstructor
 @ToString
 public class Alias extends UnresolvedExpression {
 
@@ -35,7 +32,41 @@ public class Alias extends UnresolvedExpression {
   private final UnresolvedExpression delegated;
 
   /** TODO. Optional field alias. This field is OpenSearch SQL-only */
-  private String alias;
+  private final String alias;
+
+  public Alias(String name, UnresolvedExpression expr) {
+    this(name, expr, false);
+  }
+
+  public Alias(String name, UnresolvedExpression expr, String alias) {
+    this(name, expr, alias, false);
+  }
+
+  public Alias(String name, UnresolvedExpression expr, boolean metaDataFieldAllowed) {
+    this(name, expr, null, metaDataFieldAllowed);
+  }
+
+  /**
+   * @param metadataFieldAllowed Whether do we allow metadata field as alias name. Should Only be
+   *     true for SQL, see {@link Alias::newAliasAllowMetaMetaField}
+   */
+  private Alias(
+      String name, UnresolvedExpression expr, String alias, boolean metadataFieldAllowed) {
+    if (!metadataFieldAllowed && OpenSearchConstants.METADATAFIELD_TYPE_MAP.containsKey(name)) {
+      throw new IllegalArgumentException(
+          String.format("Cannot use metadata field [%s] as the alias.", name));
+    }
+    this.name = name;
+    this.delegated = expr;
+    this.alias = alias;
+  }
+
+  // TODO: Only for SQL. We never allow metadata field as alias but SQL view all select items as
+  //  alias. Need to remove this tricky logic after SQL fix it.
+  public static Alias newAliasAllowMetaMetaField(
+      String name, UnresolvedExpression expr, String alias) {
+    return new Alias(name, expr, alias, true);
+  }
 
   @Override
   public <T, C> T accept(AbstractNodeVisitor<T, C> nodeVisitor, C context) {

core/src/main/java/org/opensearch/sql/ast/expression/AllFields.java

@@ -18,7 +18,7 @@
 public class AllFields extends UnresolvedExpression {
   public static final AllFields INSTANCE = new AllFields();
 
-  private AllFields() {}
+  public AllFields() {}
 
   public static AllFields of() {
     return INSTANCE;

core/src/main/java/org/opensearch/sql/ast/expression/AllFieldsExcludeMeta.java

@@ -0,0 +1,41 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.ast.expression;
+
+import java.util.Collections;
+import java.util.List;
+import lombok.EqualsAndHashCode;
+import lombok.ToString;
+import org.opensearch.sql.ast.AbstractNodeVisitor;
+import org.opensearch.sql.ast.Node;
+
+/**
+ * Represent the All fields but excluding metadata fields if user never uses them in the previous
+ * fields command
+ */
+@ToString
+@EqualsAndHashCode(callSuper = false)
+public class AllFieldsExcludeMeta extends AllFields {
+  public static final AllFieldsExcludeMeta INSTANCE = new AllFieldsExcludeMeta();
+
+  private AllFieldsExcludeMeta() {
+    super();
+  }
+
+  public static AllFieldsExcludeMeta of() {
+    return INSTANCE;
+  }
+
+  @Override
+  public List<? extends Node> getChild() {
+    return Collections.emptyList();
+  }
+
+  @Override
+  public <R, C> R accept(AbstractNodeVisitor<R, C> nodeVisitor, C context) {
+    return nodeVisitor.visitAllFieldsExcludeMeta(this, context);
+  }
+}

core/src/main/java/org/opensearch/sql/ast/expression/Let.java

@@ -9,19 +9,28 @@
 import java.util.List;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
-import lombok.RequiredArgsConstructor;
 import lombok.ToString;
 import org.opensearch.sql.ast.AbstractNodeVisitor;
+import org.opensearch.sql.calcite.plan.OpenSearchConstants;
 
 /** Represent the assign operation. e.g. velocity = distance/speed. */
 @Getter
 @ToString
 @EqualsAndHashCode(callSuper = false)
-@RequiredArgsConstructor
 public class Let extends UnresolvedExpression {
   private final Field var;
   private final UnresolvedExpression expression;
 
+  public Let(Field var, UnresolvedExpression expression) {
+    String varName = var.getField().toString();
+    if (OpenSearchConstants.METADATAFIELD_TYPE_MAP.containsKey(varName)) {
+      throw new IllegalArgumentException(
+          String.format("Cannot use metadata field [%s] as the eval field.", varName));
+    }
+    this.var = var;
+    this.expression = expression;
+  }
+
   @Override
   public List<UnresolvedExpression> getChild() {
     return ImmutableList.of();

core/src/main/java/org/opensearch/sql/ast/tree/Rename.java

@@ -9,22 +9,43 @@
 import java.util.List;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
-import lombok.RequiredArgsConstructor;
 import lombok.ToString;
 import org.opensearch.sql.ast.AbstractNodeVisitor;
+import org.opensearch.sql.ast.expression.Field;
 import org.opensearch.sql.ast.expression.Map;
+import org.opensearch.sql.ast.expression.UnresolvedExpression;
+import org.opensearch.sql.calcite.plan.OpenSearchConstants;
 
 @ToString
 @EqualsAndHashCode(callSuper = false)
 @Getter
-@RequiredArgsConstructor
 public class Rename extends UnresolvedPlan {
   private final List<Map> renameList;
   private UnresolvedPlan child;
 
   public Rename(List<Map> renameList, UnresolvedPlan child) {
     this.renameList = renameList;
     this.child = child;
+    validate();
+  }
+
+  public Rename(List<Map> renameList) {
+    this.renameList = renameList;
+    validate();
+  }
+
+  private void validate() {
+    renameList.forEach(rename -> validate(rename.getTarget()));
+  }
+
+  private void validate(UnresolvedExpression expr) {
+    if (expr instanceof Field field) {
+      String name = field.getField().toString();
+      if (OpenSearchConstants.METADATAFIELD_TYPE_MAP.containsKey(name)) {
+        throw new IllegalArgumentException(
+            String.format("Cannot use metadata field [%s] in Rename command.", name));
+      }
+    }
   }
 
   @Override

core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java

@@ -32,7 +32,16 @@ public class CalcitePlanContext {
   public final QueryType queryType;
 
   @Getter @Setter private boolean isResolvingJoinCondition = false;
-  @Getter @Setter private boolean isResolvingExistsSubquery = false;
+  @Getter @Setter private boolean isResolvingSubquery = false;
+
+  /**
+   * The flag used to determine whether we do metadata field projection for user 1. If a project is
+   * never visited, we will do metadata field projection for user 2. Else not because user may
+   * intend to show the metadata field themselves. // TODO: use stack here if we want to do similar
+   * projection for subquery.
+   */
+  @Getter @Setter private boolean isProjectVisited = false;
+
   private final Stack<RexCorrelVariable> correlVar = new Stack<>();
 
   private CalcitePlanContext(FrameworkConfig config, QueryType queryType) {

core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java

@@ -40,6 +40,7 @@
 import org.opensearch.sql.ast.AbstractNodeVisitor;
 import org.opensearch.sql.ast.Node;
 import org.opensearch.sql.ast.expression.AllFields;
+import org.opensearch.sql.ast.expression.AllFieldsExcludeMeta;
 import org.opensearch.sql.ast.expression.Argument;
 import org.opensearch.sql.ast.expression.Field;
 import org.opensearch.sql.ast.expression.Let;
@@ -73,6 +74,7 @@
 import org.opensearch.sql.ast.tree.TableFunction;
 import org.opensearch.sql.ast.tree.Trendline;
 import org.opensearch.sql.ast.tree.UnresolvedPlan;
+import org.opensearch.sql.calcite.plan.OpenSearchConstants;
 import org.opensearch.sql.calcite.utils.JoinAndLookupUtils;
 import org.opensearch.sql.exception.CalciteUnsupportedException;
 import org.opensearch.sql.exception.SemanticCheckException;
@@ -150,8 +152,10 @@ private boolean containsSubqueryExpression(Node expr) {
   public RelNode visitProject(Project node, CalcitePlanContext context) {
     visitChildren(node, context);
     List<RexNode> projectList;
-    if (node.getProjectList().stream().anyMatch(e -> e instanceof AllFields)) {
+    if (node.getProjectList().size() == 1
+        && node.getProjectList().getFirst() instanceof AllFields allFields) {
       tryToRemoveNestedFields(context);
+      tryToRemoveMetaFields(context, allFields instanceof AllFieldsExcludeMeta);
       return context.relBuilder.peek();
     } else {
       projectList =
@@ -162,6 +166,10 @@ public RelNode visitProject(Project node, CalcitePlanContext context) {
     if (node.isExcluded()) {
       context.relBuilder.projectExcept(projectList);
     } else {
+      // Only set when not resolving subquery and it's not projectExcept.
+      if (!context.isResolvingSubquery()) {
+        context.setProjectVisited(true);
+      }
       context.relBuilder.project(projectList);
     }
     return context.relBuilder.peek();
@@ -184,6 +192,31 @@ private void tryToRemoveNestedFields(CalcitePlanContext context) {
     }
   }
 
+  /**
+   * Try to remove metadata fields in two cases:
+   *
+   * <p>1. It's explicitly specified excluding by force, usually for join or subquery.
+   *
+   * <p>2. There is no other project ever visited in the main query
+   *
+   * @param context CalcitePlanContext
+   * @param excludeByForce whether exclude metadata fields by force
+   */
+  private static void tryToRemoveMetaFields(CalcitePlanContext context, boolean excludeByForce) {
+    if (excludeByForce || !context.isProjectVisited()) {
+      List<String> originalFields = context.relBuilder.peek().getRowType().getFieldNames();
+      List<RexNode> metaFieldsRef =
+          originalFields.stream()
+              .filter(OpenSearchConstants.METADATAFIELD_TYPE_MAP::containsKey)
+              .map(metaField -> (RexNode) context.relBuilder.field(metaField))
+              .toList();
+      // Remove metadata fields if there is and ensure there are other fields.
+      if (!metaFieldsRef.isEmpty() && metaFieldsRef.size() != originalFields.size()) {
+        context.relBuilder.projectExcept(metaFieldsRef);
+      }
+    }
+  }
+
   @Override
   public RelNode visitRename(Rename node, CalcitePlanContext context) {
     visitChildren(node, context);

core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java

@@ -398,6 +398,8 @@ public RexNode visitExistsSubquery(ExistsSubquery node, CalcitePlanContext conte
   }
 
   private RelNode resolveSubqueryPlan(UnresolvedPlan subquery, CalcitePlanContext context) {
+    boolean isNestedSubquery = context.isResolvingSubquery();
+    context.setResolvingSubquery(true);
     // clear and store the outer state
     boolean isResolvingJoinConditionOuter = context.isResolvingJoinCondition();
     if (isResolvingJoinConditionOuter) {
@@ -411,6 +413,10 @@ private RelNode resolveSubqueryPlan(UnresolvedPlan subquery, CalcitePlanContext
     if (isResolvingJoinConditionOuter) {
       context.setResolvingJoinCondition(true);
     }
+    // Only need to set isResolvingSubquery to false if it's not nested subquery.
+    if (!isNestedSubquery) {
+      context.setResolvingSubquery(false);
+    }
     return subqueryRel;
   }
 

core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java

@@ -311,7 +311,9 @@ public static ExprValue getExprValueByExprType(ExprType type, Object value) {
   public static RelDataType convertSchema(Table table) {
     List<String> fieldNameList = new ArrayList<>();
     List<RelDataType> typeList = new ArrayList<>();
-    for (Entry<String, ExprType> entry : table.getFieldTypes().entrySet()) {
+    Map<String, ExprType> fieldTypes = table.getFieldTypes();
+    fieldTypes.putAll(table.getReservedFieldTypes());
+    for (Entry<String, ExprType> entry : fieldTypes.entrySet()) {
       fieldNameList.add(entry.getKey());
       typeList.add(OpenSearchTypeFactory.convertExprTypeToRelDataType(entry.getValue()));
     }

docs/dev/intro-v3-engine.md

@@ -55,6 +55,13 @@ As v3 engine is experimental in 3.0.0-beta, not all PPL commands could work unde
 
 ### 3.3 Limitations
 
+For the following commands or functions, we add some defensive restrictions to ensure security.
+
+#### New Restrictions
+- `EVAL` won't allow to use [Metadata Fields of OpenSearch](https://docs.opensearch.org/docs/latest/field-types/metadata-fields/index/) as the fields
+- `RENAME` won't allow renaming to a [Metadata Fields of OpenSearch](https://docs.opensearch.org/docs/latest/field-types/metadata-fields/index/)
+- `as` won't allow to use [Metadata Fields of OpenSearch](https://docs.opensearch.org/docs/latest/field-types/metadata-fields/index/) as the alias name
+
 For the following functionalities in V3 engine, the query will be forwarded to the V2 query engine and thus you cannot use new features in [2. What's New](#2-whats-new).
 
 #### Unsupported functionalities
@@ -101,4 +108,3 @@ The following items are on our roadmap with high priority:
 - Backport to 2.19.x
 - Unified the PPL syntax between [PPL-on-OpenSearch](https://github.com/opensearch-project/sql/blob/main/ppl/src/main/antlr/OpenSearchPPLParser.g4) and [PPL-on-Spark](https://github.com/opensearch-project/opensearch-spark/blob/main/ppl-spark-integration/src/main/antlr4/OpenSearchPPLParser.g4)
 - Support more DSL aggregation
-

integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteFieldsCommandIT.java

@@ -5,7 +5,6 @@
 
 package org.opensearch.sql.calcite.remote;
 
-import java.io.IOException;
 import org.opensearch.sql.ppl.FieldsCommandIT;
 
 public class CalciteFieldsCommandIT extends FieldsCommandIT {
@@ -15,30 +14,4 @@ public void init() throws Exception {
     enableCalcite();
     disallowCalciteFallback();
   }
-
-  @Override
-  public void testDelimitedMetadataFields() throws IOException {
-    withFallbackEnabled(
-        () -> {
-          try {
-            super.testDelimitedMetadataFields();
-          } catch (IOException e) {
-            throw new RuntimeException(e);
-          }
-        },
-        "Calcite doesn't support metadata fields in fields yet");
-  }
-
-  @Override
-  public void testMetadataFields() throws IOException {
-    withFallbackEnabled(
-        () -> {
-          try {
-            super.testMetadataFields();
-          } catch (IOException e) {
-            throw new RuntimeException(e);
-          }
-        },
-        "Calcite doesn't support metadata fields in fields yet");
-  }
 }