[Backport 2.19-dev] Enable Calcite by default and implicit fallback the unsupported commands (#4372)

core/src/main/java/org/opensearch/sql/analysis/Analyzer.java

@@ -87,6 +87,7 @@
 import org.opensearch.sql.ast.tree.Rename;
 import org.opensearch.sql.ast.tree.Reverse;
 import org.opensearch.sql.ast.tree.Rex;
+import org.opensearch.sql.ast.tree.SPath;
 import org.opensearch.sql.ast.tree.Search;
 import org.opensearch.sql.ast.tree.Sort;
 import org.opensearch.sql.ast.tree.Sort.SortOption;
@@ -96,6 +97,7 @@
 import org.opensearch.sql.ast.tree.Trendline;
 import org.opensearch.sql.ast.tree.UnresolvedPlan;
 import org.opensearch.sql.ast.tree.Values;
+import org.opensearch.sql.ast.tree.Window;
 import org.opensearch.sql.common.antlr.SyntaxCheckException;
 import org.opensearch.sql.data.model.ExprMissingValue;
 import org.opensearch.sql.data.type.ExprCoreType;
@@ -755,11 +757,21 @@ public LogicalPlan visitReverse(Reverse node, AnalysisContext context) {
     throw getOnlyForCalciteException("Reverse");
   }
 
+  @Override
+  public LogicalPlan visitSpath(SPath node, AnalysisContext context) {
+    throw getOnlyForCalciteException("Spath");
+  }
+
   @Override
   public LogicalPlan visitTimechart(Timechart node, AnalysisContext context) {
     throw getOnlyForCalciteException("Timechart");
   }
 
+  @Override
+  public LogicalPlan visitWindow(Window node, AnalysisContext context) {
+    throw getOnlyForCalciteException("Window");
+  }
+
   @Override
   public LogicalPlan visitRegex(Regex node, AnalysisContext context) {
     throw getOnlyForCalciteException("Regex");

core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java

@@ -38,6 +38,7 @@
 import org.opensearch.sql.ast.expression.HighlightFunction;
 import org.opensearch.sql.ast.expression.In;
 import org.opensearch.sql.ast.expression.Interval;
+import org.opensearch.sql.ast.expression.LambdaFunction;
 import org.opensearch.sql.ast.expression.Literal;
 import org.opensearch.sql.ast.expression.Not;
 import org.opensearch.sql.ast.expression.Or;
@@ -479,6 +480,11 @@ public Expression visitInSubquery(InSubquery node, AnalysisContext context) {
     throw getOnlyForCalciteException("Subsearch");
   }
 
+  @Override
+  public Expression visitLambdaFunction(LambdaFunction node, AnalysisContext context) {
+    throw getOnlyForCalciteException("Lambda function");
+  }
+
   /**
    * If QualifiedName is actually a reserved metadata field, return the expr type associated with
    * the metadata field.

core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java

@@ -6,6 +6,7 @@
 package org.opensearch.sql.calcite;
 
 import static org.apache.calcite.sql.SqlKind.AS;
+import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.INFORMATION_SCHEMA_NAME;
 import static org.opensearch.sql.ast.tree.Join.JoinType.ANTI;
 import static org.opensearch.sql.ast.tree.Join.JoinType.SEMI;
 import static org.opensearch.sql.ast.tree.Sort.NullOrder.NULL_FIRST;
@@ -20,6 +21,7 @@
 import static org.opensearch.sql.calcite.utils.PlanUtils.getRelation;
 import static org.opensearch.sql.calcite.utils.PlanUtils.getRexCall;
 import static org.opensearch.sql.calcite.utils.PlanUtils.transformPlanToAttachChild;
+import static org.opensearch.sql.utils.SystemIndexUtils.DATASOURCES_TABLE_NAME;
 
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
@@ -66,6 +68,7 @@
 import org.apache.commons.lang3.ArrayUtils;
 import org.apache.commons.lang3.tuple.Pair;
 import org.checkerframework.checker.nullness.qual.Nullable;
+import org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver;
 import org.opensearch.sql.ast.AbstractNodeVisitor;
 import org.opensearch.sql.ast.EmptySourcePropagateVisitor;
 import org.opensearch.sql.ast.Node;
@@ -139,6 +142,7 @@
 import org.opensearch.sql.calcite.utils.WildcardUtils;
 import org.opensearch.sql.common.patterns.PatternUtils;
 import org.opensearch.sql.common.utils.StringUtils;
+import org.opensearch.sql.datasource.DataSourceService;
 import org.opensearch.sql.exception.CalciteUnsupportedException;
 import org.opensearch.sql.exception.SemanticCheckException;
 import org.opensearch.sql.expression.function.BuiltinFunctionName;
@@ -151,10 +155,12 @@ public class CalciteRelNodeVisitor extends AbstractNodeVisitor<RelNode, CalciteP
 
   private final CalciteRexNodeVisitor rexVisitor;
   private final CalciteAggCallVisitor aggVisitor;
+  private final DataSourceService dataSourceService;
 
-  public CalciteRelNodeVisitor() {
+  public CalciteRelNodeVisitor(DataSourceService dataSourceService) {
     this.rexVisitor = new CalciteRexNodeVisitor(this);
     this.aggVisitor = new CalciteAggCallVisitor(rexVisitor);
+    this.dataSourceService = dataSourceService;
   }
 
   public RelNode analyze(UnresolvedPlan unresolved, CalcitePlanContext context) {
@@ -163,6 +169,21 @@ public RelNode analyze(UnresolvedPlan unresolved, CalcitePlanContext context) {
 
   @Override
   public RelNode visitRelation(Relation node, CalcitePlanContext context) {
+    DataSourceSchemaIdentifierNameResolver nameResolver =
+        new DataSourceSchemaIdentifierNameResolver(
+            dataSourceService, node.getTableQualifiedName().getParts());
+    if (!nameResolver
+        .getDataSourceName()
+        .equals(DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME)) {
+      throw new CalciteUnsupportedException(
+          "Datasource " + nameResolver.getDataSourceName() + " is unsupported in Calcite");
+    }
+    if (nameResolver.getIdentifierName().equals(DATASOURCES_TABLE_NAME)) {
+      throw new CalciteUnsupportedException("SHOW DATASOURCES is unsupported in Calcite");
+    }
+    if (nameResolver.getSchemaName().equals(INFORMATION_SCHEMA_NAME)) {
+      throw new CalciteUnsupportedException("information_schema is unsupported in Calcite");
+    }
     context.relBuilder.scan(node.getTableQualifiedName().getParts());
     return context.relBuilder.peek();
   }
@@ -1385,7 +1406,7 @@ public RelNode visitDedupe(Dedupe node, CalcitePlanContext context) {
       throw new IllegalArgumentException("Number of duplicate events must be greater than 0");
     }
     if (consecutive) {
-      throw new UnsupportedOperationException("Consecutive deduplication is not supported");
+      throw new CalciteUnsupportedException("Consecutive deduplication is unsupported in Calcite");
     }
     // Columns to deduplicate
     List<RexNode> dedupeFields =

core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java

@@ -48,6 +48,7 @@
 import org.opensearch.sql.ast.expression.Compare;
 import org.opensearch.sql.ast.expression.EqualTo;
 import org.opensearch.sql.ast.expression.Function;
+import org.opensearch.sql.ast.expression.HighlightFunction;
 import org.opensearch.sql.ast.expression.In;
 import org.opensearch.sql.ast.expression.Interval;
 import org.opensearch.sql.ast.expression.LambdaFunction;
@@ -57,6 +58,7 @@
 import org.opensearch.sql.ast.expression.Or;
 import org.opensearch.sql.ast.expression.QualifiedName;
 import org.opensearch.sql.ast.expression.RelevanceFieldList;
+import org.opensearch.sql.ast.expression.ScoreFunction;
 import org.opensearch.sql.ast.expression.Span;
 import org.opensearch.sql.ast.expression.SpanUnit;
 import org.opensearch.sql.ast.expression.UnresolvedArgument;
@@ -651,6 +653,16 @@ public RexNode visitWhen(When node, CalcitePlanContext context) {
     throw new CalciteUnsupportedException("CastWhen function is unsupported in Calcite");
   }
 
+  @Override
+  public RexNode visitHighlightFunction(HighlightFunction node, CalcitePlanContext context) {
+    throw new CalciteUnsupportedException("Highlight function is unsupported in Calcite");
+  }
+
+  @Override
+  public RexNode visitScoreFunction(ScoreFunction node, CalcitePlanContext context) {
+    throw new CalciteUnsupportedException("Score function is unsupported in Calcite");
+  }
+
   @Override
   public RexNode visitRelevanceFieldList(RelevanceFieldList node, CalcitePlanContext context) {
     List<RexNode> varArgRexNodeList = new ArrayList<>();

core/src/main/java/org/opensearch/sql/executor/QueryService.java

@@ -1,9 +1,6 @@
 /*
+ * Copyright OpenSearch Contributors
  * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
  */
 
 package org.opensearch.sql.executor;
@@ -12,6 +9,7 @@
 import java.security.PrivilegedAction;
 import java.util.List;
 import java.util.Optional;
+import javax.annotation.Nullable;
 import lombok.AllArgsConstructor;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
@@ -57,13 +55,12 @@ public class QueryService {
   private final Analyzer analyzer;
   private final ExecutionEngine executionEngine;
   private final Planner planner;
-
-  @Getter(lazy = true)
-  private final CalciteRelNodeVisitor relNodeVisitor = new CalciteRelNodeVisitor();
-
   private DataSourceService dataSourceService;
   private Settings settings;
 
+  @Getter(lazy = true)
+  private final CalciteRelNodeVisitor relNodeVisitor = new CalciteRelNodeVisitor(dataSourceService);
+
   /** Execute the {@link UnresolvedPlan}, using {@link ResponseListener} to get response.<br> */
   public void execute(
       UnresolvedPlan plan,
@@ -107,7 +104,7 @@ public void executeWithCalcite(
                 return null;
               });
     } catch (Throwable t) {
-      if (isCalciteFallbackAllowed() && !(t instanceof NonFallbackCalciteException)) {
+      if (isCalciteFallbackAllowed(t) && !(t instanceof NonFallbackCalciteException)) {
         log.warn("Fallback to V2 query engine since got exception", t);
         executeWithLegacy(plan, queryType, listener, Optional.of(t));
       } else {
@@ -143,7 +140,7 @@ public void explainWithCalcite(
                 return null;
               });
     } catch (Throwable t) {
-      if (isCalciteFallbackAllowed()) {
+      if (isCalciteFallbackAllowed(t)) {
         log.warn("Fallback to V2 query engine since got exception", t);
         explainWithLegacy(plan, queryType, listener, format, Optional.of(t));
       } else {
@@ -165,7 +162,7 @@ public void executeWithLegacy(
     try {
       executePlan(analyze(plan, queryType), PlanContext.emptyPlanContext(), listener);
     } catch (Exception e) {
-      if (shouldUseCalcite(queryType) && isCalciteFallbackAllowed()) {
+      if (shouldUseCalcite(queryType) && isCalciteFallbackAllowed(null)) {
         // if there is a failure thrown from Calcite and execution after fallback V2
         // keeps failure, we should throw the failure from Calcite.
         calciteFailure.ifPresentOrElse(
@@ -198,7 +195,7 @@ public void explainWithLegacy(
       }
       executionEngine.explain(plan(analyze(plan, queryType)), listener);
     } catch (Exception e) {
-      if (shouldUseCalcite(queryType) && isCalciteFallbackAllowed()) {
+      if (shouldUseCalcite(queryType) && isCalciteFallbackAllowed(null)) {
         // if there is a failure thrown from Calcite and execution after fallback V2
         // keeps failure, we should throw the failure from Calcite.
         calciteFailure.ifPresentOrElse(
@@ -263,11 +260,21 @@ public RelNode optimize(RelNode plan, CalcitePlanContext context) {
         SystemLimitType.QUERY_SIZE_LIMIT, plan, context.relBuilder.literal(context.querySizeLimit));
   }
 
-  private boolean isCalciteFallbackAllowed() {
-    if (settings != null) {
-      return settings.getSettingValue(Settings.Key.CALCITE_FALLBACK_ALLOWED);
-    } else {
+  private boolean isCalciteFallbackAllowed(@Nullable Throwable t) {
+    // We always allow fallback the query failed with CalciteUnsupportedException.
+    // This is for avoiding breaking changes when enable Calcite by default.
+    if (t instanceof CalciteUnsupportedException) {
       return true;
+    } else {
+      if (settings != null) {
+        Boolean fallback_allowed = settings.getSettingValue(Settings.Key.CALCITE_FALLBACK_ALLOWED);
+        if (fallback_allowed == null) {
+          return false;
+        }
+        return fallback_allowed;
+      } else {
+        return true;
+      }
     }
   }
 

core/src/test/java/org/opensearch/sql/calcite/CalciteRelNodeVisitorSearchSimpleTest.java

@@ -11,9 +11,11 @@
 
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.mockito.Mock;
 import org.opensearch.sql.ast.dsl.AstDSL;
 import org.opensearch.sql.ast.tree.Relation;
 import org.opensearch.sql.ast.tree.Search;
+import org.opensearch.sql.datasource.DataSourceService;
 
 /**
  * Simple tests for CalciteRelNodeVisitor.visitSearch method. Tests basic functionality without
@@ -22,10 +24,11 @@
 public class CalciteRelNodeVisitorSearchSimpleTest {
 
   private CalciteRelNodeVisitor visitor;
+  @Mock DataSourceService dataSourceService;
 
   @BeforeEach
   public void setUp() {
-    visitor = new CalciteRelNodeVisitor();
+    visitor = new CalciteRelNodeVisitor(dataSourceService);
   }
 
   @Test

core/src/test/java/org/opensearch/sql/calcite/CalciteRexNodeVisitorTest.java

@@ -29,6 +29,7 @@
 import org.opensearch.sql.ast.expression.LambdaFunction;
 import org.opensearch.sql.ast.expression.QualifiedName;
 import org.opensearch.sql.calcite.utils.CalciteToolsHelper;
+import org.opensearch.sql.datasource.DataSourceService;
 import org.opensearch.sql.executor.QueryType;
 
 @ExtendWith(MockitoExtension.class)
@@ -44,6 +45,7 @@ public class CalciteRexNodeVisitorTest {
   @Mock RelDataType accType;
   @Mock QualifiedName functionArg1;
   @Mock QualifiedName functionArg2;
+  @Mock DataSourceService dataSourceService;
 
   static CalciteRexNodeVisitor visitor;
   static CalciteRelNodeVisitor relNodeVisitor;
@@ -57,7 +59,7 @@ public class CalciteRexNodeVisitorTest {
 
   @BeforeEach
   public void setUpContext() {
-    relNodeVisitor = new CalciteRelNodeVisitor();
+    relNodeVisitor = new CalciteRelNodeVisitor(dataSourceService);
     visitor = new CalciteRexNodeVisitor(relNodeVisitor);
     when(relBuilder.getRexBuilder()).thenReturn(rexBuilder);
     when(rexBuilder.getTypeFactory()).thenReturn(TYPE_FACTORY);

docs/category.json

@@ -11,7 +11,6 @@
     "user/ppl/cmd/dedup.rst",
     "user/ppl/cmd/describe.rst",
     "user/ppl/cmd/showdatasources.rst",
-    "user/ppl/cmd/information_schema.rst",
     "user/ppl/cmd/eval.rst",
     "user/ppl/cmd/head.rst",
     "user/ppl/cmd/rare.rst",

docs/dev/intro-v3-engine.md

@@ -1,4 +1,4 @@
-# PPL Engine V3 (for 3.0.0-beta)
+# PPL Engine V3
 
 ---
 ## 1. Motivations
@@ -24,7 +24,7 @@ Find more details in [V3 Architecture](./intro-v3-architecture.md).
 ---
 ## 2. What's New
 
-In the initial release of the V3 engine (3.0.0-beta), the main new features focus on enhancing the PPL language while maintaining maximum compatibility with V2 behavior.
+In the initial release of the V3 engine (3.0.0), the main new features focus on enhancing the PPL language while maintaining maximum compatibility with V2 behavior.
 
 * **[Join](../user/ppl/cmd/join.rst) Command**
 * **[Lookup](../user/ppl/cmd/lookup.rst) Command**
@@ -51,40 +51,22 @@ Because of implementation changed internally, following behaviors are changed fr
 
 ### 3.2 Fallback Mechanism
 
-As v3 engine is experimental in 3.0.0-beta, not all PPL commands could work under this new engine. Those unsupported queries will be forwarded to V2 engine by fallback mechanism. To avoid impact on your side, normally you won't see any difference in a query response. If you want to check if and why your query falls back to be handled by V2 engine, please check OpenSearch log for "Fallback to V2 query engine since ...".
+- As v3 engine is experimental in 3.0.0, not all PPL commands could work under this new engine. Those unsupported queries will be forwarded to V2 engine by fallback mechanism. To avoid impact on your side, normally you won't see any difference in a query response. If you want to check if and why your query falls back to be handled by V2 engine, please check OpenSearch log for "Fallback to V2 query engine since ...".
+- Since 3.2.0, the fallback mechanism is disabled by default, to enable fallback, set `plugins.calcite.fallback.allowed=true`.
 
 ### 3.3 Limitations
 
 For the following functionalities in V3 engine, the query will be forwarded to the V2 query engine and thus you cannot use new features in [2. What's New](#2-whats-new).
 
-#### Unsupported functionalities
+#### Unsupported functionalities (up to latest)
 - All SQL queries
-- `trendline`
-- `show datasource`
-- `explain`
-- `describe`
-- `top` and `rare`
-- `fillnull`
-- `patterns`
+- PPL queries against non-OpenSearch data sources
 - `dedup` with `consecutive=true`
 - Search relevant commands
   - AD
   - ML
   - Kmeans
 - Commands with `fetch_size` parameter
-- query with metadata fields, `_id`, `_doc`, etc.
-- Json relevant functions
-  - cast to json
-  - json
-  - json_valid
-- Search relevant functions
-  - match
-  - match_phrase
-  - match_bool_prefix
-  - match_phrase_prefix
-  - simple_query_string
-  - query_string
-  - multi_match
 - [Existed limitations of V2](intro-v2-engine.md#33-limitations)
 
 ---

docs/user/admin/settings.rst

@@ -891,9 +891,10 @@ Description
 
 You can enable Calcite as new query optimizer and execution engine to all coming requests.
 
-1. The default value is false since 3.0.0.
-2. This setting is node scope.
-3. This setting can be updated dynamically.
+1. The default value is false in 3.0, 3.1 and 3.2.
+2. The default value is true since 3.3.0.
+3. This setting is node scope.
+4. This setting can be updated dynamically.
 
 Check `introduce v3 engine <../../../dev/intro-v3-engine.md>`_ for more details.
 Check `join doc <../../ppl/cmd/join.rst>`_ for example.