[WIP] Calcite upgrade 1.33 without SEARCH operators

beeline/pom.xml

@@ -127,6 +127,11 @@
       <groupId>org.apache.logging.log4j</groupId>
       <artifactId>log4j-slf4j-impl</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.locationtech.jts</groupId>
+      <artifactId>jts-core</artifactId>
+      <version>1.19.0</version>
+    </dependency>
     <!-- test intra-project -->
     <dependency>
       <groupId>org.apache.hive</groupId>

itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestMaterializedViewRebuild.java

@@ -55,7 +55,7 @@ public class TestMaterializedViewRebuild extends CompactorOnTezTest {
   private static final List<String> INCREMENTAL_REBUILD_PLAN = Arrays.asList(
           "CBO PLAN:",
           "HiveProject(a=[$0], b=[$1], c=[$2])",
-          "  HiveFilter(condition=[AND(<(2, $5.writeid), OR(>($0, 0), IS NULL($0)))])",
+          "  HiveFilter(condition=[AND(>($5.writeid, 2), OR(IS NULL($0), >($0, 0)))])",
           "    HiveTableScan(table=[[default, t1]], table:alias=[t1])",
           ""
   );

packaging/pom.xml

@@ -173,6 +173,10 @@
                   \Qhttps://www.eclipse.org/org/documents/epl-v10.php\E
                   \Qhttp://www.eclipse.org/org/documents/epl-v10.php\E
                 </EPL-1.0>
+                <EPL-2.0>
+                  \Qhttps://raw.githubusercontent.com/locationtech/jts/master/LICENSE_EPLv2.txt\E
+                  \Qhttps://www.eclipse.org/org/documents/epl-2.0/EPL-2.0.txt\E
+                </EPL-2.0>
                 <MIT>
                   https?://(www\.)?opensource\.org/licenses/mit(-license.php)?
                 </MIT>

pom.xml

@@ -109,10 +109,10 @@
     <apache-directory-server.version>1.5.7</apache-directory-server.version>
     <!-- Include arrow for LlapOutputFormatService -->
     <arrow.version>12.0.0</arrow.version>
-    <avatica.version>1.12.0</avatica.version>
     <avro.version>1.11.3</avro.version>
+    <avatica.version>1.23.0</avatica.version>
     <bcprov-jdk18on.version>1.77</bcprov-jdk18on.version>
-    <calcite.version>1.25.0</calcite.version>
+    <calcite.version>1.33.0</calcite.version>
     <datanucleus-api-jdo.version>5.2.8</datanucleus-api-jdo.version>
     <datanucleus-core.version>5.2.10</datanucleus-core.version>
     <datanucleus-jdo.version>3.2.0-release</datanucleus-jdo.version>
@@ -150,6 +150,7 @@
     <!-- httpcomponents are not always in version sync -->
     <httpcomponents.client.version>4.5.13</httpcomponents.client.version>
     <httpcomponents.core.version>4.4.13</httpcomponents.core.version>
+    <immutables.version>2.9.2</immutables.version>
     <ivy.version>2.5.2</ivy.version>
     <jackson.version>2.16.1</jackson.version>
     <jamon.plugin.version>2.3.4</jamon.plugin.version>
@@ -216,7 +217,7 @@
     <re2j.version>1.2</re2j.version>
     <rs-api.version>2.0.1</rs-api.version>
     <json-path.version>2.9.0</json-path.version>
-    <janino.version>3.0.11</janino.version>
+    <janino.version>3.1.8</janino.version>
     <datasketches.version>1.2.0</datasketches.version>
     <spotbugs.version>4.0.3</spotbugs.version>
     <validation-api.version>1.1.0.Final</validation-api.version>
@@ -344,6 +345,12 @@
         <artifactId>parquet-hadoop-bundle</artifactId>
         <version>${parquet.version}</version>
       </dependency>
+      <dependency>
+        <groupId>org.immutables</groupId>
+        <artifactId>value-annotations</artifactId>
+        <version>${immutables.version}</version>
+        <scope>provided</scope>
+      </dependency>
       <dependency>
         <groupId>com.sun.jersey</groupId>
         <artifactId>jersey-core</artifactId>

ql/pom.xml

@@ -439,6 +439,11 @@
       <groupId>org.datanucleus</groupId>
       <artifactId>datanucleus-core</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.locationtech.jts</groupId>
+      <artifactId>jts-core</artifactId>
+      <version>1.19.0</version>
+    </dependency>
     <dependency>
       <groupId>org.apache.calcite</groupId>
       <artifactId>calcite-core</artifactId>
@@ -854,6 +859,10 @@
       <groupId>org.codehaus.janino</groupId>
       <artifactId>janino</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.immutables</groupId>
+      <artifactId>value-annotations</artifactId>
+    </dependency>
     <dependency>
       <groupId>com.sun.jersey.contribs</groupId>
       <artifactId>jersey-multipart</artifactId>

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java

@@ -27,6 +27,7 @@
 import java.util.HashSet;
 
 import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptPredicateList;
 import org.apache.calcite.plan.RelOptUtil;
 import org.apache.calcite.plan.RelTraitSet;
 import org.apache.calcite.plan.RelOptUtil.InputFinder;
@@ -45,6 +46,7 @@
 import org.apache.calcite.rex.RexCall;
 import org.apache.calcite.rex.RexCorrelVariable;
 import org.apache.calcite.rex.RexDynamicParam;
+import org.apache.calcite.rex.RexExecutor;
 import org.apache.calcite.rex.RexFieldAccess;
 import org.apache.calcite.rex.RexInputRef;
 import org.apache.calcite.rex.RexLiteral;
@@ -53,6 +55,7 @@
 import org.apache.calcite.rex.RexOver;
 import org.apache.calcite.rex.RexPatternFieldRef;
 import org.apache.calcite.rex.RexShuttle;
+import org.apache.calcite.rex.RexSimplify;
 import org.apache.calcite.rex.RexTableInputRef;
 import org.apache.calcite.rex.RexRangeRef;
 import org.apache.calcite.rex.RexSubQuery;
@@ -656,7 +659,9 @@ public static ImmutableList<RexNode> getPredsNotPushedAlready(Set<String> predic
     }
     // Build map to not convert multiple times, further remove already included predicates
     Map<String,RexNode> stringToRexNode = Maps.newLinkedHashMap();
+    RexSimplify simplify = new RexSimplify(inp.getCluster().getRexBuilder(), RelOptPredicateList.EMPTY, RexUtil.EXECUTOR);
     for (RexNode r : predsToPushDown) {
+      r = simplify.simplify(r);
       String rexNodeString = r.toString();
       if (predicatesToExclude.add(rexNodeString)) {
         stringToRexNode.put(rexNodeString, r);

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveMaterializedViewASTSubQueryRewriteRexShuttle.java

@@ -40,7 +40,7 @@ public RexNode visitSubQuery(RexSubQuery subQuery) {
     RelNode newSubquery = subQuery.rel.accept(relShuttle);
 
     RexNode newSubQueryRex;
-    switch (subQuery.op.kind) {
+    switch (subQuery.op.getKind()) {
       case IN:
         newSubQueryRex = RexSubQuery.in(newSubquery, subQuery.operands);
         break;
@@ -59,7 +59,7 @@ public RexNode visitSubQuery(RexSubQuery subQuery) {
         break;
 
       default:
-        throw new RuntimeException("Unsupported op.kind " + subQuery.op.kind);
+        throw new RuntimeException("Unsupported op.kind " + subQuery.op.getKind());
     }
 
     return newSubQueryRex;

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelFactories.java

@@ -114,8 +114,11 @@ private HiveRelFactories() {
    */
   private static class HiveProjectFactoryImpl implements ProjectFactory {
     @Override
-    public RelNode createProject(RelNode child, List<RelHint> hints,
-        List<? extends RexNode> childExprs, List<String> fieldNames) {
+    public RelNode createProject(RelNode child, List<RelHint> hints, List<? extends RexNode> childExprs,
+        List<? extends String> fieldNames, Set<CorrelationId> variablesSet) {
+      if (!variablesSet.isEmpty()) {
+        throw new IllegalStateException("Setting variables is not supported");
+      }
       RelOptCluster cluster = child.getCluster();
       RelDataType rowType = RexUtil.createStructType(
           cluster.getTypeFactory(), childExprs, fieldNames, SqlValidatorUtil.EXPR_SUGGESTER);

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java

@@ -76,6 +76,7 @@
 import org.apache.calcite.util.Pair;
 import org.apache.calcite.util.mapping.Mappings;
 import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveIn;
 import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
 import org.apache.hadoop.hive.ql.optimizer.calcite.translator.TypeConverter;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
@@ -1163,4 +1164,12 @@ public static List<Integer> getNewRelDistributionKeys(
     }
     return distributionKeys;
   }
+
+  public static boolean isHiveIN(RexNode node) {
+    if (!(node instanceof RexCall)) {
+      return false;
+    }
+
+     return ((RexCall) node).getOperator() == HiveIn.INSTANCE;
+  }
 }

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveIn.java

@@ -37,14 +37,19 @@ public class HiveIn extends SqlSpecialOperator {
   private HiveIn() {
     super(
         "IN",
-        SqlKind.IN,
+        SqlKind.OTHER,
         30,
         true,
         ReturnTypes.BOOLEAN_NULLABLE,
         InferTypes.FIRST_KNOWN,
         null);
   }
 
+  @Override
+  public SqlKind getKind() {
+    return SqlKind.IN;
+  }
+
   @Override
   public void unparse(
       SqlWriter writer,

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/jdbc/JdbcHiveTableScan.java

@@ -29,6 +29,9 @@
 
 import org.apache.calcite.rel.RelWriter;
 import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan;
+
+import com.google.common.collect.ImmutableList;
+
 /**
  * Relational expression representing a scan of a HiveDB collection.
  *
@@ -42,7 +45,7 @@ public class JdbcHiveTableScan extends JdbcTableScan {
 
   public JdbcHiveTableScan(RelOptCluster cluster, RelOptTable table, JdbcTable jdbcTable,
       JdbcConvention jdbcConvention, HiveTableScan hiveTableScan) {
-    super(cluster, table, jdbcTable, jdbcConvention);
+    super(cluster, ImmutableList.of(), table, jdbcTable, jdbcConvention);
     this.hiveTableScan= hiveTableScan;
   }
 

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/BaseMutableHiveConfig.java

@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
+
+import org.apache.calcite.plan.RelRule;
+import org.apache.calcite.tools.RelBuilderFactory;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories;
+
+import java.util.Objects;
+
+public abstract class BaseMutableHiveConfig implements RelRule.Config {
+  private RelBuilderFactory factory = HiveRelFactories.HIVE_BUILDER;
+  private String description;
+  private RelRule.OperandTransform operandSupplier;
+
+  @Override
+  public RelBuilderFactory relBuilderFactory() {
+    return factory;
+  }
+
+  @Override
+  public RelRule.Config withRelBuilderFactory(RelBuilderFactory factory) {
+    this.factory = Objects.requireNonNull(factory);
+    return this;
+  }
+
+  @Override
+  public String description() {
+    return this.description;
+  }
+
+  @Override
+  public RelRule.Config withDescription(String description) {
+    if (description == null || description.equals("")) {
+      throw new IllegalArgumentException("Description can not be null/empty");
+    }
+    this.description = description;
+    return this;
+  }
+
+  @Override
+  public RelRule.OperandTransform operandSupplier() {
+    return operandSupplier;
+  }
+
+  @Override
+  public RelRule.Config withOperandSupplier(RelRule.OperandTransform transform) {
+    this.operandSupplier = Objects.requireNonNull(transform);
+    return this;
+  }
+
+}

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateJoinTransposeRule.java

@@ -224,7 +224,7 @@ public void onMatch(RelOptRuleCall call) {
             }
           }
           side.newInput = relBuilder.push(joinInput)
-              .aggregate(relBuilder.groupKey(belowAggregateKey, null),
+              .aggregate(relBuilder.groupKey(belowAggregateKey, ImmutableList.of(belowAggregateKey)),
                   belowAggCalls)
               .build();
         }