HIVE-24167: TPC-DS query 14 fails while generating plan for the filter

Author: okumin

ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/GroupTransformer.java

@@ -21,7 +21,6 @@
 import java.lang.reflect.Modifier;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.IdentityHashMap;
@@ -200,54 +199,30 @@ public void merge(Object o1, Object o2) {
   }
 
   private void link(Object o1, Object o2, boolean mayMerge) {
-
-    Set<Object> keySet = Collections.newSetFromMap(new IdentityHashMap<Object, Boolean>());
-    keySet.add(o1);
-    keySet.add(o2);
-    keySet.add(getKeyFor(o1));
-    keySet.add(getKeyFor(o2));
-
-    Set<EquivGroup> mGroups = Collections.newSetFromMap(new IdentityHashMap<EquivGroup, Boolean>());
-
-    for (Object object : keySet) {
-      EquivGroup group = objectMap.get(object);
-      if (group != null) {
-        mGroups.add(group);
-      }
-    }
-    if (mGroups.size() > 1) {
+    final EquivGroup group1 = objectMap.get(o1);
+    final EquivGroup group2 = objectMap.get(o2);
+    if (group1 != null && group2 != null && group1 != group2) {
       if (!mayMerge) {
         throw new RuntimeException("equivalence mapping violation");
       }
       EquivGroup newGrp = new EquivGroup();
       newGrp.add(o1);
       newGrp.add(o2);
-      for (EquivGroup g : mGroups) {
-        for (Object o : g.members) {
-          newGrp.add(o);
-        }
-      }
+      group1.members.forEach(newGrp::add);
+      group2.members.forEach(newGrp::add);
       groups.add(newGrp);
-      groups.removeAll(mGroups);
+      groups.remove(group1);
+      groups.remove(group2);
     } else {
-      EquivGroup targetGroup = mGroups.isEmpty() ? new EquivGroup() : mGroups.iterator().next();
+      EquivGroup targetGroup = group1 != null ? group1 : (group2 != null ? group2 : new EquivGroup());
       groups.add(targetGroup);
       targetGroup.add(o1);
       targetGroup.add(o2);
     }
-
   }
 
   private OpTreeSignatureFactory signatureCache = OpTreeSignatureFactory.newCache();
 
-  private Object getKeyFor(Object o) {
-    if (o instanceof Operator) {
-      Operator<?> operator = (Operator<?>) o;
-      return signatureCache.getSignature(operator);
-    }
-    return o;
-  }
-
   public <T> List<T> getAll(Class<T> clazz) {
     List<T> ret = new ArrayList<>();
     for (EquivGroup g : groups) {
@@ -256,12 +231,6 @@ public <T> List<T> getAll(Class<T> clazz) {
     return ret;
   }
 
-  public void runMapper(GroupTransformer mapper) {
-    for (EquivGroup equivGroup : groups) {
-      mapper.map(equivGroup);
-    }
-  }
-
   public <T> List<T> lookupAll(Class<T> clazz, Object key) {
     EquivGroup group = objectMap.get(key);
     if (group == null) {

ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/PlanMapper.java

@@ -0,0 +1,27 @@
+--! qt:dataset:src
+
+set hive.optimize.cte.materialize.threshold=1;
+set hive.optimize.cte.materialize.full.aggregate.only=false;
+
+EXPLAIN CBO
+WITH materialized_cte AS (
+  SELECT key, value FROM src WHERE key != '100'
+),
+another_materialized_cte AS (
+  SELECT key, value FROM src WHERE key != '100'
+)
+SELECT a.key, a.value, b.key, b.value
+FROM materialized_cte a
+JOIN another_materialized_cte b ON a.key = b.key
+ORDER BY a.key;
+
+WITH materialized_cte AS (
+  SELECT key, value FROM src WHERE key != '100'
+),
+another_materialized_cte AS (
+  SELECT key, value FROM src WHERE key != '100'
+)
+SELECT a.key, a.value, b.key, b.value
+FROM materialized_cte a
+JOIN another_materialized_cte b ON a.key = b.key
+ORDER BY a.key;

ql/src/test/queries/clientpositive/cbo_cte_materialization.q

@@ -1,4 +1,3 @@
---! qt:disabled:HIVE-24167
 set hive.mapred.mode=nonstrict;
 -- start query 1 in stream 0 using template query14.tpl and seed 1819994127
 explain cbo

ql/src/test/queries/clientpositive/perf/cbo_query14.q

@@ -1,4 +1,3 @@
---! qt:disabled:HIVE-24167
 set hive.mapred.mode=nonstrict;
 -- start query 1 in stream 0 using template query14.tpl and seed 1819994127
 explain