Introduce new execution sequence builder and work unit builder

omniscidb/IR/Expr.cpp

@@ -934,6 +934,9 @@ ExprPtr UOper::cast(const Type* new_type, bool is_dict_intersection) const {
       }
     }
   }
+  if (type_->isExtDictionary() && new_type->isExtDictionary()) {
+    return operand_->cast(new_type);
+  }
   return Expr::cast(new_type, is_dict_intersection);
 }
 

omniscidb/IR/ExprRewriter.h

@@ -39,6 +39,11 @@ class ExprRewriter : public ExprVisitor<ExprPtr> {
   ExprPtr visitUOper(const hdk::ir::UOper* uoper) override {
     auto new_op = visit(uoper->operand());
     if (new_op.get() != uoper->operand()) {
+      // For casts it is better to use cast method to benefit
+      // from additional folding.
+      if (uoper->isCast()) {
+        return new_op->cast(uoper->type());
+      }
       return hdk::ir::makeExpr<hdk::ir::UOper>(
           uoper->type(), uoper->containsAgg(), uoper->opType(), new_op);
     }

omniscidb/IR/Node.cpp

@@ -571,7 +571,8 @@ size_t getNodeColumnCount(const Node* node) {
                 Compound,
                 TableFunction,
                 LogicalUnion,
-                LogicalValues>(node)) {
+                LogicalValues,
+                LeftDeepInnerJoin>(node)) {
     return node->size();
   }
 
@@ -614,21 +615,22 @@ ExprPtrVector getNodeColumnRefs(const Node* node) {
     return genColumnRefs(node, getNodeColumnCount(node));
   }
 
-  if (is_one_of<Join>(node)) {
-    CHECK_EQ(size_t(2), node->inputCount());
-    auto lhs_out =
-        genColumnRefs(node->getInput(0), getNodeColumnCount(node->getInput(0)));
-    const auto rhs_out =
-        genColumnRefs(node->getInput(1), getNodeColumnCount(node->getInput(1)));
-    lhs_out.insert(lhs_out.end(), rhs_out.begin(), rhs_out.end());
-    return lhs_out;
+  if (is_one_of<Join, LeftDeepInnerJoin>(node)) {
+    auto res = genColumnRefs(node->getInput(0), node->getInput(0)->size());
+    for (size_t i = 1; i < node->inputCount(); ++i) {
+      auto input_refs = genColumnRefs(node->getInput(i), node->getInput(i)->size());
+      res.insert(res.end(), input_refs.begin(), input_refs.end());
+    }
+    return res;
   }
 
   LOG(FATAL) << "Unhandled ra_node type: " << ::toString(node);
   return {};
 }
 
 ExprPtr getNodeColumnRef(const Node* node, unsigned index) {
+  CHECK_LT(index, node->size());
+
   if (is_one_of<Scan,
                 Project,
                 Aggregate,
@@ -638,17 +640,19 @@ ExprPtr getNodeColumnRef(const Node* node, unsigned index) {
                 LogicalValues,
                 Filter,
                 Sort>(node)) {
-    CHECK_LT(index, node->size());
     return makeExpr<ColumnRef>(getColumnType(node, index), node, index);
   }
 
-  if (is_one_of<Join>(node)) {
-    CHECK_EQ(size_t(2), node->inputCount());
-    auto lhs_size = node->getInput(0)->size();
-    if (index < lhs_size) {
-      return getNodeColumnRef(node->getInput(0), index);
+  if (is_one_of<Join, LeftDeepInnerJoin>(node)) {
+    unsigned offs = 0;
+    for (size_t i = 0; i < node->inputCount(); ++i) {
+      auto input = node->getInput(i);
+      if (index - offs < input->size()) {
+        return getNodeColumnRef(input, index - offs);
+      }
+      offs += input->size();
     }
-    return getNodeColumnRef(node->getInput(1), index - lhs_size);
+    UNREACHABLE();
   }
 
   LOG(FATAL) << "Unhandled node type: " << ::toString(node);

omniscidb/IR/Node.h

@@ -81,6 +81,11 @@ class Node {
     return dynamic_cast<const T*>(this) != nullptr;
   }
 
+  template <typename T>
+  const T* as() const {
+    return dynamic_cast<const T*>(this);
+  }
+
   void resetQueryExecutionState() {
     context_data_ = nullptr;
     targets_metainfo_ = {};
@@ -1021,13 +1026,13 @@ class TableFunction : public Node {
  public:
   TableFunction(const std::string& function_name,
                 NodeInputs inputs,
-                std::vector<std::string>& fields,
+                std::vector<std::string> fields,
                 ExprPtrVector col_input_exprs,
                 ExprPtrVector table_func_input_exprs,
                 std::vector<TargetMetaInfo> tuple_type)
       : Node(std::move(inputs))
       , function_name_(function_name)
-      , fields_(fields)
+      , fields_(std::move(fields))
       , col_input_exprs_(col_input_exprs)
       , table_func_input_exprs_(table_func_input_exprs)
       , tuple_type_(std::move(tuple_type)) {}

omniscidb/OSDependent/Unix/CMakeLists.txt

@@ -3,5 +3,6 @@ if(BUILD_SHARED_LIBS)
 endif()
 
 add_library(OSDependent ${OSDEPENDENT_SOURCE_FILES})
+set_target_properties(OSDependent PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/..)
 
 target_link_libraries(OSDependent Logger)

omniscidb/QueryEngine/CMakeLists.txt

@@ -86,6 +86,7 @@ set(query_engine_source_files
     QueryRewrite.cpp
     QueryTemplateGenerator.cpp
     QueryExecutionContext.cpp
+    QueryExecutionSequence.cpp
     QueryMemoryInitializer.cpp
     RelAlgDagBuilder.cpp
     RelAlgExecutor.cpp
@@ -132,6 +133,7 @@ set(query_engine_source_files
     DataRecycler/HashtableRecycler.cpp
     DataRecycler/HashingSchemeRecycler.cpp
     Visitors/QueryPlanDagChecker.cpp
+    WorkUnitBuilder.cpp
 
     Codec.h
     Execute.h

omniscidb/QueryEngine/CastIR.cpp

@@ -273,6 +273,7 @@ llvm::Value* CodeGenerator::codegenCastFromString(llvm::Value* operand_lv,
                   executor()->getRowSetMemoryOwner()->getLiteralStringDictProxy())
             : reinterpret_cast<int64_t>(executor()->getStringDictionaryProxy(
                   operand_dict_id, executor()->getRowSetMemoryOwner(), true));
+    CHECK(string_dictionary_ptr);
     return cgen_state_->emitExternalCall(
         "string_decompress",
         get_int_type(64, cgen_state_->context_),

omniscidb/QueryEngine/CompilationOptions.h

@@ -141,6 +141,12 @@ struct ExecutionOptions {
     return eo;
   }
 
+  ExecutionOptions with_just_validate(bool enable = true) const {
+    ExecutionOptions eo = *this;
+    eo.just_validate = enable;
+    return eo;
+  }
+
  private:
   ExecutionOptions() {}
 };

omniscidb/QueryEngine/Descriptors/InputDescriptors.h

@@ -107,7 +107,7 @@ class InputColDescriptor {
   std::string toString() const {
     return ::typeName(this) + "(table_id=" + std::to_string(getTableId()) +
            ", nest_level=" + std::to_string(getNestLevel()) +
-           "col_id=" + std::to_string(getColId()) + (isVirtual() ? "[virt])" : ")");
+           ", col_id=" + std::to_string(getColId()) + (isVirtual() ? "[virt])" : ")");
   }
 
  private:

omniscidb/QueryEngine/Execute.cpp

@@ -755,11 +755,15 @@ std::vector<int8_t> Executor::serializeLiterals(
         const auto p = boost::get<std::pair<std::string, int>>(&lit);
         CHECK(p);
         const auto str_id =
-            config_->exec.enable_experimental_string_functions
+            (config_->exec.enable_experimental_string_functions ||
+             // With WorkUnitBuilder we might produce complex expressions where encoded
+             // literals are decoded back to plain strings. In this case, we cannot use
+             // INVALID_STR_ID and should add the literal to the dictionary.
+             config_->exec.use_legacy_work_unit_builder)
                 ? getStringDictionaryProxy(p->second, row_set_mem_owner_, true)
-                      ->getOrAddTransient(p->first)
+                      ->getIdOfString(p->first)
                 : getStringDictionaryProxy(p->second, row_set_mem_owner_, true)
-                      ->getIdOfString(p->first);
+                      ->getOrAddTransient(p->first);
         memcpy(&serialized[off - lit_bytes], &str_id, lit_bytes);
         break;
       }
@@ -2171,6 +2175,12 @@ void Executor::addTransientStringLiterals(
     visit_expr(group_expr.get());
   }
 
+  for (const auto& quals : ra_exe_unit.join_quals) {
+    for (const auto& qual_expr : quals.quals) {
+      visit_expr(qual_expr.get());
+    }
+  }
+
   for (const auto& group_expr : ra_exe_unit.simple_quals) {
     visit_expr(group_expr.get());
   }