Enable folly

.github/workflows/main.yml

@@ -29,12 +29,12 @@ jobs:
       cache-suffix: ${{ needs.build.outputs.cache-suffix }}
       test: style
 
-  smoke:
+  sanity:
     needs: build
     uses: ./.github/workflows/test.yml
     with:
       cache-suffix: ${{ needs.build.outputs.cache-suffix }}
-      test: smoke
+      test: sanity
 
   asan:
     needs: build
@@ -78,8 +78,9 @@ jobs:
       name: l0
       reset-cache: ${{ !!inputs.reset-cache }}
 
-  sanity-test:
-    uses: ./.github/workflows/build-docker-hdk.yml
+#
+#  sanity-test:
+#    uses: ./.github/workflows/build-docker-hdk.yml
 
   pytest:
     uses: ./.github/workflows/pytest.yml

.github/workflows/test-docker.yml

@@ -50,7 +50,7 @@ jobs:
           docker exec hdk-build.${{ inputs.name }} chown -R ghrunner:ghrunner /_work/
           docker exec -u ghrunner hdk-build.${{ inputs.name }} tar -zxf /_work/build.tgz -C /_work/
 
-      - name: Smoke test
+      - name: sanity test
         if: inputs.name == 'cuda'
         run: |
           docker exec -u ghrunner hdk-build.${{ inputs.name }} dpkg -l

.github/workflows/test-l0-docker.yml

@@ -52,7 +52,7 @@ jobs:
           docker exec hdk-build.${{ inputs.name }} chown -R ghrunner:ghrunner /_work/
           docker exec -u ghrunner hdk-build.${{ inputs.name }} tar -zxf /_work/build.tgz -C /_work/
 
-      - name: Smoke test
+      - name: sanity test
         if: inputs.name == 'l0'
         run: |
           docker exec -u ghrunner hdk-build.${{ inputs.name }} dpkg -l

.github/workflows/test.yml

@@ -50,16 +50,16 @@ jobs:
       #   NoCatalogSqlTest
       # that's not possible to manage instance memory with standard actions
       - name: Set Swap Space
-        if: inputs.test == 'smoke' || inputs.test == 'asan'
+        if: inputs.test == 'asan'
         uses: pierotofy/set-swap-space@master
         with:
           swap-size-gb: 12
 
-      - name: Run smoke tests
-        if: inputs.test == 'smoke'
+      - name: Run sanity tests
+        if: inputs.test == 'sanity'
+        # skip large buffer tests due to limited memory on ghrunnners instances 
         run: |
-          sudo sh -c 'echo 2 >/proc/sys/vm/overcommit_memory'
-          conda run --no-capture-output -n omnisci-dev bash omniscidb/scripts/conda/test.sh
+          conda run --no-capture-output -n omnisci-dev bash omniscidb/scripts/conda/test.sh -s 
 
       - name: Run ASAN test
         if: inputs.test == 'asan'

CMakeLists.txt

@@ -49,6 +49,10 @@ if (NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
             "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
 endif ()
 
+# External Dependencies
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake_modules")
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/omniscidb/cmake/Modules")
+
 set(ENABLE_CONDA OFF)
 if(DEFINED ENV{CONDA_PREFIX})
   set(ENABLE_CONDA ON)
@@ -83,6 +87,25 @@ configure_file(
     )
 include_directories(${CMAKE_BINARY_DIR})
 
+# Folly
+option(ENABLE_FOLLY "Use Folly" ON)
+if(ENABLE_FOLLY)
+  find_package(Folly)
+  if(NOT Folly_FOUND)
+    set(ENABLE_FOLLY OFF CACHE BOOL "Use Folly" FORCE)
+  else()
+    set(FOLLY_LIBRARIES "")
+    add_definitions("-DHAVE_FOLLY")
+    list(APPEND Folly_LIBRARIES Folly::folly ${Glog_LIBRARIES})
+    # TODO: use Folly::folly_deps?
+    if(MSVC)
+      find_package(Libevent COMPONENTS core REQUIRED)
+      list(APPEND Folly_LIBRARIES libevent::core)
+    endif()
+  endif()
+endif()
+
+
 # SQLite
 include_directories(omniscidb/ThirdParty/sqlite3)
 add_subdirectory(omniscidb/ThirdParty/sqlite3)
@@ -100,10 +123,6 @@ endif()
 # Copy ThirdParty to build dir so OmniSciDB dependencies can be copied over. Note that third_party is available internally for HDK specific dependencies.
 file(COPY "${CMAKE_SOURCE_DIR}/ThirdParty" DESTINATION "${CMAKE_BINARY_DIR}/")
 
-# External Dependencies
-list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake_modules")
-list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/omniscidb/cmake/Modules")
-
 # Google log
 add_subdirectory(third_party/glog-0.5.0 EXCLUDE_FROM_ALL)
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/third_party/glog-0.5.0/cmake")

omniscidb/CMakeLists.txt

@@ -313,9 +313,9 @@ if(ENABLE_FOLLY)
   if(NOT Folly_FOUND)
     set(ENABLE_FOLLY OFF CACHE BOOL "Use Folly" FORCE)
   else()
-    include_directories(${Folly_INCLUDE_DIRS})
+    set(FOLLY_LIBRARIES "")
     add_definitions("-DHAVE_FOLLY")
-    list(APPEND Folly_LIBRARIES ${Glog_LIBRARIES})
+    list(APPEND Folly_LIBRARIES Folly::folly ${Glog_LIBRARIES})
     # TODO: use Folly::folly_deps?
     if(MSVC)
       find_package(Libevent COMPONENTS core REQUIRED)

omniscidb/DataMgr/CMakeLists.txt

@@ -43,7 +43,11 @@ endif()
 
 add_library(DataMgr ${datamgr_source_files})
 
-target_link_libraries(DataMgr CudaMgr L0Mgr Shared SchemaMgr IR ${Boost_THREAD_LIBRARY} TBB::tbb ${CMAKE_DL_LIBS})
+set(DataMgrLibs CudaMgr L0Mgr Shared SchemaMgr IR ${Boost_THREAD_LIBRARY} TBB::tbb ${CMAKE_DL_LIBS})
+if(ENABLE_FOLLY)
+  list(APPEND DataMgrLibs ${Folly_LIBRARIES})
+endif()
+target_link_libraries(DataMgr ${DataMgrLibs})
 
 option(ENABLE_CRASH_CORRUPTION_TEST "Enable crash using SIGUSR2 during page deletion to faster and affirmative test/repro db corruption" OFF)
 if(ENABLE_CRASH_CORRUPTION_TEST)

omniscidb/Tests/ResultSetTest.cpp

@@ -37,6 +37,7 @@
 
 #include <gtest/gtest.h>
 #include <algorithm>
+#include <cstdlib>
 #include <filesystem>
 #include <queue>
 #include <random>
@@ -1901,9 +1902,16 @@ TEST(Reduce, BaselineHashColumnar) {
   test_reduce(target_infos, query_mem_desc, generator1, generator2, 1, true);
 }
 
+#define SKIP_LARGE_BUFFERS()               \
+  if (std::getenv("SKIP_LARGE_BUFFERS")) { \
+    GTEST_SKIP();                          \
+  }
+
 #ifndef HAVE_TSAN
 // The large buffers tests allocate too much memory to instrument under TSAN
 TEST(ReduceLargeBuffers, PerfectHashOne_Overflow32) {
+  SKIP_LARGE_BUFFERS();
+
   try {
     const auto target_infos = generate_random_groups_nullable_target_infos();
     auto query_mem_desc = perfect_hash_one_col_desc(target_infos, 8, 0, 222208903, {8});
@@ -1917,6 +1925,8 @@ TEST(ReduceLargeBuffers, PerfectHashOne_Overflow32) {
 }
 
 TEST(ReduceLargeBuffers, PerfectHashColumnarOne_Overflow32) {
+  SKIP_LARGE_BUFFERS();
+
   try {
     const auto target_infos = generate_random_groups_nullable_target_infos();
     auto query_mem_desc = perfect_hash_one_col_desc(target_infos, 8, 0, 222208903, {8});
@@ -1932,6 +1942,8 @@ TEST(ReduceLargeBuffers, PerfectHashColumnarOne_Overflow32) {
 }
 
 TEST(ReduceLargeBuffers, BaselineHash_Overflow32) {
+  SKIP_LARGE_BUFFERS();
+
   try {
     const auto target_infos = generate_random_groups_nullable_target_infos();
     auto query_mem_desc = baseline_hash_two_col_desc_overflow32(target_infos, 8);
@@ -1945,6 +1957,8 @@ TEST(ReduceLargeBuffers, BaselineHash_Overflow32) {
 }
 
 TEST(ReduceLargeBuffers, BaselineHashColumnar_Overflow32) {
+  SKIP_LARGE_BUFFERS();
+
   try {
     const auto target_infos = generate_random_groups_nullable_target_infos();
     auto query_mem_desc = baseline_hash_two_col_desc_overflow32(target_infos, 8);

omniscidb/scripts/conda/test.sh

@@ -1,7 +1,25 @@
 #!/bin/bash
 
+usage() { echo "Usage: $0 [-s (skip large buffers)]" 1>&2; exit 1; }
+
+TEST_FLAGS=''
+
+while getopts "s" o; do
+    case "${o}" in
+        s)
+            TEST_FLAGS='SKIP_LARGE_BUFFERS=true'
+            ;;
+        *)
+            usage
+            ;;
+    esac
+done
+shift $((OPTIND-1))
+
 set -ex
 
+export ${TEST_FLAGS}
+
 cd $(dirname "$0")
 
 # Omnisci UDF support uses CLangTool for parsing Load-time UDF C++

python/CMakeLists.txt

@@ -18,6 +18,10 @@ set(pydeps
 set(SETUP_LDFLAGS "-L$<TARGET_FILE_DIR:Calcite> -L$<TARGET_FILE_DIR:IR> -L$<TARGET_FILE_DIR:ArrowStorage> -L$<TARGET_FILE_DIR:QueryBuilder> -L$<TARGET_FILE_DIR:QueryEngine> -L$<TARGET_FILE_DIR:SchemaMgr> -L$<TARGET_FILE_DIR:ConfigBuilder> -L$<TARGET_FILE_DIR:Logger> -L$<TARGET_FILE_DIR:Shared> -L$<TARGET_FILE_DIR:DataMgr>")
 set(SETUP_FLAGS -g -f -I ${CMAKE_CURRENT_SOURCE_DIR})
 
+if(ENABLE_FOLLY)
+   list(APPEND SETUP_FLAGS "-DHAVE_FOLLY") 
+endif()
+
 set(SETUP_INSTALL_FLAGS "--prefix \${CMAKE_INSTALL_PREFIX}")
 
 add_custom_target(pyhdk ALL

python/setup.py.in

@@ -119,5 +119,6 @@ setup(
             "language_level": "3",
         },
         include_path=["@CMAKE_CURRENT_SOURCE_DIR@"],
+        gdb_debug=False, # TODO: drive this via cmake build type 
     ),
 )