ORC-1356: [C++] Use Intel AVX-512 instructions to accelerate the Rle-bit-packing decode

CMakeLists.txt

@@ -67,6 +67,10 @@ option(BUILD_CPP_ENABLE_METRICS
     "Enable the metrics collection at compile phase"
     OFF)
 
+option(BUILD_ENABLE_AVX512
+    "Enable AVX512 vector decode of bit-packing"
+    ON)
+
 # Make sure that a build type is selected
 if (NOT CMAKE_BUILD_TYPE)
   message(STATUS "No build type selected, default to ReleaseWithDebugInfo")
@@ -87,6 +91,17 @@ if (BUILD_POSITION_INDEPENDENT_LIB)
   set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 endif ()
 
+if(NOT DEFINED ORC_SIMD_LEVEL)
+  set(ORC_SIMD_LEVEL
+      "DEFAULT"
+      CACHE STRING "Compile time SIMD optimization level")
+endif()
+if(NOT DEFINED ORC_RUNTIME_SIMD_LEVEL)
+  set(ORC_RUNTIME_SIMD_LEVEL
+      "MAX"
+      CACHE STRING "Max runtime SIMD optimization level")
+endif()
+
 #
 # Compiler specific flags
 #
@@ -116,7 +131,7 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
   set (WARN_FLAGS "${WARN_FLAGS} -Wno-covered-switch-default")
   set (WARN_FLAGS "${WARN_FLAGS} -Wno-missing-noreturn -Wno-unknown-pragmas")
   set (WARN_FLAGS "${WARN_FLAGS} -Wno-gnu-zero-variadic-macro-arguments")
-  set (WARN_FLAGS "${WARN_FLAGS} -Wconversion")
+  set (WARN_FLAGS "${WARN_FLAGS} -Wno-conversion")
   if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "13.0")
     set (WARN_FLAGS "${WARN_FLAGS} -Wno-reserved-identifier -Wno-suggest-destructor-override -Wno-suggest-override")
   endif()
@@ -135,7 +150,7 @@ elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
   else ()
     set (CXX17_FLAGS "-std=c++17")
   endif ()
-  set (WARN_FLAGS "-Wall -Wno-unknown-pragmas -Wconversion")
+  set (WARN_FLAGS "-Wall -Wno-unknown-pragmas -Wno-conversion")
   if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER "12.0")
     set (WARN_FLAGS "${WARN_FLAGS} -Wno-array-bounds -Wno-stringop-overread") # To compile protobuf in Fedora37
   endif ()
@@ -157,6 +172,134 @@ elseif (MSVC)
   set (WARN_FLAGS "${WARN_FLAGS} -wd4146") # unary minus operator applied to unsigned type, result still unsigned
 endif ()
 
+include(CheckCXXCompilerFlag)
+include(CheckCXXSourceCompiles)
+message(STATUS "System processor: ${CMAKE_SYSTEM_PROCESSOR}")
+
+if(NOT DEFINED ORC_CPU_FLAG)
+  if(CMAKE_SYSTEM_PROCESSOR MATCHES "AMD64|X86|x86|i[3456]86|x64")
+    set(ORC_CPU_FLAG "x86")
+  elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|ARM64|arm64")
+    set(ORC_CPU_FLAG "aarch64")
+  elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^arm$|armv[4-7]")
+    set(ORC_CPU_FLAG "aarch32")
+  elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "powerpc|ppc")
+    set(ORC_CPU_FLAG "ppc")
+  elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "s390x")
+    set(ORC_CPU_FLAG "s390x")
+  elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "riscv64")
+    set(ORC_CPU_FLAG "riscv64")
+  else()
+    message(FATAL_ERROR "Unknown system processor")
+  endif()
+endif()
+
+# Check architecture specific compiler flags
+if(ORC_CPU_FLAG STREQUAL "x86")
+  # x86/amd64 compiler flags, msvc/gcc/clang
+  if(MSVC)
+    set(ORC_SSE4_2_FLAG "")
+    set(ORC_AVX2_FLAG "/arch:AVX2")
+    set(ORC_AVX512_FLAG "/arch:AVX512")
+    set(CXX_SUPPORTS_SSE4_2 TRUE)
+  else()
+    set(ORC_SSE4_2_FLAG "-msse4.2")
+    set(ORC_AVX2_FLAG "-march=haswell")
+    # skylake-avx512 consists of AVX512F,AVX512BW,AVX512VL,AVX512CD,AVX512DQ
+    set(ORC_AVX512_FLAG "-march=native -mbmi2")
+    # Append the avx2/avx512 subset option also, fix issue ORC-9877 for homebrew-cpp
+    set(ORC_AVX2_FLAG "${ORC_AVX2_FLAG} -mavx2")
+    set(ORC_AVX512_FLAG
+        "${ORC_AVX512_FLAG} -mavx512f -mavx512cd -mavx512vl -mavx512dq -mavx512bw -mavx512vbmi")
+  endif()
+  check_cxx_compiler_flag(${ORC_AVX512_FLAG} CXX_SUPPORTS_AVX512)
+  if(MINGW)
+    # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65782
+    message(STATUS "Disable AVX512 support on MINGW for now")
+  else()
+    # Check for AVX512 support in the compiler.
+    set(OLD_CMAKE_REQURED_FLAGS ${CMAKE_REQUIRED_FLAGS})
+    set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${ORC_AVX512_FLAG}")
+    check_cxx_source_compiles("
+      #ifdef _MSC_VER
+      #include <intrin.h>
+      #else
+      #include <immintrin.h>
+      #endif
+
+      int main() {
+        __m512i mask = _mm512_set1_epi32(0x1);
+        char out[32];
+        _mm512_storeu_si512(out, mask);
+        return 0;
+      }"
+      CXX_SUPPORTS_AVX512)
+    set(CMAKE_REQUIRED_FLAGS ${OLD_CMAKE_REQURED_FLAGS})
+  endif()
+  # Runtime SIMD level it can get from compiler and ORC_RUNTIME_SIMD_LEVEL
+  if(CXX_SUPPORTS_SSE4_2 AND ORC_RUNTIME_SIMD_LEVEL MATCHES
+                             "^(SSE4_2|AVX2|AVX512|MAX)$")
+    set(ORC_HAVE_RUNTIME_SSE4_2 ON)
+    add_definitions(-DORC_HAVE_RUNTIME_SSE4_2)
+  endif()
+  if(CXX_SUPPORTS_AVX2 AND ORC_RUNTIME_SIMD_LEVEL MATCHES "^(AVX2|AVX512|MAX)$")
+    set(ORC_HAVE_RUNTIME_AVX2 ON)
+    add_definitions(-DORC_HAVE_RUNTIME_AVX2 -DORC_HAVE_RUNTIME_BMI2)
+  endif()
+  if(CXX_SUPPORTS_AVX512 AND ORC_RUNTIME_SIMD_LEVEL MATCHES "^(AVX512|MAX)$")
+    set(ORC_HAVE_RUNTIME_AVX512 ON)
+    add_definitions(-DORC_HAVE_RUNTIME_AVX512 -DORC_HAVE_RUNTIME_BMI2)
+  endif()
+  if(ORC_SIMD_LEVEL STREQUAL "DEFAULT")
+    set(ORC_SIMD_LEVEL "AVX512")
+  endif()
+
+elseif(ORC_CPU_FLAG STREQUAL "ppc")
+  # power compiler flags, gcc/clang only
+  set(ORC_ALTIVEC_FLAG "-maltivec")
+  check_cxx_compiler_flag(${ORC_ALTIVEC_FLAG} CXX_SUPPORTS_ALTIVEC)
+  if(ORC_SIMD_LEVEL STREQUAL "DEFAULT")
+    set(ORC_SIMD_LEVEL "NONE")
+  endif()
+elseif(ORC_CPU_FLAG STREQUAL "aarch64")
+  # Arm64 compiler flags, gcc/clang only
+  set(ORC_ARMV8_MARCH "armv8-a")
+  check_cxx_compiler_flag("-march=${ORC_ARMV8_MARCH}+sve" CXX_SUPPORTS_SVE)
+  if(ORC_SIMD_LEVEL STREQUAL "DEFAULT")
+    set(ORC_SIMD_LEVEL "NEON")
+  endif()
+endif()
+
+# Only enable additional instruction sets if they are supported
+if(ORC_CPU_FLAG STREQUAL "x86")
+  if(MINGW)
+    # Enable _xgetbv() intrinsic to query OS support for ZMM register saves
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mxsave")
+  endif()
+  if(ORC_SIMD_LEVEL STREQUAL "AVX512")
+    if(NOT CXX_SUPPORTS_AVX512)
+      message(FATAL_ERROR "AVX512 required but compiler doesn't support it.")
+    endif()
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${ORC_AVX512_FLAG}")
+    add_definitions(-DORC_HAVE_AVX512 -DORC_HAVE_AVX2 -DORC_HAVE_BMI2
+                    -DORC_HAVE_SSE4_2)
+  elseif(ORC_SIMD_LEVEL STREQUAL "AVX2")
+    if(NOT CXX_SUPPORTS_AVX2)
+      message(FATAL_ERROR "AVX2 required but compiler doesn't support it.")
+    endif()
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${ORC_AVX2_FLAG}")
+    add_definitions(-DORC_HAVE_AVX2 -DORC_HAVE_BMI2 -DORC_HAVE_SSE4_2)
+  elseif(ORC_SIMD_LEVEL STREQUAL "SSE4_2")
+    if(NOT CXX_SUPPORTS_SSE4_2)
+      message(FATAL_ERROR "SSE4.2 required but compiler doesn't support it.")
+    endif()
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${ORC_SSE4_2_FLAG}")
+    add_definitions(-DORC_HAVE_SSE4_2)
+  elseif(NOT ORC_SIMD_LEVEL STREQUAL "NONE")
+    message(WARNING "ORC_SIMD_LEVEL=${ORC_SIMD_LEVEL} not supported by x86.")
+  endif()
+endif()
+
 if (BUILD_CPP_ENABLE_METRICS)
   message(STATUS "Enable the metrics collection")
   add_compile_definitions(ENABLE_METRICS=1)
@@ -165,6 +308,14 @@ else ()
   add_compile_definitions(ENABLE_METRICS=0)
 endif ()
 
+if (BUILD_ENABLE_AVX512 AND CXX_SUPPORTS_AVX512 AND ORC_SIMD_LEVEL STREQUAL "AVX512")
+  message(STATUS "Enable the AVX512 vector decode of bit-packing")
+  add_compile_definitions(ENABLE_AVX512=1)
+else ()
+  message(STATUS "Disable the AVX512 vector decode of bit-packing")
+  add_compile_definitions(ENABLE_AVX512=0)
+endif ()
+
 enable_testing()
 
 INCLUDE(CheckSourceCompiles)

c++/src/DetectPlatform.hh

@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ORC_DETECTPLATFORM_HH
+#define ORC_DETECTPLATFORM_HH
+
+#if defined(__GNUC__) || defined(__clang__)
+  DIAGNOSTIC_IGNORE("-Wold-style-cast")
+#endif
+
+namespace orc
+{
+#ifdef _WIN32
+
+#include "intrin.h"
+//  Windows CPUID
+#define cpuid(info, x)    __cpuidex(info, x, 0)
+#else
+//  GCC Intrinsics 
+#include <cpuid.h>
+#include <dlfcn.h>
+
+  void cpuid(int info[4], int InfoType) {
+    __cpuid_count(InfoType, 0, info[0], info[1], info[2], info[3]);
+  }
+
+  unsigned long long xgetbv(unsigned int index) {
+    unsigned int eax, edx;
+    __asm__ __volatile__(
+    "xgetbv;"
+    : "=a" (eax), "=d"(edx)
+    : "c" (index)
+    );
+    return ((unsigned long long) edx << 32) | eax;
+  }
+
+#endif
+
+  #define CPUID_AVX512F       0x00100000
+  #define CPUID_AVX512CD      0x00200000
+  #define CPUID_AVX512VL      0x04000000
+  #define CPUID_AVX512BW      0x01000000
+  #define CPUID_AVX512DQ      0x02000000
+  #define EXC_OSXSAVE         0x08000000 // 27th  bit
+
+  #define CPUID_AVX512_MASK (CPUID_AVX512F | CPUID_AVX512CD | CPUID_AVX512VL | CPUID_AVX512BW | CPUID_AVX512DQ)
+
+  enum class Arch {
+    PX_ARCH     = 0,
+    AVX2_ARCH   = 1,
+    AVX512_ARCH = 2
+  };
+
+  Arch detectPlatform() {
+    Arch detected_platform = Arch::PX_ARCH;
+    int    cpuInfo[4];
+    cpuid(cpuInfo, 1);
+
+    bool avx512_support_cpu   = cpuInfo[1] & CPUID_AVX512_MASK;
+    bool os_uses_XSAVE_XSTORE = cpuInfo[2] & EXC_OSXSAVE;
+
+    if (avx512_support_cpu && os_uses_XSAVE_XSTORE) {
+      // Check if XMM state and YMM state are saved
+#ifdef _WIN32
+      unsigned long long xcr_feature_mask = _xgetbv(0);  /* min VS2010 SP1 compiler is required */
+#else
+      unsigned long long xcr_feature_mask = xgetbv(0);
+#endif
+
+      if ((xcr_feature_mask & 0x6) == 0x6) { // AVX2 is supported now
+        if ((xcr_feature_mask & 0xe0) == 0xe0) { // AVX512 is supported now
+            detected_platform = Arch::AVX512_ARCH;
+        }
+      }
+    }
+
+    return detected_platform;
+  }
+}
+
+#endif

c++/src/RLEv2.hh

@@ -25,6 +25,9 @@
 
 #include <vector>
 
+#define MAX_VECTOR_BUF_8BIT_LENGTH 64
+#define MAX_VECTOR_BUF_16BIT_LENGTH 32
+#define MAX_VECTOR_BUF_32BIT_LENGTH 16
 #define MAX_LITERAL_SIZE 512
 #define MIN_REPEAT 3
 #define HIST_LEN 32
@@ -189,13 +192,45 @@ namespace orc {
       resetReadLongs();
     }
 
+    void resetBufferStart(uint64_t len, bool resetBuf, uint32_t backupLen);
     unsigned char readByte();
 
     int64_t readLongBE(uint64_t bsz);
     int64_t readVslong();
     uint64_t readVulong();
     void readLongs(int64_t* data, uint64_t offset, uint64_t len, uint64_t fbs);
-    void plainUnpackLongs(int64_t* data, uint64_t offset, uint64_t len, uint64_t fbs);
+    void plainUnpackLongs(int64_t *data, uint64_t offset, uint64_t len, uint64_t fbs,
+                        uint64_t& startBit);
+
+#if ENABLE_AVX512
+    void unrolledUnpackVector1(int64_t *data, uint64_t offset, uint64_t len);
+    void unrolledUnpackVector2(int64_t *data, uint64_t offset, uint64_t len);
+    void unrolledUnpackVector3(int64_t *data, uint64_t offset, uint64_t len);
+    void unrolledUnpackVector4(int64_t *data, uint64_t offset, uint64_t len);
+    void unrolledUnpackVector5(int64_t *data, uint64_t offset, uint64_t len);
+    void unrolledUnpackVector6(int64_t *data, uint64_t offset, uint64_t len);
+    void unrolledUnpackVector7(int64_t *data, uint64_t offset, uint64_t len);
+    void unrolledUnpackVector9(int64_t *data, uint64_t offset, uint64_t len);
+    void unrolledUnpackVector10(int64_t *data, uint64_t offset, uint64_t len);
+    void unrolledUnpackVector11(int64_t *data, uint64_t offset, uint64_t len);
+    void unrolledUnpackVector12(int64_t *data, uint64_t offset, uint64_t len);
+    void unrolledUnpackVector13(int64_t *data, uint64_t offset, uint64_t len);
+    void unrolledUnpackVector14(int64_t *data, uint64_t offset, uint64_t len);
+    void unrolledUnpackVector15(int64_t *data, uint64_t offset, uint64_t len);
+    void unrolledUnpackVector16(int64_t *data, uint64_t offset, uint64_t len);
+    void unrolledUnpackVector17(int64_t *data, uint64_t offset, uint64_t len);
+    void unrolledUnpackVector18(int64_t *data, uint64_t offset, uint64_t len);
+    void unrolledUnpackVector19(int64_t *data, uint64_t offset, uint64_t len);
+    void unrolledUnpackVector20(int64_t *data, uint64_t offset, uint64_t len);
+    void unrolledUnpackVector21(int64_t *data, uint64_t offset, uint64_t len);
+    void unrolledUnpackVector22(int64_t *data, uint64_t offset, uint64_t len);
+    void unrolledUnpackVector23(int64_t *data, uint64_t offset, uint64_t len);
+    void unrolledUnpackVector24(int64_t *data, uint64_t offset, uint64_t len);
+    void unrolledUnpackVector26(int64_t *data, uint64_t offset, uint64_t len);
+    void unrolledUnpackVector28(int64_t *data, uint64_t offset, uint64_t len);
+    void unrolledUnpackVector30(int64_t *data, uint64_t offset, uint64_t len);
+    void unrolledUnpackVector32(int64_t *data, uint64_t offset, uint64_t len);
+#endif
 
     void unrolledUnpack4(int64_t* data, uint64_t offset, uint64_t len);
     void unrolledUnpack8(int64_t* data, uint64_t offset, uint64_t len);
@@ -230,6 +265,11 @@ namespace orc {
     uint32_t curByte;                   // Used by anything that uses readLongs
     DataBuffer<int64_t> unpackedPatch;  // Used by PATCHED_BASE
     DataBuffer<int64_t> literals;       // Values of the current run
+#if ENABLE_AVX512
+    uint8_t vectorBuf8[MAX_VECTOR_BUF_8BIT_LENGTH + 1]; // Used by vectorially 1~8 bit-unpacking data
+    uint16_t vectorBuf16[MAX_VECTOR_BUF_16BIT_LENGTH + 1]; // Used by vectorially 9~16 bit-unpacking data
+    uint32_t vectorBuf32[MAX_VECTOR_BUF_32BIT_LENGTH + 1]; // Used by vectorially 17~32 bit-unpacking data
+#endif
   };
 }  // namespace orc