ARROW-6549: [C++] Switch to jemalloc 5.2.x

Revert "ARROW-6478: [C++] Revert to jemalloc stable-4 until we understand 5.2.x performance issues" This reverts commit 53c5af0. In addition, configure jemalloc to fix the performance regression.
apache · Sep 12, 2019 · e845e32 · e845e32
1 parent 5a1d986
commit e845e32
Show file tree

Hide file tree

Showing 5 changed files with 25 additions and 27 deletions.
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -1408,16 +1408,20 @@ if(ARROW_JEMALLOC)
       "${JEMALLOC_PREFIX}/lib/libjemalloc_pic${CMAKE_STATIC_LIBRARY_SUFFIX}")
   externalproject_add(
     jemalloc_ep
-    URL ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/jemalloc/${JEMALLOC_VERSION}.tar.gz
+    URL ${JEMALLOC_SOURCE_URL}
     PATCH_COMMAND touch doc/jemalloc.3 doc/jemalloc.html
-    CONFIGURE_COMMAND ./autogen.sh
+    # The prefix "je_arrow_" must be kept in sync with the value in memory_pool.cc
+    CONFIGURE_COMMAND ./configure
                       "AR=${CMAKE_AR}"
                       "CC=${CMAKE_C_COMPILER}"
                       "--prefix=${JEMALLOC_PREFIX}"
                       "--with-jemalloc-prefix=je_arrow_"
                       "--with-private-namespace=je_arrow_private_"
-                      "--disable-tls"
-                      ${EP_LOG_OPTIONS}
+                      "--without-export"
+                      # Don't override operator new()
+                      "--disable-cxx" "--disable-libdl"
+                      # See https://github.com/jemalloc/jemalloc/issues/1237
+                      "--disable-initial-exec-tls" ${EP_LOG_OPTIONS}
     BUILD_IN_SOURCE 1
     BUILD_COMMAND ${MAKE} ${MAKE_BUILD_ARGS}
     BUILD_BYPRODUCTS "${JEMALLOC_STATIC_LIB}"

diff --git a/cpp/src/arrow/memory_pool.cc b/cpp/src/arrow/memory_pool.cc
@@ -38,6 +38,21 @@
 #include <mimalloc.h>
 #endif
 
+#ifdef ARROW_JEMALLOC
+
+// Compile-time configuration for jemalloc options.
+// Note the prefix ("je_arrow_") must match the symbol prefix given when
+// building jemalloc.
+// See discussion in https://github.com/jemalloc/jemalloc/issues/1621
+
+#ifdef NDEBUG
+const char* je_arrow_malloc_conf = "oversize_threshold:0";
+#else
+// In debug mode, add memory poisoning on alloc / free
+const char* je_arrow_malloc_conf = "oversize_threshold:0,junk:true";
+#endif
+#endif
+
 namespace arrow {
 
 constexpr size_t kAlignment = 64;

diff --git a/cpp/thirdparty/jemalloc/17c897976c60b0e6e4f4a365c751027244dada7a.tar.gz b/cpp/thirdparty/jemalloc/17c897976c60b0e6e4f4a365c751027244dada7a.tar.gz
diff --git a/cpp/thirdparty/jemalloc/README.md b/cpp/thirdparty/jemalloc/README.md
diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt
@@ -35,7 +35,7 @@ GFLAGS_VERSION=v2.2.0
 GLOG_VERSION=v0.3.5
 GRPC_VERSION=v1.20.0
 GTEST_VERSION=1.8.1
-JEMALLOC_VERSION=17c897976c60b0e6e4f4a365c751027244dada7a
+JEMALLOC_VERSION=5.2.1
 LZ4_VERSION=v1.8.3
 # Need post-v1.0.6 version for https://github.com/microsoft/mimalloc/pull/140,
 # https://github.com/microsoft/mimalloc/pull/145 and
@@ -72,6 +72,7 @@ DEPENDENCIES=(
   "ARROW_GLOG_URL glog-${GLOG_VERSION}.tar.gz https://github.com/google/glog/archive/${GLOG_VERSION}.tar.gz"
   "ARROW_GRPC_URL grpc-${GRPC_VERSION}.tar.gz https://github.com/grpc/grpc/archive/${GRPC_VERSION}.tar.gz"
   "ARROW_GTEST_URL gtest-${GTEST_VERSION}.tar.gz https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz"
+  "ARROW_JEMALLOC_URL jemalloc-${JEMALLOC_VERSION}.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2"
   "ARROW_LZ4_URL lz4-${LZ4_VERSION}.tar.gz https://github.com/lz4/lz4/archive/${LZ4_VERSION}.tar.gz"
   "ARROW_MIMALLOC_URL mimalloc-${MIMALLOC_VERSION}.tar.gz https://github.com/microsoft/mimalloc/archive/${MIMALLOC_VERSION}.tar.gz"
   "ARROW_ORC_URL orc-${ORC_VERSION}.tar.gz https://github.com/apache/orc/archive/rel/release-${ORC_VERSION}.tar.gz"