diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 05e5e8e0cd2f6..e8e4fa6f7bc7b 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -1408,16 +1408,21 @@ if(ARROW_JEMALLOC)
       "${JEMALLOC_PREFIX}/lib/libjemalloc_pic${CMAKE_STATIC_LIBRARY_SUFFIX}")
   externalproject_add(
     jemalloc_ep
-    URL ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/jemalloc/${JEMALLOC_VERSION}.tar.gz
-    PATCH_COMMAND touch doc/jemalloc.3 doc/jemalloc.html
-    CONFIGURE_COMMAND ./autogen.sh
+    URL ${JEMALLOC_SOURCE_URL}
+    PATCH_COMMAND
+      touch doc/jemalloc.3 doc/jemalloc.html
+      # The prefix "je_arrow_" must be kept in sync with the value in memory_pool.cc
+    CONFIGURE_COMMAND ./configure
                       "AR=${CMAKE_AR}"
                       "CC=${CMAKE_C_COMPILER}"
                       "--prefix=${JEMALLOC_PREFIX}"
                       "--with-jemalloc-prefix=je_arrow_"
                       "--with-private-namespace=je_arrow_private_"
-                      "--disable-tls"
-                      ${EP_LOG_OPTIONS}
+                      "--without-export"
+                      # Don't override operator new()
+                      "--disable-cxx" "--disable-libdl"
+                      # See https://github.com/jemalloc/jemalloc/issues/1237
+                      "--disable-initial-exec-tls" ${EP_LOG_OPTIONS}
     BUILD_IN_SOURCE 1
     BUILD_COMMAND ${MAKE} ${MAKE_BUILD_ARGS}
     BUILD_BYPRODUCTS "${JEMALLOC_STATIC_LIB}"
diff --git a/cpp/src/arrow/memory_pool.cc b/cpp/src/arrow/memory_pool.cc
index e77d4922ca4c0..6eea51cb2e29c 100644
--- a/cpp/src/arrow/memory_pool.cc
+++ b/cpp/src/arrow/memory_pool.cc
@@ -38,6 +38,21 @@
 #include <mimalloc.h>
 #endif
 
+#ifdef ARROW_JEMALLOC
+
+// Compile-time configuration for jemalloc options.
+// Note the prefix ("je_arrow_") must match the symbol prefix given when
+// building jemalloc.
+// See discussion in https://github.com/jemalloc/jemalloc/issues/1621
+
+#ifdef NDEBUG
+const char* je_arrow_malloc_conf = "oversize_threshold:0";
+#else
+// In debug mode, add memory poisoning on alloc / free
+const char* je_arrow_malloc_conf = "oversize_threshold:0,junk:true";
+#endif
+#endif
+
 namespace arrow {
 
 constexpr size_t kAlignment = 64;
diff --git a/cpp/thirdparty/jemalloc/17c897976c60b0e6e4f4a365c751027244dada7a.tar.gz b/cpp/thirdparty/jemalloc/17c897976c60b0e6e4f4a365c751027244dada7a.tar.gz
deleted file mode 100644
index 29d9266a12ded..0000000000000
Binary files a/cpp/thirdparty/jemalloc/17c897976c60b0e6e4f4a365c751027244dada7a.tar.gz and /dev/null differ
diff --git a/cpp/thirdparty/jemalloc/README.md b/cpp/thirdparty/jemalloc/README.md
deleted file mode 100644
index 272ff9c730be1..0000000000000
--- a/cpp/thirdparty/jemalloc/README.md
+++ /dev/null
@@ -1,22 +0,0 @@
-<!---
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing,
-  software distributed under the License is distributed on an
-  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  KIND, either express or implied.  See the License for the
-  specific language governing permissions and limitations
-  under the License.
--->
-
-This directory contains a vendored commit from the jemalloc stable-4 branch.
-You can bump the version by downloading
-https://github.com/jemalloc/jemalloc/archive/{{ commit }}.tar.gz
diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt
index c60c4be28e5e0..5add0d6e41956 100644
--- a/cpp/thirdparty/versions.txt
+++ b/cpp/thirdparty/versions.txt
@@ -35,7 +35,7 @@ GFLAGS_VERSION=v2.2.0
 GLOG_VERSION=v0.3.5
 GRPC_VERSION=v1.20.0
 GTEST_VERSION=1.8.1
-JEMALLOC_VERSION=17c897976c60b0e6e4f4a365c751027244dada7a
+JEMALLOC_VERSION=5.2.1
 LZ4_VERSION=v1.8.3
 # Need post-v1.0.6 version for https://github.com/microsoft/mimalloc/pull/140,
 # https://github.com/microsoft/mimalloc/pull/145 and
@@ -72,6 +72,7 @@ DEPENDENCIES=(
   "ARROW_GLOG_URL glog-${GLOG_VERSION}.tar.gz https://github.com/google/glog/archive/${GLOG_VERSION}.tar.gz"
   "ARROW_GRPC_URL grpc-${GRPC_VERSION}.tar.gz https://github.com/grpc/grpc/archive/${GRPC_VERSION}.tar.gz"
   "ARROW_GTEST_URL gtest-${GTEST_VERSION}.tar.gz https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz"
+  "ARROW_JEMALLOC_URL jemalloc-${JEMALLOC_VERSION}.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2"
   "ARROW_LZ4_URL lz4-${LZ4_VERSION}.tar.gz https://github.com/lz4/lz4/archive/${LZ4_VERSION}.tar.gz"
   "ARROW_MIMALLOC_URL mimalloc-${MIMALLOC_VERSION}.tar.gz https://github.com/microsoft/mimalloc/archive/${MIMALLOC_VERSION}.tar.gz"
   "ARROW_ORC_URL orc-${ORC_VERSION}.tar.gz https://github.com/apache/orc/archive/rel/release-${ORC_VERSION}.tar.gz"