diff --git a/cpp/cmake_modules/FindPython3Alt.cmake b/cpp/cmake_modules/FindPython3Alt.cmake
index 11f26f5a1e4..131a0d395fc 100644
--- a/cpp/cmake_modules/FindPython3Alt.cmake
+++ b/cpp/cmake_modules/FindPython3Alt.cmake
@@ -36,16 +36,26 @@ if(${CMAKE_VERSION} VERSION_LESS "3.15.0")
   find_package_handle_standard_args(Python3Alt
                                     REQUIRED_VARS
                                     PYTHON_EXECUTABLE
-                                    PYTHON_LIBRARIES
                                     PYTHON_INCLUDE_DIRS
                                     NUMPY_INCLUDE_DIRS)
   return()
 endif()
 
-if(Python3Alt_FIND_REQUIRED)
-  find_package(Python3 COMPONENTS Interpreter Development NumPy REQUIRED)
+if(${CMAKE_VERSION} VERSION_LESS "3.18.0" OR ARROW_BUILD_TESTS)
+  # When building arrow-python-test, we need libpython to be present, so ask for
+  # the full "Development" component.  Also ask for it on CMake < 3.18,
+  # where "Development.Module" is not available.
+  if(Python3Alt_FIND_REQUIRED)
+    find_package(Python3 COMPONENTS Interpreter Development NumPy REQUIRED)
+  else()
+    find_package(Python3 COMPONENTS Interpreter Development NumPy)
+  endif()
 else()
-  find_package(Python3 COMPONENTS Interpreter Development NumPy)
+  if(Python3Alt_FIND_REQUIRED)
+    find_package(Python3 COMPONENTS Interpreter Development.Module NumPy REQUIRED)
+  else()
+    find_package(Python3 COMPONENTS Interpreter Development.Module NumPy)
+  endif()
 endif()
 
 if(NOT Python3_FOUND)
@@ -85,6 +95,5 @@ endfunction()
 find_package_handle_standard_args(Python3Alt
                                   REQUIRED_VARS
                                   PYTHON_EXECUTABLE
-                                  PYTHON_LIBRARIES
                                   PYTHON_INCLUDE_DIRS
                                   NUMPY_INCLUDE_DIRS)
diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
index 5ffd25c5009..ab67fced573 100755
--- a/python/manylinux1/build_arrow.sh
+++ b/python/manylinux1/build_arrow.sh
@@ -71,11 +71,6 @@ PIP="${CPYTHON_PATH}/bin/pip"
 # Put our Python first to avoid picking up an antiquated Python from CMake
 PATH="${CPYTHON_PATH}/bin:${PATH}"
 
-# XXX The Docker image doesn't include Python libs, this confuses CMake
-# (https://github.com/pypa/manylinux/issues/484)
-py_libname=$(${PYTHON_INTERPRETER} -c "import sysconfig; print(sysconfig.get_config_var('LDLIBRARY'))")
-touch ${CPYTHON_PATH}/lib/${py_libname}
-
 echo "=== (${PYTHON_VERSION}) Install the wheel build dependencies ==="
 $PIP install -r requirements-wheel-build.txt
 
@@ -87,9 +82,6 @@ export BUILD_ARROW_DATASET=ON
 export BUILD_ARROW_FLIGHT=ON
 export BUILD_ARROW_GANDIVA=OFF
 
-# ARROW-3052(wesm): ORC is being bundled until it can be added to the
-# manylinux1 image
-
 echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ==="
 ARROW_BUILD_DIR=/tmp/build-PY${PYTHON_VERSION}
 mkdir -p "${ARROW_BUILD_DIR}"
diff --git a/python/manylinux1/scripts/install_cmake.sh b/python/manylinux1/scripts/install_cmake.sh
index c105aae9d74..5a51b3561c7 100755
--- a/python/manylinux1/scripts/install_cmake.sh
+++ b/python/manylinux1/scripts/install_cmake.sh
@@ -16,7 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-export CMAKE_VERSION=3.17.3
+export CMAKE_VERSION=3.18.2.post1
 /opt/python/cp37-cp37m/bin/pip install cmake==${CMAKE_VERSION} ninja
 ln -s /opt/python/cp37-cp37m/bin/cmake /usr/bin/cmake
 ln -s /opt/python/cp37-cp37m/bin/ninja /usr/bin/ninja
diff --git a/python/manylinux201x/build_arrow.sh b/python/manylinux201x/build_arrow.sh
index 2f5134ba2f1..3913b46b680 100755
--- a/python/manylinux201x/build_arrow.sh
+++ b/python/manylinux201x/build_arrow.sh
@@ -73,11 +73,6 @@ PATH="${PATH}:${CPYTHON_PATH}"
 # Will be "manylinux2010" or "manylinux2014"
 manylinux_kind=$(${PYTHON_INTERPRETER} -c "import os; print(os.environ['AUDITWHEEL_PLAT'].split('_')[0], end='')")
 
-# XXX The Docker image doesn't include Python libs, this confuses CMake
-# (https://github.com/pypa/manylinux/issues/484)
-py_libname=$(${PYTHON_INTERPRETER} -c "import sysconfig; print(sysconfig.get_config_var('LDLIBRARY'))")
-touch ${CPYTHON_PATH}/lib/${py_libname}
-
 echo "=== (${PYTHON_VERSION}) Install the wheel build dependencies ==="
 $PIP install -r requirements-wheel-build.txt
 
@@ -89,9 +84,6 @@ export BUILD_ARROW_DATASET=ON
 export BUILD_ARROW_FLIGHT=ON
 export BUILD_ARROW_GANDIVA=OFF
 
-# ARROW-3052(wesm): ORC is being bundled until it can be added to the
-# manylinux1 image
-
 echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ==="
 ARROW_BUILD_DIR=/tmp/build-PY${PYTHON_VERSION}
 mkdir -p "${ARROW_BUILD_DIR}"
diff --git a/python/manylinux201x/scripts/install_cmake.sh b/python/manylinux201x/scripts/install_cmake.sh
index 637ea05340e..2afef8e70c9 100755
--- a/python/manylinux201x/scripts/install_cmake.sh
+++ b/python/manylinux201x/scripts/install_cmake.sh
@@ -16,7 +16,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
-/opt/python/cp37-cp37m/bin/pip install cmake ninja
+export CMAKE_VERSION=3.18.2.post1
+/opt/python/cp37-cp37m/bin/pip install cmake==${CMAKE_VERSION} ninja
 ln -s /opt/python/cp37-cp37m/bin/cmake /usr/local/bin/cmake
 ln -s /opt/python/cp37-cp37m/bin/ninja /usr/local/bin/ninja
 strip /opt/_internal/cpython-3.*/lib/python3.7/site-packages/cmake/data/bin/*