diff --git a/cpp/apidoc/HDFS.md b/cpp/apidoc/HDFS.md
index d54ad270c05..d3671fb7691 100644
--- a/cpp/apidoc/HDFS.md
+++ b/cpp/apidoc/HDFS.md
@@ -50,6 +50,10 @@ export CLASSPATH=`$HADOOP_HOME/bin/hadoop classpath --glob`
 * `ARROW_LIBHDFS_DIR` (optional): explicit location of `libhdfs.so` if it is
 installed somewhere other than `$HADOOP_HOME/lib/native`.
 
+To accommodate distribution-specific nuances, the `JAVA_HOME` variable may be
+set to the root path for the Java SDK, the JRE path itself, or to the directory
+containing the `libjvm` library.
+
 ### Mac Specifics
 
 The installed location of Java on OS X can vary, however the following snippet
diff --git a/cpp/src/arrow/io/hdfs-internal.cc b/cpp/src/arrow/io/hdfs-internal.cc
index 9cd1c5052fe..545b2d17d2e 100644
--- a/cpp/src/arrow/io/hdfs-internal.cc
+++ b/cpp/src/arrow/io/hdfs-internal.cc
@@ -147,7 +147,7 @@ static std::vector<fs::path> get_potential_libjvm_paths() {
   file_name = "jvm.dll";
 #elif __APPLE__
   search_prefixes = {""};
-  search_suffixes = {"", "/jre/lib/server"};
+  search_suffixes = {"", "/jre/lib/server", "/lib/server"};
   file_name = "libjvm.dylib";
 
 // SFrame uses /usr/libexec/java_home to find JAVA_HOME; for now we are
@@ -175,7 +175,7 @@ static std::vector<fs::path> get_potential_libjvm_paths() {
       "/usr/lib/jvm/default",                     // alt centos
       "/usr/java/latest",                         // alt centos
   };
-  search_suffixes = {"/jre/lib/amd64/server"};
+  search_suffixes = {"", "/jre/lib/amd64/server", "/lib/amd64/server"};
   file_name = "libjvm.so";
 #endif
   // From direct environment variable