HDDS-10560. Link rocksdb lib to Ozone rocksdb tools lib relative path instead of absolute path

hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/JniLibNamePropertyWriter.java

@@ -17,7 +17,6 @@
  */
 package org.apache.hadoop.hdds.utils.db.managed;
 
-import org.rocksdb.util.Environment;
 
 import java.io.IOException;
 import java.io.OutputStreamWriter;
@@ -39,8 +38,8 @@ public static void main(String[] args) {
     String filePath = args[0];
     try (Writer writer = new OutputStreamWriter(
         Files.newOutputStream(Paths.get(filePath)), StandardCharsets.UTF_8)) {
-      writer.write("rocksdbLibName=" +
-          Environment.getJniLibraryFileName("rocksdb"));
+      String libName = ManagedRocksObjectUtils.getRocksDBLibFileName();
+      writer.write("rocksdbLibName=" + libName);
     } catch (IOException e) {
       e.printStackTrace();
     }

hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksObjectUtils.java

@@ -23,6 +23,7 @@
 import org.apache.hadoop.hdds.utils.LeakDetector;
 import org.apache.ratis.util.UncheckedAutoCloseable;
 import org.rocksdb.RocksDB;
+import org.rocksdb.util.Environment;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -94,4 +95,11 @@ public static void waitForFileDelete(File file, Duration maxDuration)
   public static void loadRocksDBLibrary() {
     RocksDB.loadLibrary();
   }
+
+  /**
+   * Returns RocksDB library file name.
+   */
+  public static String getRocksDBLibFileName() {
+    return Environment.getJniLibraryFileName("rocksdb");
+  }
 }

hadoop-hdds/rocks-native/pom.xml

@@ -252,6 +252,8 @@
                 <configuration>
                   <target>
                     <mkdir dir="${project.build.directory}/native/rocksdb"/>
+                    <copy file="${project.build.directory}/rocksdbjni/${rocksdbLibName}"
+                          tofile="${project.build.directory}/native/rocksdb/${rocksdbLibName}" />
                     <exec executable="cmake" failonerror="true" dir="${project.build.directory}/native/rocksdb">
                       <env key="CFLAGS" value="-fPIC"/>
                       <env key="CXXFLAGS" value="-fPIC"/>
@@ -262,7 +264,7 @@
                       <arg line="-DCMAKE_STANDARDS=${cmake.standards}"/>
                       <arg line="-DROCKSDB_HEADERS=${project.build.directory}/rocksdb/rocksdb-${rocksdb.version}/include"/>
                       <arg line="-DROCKSDB_TOOLS_LIB=${project.build.directory}/rocksdb/rocksdb-${rocksdb.version}"/>
-                      <arg line="-DROCKSDB_LIB=${project.build.directory}/rocksdbjni/${rocksdbLibName}"/>
+                      <arg line="-DROCKSDB_LIB=${project.build.directory}/native/rocksdb/${rocksdbLibName}"/>
                     </exec>
                     <exec executable="make" dir="${project.build.directory}/native/rocksdb" failonerror="true"/>
                     <delete dir="${project.build.directory}/rocksdb/rocksdb-${rocksdb.version}"

hadoop-hdds/rocks-native/src/CMakeLists.txt

@@ -30,25 +30,33 @@ find_package(JNI REQUIRED)
 include_directories(${JNI_INCLUDE_DIRS})
 set(CMAKE_CXX_STANDARD ${CMAKE_STANDARDS})
 
-set(linked_libraries "")
+set(CMAKE_SKIP_BUILD_RPATH FALSE)
+
+set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE)
+
+set(CMAKE_INSTALL_RPATH "")
+
+set(CMAKE_INSTALL_RPATH_USE_LINK_PATH FALSE)
+
 if(NOT GENERATED_JAVAH)
     message(FATAL_ERROR "You must set the CMake variable GENERATED_JAVAH")
 endif()
 include_directories(${GENERATED_JAVAH})
 if(${SST_DUMP_INCLUDE})
     include_directories(${ROCKSDB_HEADERS})
     set(SOURCE_FILES ${NATIVE_DIR}/ManagedRawSSTFileReader.cpp ${NATIVE_DIR}/ManagedRawSSTFileIterator.cpp ${NATIVE_DIR}/cplusplus_to_java_convert.h)
-    ADD_LIBRARY(rocksdb SHARED IMPORTED)
-    set_target_properties(
-            rocksdb
-            PROPERTIES
-            IMPORTED_LOCATION ${ROCKSDB_LIB})
     ADD_LIBRARY(rocks_tools STATIC IMPORTED)
     set_target_properties(
             rocks_tools
             PROPERTIES
             IMPORTED_LOCATION ${ROCKSDB_TOOLS_LIB}/librocksdb_tools.a)
-    set(linked_libraries ${linked_libraries} rocks_tools rocksdb)
 endif()
+
 add_library(ozone_rocksdb_tools SHARED ${SOURCE_FILES})
-target_link_libraries(ozone_rocksdb_tools ${linked_libraries})
+
+
+target_link_libraries(ozone_rocksdb_tools PRIVATE ${ROCKSDB_LIB})
+target_link_libraries(ozone_rocksdb_tools PRIVATE rocks_tools)
+set_target_properties(ozone_rocksdb_tools PROPERTIES
+        BUILD_WITH_INSTALL_RPATH FALSE
+        LINK_FLAGS "-Wl,-rpath -Wl,'$ORIGIN'")

hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java

@@ -19,6 +19,7 @@
 package org.apache.hadoop.hdds.utils;
 
 import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang3.tuple.Pair;
 import org.apache.hadoop.ozone.util.ShutdownHookManager;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -28,6 +29,8 @@
 import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.StandardCopyOption;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
@@ -101,7 +104,7 @@ public static boolean isLibraryLoaded(final String libraryName) {
         .getOrDefault(libraryName, false);
   }
 
-  public synchronized boolean loadLibrary(final String libraryName) {
+  public synchronized boolean loadLibrary(final String libraryName, final List<String> dependentFiles) {
     if (isLibraryLoaded(libraryName)) {
       return true;
     }
@@ -116,9 +119,9 @@ public synchronized boolean loadLibrary(final String libraryName) {
 
       }
       if (!loaded) {
-        Optional<File> file = copyResourceFromJarToTemp(libraryName);
-        if (file.isPresent()) {
-          System.load(file.get().getAbsolutePath());
+        Pair<Optional<File>, List<File>> files = copyResourceFromJarToTemp(libraryName, dependentFiles);
+        if (files.getKey().isPresent()) {
+          System.load(files.getKey().get().getAbsolutePath());
           loaded = true;
         }
       }
@@ -137,19 +140,20 @@ static String getSystemProperty(String property) {
 
   // Added function to make this testable
   @VisibleForTesting
-  static InputStream getResourceStream(String libraryFileName) {
+  static InputStream getResourceStream(String libraryFileName) throws IOException {
     return NativeLibraryLoader.class.getClassLoader()
         .getResourceAsStream(libraryFileName);
   }
 
-  private Optional<File> copyResourceFromJarToTemp(final String libraryName)
+  private Pair<Optional<File>, List<File>> copyResourceFromJarToTemp(final String libraryName,
+                                                                     final List<String> dependentFileNames)
       throws IOException {
     final String libraryFileName = getJniLibraryFileName(libraryName);
     InputStream is = null;
     try {
       is = getResourceStream(libraryFileName);
       if (is == null) {
-        return Optional.empty();
+        return Pair.of(Optional.empty(), null);
       }
 
       final String nativeLibDir =
@@ -160,15 +164,28 @@ private Optional<File> copyResourceFromJarToTemp(final String libraryName)
       // create a temporary file to copy the library to
       final File temp = File.createTempFile(libraryName, getLibOsSuffix(), dir);
       if (!temp.exists()) {
-        return Optional.empty();
+        return Pair.of(Optional.empty(), null);
       } else {
         temp.deleteOnExit();
       }
 
       Files.copy(is, temp.toPath(), StandardCopyOption.REPLACE_EXISTING);
+      List<File> dependentFiles = new ArrayList<>();
+      for (String fileName : dependentFileNames) {
+        if (is != null) {
+          is.close();
+        }
+        is = getResourceStream(fileName);
+        File file = new File(dir, fileName);
+        Files.copy(is, file.toPath(), StandardCopyOption.REPLACE_EXISTING);
+        if (file.exists()) {
+          file.deleteOnExit();
+        }
+        dependentFiles.add(file);
+      }
       ShutdownHookManager.get().addShutdownHook(temp::delete,
           LIBRARY_SHUTDOWN_HOOK_PRIORITY);
-      return Optional.of(temp);
+      return Pair.of(Optional.of(temp), dependentFiles);
     } finally {
       if (is != null) {
         is.close();

hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReader.java

@@ -23,6 +23,7 @@
 import org.slf4j.LoggerFactory;
 
 import java.io.Closeable;
+import java.util.Arrays;
 import java.util.function.Function;
 
 import static org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_LIBRARY_NAME;
@@ -34,7 +35,8 @@ public class ManagedRawSSTFileReader<T> implements Closeable {
 
   public static boolean loadLibrary() throws NativeLibraryNotLoadedException {
     ManagedRocksObjectUtils.loadRocksDBLibrary();
-    if (!NativeLibraryLoader.getInstance().loadLibrary(ROCKS_TOOLS_NATIVE_LIBRARY_NAME)) {
+    if (!NativeLibraryLoader.getInstance().loadLibrary(ROCKS_TOOLS_NATIVE_LIBRARY_NAME, Arrays.asList(
+        ManagedRocksObjectUtils.getRocksDBLibFileName()))) {
       throw new NativeLibraryNotLoadedException(ROCKS_TOOLS_NATIVE_LIBRARY_NAME);
     }
     return true;

hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/TestNativeLibraryLoader.java

@@ -28,6 +28,7 @@
 import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.nio.file.Path;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.stream.Stream;
@@ -72,7 +73,7 @@ public void testNativeLibraryLoader(String nativeLibraryDirectoryLocation) throw
       mockedNativeLibraryLoader.when(() -> NativeLibraryLoader.getResourceStream(anyString()))
           .thenReturn(new ByteArrayInputStream(new byte[]{0, 1, 2, 3}));
       String dummyLibraryName = "dummy_lib";
-      NativeLibraryLoader.getInstance().loadLibrary(dummyLibraryName);
+      NativeLibraryLoader.getInstance().loadLibrary(dummyLibraryName, Collections.emptyList());
       NativeLibraryLoader.isLibraryLoaded(dummyLibraryName);
       // Checking if the resource with random was copied to a temp file.
       File[] libPath = new File(nativeLibraryDirectoryLocation == null ? "" : nativeLibraryDirectoryLocation)