HDDS-8028. JNI for RocksDB SST Dump tool

hadoop-hdds/pom.xml

@@ -49,6 +49,7 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
     <module>config</module>
     <module>test-utils</module>
     <module>erasurecode</module>
+    <module>rocks-native</module>
   </modules>
 
   <repositories>
@@ -248,6 +249,11 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
         <artifactId>bcpkix-jdk15on</artifactId>
         <version>${bouncycastle.version}</version>
       </dependency>
+      <dependency>
+        <groupId>org.apache.ozone</groupId>
+        <artifactId>hdds-rocks-native</artifactId>
+        <version>${hdds.version}</version>
+      </dependency>
     </dependencies>
   </dependencyManagement>
   <dependencies>

hadoop-hdds/rocks-native/src/CMakeLists.txt

@@ -0,0 +1,78 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# CMake configuration.
+#
+
+cmake_minimum_required(VERSION 2.8)
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC")
+project(ozone_native)
+set(CMAKE_BUILD_TYPE Release)
+find_package(JNI REQUIRED)
+include_directories(${JNI_INCLUDE_DIRS})
+set(CMAKE_CXX_STANDARD ${CMAKE_STANDARDS})
+
+set(linked_libraries "")
+if(NOT GENERATED_JAVAH)
+    message(FATAL_ERROR "You must set the CMake variable GENERATED_JAVAH")
+endif()
+include_directories(${GENERATED_JAVAH})
+if(${SST_DUMP_INCLUDE})
+    include_directories(${ROCKSDB_HEADERS})
+    set(SOURCE_FILES ${NATIVE_DIR}/SSTDumpTool.cpp ${NATIVE_DIR}/PipeInputStream.cpp ${NATIVE_DIR}/Pipe.h ${NATIVE_DIR}/Pipe.cpp ${NATIVE_DIR}/cplusplus_to_java_convert.h)
+    ADD_LIBRARY(rocksdb STATIC IMPORTED)
+    set_target_properties(
+            rocksdb
+            PROPERTIES
+            IMPORTED_LOCATION ${ROCKSDB_LIB}/librocksdb_debug.a)
+    ADD_LIBRARY(rocks_tools STATIC IMPORTED)
+    set_target_properties(
+            rocks_tools
+            PROPERTIES
+            IMPORTED_LOCATION ${ROCKSDB_LIB}/librocksdb_tools_debug.a)
+    ADD_LIBRARY(bz2 STATIC IMPORTED)
+    set_target_properties(
+            bz2
+            PROPERTIES
+            IMPORTED_LOCATION ${BZIP2_LIB}/libbz2.a)
+    ADD_LIBRARY(zlib STATIC IMPORTED)
+    set_target_properties(
+            zlib
+            PROPERTIES
+            IMPORTED_LOCATION ${ZLIB_LIB}/libz.a)
+    ADD_LIBRARY(lz4 STATIC IMPORTED)
+    set_target_properties(
+            lz4
+            PROPERTIES
+            IMPORTED_LOCATION ${LZ4_LIB}/liblz4.a)
+    ADD_LIBRARY(snappy STATIC IMPORTED)
+    set_target_properties(
+            snappy
+            PROPERTIES
+            IMPORTED_LOCATION ${SNAPPY_LIB}/libsnappy.a)
+    ADD_LIBRARY(zstd STATIC IMPORTED)
+        set_target_properties(
+                zstd
+                PROPERTIES
+                IMPORTED_LOCATION ${ZSTD_LIB}/libzstd.a)
+    set(linked_libraries ${linked_libraries} bz2 zlib rocks_tools rocksdb lz4 snappy zstd)
+endif()
+add_library(ozone_rocksdb_tools SHARED ${SOURCE_FILES})
+target_link_libraries(ozone_rocksdb_tools ${linked_libraries})

hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeConstants.java

@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.utils;
+
+/**
+ * Native Constants.
+ */
+public final class NativeConstants {
+
+  private NativeConstants() {
+
+  }
+  public static final String ROCKS_TOOLS_NATIVE_LIBRARY_NAME
+          = "ozone_rocksdb_tools";
+}

hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java

@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.utils;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Class to load Native Libraries.
+ */
+public class NativeLibraryLoader {
+
+  private static final Logger LOG =
+          LoggerFactory.getLogger(NativeLibraryLoader.class);
+  private static final String OS = System.getProperty("os.name").toLowerCase();
+  private Map<String, Boolean> librariesLoaded;
+  private static volatile NativeLibraryLoader instance;
+
+  public NativeLibraryLoader(final Map<String, Boolean> librariesLoaded) {
+    this.librariesLoaded = librariesLoaded;
+  }
+
+  private static synchronized void initNewInstance() {
+    if (instance == null) {
+      instance = new NativeLibraryLoader(new ConcurrentHashMap<>());
+    }
+  }
+
+  public static NativeLibraryLoader getInstance() {
+    if (instance == null) {
+      initNewInstance();
+    }
+    return instance;
+  }
+
+  public static String getJniLibraryFileName(String libraryName) {
+    return appendLibOsSuffix("lib" + libraryName);
+  }
+
+  public static boolean isMac() {
+    return OS.startsWith("mac");
+  }
+
+  public static boolean isWindows() {
+    return OS.startsWith("win");
+  }
+
+  public static boolean isLinux() {
+    return OS.startsWith("linux");
+  }
+
+  private static String getLibOsSuffix() {
+    if (isMac()) {
+      return ".dylib";
+    } else if (isWindows()) {
+      return ".dll";
+    } else if (isLinux()) {
+      return ".so";
+    }
+    throw new UnsatisfiedLinkError(String.format("Unsupported OS %s", OS));
+  }
+
+  private static String appendLibOsSuffix(String libraryFileName) {
+    return libraryFileName + getLibOsSuffix();
+  }
+
+  public static boolean isLibraryLoaded(final String libraryName) {
+    return getInstance().librariesLoaded
+            .getOrDefault(libraryName, false);
+  }
+
+  public synchronized boolean loadLibrary(final String libraryName) {
+    if (isLibraryLoaded(libraryName)) {
+      return true;
+    }
+    boolean loaded = false;
+    try {
+      loaded = false;
+      try {
+        System.loadLibrary(libraryName);
+        loaded = true;
+      } catch (Throwable e) {
+
+      }
+      if (!loaded) {
+        Optional<File> file = copyResourceFromJarToTemp(libraryName);
+        if (file.isPresent()) {
+          System.load(file.get().getAbsolutePath());
+          loaded = true;
+        }
+      }
+    } catch (Throwable e) {
+      LOG.warn("Unable to load library: {}", libraryName, e);
+    }
+    this.librariesLoaded.put(libraryName, loaded);
+    return isLibraryLoaded(libraryName);
+  }
+
+  private Optional<File> copyResourceFromJarToTemp(final String libraryName)
+          throws IOException {
+    final String libraryFileName = getJniLibraryFileName(libraryName);
+    InputStream is = null;
+    try {
+      is = getClass().getClassLoader().getResourceAsStream(libraryFileName);
+      if (is == null) {
+        return Optional.empty();
+      }
+
+      // create a temporary file to copy the library to
+      final File temp = File.createTempFile(libraryName, getLibOsSuffix());
+      if (!temp.exists()) {
+        return Optional.empty();
+      } else {
+        temp.deleteOnExit();
+      }
+
+      Files.copy(is, temp.toPath(), StandardCopyOption.REPLACE_EXISTING);
+      return Optional.ofNullable(temp);
+    } finally {
+      if (is != null) {
+        is.close();
+      }
+    }
+  }
+}

hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryNotLoadedException.java

@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.utils;
+
+/**
+  Exception when native library not loaded.
+ */
+public class NativeLibraryNotLoadedException extends Exception {
+  public NativeLibraryNotLoadedException(String libraryName) {
+    super(String.format("Unable to load library %s from both " +
+            "java.library.path & resource file %s from jar.", libraryName,
+            NativeLibraryLoader.getJniLibraryFileName(libraryName)));
+  }
+}