diff --git a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java index d93933dee362..08e397d0683f 100644 --- a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java +++ b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java @@ -20,6 +20,7 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.ozone.util.ShutdownHookManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,6 +29,7 @@ import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.List; @@ -170,31 +172,38 @@ private Pair, List> copyResourceFromJarToTemp(final String getSystemProperty(NATIVE_LIB_TMP_DIR) : ""; final File dir = new File(nativeLibDir).getAbsoluteFile(); - // create a temporary file to copy the library to - final File temp = File.createTempFile(libraryName, getLibOsSuffix(), dir); - if (!temp.exists()) { + // create a temporary dir to copy the library to + final Path tempPath = Files.createTempDirectory(dir.toPath(), libraryName); + final File tempDir = tempPath.toFile(); + if (!tempDir.exists()) { return Pair.of(Optional.empty(), null); - } else { - temp.deleteOnExit(); } - Files.copy(is, temp.toPath(), StandardCopyOption.REPLACE_EXISTING); + Path libPath = tempPath.resolve(libraryFileName); + Files.copy(is, libPath, StandardCopyOption.REPLACE_EXISTING); + File libFile = libPath.toFile(); + if (libFile.exists()) { + libFile.deleteOnExit(); + } + List dependentFiles = new ArrayList<>(); for (String fileName : dependentFileNames) { if (is != null) { is.close(); } is = getResourceStream(fileName); - File file = new File(dir, fileName); - Files.copy(is, file.toPath(), StandardCopyOption.REPLACE_EXISTING); + Path path = tempPath.resolve(fileName); + Files.copy(is, path, StandardCopyOption.REPLACE_EXISTING); + File file = path.toFile(); if (file.exists()) { file.deleteOnExit(); } dependentFiles.add(file); } - ShutdownHookManager.get().addShutdownHook(temp::delete, + ShutdownHookManager.get().addShutdownHook( + () -> FileUtil.fullyDelete(tempDir), LIBRARY_SHUTDOWN_HOOK_PRIORITY); - return Pair.of(Optional.of(temp), dependentFiles); + return Pair.of(Optional.of(libFile), dependentFiles); } finally { if (is != null) { is.close(); diff --git a/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/TestNativeLibraryLoader.java b/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/TestNativeLibraryLoader.java index f0074e0a1ac9..6e1622ebd7cf 100644 --- a/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/TestNativeLibraryLoader.java +++ b/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/TestNativeLibraryLoader.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdds.utils; +import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader; import org.apache.ozone.test.tag.Native; import org.junit.jupiter.api.io.TempDir; @@ -28,15 +29,16 @@ import java.io.ByteArrayInputStream; import java.io.File; import java.nio.file.Path; -import java.util.Collections; +import java.util.Arrays; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.stream.Stream; import static org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_LIBRARY_NAME; import static org.apache.hadoop.hdds.utils.NativeLibraryLoader.NATIVE_LIB_TMP_DIR; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.apache.hadoop.hdds.utils.NativeLibraryLoader.getJniLibraryFileName; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.CALLS_REAL_METHODS; import static org.mockito.Mockito.anyString; @@ -68,21 +70,45 @@ public void testNativeLibraryLoader(String nativeLibraryDirectoryLocation) throw mockedNativeLibraryLoader.when(() -> NativeLibraryLoader.getInstance()).thenReturn(loader); ManagedRawSSTFileReader.loadLibrary(); assertTrue(NativeLibraryLoader.isLibraryLoaded(ROCKS_TOOLS_NATIVE_LIBRARY_NAME)); + } + } + + @ParameterizedTest + @MethodSource("nativeLibraryDirectoryLocations") + public void testDummyLibrary(String nativeLibraryDirectoryLocation) { + Map libraryLoadedMap = new HashMap<>(); + NativeLibraryLoader loader = new NativeLibraryLoader(libraryLoadedMap); + try (MockedStatic mockedNativeLibraryLoader = mockStatic(NativeLibraryLoader.class, + CALLS_REAL_METHODS)) { + mockedNativeLibraryLoader.when(() -> NativeLibraryLoader.getSystemProperty(same(NATIVE_LIB_TMP_DIR))) + .thenReturn(nativeLibraryDirectoryLocation); + mockedNativeLibraryLoader.when(NativeLibraryLoader::getInstance).thenReturn(loader); // Mocking to force copy random bytes to create a lib file to // nativeLibraryDirectoryLocation. But load library will fail. mockedNativeLibraryLoader.when(() -> NativeLibraryLoader.getResourceStream(anyString())) .thenReturn(new ByteArrayInputStream(new byte[]{0, 1, 2, 3})); String dummyLibraryName = "dummy_lib"; - NativeLibraryLoader.getInstance().loadLibrary(dummyLibraryName, Collections.emptyList()); - NativeLibraryLoader.isLibraryLoaded(dummyLibraryName); + List dependencies = Arrays.asList("dep1", "dep2"); + File absDir = new File(nativeLibraryDirectoryLocation == null ? "" : nativeLibraryDirectoryLocation) + .getAbsoluteFile(); + + NativeLibraryLoader.getInstance().loadLibrary(dummyLibraryName, dependencies); + // Checking if the resource with random was copied to a temp file. - File[] libPath = new File(nativeLibraryDirectoryLocation == null ? "" : nativeLibraryDirectoryLocation) - .getAbsoluteFile().listFiles((dir, name) -> name.startsWith(dummyLibraryName) && - name.endsWith(NativeLibraryLoader.getLibOsSuffix())); - assertNotNull(libPath); - assertEquals(1, libPath.length); - assertTrue(libPath[0].delete()); + File[] libPath = absDir + .listFiles((dir, name) -> name.startsWith(dummyLibraryName)); + assertThat(libPath) + .isNotNull() + .isNotEmpty(); + assertThat(libPath[0]) + .isDirectory(); + try { + assertThat(new File(libPath[0], getJniLibraryFileName(dummyLibraryName))) + .isFile(); + dependencies.forEach(dep -> assertThat(new File(libPath[0], dep)).isFile()); + } finally { + FileUtil.fullyDelete(libPath[0]); + } } - } }