Skip to content

Commit d8dc9be

Browse files
staticlibsstaticlibs
committed
Support loading native lib directly from FS
This PR is a continuation of duckdb#447 PR to allow using `duckdb_jdbc-x.x.x.x-nolib.jar` along with a JNI native library, that is loaded directly from file system. It extends the idea from duckdb#421 (and supersedes it) implementing the following logic: 1. if the driver JAR has a bundled native library (for current JVM os/arch), then this library will be unpacked to the temporary directory and loaded from there. If the library cannot be unpacked or loaded - there is no fallback to other methods (it is expected that `-nolib` JAR is used for other loading methods) 2. if the driver JAR does not hava a native library bundled inside it, then it will check whether a JNI native libary with the DuckDB internal naming (like `libduckdb_java.so_linux_amd64`) exists in file system next to the driver JAR (in the same directory). If library file is found there - then the driver will attempt to load it. If the file is found in file system, then it is expected that is can be loaded and there is no fallback to loading by name. 3. if the native lib is not found in the same directory, then, like in duckdb#421, the driver tries to load it using `duckdb_java` name (that will be translated by JVM to a platform-specific name like `libduckdb_java.so`). Testing: new test added that covers loading from the same dir and loading by name. Fixes: duckdb#444
1 parent d00260e commit d8dc9be

File tree

6 files changed

+195
-56
lines changed

6 files changed

+195
-56
lines changed

‎CMakeLists.txt‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -560,10 +560,10 @@ file(GLOB_RECURSE JAVA_SRC_FILES src/main/java/org/duckdb/*.java)
560560
file(GLOB_RECURSE JAVA_TEST_FILES src/test/java/org/duckdb/*.java)
561561
set(CMAKE_JAVA_COMPILE_FLAGS -encoding utf-8 -g -Xlint:all)
562562

563-
add_jar(duckdb_jdbc ${JAVA_SRC_FILES} META-INF/services/java.sql.Driver
563+
add_jar(duckdb_jdbc_nolib ${JAVA_SRC_FILES} META-INF/services/java.sql.Driver
564564
MANIFEST META-INF/MANIFEST.MF
565565
GENERATE_NATIVE_HEADERS duckdb-native)
566-
add_jar(duckdb_jdbc_tests ${JAVA_TEST_FILES} INCLUDE_JARS duckdb_jdbc)
566+
add_jar(duckdb_jdbc_tests ${JAVA_TEST_FILES} INCLUDE_JARS duckdb_jdbc_nolib)
567567

568568

569569
# main shared lib compilation
@@ -654,7 +654,10 @@ set_target_properties(duckdb_java PROPERTIES PREFIX "lib")
654654

655655
add_custom_command(
656656
OUTPUT dummy_jdbc_target
657-
DEPENDS duckdb_java duckdb_jdbc duckdb_jdbc_tests
657+
DEPENDS duckdb_java duckdb_jdbc_nolib duckdb_jdbc_tests
658+
COMMAND ${CMAKE_COMMAND} -E copy
659+
duckdb_jdbc_nolib.jar
660+
duckdb_jdbc.jar
658661
COMMAND ${Java_JAR_EXECUTABLE} uf duckdb_jdbc.jar -C
659662
$<TARGET_FILE_DIR:duckdb_java> $<TARGET_FILE_NAME:duckdb_java>)
660663

‎CMakeLists.txt.in‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -86,10 +86,10 @@ file(GLOB_RECURSE JAVA_SRC_FILES src/main/java/org/duckdb/*.java)
8686
file(GLOB_RECURSE JAVA_TEST_FILES src/test/java/org/duckdb/*.java)
8787
set(CMAKE_JAVA_COMPILE_FLAGS -encoding utf-8 -g -Xlint:all)
8888

89-
add_jar(duckdb_jdbc ${JAVA_SRC_FILES} META-INF/services/java.sql.Driver
89+
add_jar(duckdb_jdbc_nolib ${JAVA_SRC_FILES} META-INF/services/java.sql.Driver
9090
MANIFEST META-INF/MANIFEST.MF
9191
GENERATE_NATIVE_HEADERS duckdb-native)
92-
add_jar(duckdb_jdbc_tests ${JAVA_TEST_FILES} INCLUDE_JARS duckdb_jdbc)
92+
add_jar(duckdb_jdbc_tests ${JAVA_TEST_FILES} INCLUDE_JARS duckdb_jdbc_nolib)
9393

9494

9595
# main shared lib compilation
@@ -180,7 +180,10 @@ set_target_properties(duckdb_java PROPERTIES PREFIX "lib")
180180

181181
add_custom_command(
182182
OUTPUT dummy_jdbc_target
183-
DEPENDS duckdb_java duckdb_jdbc duckdb_jdbc_tests
183+
DEPENDS duckdb_java duckdb_jdbc_nolib duckdb_jdbc_tests
184+
COMMAND ${CMAKE_COMMAND} -E copy
185+
duckdb_jdbc_nolib.jar
186+
duckdb_jdbc.jar
184187
COMMAND ${Java_JAR_EXECUTABLE} uf duckdb_jdbc.jar -C
185188
$<TARGET_FILE_DIR:duckdb_java> $<TARGET_FILE_NAME:duckdb_java>)
186189

‎src/main/java/org/duckdb/DuckDBNative.java‎

Lines changed: 107 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -1,66 +1,129 @@
11
package org.duckdb;
22

3-
import java.io.File;
3+
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
4+
45
import java.io.IOException;
56
import java.io.InputStream;
67
import java.math.BigDecimal;
8+
import java.net.URI;
79
import java.net.URL;
810
import java.nio.ByteBuffer;
911
import java.nio.file.Files;
1012
import java.nio.file.Path;
1113
import java.nio.file.Paths;
12-
import java.nio.file.StandardCopyOption;
14+
import java.security.CodeSource;
15+
import java.security.ProtectionDomain;
1316
import java.sql.SQLException;
1417
import java.util.Properties;
1518

1619
final class DuckDBNative {
20+
21+
private static final String ARCH_X86_64 = "amd64";
22+
private static final String ARCH_AARCH64 = "arm64";
23+
private static final String ARCH_UNIVERSAL = "universal";
24+
25+
private static final String OS_WINDOWS = "windows";
26+
private static final String OS_MACOS = "osx";
27+
private static final String OS_LINUX = "linux";
28+
1729
static {
1830
try {
19-
String os_name = "";
20-
String os_arch;
21-
String os_name_detect = System.getProperty("os.name").toLowerCase().trim();
22-
String os_arch_detect = System.getProperty("os.arch").toLowerCase().trim();
23-
switch (os_arch_detect) {
24-
case "x86_64":
25-
case "amd64":
26-
os_arch = "amd64";
27-
break;
28-
case "aarch64":
29-
case "arm64":
30-
os_arch = "arm64";
31-
break;
32-
case "i386":
33-
os_arch = "i386";
34-
break;
35-
default:
36-
throw new IllegalStateException("Unsupported system architecture");
37-
}
38-
if (os_name_detect.startsWith("windows")) {
39-
os_name = "windows";
40-
} else if (os_name_detect.startsWith("mac")) {
41-
os_name = "osx";
42-
os_arch = "universal";
43-
} else if (os_name_detect.startsWith("linux")) {
44-
os_name = "linux";
45-
}
46-
String lib_res_name = "/libduckdb_java.so"
47-
+ "_" + os_name + "_" + os_arch;
48-
49-
Path lib_file = Files.createTempFile("libduckdb_java", ".so");
50-
URL lib_res = DuckDBNative.class.getResource(lib_res_name);
51-
if (lib_res == null) {
52-
System.load(Paths.get("build/debug", lib_res_name).normalize().toAbsolutePath().toString());
53-
} else {
54-
try (final InputStream lib_res_input_stream = lib_res.openStream()) {
55-
Files.copy(lib_res_input_stream, lib_file, StandardCopyOption.REPLACE_EXISTING);
56-
}
57-
new File(lib_file.toString()).deleteOnExit();
58-
System.load(lib_file.toAbsolutePath().toString());
59-
}
60-
} catch (IOException e) {
31+
loadNativeLibrary();
32+
} catch (Exception e) {
6133
throw new RuntimeException(e);
6234
}
6335
}
36+
37+
private static void loadNativeLibrary() throws Exception {
38+
String libName = nativeLibName();
39+
URL libRes = DuckDBNative.class.getResource("/" + libName);
40+
41+
// The current JAR has a native library bundled, in this case we unpack and load it.
42+
// There is no fallback if the unpacking or loading fails. We expect that only
43+
// the '-nolib' JAR can be used with external native lib.
44+
if (null != libRes) {
45+
unpackAndLoad(libRes);
46+
return;
47+
}
48+
49+
// External native library must be loaded, first we try to load it from the
50+
// same directory where this JAR resides.
51+
Path dir = currentJarDir();
52+
if (null != dir) {
53+
Path libPath = dir.resolve(libName);
54+
if (Files.exists(libPath)) {
55+
// Native lib is found next to this JAR, we expect it to be loadable
56+
// and do NOT fall back to load-by-name if the loading fails.
57+
System.load(libPath.toAbsolutePath().toString());
58+
return;
59+
}
60+
}
61+
62+
// There is no native library next to the JAR file, so we try to load it by name.
63+
System.loadLibrary("duckdb_java");
64+
}
65+
66+
private static String cpuArch() throws SQLException {
67+
String prop = System.getProperty("os.arch").toLowerCase().trim();
68+
switch (prop) {
69+
case "x86_64":
70+
case "amd64":
71+
return ARCH_X86_64;
72+
case "aarch64":
73+
case "arm64":
74+
return ARCH_AARCH64;
75+
default:
76+
throw new SQLException("Unsupported system architecture: '" + prop + "'");
77+
}
78+
}
79+
80+
static String osName() throws SQLException {
81+
String prop = System.getProperty("os.name").toLowerCase().trim();
82+
if (prop.startsWith("windows")) {
83+
return OS_WINDOWS;
84+
} else if (prop.startsWith("mac")) {
85+
return OS_MACOS;
86+
} else if (prop.startsWith("linux")) {
87+
return OS_LINUX;
88+
} else {
89+
throw new SQLException("Unsupported OS: '" + prop + "'");
90+
}
91+
}
92+
93+
static String nativeLibName() throws SQLException {
94+
String os = osName();
95+
final String arch;
96+
if (OS_MACOS.equals(os)) {
97+
arch = ARCH_UNIVERSAL;
98+
} else {
99+
arch = cpuArch();
100+
}
101+
return "libduckdb_java.so_" + os + "_" + arch;
102+
}
103+
104+
static Path currentJarDir() {
105+
try {
106+
ProtectionDomain pd = DuckDBNative.class.getProtectionDomain();
107+
CodeSource cs = pd.getCodeSource();
108+
URL loc = cs.getLocation();
109+
URI uri = loc.toURI();
110+
Path jarPath = Paths.get(uri);
111+
Path dirPath = jarPath.getParent();
112+
return dirPath.toRealPath();
113+
} catch (Exception e) {
114+
return null;
115+
}
116+
}
117+
118+
private static void unpackAndLoad(URL nativeLibRes) throws IOException {
119+
Path tmpFile = Files.createTempFile("libduckdb_java", ".so");
120+
try (InputStream is = nativeLibRes.openStream()) {
121+
Files.copy(is, tmpFile, REPLACE_EXISTING);
122+
}
123+
tmpFile.toFile().deleteOnExit();
124+
System.load(tmpFile.toAbsolutePath().toString());
125+
}
126+
64127
// We use zero-length ByteBuffer-s as a hacky but cheap way to pass C++ pointers
65128
// back and forth
66129

‎src/test/java/org/duckdb/TestDuckDBJDBC.java‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3155,11 +3155,12 @@ public static void main(String[] args) throws Exception {
31553155
Class<?> clazz = Class.forName("org.duckdb." + arg1);
31563156
statusCode = runTests(new String[0], clazz);
31573157
} else {
3158-
statusCode = runTests(args, TestDuckDBJDBC.class, TestAppender.class, TestAppenderCollection.class,
3159-
TestAppenderCollection2D.class, TestAppenderComposite.class,
3160-
TestSingleValueAppender.class, TestBatch.class, TestBindings.class, TestClosure.class,
3161-
TestExtensionTypes.class, TestSpatial.class, TestParameterMetadata.class,
3162-
TestPrepare.class, TestResults.class, TestSessionInit.class, TestTimestamp.class);
3158+
statusCode =
3159+
runTests(args, TestDuckDBJDBC.class, TestAppender.class, TestAppenderCollection.class,
3160+
TestAppenderCollection2D.class, TestAppenderComposite.class, TestSingleValueAppender.class,
3161+
TestBatch.class, TestBindings.class, TestClosure.class, TestExtensionTypes.class,
3162+
TestNoLib.class, TestSpatial.class, TestParameterMetadata.class, TestPrepare.class,
3163+
TestResults.class, TestSessionInit.class, TestTimestamp.class);
31633164
}
31643165
System.exit(statusCode);
31653166
}

‎src/test/java/org/duckdb/TestNoLib.java‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
package org.duckdb;
2+
3+
import java.io.File;
4+
import java.nio.file.Files;
5+
import java.nio.file.Path;
6+
import java.nio.file.Paths;
7+
import java.sql.SQLException;
8+
9+
public class TestNoLib {
10+
11+
private static Path javaExe() throws Exception {
12+
String javaHomeProp = System.getProperty("java.home");
13+
Path javaHome = Paths.get(javaHomeProp);
14+
boolean isWindows = "windows".equals(DuckDBNative.osName());
15+
return isWindows ? javaHome.resolve("bin/java.exe") : javaHome.resolve("bin/java");
16+
}
17+
18+
private static void runQuickTest(Path currentJarDir) throws Exception {
19+
String dir = currentJarDir.toAbsolutePath().toString();
20+
ProcessBuilder pb = new ProcessBuilder(javaExe().toAbsolutePath().toString(), "-cp",
21+
dir + File.separator + "duckdb_jdbc_tests.jar" + File.pathSeparator +
22+
dir + File.separator + "duckdb_jdbc_nolib.jar",
23+
"org.duckdb.TestDuckDBJDBC", "test_spatial_POINT_2D")
24+
.inheritIO();
25+
pb.environment().put("LD_LIBRARY_PATH", currentJarDir.toAbsolutePath().toString());
26+
int code = pb.start().waitFor();
27+
if (0 != code) {
28+
throw new RuntimeException("Spawned test failure, code: " + code);
29+
}
30+
}
31+
32+
private static String platformLibName() throws Exception {
33+
String os = DuckDBNative.osName();
34+
switch (os) {
35+
case "windows":
36+
return "duckdb_java.dll";
37+
case "osx":
38+
return "libduckdb_java.dylib";
39+
case "linux":
40+
return "libduckdb_java.so";
41+
default:
42+
throw new SQLException("Unsupported OS: " + os);
43+
}
44+
}
45+
46+
public static void test_nolib() throws Exception {
47+
Path dir = DuckDBNative.currentJarDir();
48+
Path nativeLib = dir.resolve(DuckDBNative.nativeLibName());
49+
Path platformNamedLib = dir.resolve(platformLibName());
50+
51+
// test next to JAR
52+
if (Files.exists(platformNamedLib)) {
53+
Files.delete(platformNamedLib);
54+
}
55+
System.out.println("\n----");
56+
runQuickTest(dir);
57+
System.out.println("----");
58+
59+
// test by name
60+
try {
61+
Files.move(nativeLib, platformNamedLib);
62+
System.out.println("----");
63+
runQuickTest(dir);
64+
System.out.println("----");
65+
} finally {
66+
Files.move(platformNamedLib, nativeLib);
67+
}
68+
}
69+
}

‎src/test/java/org/duckdb/test/Runner.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ public static int runTests(String[] args, Class<?>... testClasses) {
4040
boolean anyFailed = false;
4141
for (Method m : methods) {
4242
if (m.getName().startsWith("test_")) {
43-
if (quick_run && m.getName().startsWith("test_lots_")) {
43+
if (quick_run && (m.getName().startsWith("test_lots_") || m.getName().startsWith("test_nolib"))) {
4444
continue;
4545
}
4646
if (specific_test != null && !m.getName().contains(specific_test)) {

0 commit comments

Comments
 (0)