[java-native-accessGH-1313] Move default darwin prefix to darwin-$arc…

…h and support darwin as fallback The darwin platform traditionally used fatbinaries to support multiple architectures. This works, but is a problem: a) when apple decides, that only certain architectures may be bundled (it was reported, that the presence of x86 caused appstore validation to fail) b) when upstream does not bundle the artifacts, but builds them in isolation. Given, that JNA also hit (b) and needed post processing to merge binaries, moving darwin to the same scheme as other OSes (<os>-<arch>) is considered sensible. While the resource prefix is changed, the old "darwin" prefix is still considered for extraction and thus old libraries will continue to work.
matthiasblaesing · Feb 25, 2021 · ab2cb98 · ab2cb98
1 parent acee007
commit ab2cb98
Show file tree

Hide file tree

Showing 14 changed files with 116 additions and 91 deletions.
diff --git a/CHANGES.md b/CHANGES.md
@@ -7,6 +7,7 @@ Next Release (5.8.0)
 
 Features
 --------
+* [#1313](https://github.com/java-native-access/jna/issues/1313): Normalize `RESOURCE_PREFIX` for darwin to `darwin-$arch` and split jnidispatch library per architecture - [@matthiasblaesing](https://github.com/matthiasblaesing).
 
 Bug Fixes
 ---------
@@ -20,6 +21,15 @@ Important Changes
   the platform artifacts depend on the jna artifacts and need to pull in the
   right variant. This is not possible if the classifier is used.
 
+* `RESOURCE_PREFIX` for darwin (mac OS) was changed from `darwin` to
+  `darwin-$arch` as the fat binaries on mac OS causes various problems:
+  It was reported, that binaries were rejected from the appstore because x86
+  binaries were found in the application (jnidispatch for mac OS x86) and that
+  builds needed to be special cased so that the native library can be
+  assembled. The latter is also true for JNA.<br />
+  While the prefix is changed, the old prefix is still searched as a fallback
+  location, so if only a fat binary is present, it can still be loaded.
+
 Release 5.7.0
 =============
 

diff --git a/build.xml b/build.xml
@@ -38,6 +38,8 @@
   <property name="aar" value="${name}.aar"/>
   <property name="minjar" value="${name}-min.jar"/>
   <property name="testjar" value="${name}-test.jar"/>
+  <property name="testjar2" value="${name}-test2.jar"/>
+  <property name="testjar3" value="${name}-test3.jar"/>
   <property name="jar-jpms" value="${name}-jpms.jar"/>
   <property name="debug" value="true"/>
   <property name="debug.native" value="false"/>
@@ -311,13 +313,6 @@
     <condition property="os.prefix" value="kfreebsd-${jre.arch}">
       <equals arg1="${build.os.name}" arg2="GNU/kFreeBSD"/>
     </condition>
-    <!-- Darwin's resource prefix lacks arch info, use "darwin" instead -->
-    <condition property="resource.prefix" value="darwin">
-        <matches string="${os.prefix}" pattern="^darwin-"/>
-    </condition>
-    <condition property="darwin.build" value="true">
-        <matches string="${os.prefix}" pattern="^darwin-"/>
-    </condition>
     <property name="resource.prefix" value="${os.prefix}" description="fallback"/>
     <fail unless="os.prefix" message="OS/arch not supported (${os.name}/${jre.arch}), edit build.xml and native/Makefile to add it."/>
     <!-- Keep all natives separate -->
@@ -340,9 +335,6 @@
     <condition property="ld.preload.name" value="LD_PRELOAD" else="IGNORE">
       <not><equals arg1="${libjsig}" arg2=""/></not>
     </condition>
-    <condition property="native.jar" value="darwin.jar">
-      <matches string="${os.prefix}" pattern="^darwin-"/>
-    </condition>
     <property name="native.jar" value="${os.prefix}.jar"/>
     <property name="build.native" location="${build}/${native.subdir}"/>
     <property name="build.headers" location="${build}/headers"/>
@@ -542,13 +534,22 @@ com/sun/jna/freebsd-x86/libjnidispatch.so;
 processor=x86;osname=freebsd,
 com/sun/jna/freebsd-x86-64/libjnidispatch.so;
 processor=x86-64;osname=freebsd,
+
 com/sun/jna/openbsd-x86/libjnidispatch.so;
 processor=x86;osname=openbsd,
 com/sun/jna/openbsd-x86-64/libjnidispatch.so;
 processor=x86-64;osname=openbsd,
 
-com/sun/jna/darwin/libjnidispatch.jnilib;
-osname=macosx;processor=x86;processor=x86-64;processor=aarch64;processor=ppc
+com/sun/jna/darwin-ppc/libjnidispatch.jnilib;
+osname=macosx;processor=ppc,
+com/sun/jna/darwin-ppc64/libjnidispatch.jnilib;
+osname=macosx;processor=ppc64,
+com/sun/jna/darwin-x86/libjnidispatch.jnilib;
+osname=macosx;processor=x86,
+com/sun/jna/darwin-x86-64/libjnidispatch.jnilib;
+osname=macosx;processor=x86-64,
+com/sun/jna/darwin-aarch64/libjnidispatch.jnilib;
+osname=macosx;processor=aarch64
 "/>
           </manifest>
           <manifest file="@{target}" mode="update" if:true="@{module-info}">
@@ -575,7 +576,6 @@ osname=macosx;processor=x86;processor=x86-64;processor=aarch64;processor=ppc
         <exports package="com.sun.jna.internal" to="com.sun.jna.platform"/>
         <requires module="java.logging" />
         <requires module="java.desktop" />
-        <opens package="com.sun.jna.darwin" />
     </ModuleGenerator>
     <jar jarfile="${build}/${jar}" duplicate="preserve" createUnicodeExtraFields="never" encoding="UTF-8" manifest="${build}/manifest/automatic.mf">
       <fileset dir="${classes}" excludes="${jar.omitted}">
@@ -590,9 +590,21 @@ osname=macosx;processor=x86;processor=x86-64;processor=aarch64;processor=ppc
       <zipfileset src="${lib.native}/aix-ppc64.jar"
                   includes="*jnidispatch*"
                   prefix="com/sun/jna/aix-ppc64"/>
-      <zipfileset src="${lib.native}/darwin.jar"
+      <zipfileset src="${lib.native}/darwin-ppc.jar"
                   includes="*jnidispatch*"
-                  prefix="com/sun/jna/darwin"/>
+                  prefix="com/sun/jna/darwin-ppc"/>
+      <zipfileset src="${lib.native}/darwin-ppc64.jar"
+                  includes="*jnidispatch*"
+                  prefix="com/sun/jna/darwin-ppc64"/>
+      <zipfileset src="${lib.native}/darwin-x86.jar"
+                  includes="*jnidispatch*"
+                  prefix="com/sun/jna/darwin-x86"/>
+      <zipfileset src="${lib.native}/darwin-x86-64.jar"
+                  includes="*jnidispatch*"
+                  prefix="com/sun/jna/darwin-x86-64"/>
+      <zipfileset src="${lib.native}/darwin-aarch64.jar"
+                  includes="*jnidispatch*"
+                  prefix="com/sun/jna/darwin-aarch64"/>
       <zipfileset src="${lib.native}/linux-x86.jar"
                   includes="*jnidispatch*"
                   prefix="com/sun/jna/linux-x86"/>
@@ -814,7 +826,11 @@ osname=macosx;processor=x86;processor=x86-64;processor=aarch64;processor=ppc
   <!-- Invalidate native libraries when native API changes -->
   <target name="-native-api-check" depends="javah" unless="jni.valid">
     <echo>Invalidating native code, new checksum is ${md5}</echo>
-    <copy file="${lib.native}/out-of-date.jar" tofile="${lib.native}/darwin.jar" overwrite="true"/>
+    <copy file="${lib.native}/out-of-date.jar" tofile="${lib.native}/darwin-ppc.jar" overwrite="true"/>
+    <copy file="${lib.native}/out-of-date.jar" tofile="${lib.native}/darwin-ppc64.jar" overwrite="true"/>
+    <copy file="${lib.native}/out-of-date.jar" tofile="${lib.native}/darwin-x86.jar" overwrite="true"/>
+    <copy file="${lib.native}/out-of-date.jar" tofile="${lib.native}/darwin-x86-64.jar" overwrite="true"/>
+    <copy file="${lib.native}/out-of-date.jar" tofile="${lib.native}/darwin-aarch64.jar" overwrite="true"/>
     <copy file="${lib.native}/out-of-date.jar" tofile="${lib.native}/win32-x86.jar" overwrite="true"/>
     <copy file="${lib.native}/out-of-date.jar" tofile="${lib.native}/win32-x86-64.jar" overwrite="true"/>
     <copy file="${lib.native}/out-of-date.jar" tofile="${lib.native}/win32-aarch64.jar" overwrite="true"/>
@@ -903,7 +919,7 @@ osname=macosx;processor=x86;processor=x86-64;processor=aarch64;processor=ppc
     <condition property="ARCH" value="aarch64">
         <equals arg1="${os.prefix}" arg2="darwin-aarch64"/>
     </condition>
-    <condition property="ARCH" value="x86">
+    <condition property="ARCH" value="i386">
         <equals arg1="${os.prefix}" arg2="darwin-x86"/>
     </condition>
     <condition property="ARCH" value="x86-64">
@@ -1091,68 +1107,6 @@ cd ..
     </zip>
   </target>
 
-  <!-- 
-    Darwin prefers fat libraries. native/Makefile outputs a binary file for
-    one architecture. The binary is joined into a preexisting library pulled
-    from lib/native/darwin.jar using lipo.
-
-    If the preexisting library does not exist, or the library is thin and
-    only supports the target arch, the output binary will be renamed and
-    used in lieu of joining a fat lib.
-  -->
-  <macrodef name="darwin-lipo">
-    <sequential>
-      <available file="${lib.native}/${native.jar}" property="lib-available"/>
-      <unzip src="${lib.native}/${native.jar}" dest="${build.native}" if:true="${lib-available}">
-        <patternset>
-          <include name="libjnidispatch.jnilib"/>
-        </patternset>
-      </unzip>
-      <!--
-        Detect the arch that was just built.
-        Note, lipo -info will not exactly match that provided by ${ARCH}, this is OK
-      -->
-      <exec executable="lipo" dir="${build.native}" failonerror="true" outputproperty="lipo-info" if:true="${lib-available}">
-        <arg value="-info"/>
-        <arg value="libjnidispatch-${ARCH}.jnilib"/>
-      </exec>
-      <regexp id="lipo-regex" pattern="\b(\w+)$"/>
-      <loadresource property="lipo-arch" if:true="${lib-available}">
-          <propertyresource name="lipo-info"/>
-          <filterchain>
-              <tokenfilter>
-                  <filetokenizer/>
-                  <replaceregex pattern=".*\s" replace=""/>
-              </tokenfilter>
-          </filterchain>
-      </loadresource>
-      <!-- Use the detected arch to remove the old binary, and add the new binary to the fat library -->
-      <exec executable="lipo" dir="${build.native}" failonerror="false" if:true="${lib-available}">
-        <arg value="-remove"/>
-        <arg value="${lipo-arch}"/>
-        <arg value="libjnidispatch.jnilib"/>
-        <arg value="-output"/>
-        <arg value="libjnidispatch.jnilib"/>
-      </exec>
-      <exec executable="lipo" dir="${build.native}" failonerror="false" if:true="${lib-available}" resultproperty="lipo-result">
-        <arg value="-create"/>
-        <arg value="libjnidispatch.jnilib"/>
-        <arg value="libjnidispatch-${ARCH}.jnilib"/>
-        <arg value="-output"/>
-        <arg value="libjnidispatch.jnilib"/>
-      </exec>
-      <condition property="replace-lib" value="true">
-        <or>
-          <not>
-            <equals arg1="0" arg2="${lipo-result}"/>
-          </not>
-          <isfalse value="${lib-available}"/>
-        </or>
-      </condition>
-      <move file="${build.native}/libjnidispatch-${ARCH}.jnilib" tofile="${build.native}/libjnidispatch.jnilib" if:true="${replace-lib}" overwrite="true"/>
-    </sequential>
-  </macrodef>
-
   <target name="native" depends="-enable-native,javah,-native-api-check,-prepare-native" unless="-native"
           description="Build native libraries.  Use 'ant -DCC=xxx' to build using a compiler other than gcc">
     <exec executable="${make}" dir="${native}" failonerror="true">
@@ -1172,7 +1126,6 @@ cd ..
       <arg value="JNA_JNI_VERSION=${jni.version}"/>
       <arg value="CHECKSUM=${jni.md5}"/>
     </exec>
-    <darwin-lipo if:set="darwin.build"/>
     <mkdir dir="${classes}/${native.path}"/>
     <copy todir="${classes}/${native.path}">
       <fileset dir="${build.native}"
@@ -1258,6 +1211,42 @@ cd ..
         <attribute name="permissions" value="all-permissions"/>
       </manifest>
     </jar>
+
+    <!--
+        Create test case for darwin - darwin supports two paths: darwin-$arch
+        and darwin.
+
+        - The jna-test2.jar is the artifact, that contains the native library
+          in darwin-$arch
+        - The jna-test3.jar is the artifact, that contains the native library
+          in darwin
+    -->
+
+    <delete>
+        <fileset dir="${test.classes}">
+            <include name="*testlib-jar*"/>
+        </fileset>
+    </delete>
+
+    <jar jarfile="${build}/${testjar2}" createUnicodeExtraFields="never" encoding="UTF-8">
+      <fileset dir="${test.classes}">
+        <include name="**/*testlib-jar*"/>
+      </fileset>
+    </jar>
+
+    <condition property="resource.secondary-prefix" value="darwin">
+        <matches pattern="^darwin.*" string="${resource.prefix}"/>
+    </condition>
+    <property name="resource.secondary-prefix" value="${resource.prefix}" />
+
+    <move file="${test.classes}/${resource.prefix}" tofile="${test.classes}/${resource.secondary-prefix}" />
+
+    <jar jarfile="${build}/${testjar3}" createUnicodeExtraFields="never" encoding="UTF-8">
+      <fileset dir="${test.classes}">
+        <include name="**/*testlib-jar*"/>
+      </fileset>
+    </jar>
+
     <!-- Ensure jar-based library is unavailable on FS-based class path -->
     <delete>
       <fileset dir="${build}">

diff --git a/lib/native/darwin-aarch64.jar b/lib/native/darwin-aarch64.jar
diff --git a/lib/native/darwin-ppc.jar b/lib/native/darwin-ppc.jar
diff --git a/lib/native/darwin-ppc64.jar b/lib/native/darwin-ppc64.jar
diff --git a/lib/native/darwin-x86-64.jar b/lib/native/darwin-x86-64.jar
diff --git a/lib/native/darwin-x86.jar b/lib/native/darwin-x86.jar
diff --git a/lib/native/darwin.jar b/lib/native/darwin.jar
diff --git a/native/Makefile b/native/Makefile
@@ -417,7 +417,7 @@ DEFAULT_ARCH=$(shell arch)
 HOST_CONFIG=--host $(DARWIN_ARCH)-apple-darwin
 FFI_ENV += CC="$(CC)" CFLAGS="-mmacosx-version-min=$(MACOSX_DEPLOYMENT_TARGET) -arch $(DARWIN_ARCH) $(ISYSROOT) $(COPT) $(CDEBUG)" CPPFLAGS="$(CDEFINES)" LD="$(LD) -arch $(DARWIN_ARCH)"
 LIBSFX=.dylib
-JNISFX=-$(ARCH).jnilib
+JNISFX=.jnilib
 # JAWT no longer supported on OSX
 CDEFINES+=-DTARGET_RT_MAC_CFM=0 -DFFI_MMAP_EXEC_WRIT -DNO_JAWT
 

diff --git a/src/com/sun/jna/Native.java b/src/com/sun/jna/Native.java
@@ -1084,9 +1084,32 @@ public static File extractFromResourcePath(String name, ClassLoader loader) thro
             resourcePath = resourcePath.substring(1);
         }
         URL url = loader.getResource(resourcePath);
-        if (url == null && resourcePath.startsWith(Platform.RESOURCE_PREFIX)) {
-            // If not found with the standard resource prefix, try without it
-            url = loader.getResource(libname);
+        if (url == null) {
+            if (resourcePath.startsWith(Platform.RESOURCE_PREFIX)) {
+                // Fallback for legacy darwin behaviour: darwin was in the past
+                // special cased in that all architectures were mapped to the same
+                // prefix and it was expected, that a fat binary was present at that
+                // point, that contained all architectures.
+                if(Platform.RESOURCE_PREFIX.startsWith("darwin")) {
+                    url = loader.getResource("darwin/" + resourcePath.substring(Platform.RESOURCE_PREFIX.length() + 1));
+                }
+                if (url == null) {
+                    // If not found with the standard resource prefix, try without it
+                    url = loader.getResource(libname);
+                }
+            } else if (resourcePath.startsWith("com/sun/jna/" + Platform.RESOURCE_PREFIX + "/")) {
+                // Fallback for legacy darwin behaviour: darwin was in the past
+                // special cased in that all architectures were mapped to the same
+                // prefix and it was expected, that a fat binary was present at that
+                // point, that contained all architectures.
+                if(Platform.RESOURCE_PREFIX.startsWith("com/sun/jna/darwin")) {
+                    url = loader.getResource("com/sun/jna/darwin" + resourcePath.substring(("com/sun/jna/" + Platform.RESOURCE_PREFIX).length() + 1));
+                }
+                if (url == null) {
+                    // If not found with the standard resource prefix, try without it
+                    url = loader.getResource(libname);
+                }
+            }
         }
         if (url == null) {
             String path = System.getProperty("java.class.path");
@@ -1111,7 +1134,7 @@ public static File extractFromResourcePath(String name, ClassLoader loader) thro
             }
         }
         else if (!Boolean.getBoolean("jna.nounpack")) {
-            InputStream is = loader.getResourceAsStream(resourcePath);
+            InputStream is = url.openStream();
             if (is == null) {
                 throw new IOException("Can't obtain InputStream for " + resourcePath);
             }

diff --git a/src/com/sun/jna/Platform.java b/src/com/sun/jna/Platform.java
@@ -313,7 +313,7 @@ static String getNativeLibraryResourcePrefix(int osType, String arch, String nam
                 osPrefix = "w32ce-" + arch;
                 break;
             case Platform.MAC:
-                osPrefix = "darwin";
+                osPrefix = "darwin-" + arch;
                 break;
             case Platform.LINUX:
                 osPrefix = "linux-" + arch;

diff --git a/test/com/sun/jna/LibraryLoadTest.java b/test/com/sun/jna/LibraryLoadTest.java
@@ -100,7 +100,8 @@ public void testLoadFromClasspathAbsolute() throws MalformedURLException {
     }
 
     public void testLoadFromJar() throws MalformedURLException {
-        NativeLibrary.getInstance("testlib-jar", new TestLoader(new File(TESTJAR)));
+        NativeLibrary.getInstance("testlib-jar", new TestLoader(new File(TESTJAR2)));
+        NativeLibrary.getInstance("testlib-jar", new TestLoader(new File(TESTJAR3)));
     }
 
     public void testLoadFromJarAbsolute() throws MalformedURLException {

diff --git a/test/com/sun/jna/Paths.java b/test/com/sun/jna/Paths.java
@@ -54,4 +54,6 @@ public CloverLoader(ClassLoader parent) throws MalformedURLException {
         : System.getProperty("jna.nativedir",
                              BUILDDIR + "/native-" + Platform.RESOURCE_PREFIX + "/");
     String TESTJAR = BUILDDIR + "/jna-test.jar";
+    String TESTJAR2 = BUILDDIR + "/jna-test2.jar";
+    String TESTJAR3 = BUILDDIR + "/jna-test3.jar";
 }
diff --git a/test/com/sun/jna/PlatformTest.java b/test/com/sun/jna/PlatformTest.java
@@ -38,16 +38,16 @@ public void testOSPrefix() {
         assertEquals("Wrong resource path Windows CE/arm", "w32ce-arm",
                      Platform.getNativeLibraryResourcePrefix(Platform.WINDOWSCE,
                                                              "arm", "Windows CE"));
-        assertEquals("Wrong resource path Mac/x86", "darwin",
+        assertEquals("Wrong resource path Mac/x86", "darwin-x86",
                      Platform.getNativeLibraryResourcePrefix(Platform.MAC,
                                                              "x86", "Darwin"));
-        assertEquals("Wrong resource path Mac/x86", "darwin",
+        assertEquals("Wrong resource path Mac/x86", "darwin-x86",
                      Platform.getNativeLibraryResourcePrefix(Platform.MAC,
                                                              "i386", "Darwin"));
-        assertEquals("Wrong resource path Mac/x86_64", "darwin",
+        assertEquals("Wrong resource path Mac/x86_64", "darwin-x86-64",
                      Platform.getNativeLibraryResourcePrefix(Platform.MAC,
                                                              "x86_64", "Mac"));
-        assertEquals("Wrong resource path Mac/aarch64", "darwin",
+        assertEquals("Wrong resource path Mac/aarch64", "darwin-aarch64",
                     Platform.getNativeLibraryResourcePrefix(Platform.MAC,
                                                              "aarch64", "Mac"));
         assertEquals("Wrong resource path Solaris/sparc", "sunos-sparc",