Work around Android 9+ bug with extractNativeLibs flag

Fixes: dotnet#4983
Context: https://developer.android.com/reference/android/content/pm/ApplicationInfo.html#FLAG_EXTRACT_NATIVE_LIBS

Xamarin.Android has supported the [extractNativeLibs][0]
attribute (introduced by Android API 23) on the `<application>` element
in AndroidManifest.xml since 95ca102,
with a single significant modification in
feb9ea2 after we discovered that
Android build system can set the flag when constructing the APK after we
are done packaging.

feb9ea2 introduced a runtime check to see whether the
`FLAG_EXTRACT_NATIVE_LIBS` is *not* set, which meant that the native
libraries are to stay in the APK file and we need to set up our DSO
search paths to point to the inside of APK files instead of the
traditional filesystem location.

However, it appears that Android 10 (API 29, on both devices and in emulators)
and Android 9 (API 28, on *just* the devices) broke the
`FLAG_EXTRACT_NATIVE_LIBS` semantics in that it is possible for the flag
to be *set* (which means libraries are *extracted*) with the libraries
not extracted from APKs, thus breaking the logic implemented in
feb9ea2.

It is possible that other Android versions are affected as well, which
means we can no longer trust the flag value and need to implement
another way of checking whether the libraries are on the filesystem or
in the APK.  The simplest approach is to check for existence of a known
library in the filesystem location, regardless of the API level, and
assume the flag is *not* set if the shared library is missing.  This is
what this commit implements.  The check is performed once on application
startup, thus minimizing the performance impact.

[0]: https://developer.android.com/guide/topics/manifest/application-element#extractNativeLibs

src/java-runtime/java/mono/android/DebugRuntime.java

@@ -4,5 +4,5 @@ public class DebugRuntime {
 	private DebugRuntime ()
 	{}
 
-	public static native void init (String[] apks, String runtimeLibDir, String[] appDirs, String[] externalStorageDirs, int androidApiLevel, boolean embeddedDSOsEnabled);
+	public static native void init (String[] apks, String runtimeLibDir, String[] appDirs, String[] externalStorageDirs);
 }

src/java-runtime/java/mono/android/MonoPackageManager.java

@@ -18,9 +18,6 @@
 
 public class MonoPackageManager {
 
-	// Exists only since API23 onwards, we must define it here
-	static final int FLAG_EXTRACT_NATIVE_LIBS = 0x10000000;
-
 	static Object lock = new Object ();
 	static boolean initialized;
 
@@ -51,7 +48,6 @@ public static void LoadApplication (Context context, ApplicationInfo runtimePack
 				String externalLegacyDir = new java.io.File (
 							external0,
 							"../legacy/Android/data/" + context.getPackageName () + "/files/.__override__").getAbsolutePath ();
-				boolean embeddedDSOsEnabled = android.os.Build.VERSION.SDK_INT >= 23 && (runtimePackage.flags & FLAG_EXTRACT_NATIVE_LIBS) == 0;
 				String runtimeDir = getNativeLibraryPath (runtimePackage);
 				String[] appDirs = new String[] {filesDir, cacheDir, dataDir};
 				String[] externalStorageDirs = new String[] {externalDir, externalLegacyDir};
@@ -85,7 +81,7 @@ public static void LoadApplication (Context context, ApplicationInfo runtimePack
 				//
 				if (BuildConfig.Debug) {
 					System.loadLibrary ("xamarin-debug-app-helper");
-					DebugRuntime.init (apks, runtimeDir, appDirs, externalStorageDirs, android.os.Build.VERSION.SDK_INT, embeddedDSOsEnabled);
+					DebugRuntime.init (apks, runtimeDir, appDirs, externalStorageDirs);
 				} else {
 					System.loadLibrary("monosgen-2.0");
 				}
@@ -111,7 +107,6 @@ public static void LoadApplication (Context context, ApplicationInfo runtimePack
 						externalStorageDirs,
 						MonoPackageManager_Resources.Assemblies,
 						Build.VERSION.SDK_INT,
-						embeddedDSOsEnabled,
 						isEmulator ()
 					);
 

src/java-runtime/java/mono/android/Runtime.java

@@ -15,7 +15,7 @@ public class Runtime {
 	}
 
 	public static native void init (String lang, String[] runtimeApks, String runtimeDataDir, String[] appDirs, ClassLoader loader, String[] externalStorageDirs, String[] assemblies, String packageName, int apiLevel, String[] environmentVariables);
-	public static native void initInternal (String lang, String[] runtimeApks, String runtimeDataDir, String[] appDirs, ClassLoader loader, String[] externalStorageDirs, String[] assemblies, int apiLevel, boolean embeddedDSOsEnabled, boolean isEmulator);
+	public static native void initInternal (String lang, String[] runtimeApks, String runtimeDataDir, String[] appDirs, ClassLoader loader, String[] externalStorageDirs, String[] assemblies, int apiLevel, boolean isEmulator);
 	public static native void register (String managedType, java.lang.Class nativeClass, String methods);
 	public static native void notifyTimeZoneChanged ();
 	public static native int createNewContext (String[] runtimeApks, String[] assemblies, ClassLoader loader);

src/monodroid/jni/basic-android-system.cc

@@ -1,6 +1,7 @@
 #include <cerrno>
 
 #include "basic-android-system.hh"
+#include "cpp-util.hh"
 #include "globals.hh"
 
 using namespace xamarin::android;
@@ -12,13 +13,29 @@ size_t BasicAndroidSystem::app_lib_directories_size = 0;
 const char* BasicAndroidSystem::built_for_abi_name = nullptr;
 
 void
-BasicAndroidSystem::setup_app_library_directories (jstring_array_wrapper& runtimeApks, jstring_array_wrapper& appDirs, int androidApiLevel)
+BasicAndroidSystem::detect_embedded_dso_mode (jstring_array_wrapper& appDirs) noexcept
 {
-	if (androidApiLevel < 23 || !is_embedded_dso_mode_enabled ()) {
+	// appDirs[2] points to the native library directory
+	simple_pointer_guard<char[]> libmonodroid_path = utils.path_combine (appDirs[2].get_cstr (), "libmonodroid.so");
+	log_debug (LOG_ASSEMBLY, "Checking if libmonodroid was unpacked to %s", libmonodroid_path.get ());
+	if (!utils.file_exists (libmonodroid_path)) {
+		log_debug (LOG_ASSEMBLY, "%s not found, assuming application/android:extractNativeLibs == false", libmonodroid_path.get ());
+		set_embedded_dso_mode_enabled (true);
+	} else {
+		log_debug (LOG_ASSEMBLY, "Native libs extracted to %s, assuming application/android:extractNativeLibs == true", appDirs[2].get_cstr ());
+		set_embedded_dso_mode_enabled (false);
+	}
+}
+
+void
+BasicAndroidSystem::setup_app_library_directories (jstring_array_wrapper& runtimeApks, jstring_array_wrapper& appDirs)
+{
+	if (!is_embedded_dso_mode_enabled ()) {
 		log_info (LOG_DEFAULT, "Setting up for DSO lookup in app data directories");
 		BasicAndroidSystem::app_lib_directories_size = 1;
 		BasicAndroidSystem::app_lib_directories = new const char*[app_lib_directories_size]();
 		BasicAndroidSystem::app_lib_directories [0] = utils.strdup_new (appDirs[2].get_cstr ());
+		log_debug (LOG_ASSEMBLY, "Added filesystem DSO lookup location: %s", appDirs[2].get_cstr ());
 	} else {
 		log_info (LOG_DEFAULT, "Setting up for DSO lookup directly in the APK");
 		BasicAndroidSystem::app_lib_directories_size = runtimeApks.get_length ();
@@ -48,6 +65,7 @@ BasicAndroidSystem::add_apk_libdir (const char *apk, size_t index, [[maybe_unuse
 	assert (user_data != nullptr);
 	assert (index < app_lib_directories_size);
 	app_lib_directories [index] = utils.string_concat (apk, "!/lib/", static_cast<const char*>(user_data));
+	log_debug (LOG_ASSEMBLY, "Added APK DSO lookup location: %s", app_lib_directories[index]);
 }
 
 void

src/monodroid/jni/basic-android-system.hh

@@ -55,7 +55,7 @@ namespace xamarin::android::internal
 		static const char* get_built_for_abi_name ();
 
 	public:
-		void setup_app_library_directories (jstring_array_wrapper& runtimeApks, jstring_array_wrapper& appDirs, int androidApiLevel);
+		void setup_app_library_directories (jstring_array_wrapper& runtimeApks, jstring_array_wrapper& appDirs);
 		void setup_apk_directories (unsigned short running_on_cpu, jstring_array_wrapper &runtimeApks);
 
 		const char* get_override_dir (size_t index) const
@@ -79,10 +79,7 @@ namespace xamarin::android::internal
 			return embedded_dso_mode_enabled;
 		}
 
-		void set_embedded_dso_mode_enabled (bool yesno)
-		{
-			embedded_dso_mode_enabled = yesno;
-		}
+		void detect_embedded_dso_mode (jstring_array_wrapper& appDirs) noexcept;
 
 		char *get_runtime_libdir () const
 		{
@@ -111,6 +108,11 @@ namespace xamarin::android::internal
 	private:
 		char* determine_primary_override_dir (jstring_wrapper &home);
 
+		void set_embedded_dso_mode_enabled (bool yesno) noexcept
+		{
+			embedded_dso_mode_enabled = yesno;
+		}
+
 	private:
 		bool  embedded_dso_mode_enabled = false;
 		char *runtime_libdir = nullptr;

src/monodroid/jni/debug-app-helper.cc

@@ -43,6 +43,7 @@ bool maybe_load_library (const char *path);
 #define ANDROID_LOG_WARN  2
 #define ANDROID_LOG_ERROR 3
 #define ANDROID_LOG_FATAL 4
+#define ANDROID_LOG_DEBUG 5
 
 static void
 __android_log_vprint (int prio, const char* tag, const char* fmt, va_list ap)
@@ -56,7 +57,7 @@ __android_log_vprint (int prio, const char* tag, const char* fmt, va_list ap)
 
 static constexpr char TAG[] = "debug-app-helper";
 
-unsigned int log_categories = LOG_DEFAULT;
+unsigned int log_categories = LOG_DEFAULT | LOG_ASSEMBLY;
 BasicUtilities utils;
 BasicAndroidSystem androidSystem;
 
@@ -69,17 +70,15 @@ JNI_OnLoad ([[maybe_unused]] JavaVM *vm, [[maybe_unused]] void *reserved)
 JNIEXPORT void JNICALL
 Java_mono_android_DebugRuntime_init (JNIEnv *env, [[maybe_unused]] jclass klass, jobjectArray runtimeApksJava,
                                      jstring runtimeNativeLibDir, jobjectArray appDirs,
-                                     jobjectArray externalStorageDirs, jint androidApiLevel,
-                                     jboolean embeddedDSOsEnabled)
+                                     jobjectArray externalStorageDirs)
 {
-	androidSystem.set_embedded_dso_mode_enabled ((bool) embeddedDSOsEnabled);
-
 	jstring_array_wrapper applicationDirs (env, appDirs);
 	jstring_array_wrapper runtimeApks (env, runtimeApksJava);
 
+	androidSystem.detect_embedded_dso_mode (applicationDirs);
 	androidSystem.set_primary_override_dir (applicationDirs [0]);
 	androidSystem.set_override_dir (0, androidSystem.get_primary_override_dir ());
-	androidSystem.setup_app_library_directories (runtimeApks, applicationDirs, androidApiLevel);
+	androidSystem.setup_app_library_directories (runtimeApks, applicationDirs);
 
 	jstring_wrapper jstr (env);
 	jstr = env->GetObjectArrayElement (externalStorageDirs, 0);
@@ -298,6 +297,17 @@ get_libmonosgen_path ()
 	return libmonoso;
 }
 
+void
+log_debug_nocheck ([[maybe_unused]] LogCategories category, const char *format, ...)
+{
+	va_list args;
+
+	if ((log_categories & category) == 0)
+		return;
+
+	DO_LOG (ANDROID_LOG_DEBUG, TAG, format, args);
+}
+
 void
 log_info ([[maybe_unused]] LogCategories category, const char *format, ...)
 {

src/monodroid/jni/debug-app-helper.hh

@@ -12,6 +12,6 @@ extern "C" {
 	 * Signature: ([Ljava/lang/String;Ljava/lang/String;[Ljava/lang/String);[Ljava/lang/String);Ljava/lang/String;IZ)V
 	 */
 	JNIEXPORT void JNICALL Java_mono_android_DebugRuntime_init
-	(JNIEnv *, jclass, jobjectArray, jstring, jobjectArray, jobjectArray, jint, jboolean);
+	(JNIEnv *, jclass, jobjectArray, jstring, jobjectArray, jobjectArray);
 }
 #endif // _Included_mono_android_DebugRuntime

src/monodroid/jni/monodroid-glue-internal.hh

@@ -77,7 +77,7 @@ namespace xamarin::android::internal
 		void Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass klass, jstring lang, jobjectArray runtimeApksJava,
 		                                             jstring runtimeNativeLibDir, jobjectArray appDirs, jobject loader,
 		                                             jobjectArray externalStorageDirs, jobjectArray assembliesJava,
-		                                             jint apiLevel, jboolean embeddedDSOsEnabled, jboolean isEmulator);
+		                                             jint apiLevel, jboolean isEmulator);
 		jint Java_mono_android_Runtime_createNewContextWithData (JNIEnv *env, jclass klass, jobjectArray runtimeApksJava, jobjectArray assembliesJava,
 		                                                         jobjectArray assembliesBytes, jobjectArray assembliesPaths, jobject loader, jboolean force_preload_assemblies);
 		void Java_mono_android_Runtime_switchToContext (JNIEnv *env, jint contextID);

src/monodroid/jni/monodroid-glue.cc

@@ -1522,7 +1522,7 @@ inline void
 MonodroidRuntime::Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass klass, jstring lang, jobjectArray runtimeApksJava,
                                                           jstring runtimeNativeLibDir, jobjectArray appDirs, jobject loader,
                                                           [[maybe_unused]] jobjectArray externalStorageDirs, jobjectArray assembliesJava,
-                                                          jint apiLevel, jboolean embeddedDSOsEnabled, jboolean isEmulator)
+                                                          jint apiLevel, jboolean isEmulator)
 {
 	init_logging_categories ();
 
@@ -1532,8 +1532,10 @@ MonodroidRuntime::Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass kl
 		total_time.mark_start ();
 	}
 
+	jstring_array_wrapper applicationDirs (env, appDirs);
+
 	android_api_level = apiLevel;
-	androidSystem.set_embedded_dso_mode_enabled ((bool) embeddedDSOsEnabled);
+	androidSystem.detect_embedded_dso_mode (applicationDirs);
 	androidSystem.set_running_in_emulator (isEmulator);
 
 	java_TimeZone = utils.get_class_from_runtime_field (env, klass, "java_util_TimeZone", true);
@@ -1545,7 +1547,6 @@ MonodroidRuntime::Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass kl
 
 	androidSystem.setup_environment ();
 
-	jstring_array_wrapper applicationDirs (env, appDirs);
 	jstring_wrapper &home = applicationDirs[0];
 	set_environment_variable_for_directory ("TMPDIR", applicationDirs[1]);
 	set_environment_variable_for_directory ("HOME", home);
@@ -1555,7 +1556,7 @@ MonodroidRuntime::Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass kl
 	disable_external_signal_handlers ();
 
 	jstring_array_wrapper runtimeApks (env, runtimeApksJava);
-	androidSystem.setup_app_library_directories (runtimeApks, applicationDirs, apiLevel);
+	androidSystem.setup_app_library_directories (runtimeApks, applicationDirs);
 
 	init_reference_logging (androidSystem.get_primary_override_dir ());
 	androidSystem.create_update_dir (androidSystem.get_primary_override_dir ());
@@ -1755,7 +1756,6 @@ Java_mono_android_Runtime_init (JNIEnv *env, jclass klass, jstring lang, jobject
 		externalStorageDirs,
 		assembliesJava,
 		apiLevel,
-		/* embeddedDSOsEnabled */ JNI_FALSE,
 		/* isEmulator */ JNI_FALSE
 	);
 }
@@ -1764,7 +1764,7 @@ JNIEXPORT void JNICALL
 Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass klass, jstring lang, jobjectArray runtimeApksJava,
                                 jstring runtimeNativeLibDir, jobjectArray appDirs, jobject loader,
                                 jobjectArray externalStorageDirs, jobjectArray assembliesJava,
-                                jint apiLevel, jboolean embeddedDSOsEnabled, jboolean isEmulator)
+                                jint apiLevel, jboolean isEmulator)
 {
 	monodroidRuntime.Java_mono_android_Runtime_initInternal (
 		env,
@@ -1777,7 +1777,6 @@ Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass klass, jstring lang,
 		externalStorageDirs,
 		assembliesJava,
 		apiLevel,
-		embeddedDSOsEnabled,
 		isEmulator
 	);
 }

tests/MSBuildDeviceIntegration/Tests/DebuggingTest.cs

@@ -33,7 +33,7 @@ void SetTargetFrameworkAndManifest(XamarinAndroidApplicationProject proj, Builde
 
 		[Test]
 		[Retry (1)]
-		public void ApplicationRunsWithoutDebugger ([Values (false, true)] bool isRelease)
+		public void ApplicationRunsWithoutDebugger ([Values (false, true)] bool isRelease, [Values (false, true)] bool extractNativeLibs)
 		{
 			AssertHasDevices ();
 
@@ -46,6 +46,7 @@ public void ApplicationRunsWithoutDebugger ([Values (false, true)] bool isReleas
 			proj.SetDefaultTargetDevice ();
 			using (var b = CreateApkBuilder (Path.Combine ("temp", TestName))) {
 				SetTargetFrameworkAndManifest (proj, b);
+				proj.AndroidManifest = proj.AndroidManifest.Replace ("<application ", $"<application android:extractNativeLibs=\"{extractNativeLibs}\" ");
 				Assert.True (b.Install (proj), "Project should have installed.");
 				ClearAdbLogcat ();
 				if (CommercialBuildAvailable)