[runtime/tools] Implement finding native support libraries when linki…

…ng statically. Fixes #10950, #11145 and #12100. (#12323) * Add support for Mono Components. * Modify how we look up symbols from native libraries shipped with Mono: we keep track of which native libraries we linked with, and depending on how we linked to those assemblies, we look the symbols up at runtime in either the current executable (if linking statically), or the actual library (where the P/Invoke says they're supposed to be). * This means that we have to propagate how libmono is linked from the MSBuild code to the Application class so that our existing logic is able to correctly determine which native mono lib to use. * Modify how we list the P/Invokes we need to preserve by taking into account the list of native libraries from Mono we have to link with (for .NET). For legacy Xamarin, I've reverted the logic to how it was before we started adding .NET support. Fixes #10950. Fixes #11145. Fixes #12100.
xamarin · Aug 3, 2021 · eafe528 · eafe528 · vs-mobiletools-engineering-service2 · Aug 3, 2021
1 parent 1fc007c
commit eafe528
Show file tree

Hide file tree

Showing 7 changed files with 145 additions and 75 deletions.
diff --git a/dotnet/targets/Xamarin.Shared.Sdk.targets b/dotnet/targets/Xamarin.Shared.Sdk.targets
@@ -331,7 +331,40 @@
 		</Touch>
 	</Target>
 
-	<Target Name="_ComputeLinkerArguments" DependsOnTargets="_ComputeLinkMode;_ComputeFrameworkVariables;_ComputeFrameworkAssemblies;ComputeResolvedFilesToPublishList;_ParseBundlerArguments;_ComputeVariables;_CreateRuntimeConfiguration">
+	<Target Name="_ComputeMonoComponents" Condition="'$(UseMonoRuntime)' == 'true' And '$(_LibMonoLinkMode)' == 'static'" BeforeTargets="_MonoSelectRuntimeComponents" DependsOnTargets="_ComputeVariables">
+		<!-- https://github.com/dotnet/runtime/blob/main/docs/design/mono/components.md -->
+		<ItemGroup>
+			<_MonoComponent Include="hot_reload" Condition="'$(MtouchInterpreter)' == 'true'" />
+			<_MonoComponent Include="debugger" Condition="'$(_BundlerDebug)' == 'true'" />
+			<_MonoComponent Include="diagnostics_tracing" Condition="'$(_BundlerDebug)' == 'true'" />
+		</ItemGroup>
+	</Target>
+
+	<Target Name="_ComputeMonoLibraries" Condition="'$(UseMonoRuntime)' == 'true'" DependsOnTargets="_ComputeMonoComponents;_MonoSelectRuntimeComponents">
+		<!-- We only include any mono components when linking with mono statically. The components are already included in the dynamic versions of Mono (both the dylib and the framework) -->
+		<ItemGroup Condition="'$(_LibMonoLinkMode)' == 'static'">
+			<!-- Remove files mono told us not to link with -->
+			<_MonoLibrary Remove="@(_MonoRuntimeComponentDontLink -> '$(_MonoRuntimePackPath)/native/%(Identity)')" />
+
+			<!-- Add files mono told us to link with -->
+			<_MonoLibrary Include="@(_MonoRuntimeComponentLink -> '$(_MonoRuntimePackPath)/native/%(Identity)')" />
+		</ItemGroup>
+	</Target>
+
+	<PropertyGroup>
+		<_ComputeLinkerArgumentsDependsOn>
+			_ComputeLinkMode;
+			_ComputeFrameworkVariables;
+			_ComputeFrameworkAssemblies;
+			ComputeResolvedFilesToPublishList;
+			_ParseBundlerArguments;
+			_ComputeVariables;
+			_CreateRuntimeConfiguration;
+			_ComputeMonoLibraries;
+		</_ComputeLinkerArgumentsDependsOn>
+	</PropertyGroup>
+
+	<Target Name="_ComputeLinkerArguments" DependsOnTargets="$(_ComputeLinkerArgumentsDependsOn)">
 		<!-- Validate the linker mode -->
 		<Error Text="Invalid link mode: '$(_LinkMode)'. Valid link modes are: 'None', 'SdkOnly' and 'Full'" Condition="'$(_LinkMode)' != 'None' And '$(_LinkMode)' != 'SdkOnly' And '$(_LinkMode)' != 'Full'" />
 
@@ -376,6 +409,7 @@
 				LinkMode=$(_LinkMode)
 				MarshalManagedExceptionMode=$(_MarshalManagedExceptionMode)
 				MarshalObjectiveCExceptionMode=$(_MarshalObjectiveCExceptionMode)
+				@(_MonoLibrary -> 'MonoLibrary=%(Identity)')
 				Optimize=$(_BundlerOptimize)
 				PartialStaticRegistrarLibrary=$(_LibPartialStaticRegistrar)
 				Platform=$(_PlatformName)
@@ -836,9 +870,7 @@
 		<ItemGroup>
 			<_XamarinMainLibraries Include="$(_XamarinNativeLibraryDirectory)/$(_LibXamarinName)" />
 			<!-- Link with the libraries shipped with the mono runtime pack -->
-			<_XamarinMainLibraries Include="@(_MonoLibrary)" Condition="'$(_PlatformName)' != 'macOS'" />
-			<!-- But don't link with any dylibs on macOS, because they might have library initializers which may fail - instead delay any failures until whatever feature they provide is needed -->
-			<_XamarinMainLibraries Include="@(_MonoLibrary)" Condition="'$(_PlatformName)' == 'macOS' And ('%(_MonoLibrary.Extension)' == '.a' Or '%(_MonoLibrary.Filename)' == 'libcoreclr')" />
+			<_XamarinMainLibraries Include="@(_MonoLibrary)" />
 			<!-- Link with static libraries from NativeReference items ((xc)frameworks are handled in the _ComputeFrameworkFilesToPublish target, and dynamic libraries are handled in the _ComputeDynamicLibrariesToPublish target) -->
 			<_XamarinMainLibraries Include="@(_FileNativeReference)" />
 
@@ -906,7 +938,7 @@
 		</_ComputeLinkModeDependsOn>
 	</PropertyGroup>
 
-	<Target Name="_ComputePublishLocation" DependsOnTargets="_GenerateBundleName;_ParseBundlerArguments">
+	<Target Name="_ComputePublishLocation" DependsOnTargets="_GenerateBundleName;_ParseBundlerArguments;_ComputeMonoLibraries">
 		<PropertyGroup>
 			<_AssemblyPublishDir Condition="'$(_PlatformName)' == 'iOS' Or '$(_PlatformName)' == 'tvOS' Or '$(_PlatformName)' == 'watchOS'">$(MSBuildProjectDirectory)\$(_AppBundlePath)\</_AssemblyPublishDir>
 			<_AssemblyPublishDir Condition="'$(_PlatformName)' == 'macOS' Or '$(_PlatformName)' == 'MacCatalyst'">$(MSBuildProjectDirectory)\$(_AppBundlePath)\Contents\$(_CustomBundleName)\</_AssemblyPublishDir>
@@ -973,6 +1005,14 @@
 				            "
 			/>
 
+			<!-- Remove any dylibs Mono told us not to link with -->
+			<ResolvedFileToPublish
+				Remove="@(_MonoRuntimeComponentDontLink -> '$(_MonoRuntimePackPath)/native/%(Identity)')"
+				Condition=" '%(ResolvedFileToPublish.AssetType)' == 'native' And
+				            '%(ResolvedFileToPublish.RuntimeIdentifier)' == '$(RuntimeIdentifier)' And
+				            '%(ResolvedFileToPublish.NuGetPackageId)' == '$(_MonoNugetPackageId)'
+				            "
+			/>
 		</ItemGroup>
 	</Target>
 

diff --git a/runtime/runtime.m b/runtime/runtime.m
@@ -83,7 +83,11 @@
 enum XamarinLaunchMode xamarin_launch_mode = XamarinLaunchModeApp;
 bool xamarin_supports_dynamic_registration = true;
 const char *xamarin_runtime_configuration_name = NULL;
-const char *xamarin_mono_native_lib_name = "__Internal";
+
+#if DOTNET
+enum XamarinNativeLinkMode xamarin_libmono_native_link_mode = XamarinNativeLinkModeStaticObject;
+const char **xamarin_runtime_libraries = NULL;
+#endif
 
 /* Callbacks */
 
@@ -2528,31 +2532,28 @@ -(void) xamarinSetFlags: (enum XamarinGCHandleFlags) flags;
 		xamarin_assertion_message ("Failed to initialize the VM");
 }
 
+static bool
+xamarin_is_native_library (const char *libraryName)
+{
+	if (xamarin_runtime_libraries == NULL)
+		return false;
+
+	for (int i = 0; xamarin_runtime_libraries [i] != NULL; i++) {
+		if (!strcmp (xamarin_runtime_libraries [i], libraryName))
+			return true;
+	}
+
+	return false;
+}
+
 void*
 xamarin_pinvoke_override (const char *libraryName, const char *entrypointName)
 {
 
 	void* symbol = NULL;
 
-#if TARGET_OS_MACCATALYST
-	static void *monoNativeLibrary = NULL;
-#endif
-
 	if (!strcmp (libraryName, "__Internal")) {
 		symbol = dlsym (RTLD_DEFAULT, entrypointName);
-#if TARGET_OS_MACCATALYST
-	} else if (!strcmp (libraryName, "libSystem.Native") ||
-	           !strcmp (libraryName, "libSystem.Security.Cryptography.Native.Apple") ||
-	           !strcmp (libraryName, "libSystem.Net.Security.Native")) {
-		if (monoNativeLibrary == NULL) {
-			if (xamarin_mono_native_lib_name == NULL || !strcmp (xamarin_mono_native_lib_name, "__Internal")) {
-				monoNativeLibrary = RTLD_DEFAULT;
-			} else {
-				monoNativeLibrary = dlopen (xamarin_mono_native_lib_name, RTLD_LAZY);
-			}
-		}
-		symbol = dlsym (monoNativeLibrary, entrypointName);
-#endif // TARGET_OS_MACCATALYST
 #if !defined (CORECLR_RUNTIME) // we're intercepting objc_msgSend calls using the managed System.Runtime.InteropServices.ObjectiveC.Bridge.SetMessageSendCallback instead.
 #if defined (__i386__) || defined (__x86_64__) || defined (__arm64__)
 	} else if (!strcmp (libraryName, "/usr/lib/libobjc.dylib")) {
@@ -2573,6 +2574,21 @@ -(void) xamarinSetFlags: (enum XamarinGCHandleFlags) flags;
 		}
 #endif // defined (__i386__) || defined (__x86_64__) || defined (__arm64__)
 #endif // !defined (CORECLR_RUNTIME)
+	} else if (xamarin_is_native_library (libraryName)) {
+		switch (xamarin_libmono_native_link_mode) {
+		case XamarinNativeLinkModeStaticObject:
+			// lookup the symbol in loaded memory, like __Internal does.
+			symbol = dlsym (RTLD_DEFAULT, entrypointName);
+			break;
+		case XamarinNativeLinkModeDynamicLibrary:
+			// if we're not linking statically, then don't do anything at all, let mono handle whatever needs to be done
+			return NULL;
+		case XamarinNativeLinkModeFramework:
+		default:
+			// handle this as "DynamicLibrary" for now - do nothing.
+			LOG (PRODUCT ": Unhandled libmono link mode: %i when looking up %s in %s", xamarin_libmono_native_link_mode, entrypointName, libraryName);
+			return NULL;
+		}
 	} else {
 		return NULL;
 	}

diff --git a/runtime/xamarin/main.h b/runtime/xamarin/main.h
@@ -83,6 +83,13 @@ enum XamarinExceptionTypes : int {
 	XamarinExceptionTypes_System_OutOfMemoryException,
 };
 
+// Keep in sync with AssemblyBuildTarget in AssemblyBuildTarget.cs
+enum XamarinNativeLinkMode : int {
+	XamarinNativeLinkModeStaticObject,
+	XamarinNativeLinkModeDynamicLibrary,
+	XamarinNativeLinkModeFramework,
+};
+
 extern bool mono_use_llvm; // this is defined inside mono
 
 #if DEBUG
@@ -109,7 +116,8 @@ extern enum MarshalManagedExceptionMode xamarin_marshal_managed_exception_mode;
 extern enum XamarinLaunchMode xamarin_launch_mode;
 extern bool xamarin_supports_dynamic_registration;
 extern const char *xamarin_runtime_configuration_name;
-extern const char *xamarin_mono_native_lib_name;
+extern enum XamarinNativeLinkMode xamarin_libmono_native_link_mode;
+extern const char** xamarin_runtime_libraries;
 
 typedef void (*xamarin_setup_callback) ();
 typedef int (*xamarin_extension_main_callback) (int argc, char** argv);

diff --git a/tools/common/Application.cs b/tools/common/Application.cs
@@ -89,6 +89,7 @@ public partial class Application
 		public bool IsExtension;
 		public ApplePlatform Platform { get { return Driver.TargetFramework.Platform; } }
 
+		public List<string> MonoLibraries = new List<string> ();
 		public List<string> InterpretedAssemblies = new List<string> ();
 
 		// EnableMSym: only implemented for Xamarin.iOS

diff --git a/tools/common/Target.cs b/tools/common/Target.cs
@@ -743,6 +743,16 @@ void GenerateIOSMain (StringWriter sw, Abi abi)
 				sw.WriteLine ("extern \"C\" { void mono_sgen_mono_ilgen_init (void); }");
 			}
 
+#if NET
+			if (app.MonoNativeMode != MonoNativeMode.None) {
+				sw.WriteLine ("static const char *xamarin_runtime_libraries_array[] = {");
+				foreach (var lib in app.MonoLibraries)
+					sw.WriteLine ($"\t\"{Path.GetFileName (lib)}\",");
+				sw.WriteLine ($"\tNULL");
+				sw.WriteLine ("};");
+			}
+#endif
+
 			sw.WriteLine ("void xamarin_setup_impl ()");
 			sw.WriteLine ("{");
 
@@ -773,33 +783,26 @@ void GenerateIOSMain (StringWriter sw, Abi abi)
 
 			if (app.MonoNativeMode != MonoNativeMode.None) {
 #if NET
-				// Mono doesn't support dllmaps for Mac Catalyst / macOS in .NET for now:
-				// macOS: https://github.com/dotnet/runtime/issues/43204
-				// Mac Catalyst: https://github.com/dotnet/runtime/issues/48110
-				var addDllMap = App.Platform != ApplePlatform.MacCatalyst && app.Platform != ApplePlatform.MacOSX;
+				// Mono doesn't support dllmaps for Mac Catalyst / macOS in .NET, so we're using an alternative:
+				// the PINVOKE_OVERRIDE runtime option. Since we have to use it for Mac Catalyst + macOS, let's
+				// just use it everywhere to simplify code. This means that at runtime we need to know how we
+				// linked to mono, so store that in the xamarin_libmono_native_link_mode variable.
+				// Ref: https://github.com/dotnet/runtime/issues/43204 (macOS) https://github.com/dotnet/runtime/issues/48110 (Mac Catalyst)
+				sw.WriteLine ($"\txamarin_libmono_native_link_mode = XamarinNativeLinkMode{app.LibMonoNativeLinkMode};");
+				sw.WriteLine ($"\txamarin_runtime_libraries = xamarin_runtime_libraries_array;");
 #else
-				var addDllMap = true;
-#endif
 				string mono_native_lib;
-				if (app.LibMonoNativeLinkMode == AssemblyBuildTarget.StaticObject || Driver.IsDotNet)
+				if (app.LibMonoNativeLinkMode == AssemblyBuildTarget.StaticObject) {
 					mono_native_lib = "__Internal";
-				else
-					mono_native_lib = app.GetLibNativeName () + ".dylib";
-				if (addDllMap) {
-					sw.WriteLine ();
-#if NET
-					sw.WriteLine ($"\tmono_dllmap_insert (NULL, \"libSystem.Native\", NULL, \"{mono_native_lib}\", NULL);");
-					sw.WriteLine ($"\tmono_dllmap_insert (NULL, \"libSystem.Security.Cryptography.Native.Apple\", NULL, \"{mono_native_lib}\", NULL);");
-					sw.WriteLine ($"\tmono_dllmap_insert (NULL, \"libSystem.Net.Security.Native\", NULL, \"{mono_native_lib}\", NULL);");
-#else
-					sw.WriteLine ($"\tmono_dllmap_insert (NULL, \"System.Native\", NULL, \"{mono_native_lib}\", NULL);");
-					sw.WriteLine ($"\tmono_dllmap_insert (NULL, \"System.Security.Cryptography.Native.Apple\", NULL, \"{mono_native_lib}\", NULL);");
-					sw.WriteLine ($"\tmono_dllmap_insert (NULL, \"System.Net.Security.Native\", NULL, \"{mono_native_lib}\", NULL);");
-#endif
-					sw.WriteLine ();
 				} else {
-					sw.WriteLine ($"\txamarin_mono_native_lib_name = \"{mono_native_lib}\";");
+					mono_native_lib = app.GetLibNativeName () + ".dylib";
 				}
+				sw.WriteLine ();
+				sw.WriteLine ($"\tmono_dllmap_insert (NULL, \"System.Native\", NULL, \"{mono_native_lib}\", NULL);");
+				sw.WriteLine ($"\tmono_dllmap_insert (NULL, \"System.Security.Cryptography.Native.Apple\", NULL, \"{mono_native_lib}\", NULL);");
+				sw.WriteLine ($"\tmono_dllmap_insert (NULL, \"System.Net.Security.Native\", NULL, \"{mono_native_lib}\", NULL);");
+				sw.WriteLine ();
+#endif
 			}
 
 			if (app.EnableDebug)

diff --git a/tools/dotnet-linker/LinkerConfiguration.cs b/tools/dotnet-linker/LinkerConfiguration.cs
@@ -180,6 +180,9 @@ public static LinkerConfiguration GetInstance (LinkContext context, bool createI
 						Application.MarshalObjectiveCExceptions = mode;
 					}
 					break;
+				case "MonoLibrary":
+					Application.MonoLibraries.Add (value);
+					break;
 				case "Optimize":
 					user_optimize_flags = value;
 					break;
@@ -320,6 +323,8 @@ AssemblyBuildTarget ParseLinkMode (string value, string variableName)
 				return AssemblyBuildTarget.DynamicLibrary;
 			} else if (string.Equals (value, "static", StringComparison.OrdinalIgnoreCase)) {
 				return AssemblyBuildTarget.StaticObject;
+			} else if (string.Equals (value, "framework", StringComparison.OrdinalIgnoreCase)) {
+				return AssemblyBuildTarget.Framework;
 			}
 
 			throw new InvalidOperationException ($"Invalid {variableName} '{value}' in {LinkerFile}");
@@ -347,6 +352,9 @@ public void Write ()
 				Console.WriteLine ($"    LinkMode: {LinkMode}");
 				Console.WriteLine ($"    MarshalManagedExceptions: {Application.MarshalManagedExceptions} (IsDefault: {Application.IsDefaultMarshalManagedExceptionMode})");
 				Console.WriteLine ($"    MarshalObjectiveCExceptions: {Application.MarshalObjectiveCExceptions}");
+				Console.WriteLine ($"    {Application.MonoLibraries.Count} mono libraries:");
+				foreach (var lib in Application.MonoLibraries.OrderBy (v => v))
+					Console.WriteLine ($"        {lib}");
 				Console.WriteLine ($"    Optimize: {user_optimize_flags} => {Application.Optimizations}");
 				Console.WriteLine ($"    PartialStaticRegistrarLibrary: {PartialStaticRegistrarLibrary}");
 				Console.WriteLine ($"    Platform: {Platform}");

diff --git a/tools/linker/MonoTouch.Tuner/ListExportedSymbols.cs b/tools/linker/MonoTouch.Tuner/ListExportedSymbols.cs
@@ -1,5 +1,7 @@
 using System;
 using System.Collections.Generic;
+using System.IO;
+using System.Linq;
 
 using Mono.Cecil;
 using Mono.Linker;
@@ -124,48 +126,40 @@ void ProcessMethod (MethodDefinition method)
 					}
 				}
 
+#if NET
+				// Create a list of all the dynamic libraries from Mono that we'll link with
+				// We add 4 different variations for each library:
+				// * with and without a "lib" prefix
+				// * with and without the ".dylib" extension
+				var app = LinkerConfiguration.GetInstance (Context).Application;
+				var dynamicMonoLibraries = app.MonoLibraries.
+					Where (v => v.EndsWith (".dylib", StringComparison.OrdinalIgnoreCase)).
+					Select (v => Path.GetFileNameWithoutExtension (v)).
+					Select (v => v.StartsWith ("lib", StringComparison.OrdinalIgnoreCase) ? v.Substring (3) : v).ToHashSet ();
+				dynamicMonoLibraries.UnionWith (dynamicMonoLibraries.Select (v => "lib" + v).ToArray ());
+				dynamicMonoLibraries.UnionWith (dynamicMonoLibraries.Select (v => v + ".dylib").ToArray ());
+				// If the P/Invoke points to any of those libraries, then we add it as a P/Invoke symbol.
+				if (dynamicMonoLibraries.Contains (pinfo.Module.Name))
+					addPInvokeSymbol = true;
+#endif
+
 				switch (pinfo.Module.Name) {
 				case "__Internal":
 					Driver.Log (4, "Adding native reference to {0} in {1} because it's referenced by {2} in {3}.", pinfo.EntryPoint, pinfo.Module.Name, method.FullName, method.Module.Name);
 					DerivedLinkContext.RequiredSymbols.AddFunction (pinfo.EntryPoint).AddMember (method);
 					break;
 
-				case "libSystem.Net.Security.Native":
+#if !NET
 				case "System.Net.Security.Native":
-					addPInvokeSymbol = true;
-					break;
-
-				case "libSystem.Security.Cryptography.Native.Apple":
 				case "System.Security.Cryptography.Native.Apple":
-#if NET
-					// https://github.com/dotnet/runtime/issues/47533
-					if (DerivedLinkContext.App.Platform != ApplePlatform.MacOSX && DerivedLinkContext.App.Platform != ApplePlatform.MacCatalyst) {
-						Driver.Log (4, "Did not add native reference to {0} in {1} referenced by {2} in {3}.", pinfo.EntryPoint, pinfo.Module.Name, method.FullName, method.Module.Name);
-						break;
-					}
-#endif
-					addPInvokeSymbol = true;
-					break;
-
-				case "libSystem.Native":
 				case "System.Native":
-#if NET
-					// https://github.com/dotnet/runtime/issues/47533
-					if (DerivedLinkContext.App.Platform != ApplePlatform.MacOSX && pinfo.EntryPoint == "SystemNative_ConfigureTerminalForChildProcess") {
-						Driver.Log (4, "Did not add native reference to {0} in {1} referenced by {2} in {3}.", pinfo.EntryPoint, pinfo.Module.Name, method.FullName, method.Module.Name);
-						break;
-					}
-#endif
-					addPInvokeSymbol = true;
-					break;
-
-				case "libSystem.Globalization.Native":
-				case "System.Globalization.Native":
 					addPInvokeSymbol = true;
 					break;
+#endif
 
 				default:
-					Driver.Log (4, "Did not add native reference to {0} in {1} referenced by {2} in {3}.", pinfo.EntryPoint, pinfo.Module.Name, method.FullName, method.Module.Name);
+					if (!addPInvokeSymbol)
+						Driver.Log (4, "Did not add native reference to {0} in {1} referenced by {2} in {3}.", pinfo.EntryPoint, pinfo.Module.Name, method.FullName, method.Module.Name);
 					break;
 				}