Bump to mono/mono@c66141a8; NDK r22 support (dotnet#5475)

Context: mono/mono#20606
Context: https://github.com/android/ndk/wiki/Changelog-r22

Changes: mono/mono@5e9cb6d...c66141a

  * mono/mono@c66141a8c7b: [AOT] Make native linker name configurable (#20816)
  * mono/mono@64368a00d85: Bump msbuild
  * mono/mono@364c87b1a7a: Disable acceptance-tests/roslyn.mk
  * mono/mono@9b139294c04: Bump msbuild to fix build break
  * mono/mono@b9c050faf2e: [aot] Quote the -Wl,install_name argument to clang. (#20660)

Add support for using [NDK r22][0] when building Xamarin.Android apps.

The most important NDK changes:

  * GNU binutils are deprecated (but still used)
  * LLVM 11 is used for the toolchain
  * LLD is now the default linker

Mono changes:

  * Allow the `ld` name to be configurable, as we now prefer e.g.
    `aarch64-linux-android-ld.gold`, not `aarch64-linux-android-ld`,
    as the latter is LLD and not binutils.

Xamarin.Android CI changes:

  * All the binutils tools are listed in a single location, within
    `xaprepare`'s `Configurables.Defaults`.

  * Host NDK binutils are installed by `xaprepare` instead of by
    Xamarin.Android.Build.Tools

  * `UBSAN` checked builds now require `RTTI` and exceptions.

Xamarin.Android App build changes:

  * The `<Aot/>` task now supports using NDK r22

TODO:

  * Add a nightly job (916d24b) which builds Xamarin.Android apps
    against NDK r21, to ensure we don't break support.

  * NDK r22's [`cmake` doesn't work on Windows][1].
    We have worked around this by copying the fixed
    `android.toolchain.cmake` as
    `src-ThirdParty/ndk/android.toolchain.cmake.ndk_r21.3`.

    Once NDK r22 properly includes the fix, we should remove
    `src-ThirdParty/ndk/android.toolchain.cmake.ndk_r21.3`.

[0]: https://github.com/android/ndk/wiki/Changelog-r22
[1]: android/ndk#1427

.external

@@ -1,2 +1,2 @@
 xamarin/monodroid:main@7da768cf9ddbd137bbce5326dab79b139bcc59e0
-mono/mono:2020-02@5e9cb6d1c1de430965312927d5aed7fcb27bfa73
+mono/mono:2020-02@c66141a8c7ba2566c578c2dd012b2b723e006213

build-tools/cmake/xa_common.cmake

@@ -1,9 +1,3 @@
-if((CMAKE_VERSION_MAJOR EQUAL 3 AND CMAKE_VERSION_MINOR GREATER_EQUAL 7) OR CMAKE_VERSION_MAJOR GREATER 3)
-  set(CMAKE_POLICY_DEFAULT_CMP0066 NEW)
-endif()
-
-if((CMAKE_VERSION_MAJOR EQUAL 3 AND CMAKE_VERSION_MINOR GREATER_EQUAL 8) OR CMAKE_VERSION_MAJOR GREATER 3)
-  set(CMAKE_POLICY_DEFAULT_CMP0067 NEW)
-endif()
-
+set(CMAKE_POLICY_DEFAULT_CMP0066 NEW)
+set(CMAKE_POLICY_DEFAULT_CMP0067 NEW)
 set(CMAKE_OSX_DEPLOYMENT_TARGET 10.12)

build-tools/xaprepare/xaprepare/Application/BuildInfo.cs

@@ -10,7 +10,6 @@ namespace Xamarin.Android.Prepare
 	partial class BuildInfo : AppObject
 	{
 		static readonly char[] NDKPropertySeparator = new [] { '=' };
-		static readonly char[] NDKPlatformDirectorySeparator = new [] { '-' };
 
 		public string CommitOfLastVersionChange { get; private set; } = String.Empty;
 
@@ -38,9 +37,10 @@ public async Task GatherGitInfo (Context context)
 			Log.StatusLine ();
 		}
 
-		public bool GatherNDKInfo (Context context, string ndkRoot)
+		public bool GatherNDKInfo (Context context)
 		{
-			string props = Path.Combine (ndkRoot, "source.properties");
+			string ndkDir = Configurables.Paths.AndroidNdkDirectory;
+			string props = Path.Combine (ndkDir, "source.properties");
 			if (!File.Exists (props)) {
 				Log.ErrorLine ("NDK properties file does not exist: ", props, tailColor: Log.DestinationColor);
 				return false;
@@ -71,24 +71,32 @@ public bool GatherNDKInfo (Context context, string ndkRoot)
 				break;
 			}
 
+			Log.DebugLine ($"Looking for minimum API available in {ndkDir}");
 			int minimumApi = Int32.MaxValue;
-			string platforms = Path.Combine (ndkRoot, "platforms");
-			foreach (string p in Directory.EnumerateDirectories (platforms, "android-*", SearchOption.TopDirectoryOnly)) {
-				string pdir = Path.GetFileName (p);
-				string[] parts = pdir.Split (NDKPlatformDirectorySeparator, 2);
-				if (parts.Length != 2)
-					continue;
-
-				int api;
-				if (!Int32.TryParse (parts [1].Trim (), out api))
-					continue;
-
-				if (api >= minimumApi)
-					continue;
-
-				minimumApi = api;
+			foreach (var kvp in Configurables.Defaults.AndroidToolchainPrefixes) {
+				string dirName = kvp.Value;
+				string platforms = Path.Combine (Configurables.Paths.AndroidToolchainSysrootLibDirectory, dirName);
+				Log.DebugLine ($"  searching in {platforms}");
+				foreach (string p in Directory.EnumerateDirectories (platforms, "*", SearchOption.TopDirectoryOnly)) {
+					string plibc = Path.Combine (p, "libc.so");
+					if (!Utilities.FileExists (plibc)) {
+						continue;
+					}
+
+					Log.DebugLine ($"    found {p}");
+					string pdir = Path.GetFileName (p);
+					int api;
+					if (!Int32.TryParse (pdir, out api))
+						continue;
+
+					if (api >= minimumApi)
+						continue;
+
+					minimumApi = api;
+				}
 			}
 
+			Log.DebugLine ($"Detected minimum NDK API level: {minimumApi}");
 			NDKMinimumApiAvailable = minimumApi.ToString ();
 			return true;
 		}

build-tools/xaprepare/xaprepare/Application/KnownProperties.cs

@@ -22,6 +22,7 @@ static class KnownProperties
 		public const string DotNetPreviewVersionFull            = "DotNetPreviewVersionFull";
 		public const string EmulatorVersion                     = "EmulatorVersion";
 		public const string EmulatorPkgRevision                 = "EmulatorPkgRevision";
+		public const string HostOS                              = "HostOS";
 		public const string IgnoreMaxMonoVersion                = "IgnoreMaxMonoVersion";
 		public const string IgnoreMinMonoVersion                = "IgnoreMinMonoVersion";
 		public const string JavaInteropFullPath                 = "JavaInteropFullPath";

build-tools/xaprepare/xaprepare/Application/NDKTool.cs

@@ -0,0 +1,22 @@
+using System;
+
+namespace Xamarin.Android.Prepare
+{
+	class NDKTool
+	{
+		public string Name               { get; }
+		public string DestinationName    { get; } = String.Empty;
+
+		public NDKTool (string name, string? destinationName = null)
+		{
+			if (name.Trim ().Length == 0) {
+				throw new ArgumentException (nameof (name), "must not be empty");
+			}
+			Name = name;
+			if (String.IsNullOrWhiteSpace (destinationName)) {
+				return;
+			}
+			DestinationName = destinationName!;
+		}
+	}
+}

build-tools/xaprepare/xaprepare/Application/Properties.Defaults.cs.in

@@ -26,6 +26,7 @@ namespace Xamarin.Android.Prepare
 			properties.Add (KnownProperties.DotNetPreviewVersionFull,            StripQuotes ("@DotNetPreviewVersionFull@"));
 			properties.Add (KnownProperties.EmulatorVersion,                     StripQuotes ("@EmulatorVersion@"));
 			properties.Add (KnownProperties.EmulatorPkgRevision,                 StripQuotes ("@EmulatorPkgRevision@"));
+			properties.Add (KnownProperties.HostOS,                              StripQuotes ("@HostOS@"));
 			properties.Add (KnownProperties.IgnoreMaxMonoVersion,                StripQuotes ("@IgnoreMaxMonoVersion@"));
 			properties.Add (KnownProperties.IgnoreMinMonoVersion,                StripQuotes ("@IgnoreMinMonoVersion@"));
 			properties.Add (KnownProperties.JavaInteropFullPath,                 StripQuotes (@"@JavaInteropFullPath@"));

build-tools/xaprepare/xaprepare/ConfigAndData/BuildAndroidPlatforms.cs

@@ -5,8 +5,8 @@ namespace Xamarin.Android.Prepare
 {
 	class BuildAndroidPlatforms
 	{
-		public const string AndroidNdkVersion = "21d";
-		public const string AndroidNdkPkgRevision = "21.3.6528147";
+		public const string AndroidNdkVersion = "22";
+		public const string AndroidNdkPkgRevision = "22.0.7026061";
 
 		public static readonly List<AndroidPlatform> AllPlatforms = new List<AndroidPlatform> {
 			new AndroidPlatform (apiName: "",                       apiLevel: 1,  platformID: "1"),

build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs

@@ -266,6 +266,12 @@ public static partial class Defaults
 			/// </summary>
 			public static readonly List <string> BuildStatusBundleExclude = new List <string> {
 			};
+
+			public static readonly List <NDKTool> NDKTools = new List<NDKTool> {
+				new NDKTool (name: "as"),
+				new NDKTool (name: "ld.gold", destinationName: "ld"),
+				new NDKTool (name: "strip"),
+			};
 		}
 
 		public static partial class Paths
@@ -362,7 +368,12 @@ public static partial class Paths
 			public static string MonoArchiveWindowsLocalPath         => Path.Combine (ctx.Properties.GetRequiredValue (KnownProperties.AndroidToolchainCacheDirectory), MonoArchiveWindowsFileName);
 
 			// Other
-			public static string AndroidToolchainBinDirectory => EnsureAndroidToolchainBinDirectories ();
+			public static string AndroidNdkDirectory                 => ctx.Properties.GetRequiredValue (KnownProperties.AndroidNdkDirectory);
+			public static string AndroidToolchainRootDirectory       => GetCachedPath (ref androidToolchainRootDirectory,       () => Path.Combine (AndroidNdkDirectory, "toolchains", "llvm", "prebuilt", NdkToolchainOSTag));
+			public static string AndroidToolchainBinDirectory        => GetCachedPath (ref androidToolchainBinDirectory,        () => Path.Combine (AndroidToolchainRootDirectory, "bin"));
+			public static string AndroidToolchainSysrootLibDirectory => GetCachedPath (ref androidToolchainSysrootLibDirectory, () => Path.Combine (AndroidToolchainRootDirectory, "sysroot", "usr", "lib"));
+			public static string WindowsBinutilsInstallDir           => GetCachedPath (ref windowsBinutilsInstallDir,           () => Path.Combine (InstallMSBuildDir, "ndk"));
+			public static string HostBinutilsInstallDir              => GetCachedPath (ref hostBinutilsInstallDir,              () => Path.Combine (InstallMSBuildDir, ctx.Properties.GetRequiredValue (KnownProperties.HostOS), "ndk"));
 
 			// not really configurables, merely convenience aliases for more frequently used paths that come from properties
 			public static string XAInstallPrefix                => ctx.Properties.GetRequiredValue (KnownProperties.XAInstallPrefix);
@@ -410,7 +421,9 @@ static string GetCachedPath (ref string? variable, Func<string> creator)
 			static string? binDir;
 			static string? netCoreBinDir;
 			static string? monoSDKsOutputDir;
+			static string? androidToolchainRootDirectory;
 			static string? androidToolchainBinDirectory;
+			static string? androidToolchainSysrootLibDirectory;
 			static string? monoProfileDir;
 			static string? monoProfileToolsDir;
 			static string? bclTestsDestDir;
@@ -449,6 +462,8 @@ static string GetCachedPath (ref string? variable, Func<string> creator)
 			static string? openJDK8CacheDir,    openJDK11CacheDir;
 			static string? oldOpenJDKInstallDir;
 			static string? configurationPropsGeneratedPath;
+			static string? windowsBinutilsInstallDir;
+			static string? hostBinutilsInstallDir;
 		}
 	}
 }

build-tools/xaprepare/xaprepare/OperatingSystems/OS.cs

@@ -521,6 +521,11 @@ string GetExecutableWithExtension (string programPath, Func<string, string> find
 			return String.Empty;
 		}
 
+		public virtual string AppendExecutableExtension (string programName)
+		{
+			return programName;
+		}
+
 		protected static string FindProgram (string programName, List<string> directories)
 		{
 			foreach (string dir in directories) {

build-tools/xaprepare/xaprepare/OperatingSystems/Windows.cs

@@ -80,6 +80,16 @@ public override string Which (string programPath, bool required = true)
 			return base.Which (programPath, required);
 		}
 
+		public override string AppendExecutableExtension (string programName)
+		{
+			string ext = Path.GetExtension (programName);
+			if (String.Compare (".exe", ext, StringComparison.OrdinalIgnoreCase) == 0) {
+				return programName;
+			}
+
+			return $"{programName}.exe";
+		}
+
 		protected override string AssertIsExecutable (string fullPath)
 		{
 			return fullPath;

build-tools/xaprepare/xaprepare/Scenarios/Scenario_Standard.cs

@@ -31,6 +31,10 @@ protected override void AddSteps (Context context)
 			Steps.Add (new Step_DownloadMonoArchive ());
 			AddRequiredOSSpecificSteps (true);
 			Steps.Add (new Step_InstallMonoRuntimes ());
+
+			// The next two steps MUST be after InstallMonoRuntimes above since the latter cleans up the target
+			// directory where the NDK binutils are installed
+			Steps.Add (new Step_InstallNDKBinutils ());
 			Steps.Add (new Step_Get_Windows_Binutils ());
 			Steps.Add (new Step_GenerateCGManifest ());
 

build-tools/xaprepare/xaprepare/Steps/Step_Android_SDK_NDK.cs

@@ -55,7 +55,7 @@ protected override async Task<bool> Execute (Context context)
 					return false;
 				}
 				WritePackageXmls (sdkRoot);
-				return GatherNDKInfo (context, ndkRoot);
+				return GatherNDKInfo (context);
 			}
 
 			Log.MessageLine ();
@@ -101,7 +101,7 @@ protected override async Task<bool> Execute (Context context)
 
 			WritePackageXmls (sdkRoot);
 
-			return GatherNDKInfo (context, ndkRoot);
+			return GatherNDKInfo (context);
 		}
 
 		bool AcceptLicenses (Context context, string sdkRoot)
@@ -145,13 +145,22 @@ bool AcceptLicenses (Context context, string sdkRoot)
 			return true;
 		}
 
-		bool GatherNDKInfo (Context context, string ndkRoot)
+		bool GatherNDKInfo (Context context)
 		{
+			if (context.OS.IsWindows) {
+				// Quick hack to test https://github.com/android/ndk/issues/1427#issuecomment-763424992
+				Log.Info ("Copying NDK r21 CMake toolchain file to NDK r22 directory");
+				Utilities.CopyFile (
+					Path.Combine (BuildPaths.XamarinAndroidSourceRoot, "src-ThirdParty", "ndk", "android.toolchain.cmake.ndk_r21.3"),
+					Path.Combine (context.Properties.GetRequiredValue (KnownProperties.AndroidNdkDirectory), "build", "cmake", "android.toolchain.cmake")
+				);
+			}
+
 			// Ignore NDK property setting if not installing the NDK
 			if (!DependencyTypeToInstall.HasFlag (AndroidToolchainComponentType.BuildDependency))
 				return true;
 			else
-				return context.BuildInfo.GatherNDKInfo (context, ndkRoot);
+				return context.BuildInfo.GatherNDKInfo (context);
 		}
 
 		void CheckPackageStatus (Context context, string packageCacheDir, AndroidPackage pkg, List <AndroidPackage> toDownload)