Embed some data files in libxamarin-app.so

src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.After.targets

@@ -29,4 +29,5 @@ This file is imported *after* the Microsoft.NET.Sdk/Sdk.targets.
   <Import Project="Microsoft.Android.Sdk.Publish.targets" />
   <Import Project="Microsoft.Android.Sdk.RuntimeConfig.targets" />
   <Import Project="Microsoft.Android.Sdk.Tooling.targets" />
+  <Import Project="Microsoft.Android.Sdk.AssemblyStores.targets" />
 </Project>

src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyStores.targets

@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <UsingTask TaskName="Xamarin.Android.Tasks.CreateEmbeddedAssemblyStore" AssemblyFile="$(_XamarinAndroidBuildTasksAssembly)" />
+  <UsingTask TaskName="Xamarin.Android.Tasks.PrepareAbiItems" AssemblyFile="$(_XamarinAndroidBuildTasksAssembly)" />
+
+  <Target Name="_PrepareCreateEmbeddedAssemblyStoreOutputItems">
+    <PrepareAbiItems
+      BuildTargetAbis="@(_BuildTargetAbis)"
+      NativeSourcesDir="$(_NativeAssemblySourceDir)"
+      Mode="EmbeddedAssemblyStore">
+      <Output TaskParameter="AssemblySources" ItemName="_EmbeddedAssemblyStoreSourceFiles" />
+    </PrepareAbiItems>
+
+    <ItemGroup>
+      <_CreateEmbeddedAssemblyStoreAssembly Include="@(_ShrunkUserAssemblies);@(_AndroidResolvedSatellitePaths);@(_ShrunkFrameworkAssemblies)"/>
+      <_CreateEmbeddedAssemblyStoreAssembly Condition=" '@(_CreateEmbeddedAssemblyStoreAssembly->Count())' == '0' " Include="@(_ResolvedUserAssemblies);@(_ResolvedFrameworkAssemblies);@(_AndroidResolvedSatellitePaths)" />
+    </ItemGroup>
+  </Target>
+
+  <Target Name="_CreateEmbeddedAssemblyStore"
+          DependsOnTargets="_PrepareCreateEmbeddedAssemblyStoreOutputItems"
+          Inputs="@(_CreateEmbeddedAssemblyStoreAssembly)"
+          Outputs="@(_EmbeddedAssemblyStoreSourceFiles)">
+    <CreateEmbeddedAssemblyStore
+        AndroidBinUtilsDirectory="$(AndroidBinUtilsDirectory)"
+        AppSharedLibrariesDir="$(_AndroidApplicationSharedLibraryPath)"
+        AssemblySourcesDir="$(IntermediateOutputPath)android"
+        AssemblyStoreEmbeddedInRuntime="$(_AndroidEmbedAssemblyStoreInRuntime)"
+        CompressedAssembliesDir="$(_AndroidCompressedAssembliesDir)\test\"
+        Debug="$(AndroidIncludeDebugSymbols)"
+        EnableCompression="$(AndroidEnableAssemblyCompression)"
+        ProjectFullPath="$(MSBuildProjectFullPath)"
+        ResolvedUserAssemblies="@(_ShrunkUserAssemblies);@(_AndroidResolvedSatellitePaths)"
+        ResolvedFrameworkAssemblies="@(_ShrunkFrameworkAssemblies)"
+        SupportedAbis="@(_BuildTargetAbis)" />
+
+    <ItemGroup>
+      <FileWrites Include="@(_EmbeddedAssemblyStoreSourceFiles)" />
+    </ItemGroup>
+  </Target>
+</Project>

src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.BuildOrder.targets

@@ -62,7 +62,6 @@ properties that determine build ordering.
     <_PrepareBuildApkDependsOnTargets>
       _SetLatestTargetFrameworkVersion;
       _GetLibraryImports;
-      _RemoveRegisterAttribute;
       _ResolveAssemblies;
       _ResolveSatellitePaths;
       _CreatePackageWorkspace;

src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs

@@ -99,8 +99,6 @@ public class BuildApk : AndroidTask
 
 		public string CheckedBuild { get; set; }
 
-		public string RuntimeConfigBinFilePath { get; set; }
-
 		public bool UseAssemblyStore { get; set; }
 
 		public string ZipFlushFilesLimit { get; set; }
@@ -118,6 +116,12 @@ public class BuildApk : AndroidTask
 		[Required]
 		public string ProjectFullPath { get; set; }
 
+		[Required]
+		public string CompressedAssembliesDir { get; set; }
+
+		[Required]
+		public bool AssemblyStoreEmbeddedInRuntime { get; set; }
+
 		[Output]
 		public ITaskItem[] OutputFiles { get; set; }
 
@@ -218,7 +222,6 @@ void ExecuteWithAbi (DSOWrapperGenerator.Config dsoWrapperConfig, string [] supp
 					apk.Flush ();
 				}
 
-				AddRuntimeConfigBlob (dsoWrapperConfig, apk);
 				AddRuntimeLibraries (apk, supportedAbis);
 				apk.Flush();
 				AddNativeLibraries (files, supportedAbis);
@@ -354,7 +357,7 @@ public override bool RunTask ()
 
 			if (compress) {
 				string key = CompressedAssemblyInfo.GetKey (ProjectFullPath);
-				Log.LogDebugMessage ($"Retrieving assembly compression info with key '{key}'");
+				Log.LogDebugMessage ($"[{TaskPrefix}] Retrieving assembly compression info with key '{key}'");
 				compressedAssembliesInfo = BuildEngine4.UnregisterTaskObjectAssemblyLocal<IDictionary<AndroidTargetArch, Dictionary<string, CompressedAssemblyInfo>>> (key, RegisteredTaskObjectLifetime.Build);
 				if (compressedAssembliesInfo == null)
 					throw new InvalidOperationException ($"Assembly compression info not found for key '{key}'. Compression will not be performed.");
@@ -403,31 +406,17 @@ static Regex FileGlobToRegEx (string fileGlob, RegexOptions options)
 			return new Regex (sb.ToString (), options);
 		}
 
-		void AddRuntimeConfigBlob (DSOWrapperGenerator.Config dsoWrapperConfig, ZipArchiveEx apk)
-		{
-			// We will place rc.bin in the `lib` directory next to the blob, to make startup slightly faster, as we will find the config file right after we encounter
-			// our assembly store.  Not only that, but also we'll be able to skip scanning the `base.apk` archive when split configs are enabled (which they are in 99%
-			// of cases these days, since AAB enforces that split).  `base.apk` contains only ABI-agnostic file, while one of the split config files contains only
-			// ABI-specific data+code.
-			if (!String.IsNullOrEmpty (RuntimeConfigBinFilePath) && File.Exists (RuntimeConfigBinFilePath)) {
-				foreach (string abi in SupportedAbis) {
-					// Prefix it with `a` because bundletool sorts entries alphabetically, and this will place it right next to `assemblies.*.blob.so`, which is what we
-					// like since we can finish scanning the zip central directory earlier at startup.
-					string inArchivePath = MakeArchiveLibPath (abi, "libarc.bin.so");
-					string wrappedSourcePath = DSOWrapperGenerator.WrapIt (Log, dsoWrapperConfig, MonoAndroidHelper.AbiToTargetArch (abi), RuntimeConfigBinFilePath, Path.GetFileName (inArchivePath));
-					AddFileToArchiveIfNewer (apk, wrappedSourcePath, inArchivePath, compressionMethod: GetCompressionMethod (inArchivePath));
-				}
-			}
-		}
-
 		void AddAssemblies (DSOWrapperGenerator.Config dsoWrapperConfig, ZipArchiveEx apk, bool debug, bool compress, IDictionary<AndroidTargetArch, Dictionary<string, CompressedAssemblyInfo>> compressedAssembliesInfo, string assemblyStoreApkName)
 		{
 			string sourcePath;
-			AssemblyCompression.AssemblyData compressedAssembly = null;
-			string compressedOutputDir = Path.GetFullPath (Path.Combine (Path.GetDirectoryName (ApkOutputPath), "..", "lz4"));
 			AssemblyStoreBuilder? storeBuilder = null;
 
 			if (UseAssemblyStore) {
+				if (AssemblyStoreEmbeddedInRuntime) {
+					// We don't need to do anything here, the store is in `libxamarin-app.so`
+					return;
+				}
+
 				storeBuilder = new AssemblyStoreBuilder (Log);
 			}
 
@@ -501,21 +490,13 @@ void DoAddAssembliesFromArchCollection (TaskLoggingHelper log, AndroidTargetArch
 				);
 			}
 
-			void EnsureCompressedAssemblyData (string sourcePath, uint descriptorIndex)
-			{
-				if (compressedAssembly == null)
-					compressedAssembly = new AssemblyCompression.AssemblyData (sourcePath, descriptorIndex);
-				else
-					compressedAssembly.SetData (sourcePath, descriptorIndex);
-			}
-
 			string CompressAssembly (ITaskItem assembly)
 			{
 				if (!compress) {
 					return assembly.ItemSpec;
 				}
 
-				return AssemblyCompression.Compress (Log, assembly, compressedAssembliesInfo, compressedOutputDir);
+				return AssemblyCompression.Compress (Log, assembly, compressedAssembliesInfo, CompressedAssembliesDir);
 			}
 		}
 

src/Xamarin.Android.Build.Tasks/Tasks/CompileNativeAssembly.cs

@@ -2,12 +2,8 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
-using System.Threading;
 
 using Microsoft.Build.Framework;
-using Microsoft.Build.Utilities;
-
-using Xamarin.Android.Tools;
 using Microsoft.Android.Build.Tasks;
 
 namespace Xamarin.Android.Tasks
@@ -16,13 +12,6 @@ public class CompileNativeAssembly : AsyncTask
 	{
 		public override string TaskPrefix => "CNA";
 
-		sealed class Config
-		{
-			public string AssemblerPath;
-			public string AssemblerOptions;
-			public string InputSource;
-		}
-
 		[Required]
 		public ITaskItem[] Sources { get; set; }
 
@@ -37,109 +26,34 @@ sealed class Config
 
 		public override System.Threading.Tasks.Task RunTaskAsync ()
 		{
-			return this.WhenAll (GetAssemblerConfigs (), RunAssembler);
+			var context = new NativeAssemblerCompilation.AssemblerRunContext (
+				Log,
+				Path.GetFullPath (WorkingDirectory),
+				registerForCancellation: RegisterForCancellation,
+				cancel: Cancel
+			);
+
+			return this.WhenAll (
+				GetAssemblerConfigs (),
+				(NativeAssemblerCompilation.AssemblerConfig config) => NativeAssemblerCompilation.RunAssembler (context, config)
+			);
 		}
 
-		void RunAssembler (Config config)
+		void RegisterForCancellation (Process proc)
 		{
-			var stdout_completed = new ManualResetEvent (false);
-			var stderr_completed = new ManualResetEvent (false);
-			var psi = new ProcessStartInfo () {
-				FileName = config.AssemblerPath,
-				Arguments = config.AssemblerOptions,
-				UseShellExecute = false,
-				RedirectStandardOutput = true,
-				RedirectStandardError = true,
-				CreateNoWindow = true,
-				WindowStyle = ProcessWindowStyle.Hidden,
-				WorkingDirectory = WorkingDirectory,
-			};
-
-			string assemblerName = Path.GetFileName (config.AssemblerPath);
-			LogDebugMessage ($"[LLVM llc] {psi.FileName} {psi.Arguments}");
-
-			var stdoutLines = new List<string> ();
-			var stderrLines = new List<string> ();
-
-			using (var proc = new Process ()) {
-				proc.OutputDataReceived += (s, e) => {
-					if (e.Data != null) {
-						OnOutputData (assemblerName, s, e);
-						stdoutLines.Add (e.Data);
-					} else
-						stdout_completed.Set ();
-				};
-
-				proc.ErrorDataReceived += (s, e) => {
-					if (e.Data != null) {
-						OnErrorData (assemblerName, s, e);
-						stderrLines.Add (e.Data);
-					} else
-						stderr_completed.Set ();
-				};
-
-				proc.StartInfo = psi;
-				proc.Start ();
-				proc.BeginOutputReadLine ();
-				proc.BeginErrorReadLine ();
-				CancellationToken.Register (() => { try { proc.Kill (); } catch (Exception) { } });
-				proc.WaitForExit ();
-
-				if (psi.RedirectStandardError)
-					stderr_completed.WaitOne (TimeSpan.FromSeconds (30));
-
-				if (psi.RedirectStandardOutput)
-					stdout_completed.WaitOne (TimeSpan.FromSeconds (30));
-
-				if (proc.ExitCode != 0) {
-					var sb = MonoAndroidHelper.MergeStdoutAndStderrMessages (stdoutLines, stderrLines);
-					LogCodedError ("XA3006", Properties.Resources.XA3006, Path.GetFileName (config.InputSource), sb.ToString ());
-					Cancel ();
+			CancellationToken.Register (() => {
+				try {
+					proc.Kill ();
+				} catch (Exception) {
 				}
-			}
+			});
 		}
 
-		IEnumerable<Config> GetAssemblerConfigs ()
+		IEnumerable<NativeAssemblerCompilation.AssemblerConfig> GetAssemblerConfigs ()
 		{
-			const string assemblerOptions =
-				"-O2 " +
-				"--debugger-tune=lldb " + // NDK uses lldb now
-				"--debugify-level=location+variables " +
-				"--fatal-warnings " +
-				"--filetype=obj " +
-				"--relocation-model=pic";
-			string llcPath = Path.Combine (AndroidBinUtilsDirectory, "llc");
-
 			foreach (ITaskItem item in Sources) {
-				// We don't need the directory since our WorkingDirectory is where all the sources are
-				string sourceFile = Path.GetFileName (item.ItemSpec);
-				string outputFile = QuoteFileName (sourceFile.Replace (".ll", ".o"));
-				string executableDir = Path.GetDirectoryName (llcPath);
-				string executableName = MonoAndroidHelper.GetExecutablePath (executableDir, Path.GetFileName (llcPath));
-
-				yield return new Config {
-					InputSource = item.ItemSpec,
-					AssemblerPath = Path.Combine (executableDir, executableName),
-					AssemblerOptions = $"{assemblerOptions} -o={outputFile} {QuoteFileName (sourceFile)}",
-				};
+				yield return NativeAssemblerCompilation.GetAssemblerConfig (AndroidBinUtilsDirectory, item, stripFilePaths: true);
 			}
 		}
-
-		void OnOutputData (string assemblerName, object sender, DataReceivedEventArgs e)
-		{
-			LogDebugMessage ($"[{assemblerName} stdout] {e.Data}");
-		}
-
-		void OnErrorData (string assemblerName, object sender, DataReceivedEventArgs e)
-		{
-			LogMessage ($"[{assemblerName} stderr] {e.Data}", MessageImportance.High);
-		}
-
-		static string QuoteFileName (string fileName)
-		{
-			var builder = new CommandLineBuilder ();
-			builder.AppendFileNameIfNotNull (fileName);
-			return builder.ToString ();
-		}
 	}
 }