Skip to content

Commit

Permalink
[WIP] More test fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@grendello
grendello committed Sep 29, 2021
1 parent f8cc189 commit 57fac3b
Show file tree
Hide file tree
Showing 5 changed files with 121 additions and 53 deletions.
28 changes: 16 additions & 12 deletions src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ protected virtual void FixupArchive (ZipArchiveEx zip) { }

List<string> existingEntries = new List<string> ();

void ExecuteWithAbi (string [] supportedAbis, string apkInputPath, string apkOutputPath, bool debug, bool compress, IDictionary<string, CompressedAssemblyInfo> compressedAssembliesInfo)
void ExecuteWithAbi (string [] supportedAbis, string apkInputPath, string apkOutputPath, bool debug, bool compress, IDictionary<string, CompressedAssemblyInfo> compressedAssembliesInfo, string blobApkName)
{
ArchiveFileList files = new ArchiveFileList ();
bool refresh = true;
Expand Down Expand Up @@ -180,7 +180,7 @@ void ExecuteWithAbi (string [] supportedAbis, string apkInputPath, string apkOut
}

if (EmbedAssemblies && !BundleAssemblies)
AddAssemblies (apk, debug, compress, compressedAssembliesInfo);
AddAssemblies (apk, debug, compress, compressedAssembliesInfo, blobApkName);

AddRuntimeLibraries (apk, supportedAbis);
apk.Flush();
Expand Down Expand Up @@ -301,7 +301,7 @@ public override bool RunTask ()
throw new InvalidOperationException ($"Assembly compression info not found for key '{key}'. Compression will not be performed.");
}

ExecuteWithAbi (SupportedAbis, ApkInputPath, ApkOutputPath, debug, compress, compressedAssembliesInfo);
ExecuteWithAbi (SupportedAbis, ApkInputPath, ApkOutputPath, debug, compress, compressedAssembliesInfo, blobApkName: null);
outputFiles.Add (ApkOutputPath);
if (CreatePackagePerAbi && SupportedAbis.Length > 1) {
foreach (var abi in SupportedAbis) {
Expand All @@ -310,7 +310,7 @@ public override bool RunTask ()
var apk = Path.GetFileNameWithoutExtension (ApkOutputPath);
ExecuteWithAbi (new [] { abi }, String.Format ("{0}-{1}", ApkInputPath, abi),
Path.Combine (path, String.Format ("{0}-{1}.apk", apk, abi)),
debug, compress, compressedAssembliesInfo);
debug, compress, compressedAssembliesInfo, blobApkName: abi);
outputFiles.Add (Path.Combine (path, String.Format ("{0}-{1}.apk", apk, abi)));
}
}
Expand All @@ -322,7 +322,7 @@ public override bool RunTask ()
return !Log.HasLoggedErrors;
}

void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary<string, CompressedAssemblyInfo> compressedAssembliesInfo)
void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary<string, CompressedAssemblyInfo> compressedAssembliesInfo, string blobApkName)
{
var appConfState = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal<ApplicationConfigTaskState> (ApplicationConfigTaskState.RegisterTaskObjectKey, RegisteredTaskObjectLifetime.Build);
bool useAssembliesBlob = appConfState != null ? appConfState.UseAssembliesBlob : false;
Expand All @@ -332,6 +332,7 @@ void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary<str
AssemblyBlobGenerator blobGenerator;

if (useAssembliesBlob) {
Log.LogDebugMessage ("Creating AssemblyBlobGenerator instance");
blobGenerator = new AssemblyBlobGenerator (AssembliesPath, Log);
} else {
blobGenerator = null;
Expand All @@ -347,6 +348,13 @@ void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary<str
// are included in other APKs than the base one. The ID 0 blob **must** be placed in the base assembly
//

// Currently, all the assembly blobs end up in the "base" apk (the APK name is the key in the dictionary below) but the code is ready for the time when we
// partition assemblies into "feature" APKs
const string DefaultBaseApkName = "base";
if (String.IsNullOrEmpty (blobApkName)) {
blobApkName = DefaultBaseApkName;
}

// Add user assemblies
AddAssembliesFromCollection (ResolvedUserAssemblies);

Expand All @@ -358,20 +366,16 @@ void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary<str
return;
}

// Currently, all the assembly blobs end up in the "base" apk (the APK name is the key in the dictionary below) but the code is ready for the time when we
// partition assemblies into "feature" APKs
const string BaseApkName = "base";

Dictionary<string, List<string>> blobPaths = blobGenerator.Generate (Path.GetDirectoryName (ApkOutputPath));
if (blobPaths == null) {
throw new InvalidOperationException ("Blob generator did not generate any blobs");
}

if (!blobPaths.TryGetValue (BaseApkName, out List<string> baseBlobs) || baseBlobs == null || baseBlobs.Count == 0) {
if (!blobPaths.TryGetValue (blobApkName, out List<string> baseBlobs) || baseBlobs == null || baseBlobs.Count == 0) {
throw new InvalidOperationException ("Blob generator didn't generate the required base blobs");
}

string blobPrefix = $"{BaseApkName}_";
string blobPrefix = $"{blobApkName}_";
foreach (string blobPath in baseBlobs) {
string inArchiveName = Path.GetFileName (blobPath);

Expand Down Expand Up @@ -445,7 +449,7 @@ void AddAssembliesFromCollection (ITaskItem[] assemblies)
}

if (useAssembliesBlob) {
blobGenerator.Add (BaseApkName, blobAssembly);
blobGenerator.Add (blobApkName, blobAssembly);
} else {
count++;
if (count >= ZipArchiveEx.ZipFlushFilesLimit) {
Expand Down
49 changes: 48 additions & 1 deletion ...Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.TestCaseSource.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,17 @@ public partial class BuildTest : BaseTest
/* debugType */ "Full",
/* embedMdb */ !CommercialBuildAvailable, // because we don't use FastDev in the OSS repo
/* expectedRuntime */ "debug",
/* usesAssemblyBlobs */ false,
},
new object[] {
/* isRelease */ false,
/* monoSymbolArchive */ false ,
/* aotAssemblies */ false,
/* debugSymbols */ true,
/* debugType */ "Full",
/* embedMdb */ !CommercialBuildAvailable, // because we don't use FastDev in the OSS repo
/* expectedRuntime */ "debug",
/* usesAssemblyBlobs */ true,
},
new object[] {
/* isRelease */ true,
Expand All @@ -105,6 +116,17 @@ public partial class BuildTest : BaseTest
/* debugType */ "Full",
/* embedMdb */ false,
/* expectedRuntime */ "release",
/* usesAssemblyBlobs */ false,
},
new object[] {
/* isRelease */ true,
/* monoSymbolArchive */ false,
/* aotAssemblies */ false,
/* debugSymbols */ true,
/* debugType */ "Full",
/* embedMdb */ false,
/* expectedRuntime */ "release",
/* usesAssemblyBlobs */ true,
},
new object[] {
/* isRelease */ true,
Expand All @@ -114,6 +136,7 @@ public partial class BuildTest : BaseTest
/* debugType */ "Full",
/* embedMdb */ false,
/* expectedRuntime */ "release",
/* usesAssemblyBlobs */ false,
},
new object[] {
/* isRelease */ true,
Expand All @@ -123,6 +146,7 @@ public partial class BuildTest : BaseTest
/* debugType */ "Portable",
/* embedMdb */ false,
/* expectedRuntime */ "release",
/* usesAssemblyBlobs */ false,
},
new object[] {
/* isRelease */ true,
Expand All @@ -132,6 +156,7 @@ public partial class BuildTest : BaseTest
/* debugType */ "Portable",
/* embedMdb */ false,
/* expectedRuntime */ "release",
/* usesAssemblyBlobs */ false,
},
new object[] {
/* isRelease */ true,
Expand All @@ -141,15 +166,37 @@ public partial class BuildTest : BaseTest
/* debugType */ "Portable",
/* embedMdb */ false,
/* expectedRuntime */ "release",
/* usesAssemblyBlobs */ false,
},
new object[] {
/* isRelease */ true,
/* monoSymbolArchive */ false ,
/* aotAssemblies */ false,
/* debugSymbols */ true,
/* debugType */ "Portable",
/* embedMdb */ false,
/* expectedRuntime */ "release",
/* usesAssemblyBlobs */ true,
},
new object[] {
/* isRelease */ true,
/* monoSymbolArchive */ false ,
/* aotAssemblies */ true,
/* debugSymbols */ false,
/* debugType */ "",
/* embedMdb */ false,
/* expectedRuntime */ "release",
/* usesAssemblyBlobs */ false,
},
new object[] {
/* isRelease */ true,
/* monoSymbolArchive */ true ,
/* aotAssemblies */ true,
/* debugSymbols */ false,
/* debugType */ "",
/* embedMdb */ false,
/* expectedRuntime */ "release",
/* usesAssemblyBlobs */ false,
},
new object[] {
/* isRelease */ true,
Expand All @@ -159,9 +206,9 @@ public partial class BuildTest : BaseTest
/* debugType */ "",
/* embedMdb */ false,
/* expectedRuntime */ "release",
/* usesAssemblyBlobs */ true,
},
};
#pragma warning restore 414
}
}

72 changes: 35 additions & 37 deletions src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1932,7 +1932,7 @@ public void CheckWhichRuntimeIsIncluded (string supportedAbi, bool debugSymbols,
[Category ("AOT"), Category ("MonoSymbolicate")]
[TestCaseSource (nameof (SequencePointChecks))]
public void CheckSequencePointGeneration (bool isRelease, bool monoSymbolArchive, bool aotAssemblies,
bool debugSymbols, string debugType, bool embedMdb, string expectedRuntime)
bool debugSymbols, string debugType, bool embedMdb, string expectedRuntime, bool usesAssemblyBlobs)
{
var proj = new XamarinAndroidApplicationProject () {
IsRelease = isRelease,
Expand All @@ -1943,52 +1943,50 @@ public void CheckSequencePointGeneration (bool isRelease, bool monoSymbolArchive
proj.SetProperty (proj.ActiveConfigurationProperties, "MonoSymbolArchive", monoSymbolArchive);
proj.SetProperty (proj.ActiveConfigurationProperties, "DebugSymbols", debugSymbols);
proj.SetProperty (proj.ActiveConfigurationProperties, "DebugType", debugType);
proj.SetProperty (proj.ActiveConfigurationProperties, "AndroidUseAssembliesBlob", usesAssemblyBlobs.ToString ());
using (var b = CreateApkBuilder ()) {
if (aotAssemblies && !b.CrossCompilerAvailable (string.Join (";", abis)))
Assert.Ignore ("Cross compiler was not available");
Assert.IsTrue (b.Build (proj), "Build should have succeeded.");
var apk = Path.Combine (Root, b.ProjectDirectory,
proj.OutputPath, $"{proj.PackageName}-Signed.apk");
var msymarchive = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath, proj.PackageName + ".apk.mSYM");
using (var zipFile = ZipHelper.OpenZip (apk)) {
var mdbExits = ZipHelper.ReadFileFromZip (zipFile, "assemblies/UnnamedProject.dll.mdb") != null ||
ZipHelper.ReadFileFromZip (zipFile, "assemblies/UnnamedProject.pdb") != null;
Assert.AreEqual (embedMdb, mdbExits,
$"assemblies/UnnamedProject.dll.mdb or assemblies/UnnamedProject.pdb should{0}be in the {proj.PackageName}-Signed.apk", embedMdb ? " " : " not ");
if (aotAssemblies) {
foreach (var abi in abis) {
var assemblies = Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath,
"aot", abi, "libaot-UnnamedProject.dll.so");
var shouldExist = monoSymbolArchive && debugSymbols && (debugType == "PdbOnly" || debugType == "Portable");
var symbolicateFile = Directory.GetFiles (Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath,
"aot", abi), "UnnamedProject.dll.msym", SearchOption.AllDirectories).FirstOrDefault ();
if (shouldExist)
Assert.IsNotNull (symbolicateFile, "UnnamedProject.dll.msym should exist");
else
Assert.IsNull (symbolicateFile, "{0} should not exist", symbolicateFile);
if (shouldExist) {
var foundMsyms = Directory.GetFiles (Path.Combine (msymarchive), "UnnamedProject.dll.msym", SearchOption.AllDirectories).Any ();
Assert.IsTrue (foundMsyms, "UnnamedProject.dll.msym should exist in the archive {0}", msymarchive);
}
Assert.IsTrue (File.Exists (assemblies), "{0} libaot-UnnamedProject.dll.so does not exist", abi);
Assert.IsNotNull (ZipHelper.ReadFileFromZip (zipFile,
string.Format ("lib/{0}/libaot-UnnamedProject.dll.so", abi)),
$"lib/{0}/libaot-UnnamedProject.dll.so should be in the {proj.PackageName}-Signed.apk", abi);
Assert.IsNotNull (ZipHelper.ReadFileFromZip (zipFile,
"assemblies/UnnamedProject.dll"),
$"UnnamedProject.dll should be in the {proj.PackageName}-Signed.apk");
}
}
var runtimeInfo = b.GetSupportedRuntimes ();
var helper = new ArchiveAssemblyHelper (apk, usesAssemblyBlobs);
var mdbExits = helper.Exists ("assemblies/UnnamedProject.dll.mdb") || helper.Exists ("assemblies/UnnamedProject.pdb");
Assert.AreEqual (embedMdb, mdbExits,
$"assemblies/UnnamedProject.dll.mdb or assemblies/UnnamedProject.pdb should{0}be in the {proj.PackageName}-Signed.apk", embedMdb ? " " : " not ");
if (aotAssemblies) {
foreach (var abi in abis) {
var runtime = runtimeInfo.FirstOrDefault (x => x.Abi == abi && x.Runtime == expectedRuntime);
Assert.IsNotNull (runtime, "Could not find the expected runtime.");
var inApk = ZipHelper.ReadFileFromZip (apk, String.Format ("lib/{0}/{1}", abi, runtime.Name));
var inApkRuntime = runtimeInfo.FirstOrDefault (x => x.Abi == abi && x.Size == inApk.Length);
Assert.IsNotNull (inApkRuntime, "Could not find the actual runtime used.");
Assert.AreEqual (runtime.Size, inApkRuntime.Size, "expected {0} got {1}", expectedRuntime, inApkRuntime.Runtime);
var assemblies = Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath,
"aot", abi, "libaot-UnnamedProject.dll.so");
var shouldExist = monoSymbolArchive && debugSymbols && (debugType == "PdbOnly" || debugType == "Portable");
var symbolicateFile = Directory.GetFiles (Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath,
"aot", abi), "UnnamedProject.dll.msym", SearchOption.AllDirectories).FirstOrDefault ();
if (shouldExist)
Assert.IsNotNull (symbolicateFile, "UnnamedProject.dll.msym should exist");
else
Assert.IsNull (symbolicateFile, "{0} should not exist", symbolicateFile);
if (shouldExist) {
var foundMsyms = Directory.GetFiles (Path.Combine (msymarchive), "UnnamedProject.dll.msym", SearchOption.AllDirectories).Any ();
Assert.IsTrue (foundMsyms, "UnnamedProject.dll.msym should exist in the archive {0}", msymarchive);
}
Assert.IsTrue (File.Exists (assemblies), "{0} libaot-UnnamedProject.dll.so does not exist", abi);
Assert.IsTrue (helper.Exists ($"lib/{abi}/libaot-UnnamedProject.dll.so"),
$"lib/{0}/libaot-UnnamedProject.dll.so should be in the {proj.PackageName}-Signed.apk", abi);
Assert.IsTrue (helper.Exists ("assemblies/UnnamedProject.dll"),
$"UnnamedProject.dll should be in the {proj.PackageName}-Signed.apk");
}
}
var runtimeInfo = b.GetSupportedRuntimes ();
foreach (var abi in abis) {
var runtime = runtimeInfo.FirstOrDefault (x => x.Abi == abi && x.Runtime == expectedRuntime);
Assert.IsNotNull (runtime, "Could not find the expected runtime.");
var inApk = ZipHelper.ReadFileFromZip (apk, String.Format ("lib/{0}/{1}", abi, runtime.Name));
var inApkRuntime = runtimeInfo.FirstOrDefault (x => x.Abi == abi && x.Size == inApk.Length);
Assert.IsNotNull (inApkRuntime, "Could not find the actual runtime used.");
Assert.AreEqual (runtime.Size, inApkRuntime.Size, "expected {0} got {1}", expectedRuntime, inApkRuntime.Runtime);
}

b.Clean (proj);
Assert.IsTrue (!Directory.Exists (msymarchive), "{0} should have been deleted on Clean", msymarchive);
}
Expand Down
18 changes: 17 additions & 1 deletion ....Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ class ArchiveAssemblyHelper
readonly string archivePath;
readonly string assembliesRootDir;
bool useAssemblyBlobs;
List<string> archiveContents;

public ArchiveAssemblyHelper (string archivePath, bool useAssemblyBlobs)
{
Expand All @@ -45,8 +46,12 @@ public ArchiveAssemblyHelper (string archivePath, bool useAssemblyBlobs)
}
}

public List<string> ListArchiveContents (string blobEntryPrefix = DefaultBlobEntryPrefix)
public List<string> ListArchiveContents (string blobEntryPrefix = DefaultBlobEntryPrefix, bool forceRefresh = false)
{
if (!forceRefresh && archiveContents != null) {
return archiveContents;
}

if (String.IsNullOrEmpty (blobEntryPrefix)) {
throw new ArgumentException (nameof (blobEntryPrefix), "must not be null or empty");
}
Expand All @@ -58,6 +63,7 @@ public List<string> ListArchiveContents (string blobEntryPrefix = DefaultBlobEnt
}
}

archiveContents = entries;
if (!useAssemblyBlobs) {
Console.WriteLine ("Not using assembly blobs");
return entries;
Expand All @@ -81,6 +87,16 @@ public List<string> ListArchiveContents (string blobEntryPrefix = DefaultBlobEnt
return entries;
}

public bool Exists (string entryPath)
{
List<string> contents = ListArchiveContents (assembliesRootDir);
if (contents.Count == 0) {
return false;
}

return contents.Contains (entryPath);
}

public void Contains (string[] fileNames, out List<string> existingFiles, out List<string> missingFiles, out List<string> additionalFiles)
{
if (fileNames == null) {
Expand Down
7 changes: 5 additions & 2 deletions src/Xamarin.Android.Build.Tasks/Utilities/AssemblyBlob.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ abstract class AssemblyBlob
protected const string BlobExtension = ".blob";

static readonly ArrayPool<byte> bytePool = ArrayPool<byte>.Shared;
static uint id = 0;
static readonly Dictionary<string, uint> apkIds = new Dictionary<string, uint> (StringComparer.Ordinal);

protected static uint globalAssemblyIndex = 0;

Expand All @@ -52,7 +52,10 @@ protected AssemblyBlob (string apkName, string archiveAssembliesPrefix, TaskLogg
}

// NOTE: NOT thread safe, if we ever have parallel runs of BuildApk this operation must either be atomic or protected with a lock
ID = id++;
if (!apkIds.ContainsKey (apkName)) {
apkIds.Add (apkName, 0);
}
ID = apkIds[apkName]++;

this.archiveAssembliesPrefix = archiveAssembliesPrefix;
ApkName = apkName;
Expand Down

0 comments on commit 57fac3b

Please sign in to comment.