diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d47f9957bd3..5a751ef9288 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -142,10 +142,10 @@ stages: -ci -restore -test - -projects $(Build.SourcesDirectory)\tests\UnitTests.XHarness.Android.WindowsQueues.proj - /bl:$(Build.SourcesDirectory)\artifacts\log\$(_BuildConfig)\Helix.XHarness.Android.WindowsQueues.binlog + -projects $(Build.SourcesDirectory)\tests\UnitTests.XHarness.Android.Device.proj + /bl:$(Build.SourcesDirectory)\artifacts\log\$(_BuildConfig)\Helix.XHarness.Android.Device.binlog /p:RestoreUsingNuGetTargets=false - /p:XharnessTestARM64_V8A=true + /p:XHarnessTestARM64_V8A=true displayName: XHarness Android Helix Testing (Windows) env: SYSTEM_ACCESSTOKEN: $(System.AccessToken) @@ -210,11 +210,11 @@ stages: -ci -restore -test - -projects $(Build.SourcesDirectory)/tests/UnitTests.XHarness.Android.LinuxQueues.proj - /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/Helix.XHarness.Android.LinuxQueues.binlog + -projects $(Build.SourcesDirectory)/tests/UnitTests.XHarness.Android.Simulator.proj + /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/Helix.XHarness.Android.Simulator.binlog /p:RestoreUsingNuGetTargets=false - /p:XharnessTestX86=true - /p:XharnessTestX86_64=true + /p:XHarnessTestX86=true + /p:XHarnessTestX86_64=true displayName: XHarness Android Helix Testing (Linux) env: SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/CreateXHarnessAndroidWorkItemsTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/CreateXHarnessAndroidWorkItemsTests.cs index af852c7a545..8115f06826c 100644 --- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/CreateXHarnessAndroidWorkItemsTests.cs +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/CreateXHarnessAndroidWorkItemsTests.cs @@ -83,12 +83,12 @@ public void AndroidXHarnessWorkItemIsCreated() _fileSystem.FileExists(payloadArchive).Should().BeTrue(); var command = workItem.GetMetadata("Command"); - command.Should().Contain("--timeout \"00:08:55\""); + command.Should().Contain("-timeout \"00:08:55\""); _zipArchiveManager .Verify(x => x.AddResourceFileToArchive(payloadArchive, It.Is(s => s.Contains("xharness-helix-job.android.sh")), "xharness-helix-job.android.sh"), Times.AtLeastOnce); _zipArchiveManager - .Verify(x => x.AddResourceFileToArchive(payloadArchive, It.Is(s => s.Contains("xharness-helix-job.android.bat")), "xharness-helix-job.android.bat"), Times.AtLeastOnce); + .Verify(x => x.AddResourceFileToArchive(payloadArchive, It.Is(s => s.Contains("xharness-helix-job.android.ps1")), "xharness-helix-job.android.ps1"), Times.AtLeastOnce); } [Fact] @@ -123,6 +123,40 @@ public void ArchivePayloadIsOverwritten() _fileSystem.RemovedFiles.Should().Contain(payloadArchive); } + [Fact] + public void ApkIsReused() + { + var collection = CreateMockServiceCollection(); + _task.ConfigureServices(collection); + _task.Apks = new[] + { + CreateApk("item-1", "System.Foo", apkPath: "apks/System.Foo.apk"), + CreateApk("item-2", "System.Foo", apkPath: "apks/System.Foo.apk"), + }; + + // Act + using var provider = collection.BuildServiceProvider(); + _task.InvokeExecute(provider).Should().BeTrue(); + + // Verify + _task.WorkItems.Length.Should().Be(2); + _fileSystem.RemovedFiles.Should().BeEmpty(); + + var workItem1 = _task.WorkItems.Last(); + workItem1.GetMetadata("Identity").Should().Be("item-2"); + + var payloadArchive = workItem1.GetMetadata("PayloadArchive"); + payloadArchive.Should().NotBeNullOrEmpty(); + _fileSystem.FileExists(payloadArchive).Should().BeTrue(); + + var workItem2 = _task.WorkItems.First(); + workItem2.GetMetadata("Identity").Should().Be("item-1"); + + payloadArchive = workItem2.GetMetadata("PayloadArchive"); + payloadArchive.Should().NotBeNullOrEmpty(); + _fileSystem.FileExists(payloadArchive).Should().BeTrue(); + } + [Fact] public void AreDependenciesRegistered() { @@ -142,32 +176,38 @@ public void AreDependenciesRegistered() } private ITaskItem CreateApk( - string path, + string itemSpec, string apkName, string? workItemTimeout = null, string? testTimeout = null, - int expectedExitCode = 0) + int expectedExitCode = 0, + string? apkPath = null) { var mockBundle = new Mock(); - mockBundle.SetupGet(x => x.ItemSpec).Returns(path); - mockBundle.Setup(x => x.GetMetadata("AndroidPackageName")).Returns(apkName); + mockBundle.SetupGet(x => x.ItemSpec).Returns(itemSpec); + mockBundle.Setup(x => x.GetMetadata(CreateXHarnessAndroidWorkItems.MetadataNames.AndroidPackageName)).Returns(apkName); if (workItemTimeout != null) { - mockBundle.Setup(x => x.GetMetadata("WorkItemTimeout")).Returns(workItemTimeout); + mockBundle.Setup(x => x.GetMetadata(XHarnessTaskBase.MetadataName.WorkItemTimeout)).Returns(workItemTimeout); } if (testTimeout != null) { - mockBundle.Setup(x => x.GetMetadata("TestTimeout")).Returns(testTimeout); + mockBundle.Setup(x => x.GetMetadata(XHarnessTaskBase.MetadataName.TestTimeout)).Returns(testTimeout); } if (expectedExitCode != 0) { - mockBundle.Setup(x => x.GetMetadata("ExpectedExitCode")).Returns(expectedExitCode.ToString()); + mockBundle.Setup(x => x.GetMetadata(XHarnessTaskBase.MetadataName.ExpectedExitCode)).Returns(expectedExitCode.ToString()); + } + + if (apkPath != null) + { + mockBundle.Setup(x => x.GetMetadata(CreateXHarnessAndroidWorkItems.MetadataNames.ApkPath)).Returns(apkPath); } - _fileSystem.WriteToFile(path, "apk"); + _fileSystem.WriteToFile(apkPath ?? itemSpec, "apk"); return mockBundle.Object; } diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/CreateXHarnessAppleWorkItemsTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/CreateXHarnessAppleWorkItemsTests.cs index b1c76a859a2..86e5f9366b5 100644 --- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/CreateXHarnessAppleWorkItemsTests.cs +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/CreateXHarnessAppleWorkItemsTests.cs @@ -112,7 +112,7 @@ public void ArchivePayloadIsOverwritten() CreateAppBundle("apps/System.Bar.app", "ios-simulator-64_13.5"), }; - _fileSystem.Files.Add("apps/xharness-app-payload-system.foo.app.zip", "archive"); + _fileSystem.Files.Add("apps/xharness-app-payload-system.foo.zip", "archive"); // Act using var provider = collection.BuildServiceProvider(); @@ -161,6 +161,46 @@ public void CustomCommandsAreExecuted() .Verify(x => x.AddContentToArchive(payloadArchive, "command.sh", "echo foo"), Times.Once); } + [Fact] + public void AppBundleIsReused() + { + var collection = CreateMockServiceCollection(); + _task.ConfigureServices(collection); + _task.AppBundles = new[] + { + CreateAppBundle("item-1", "ios-simulator-64_13.5", appBundlePath: "apps/System.Foo.app"), + CreateAppBundle("item-2", "ios-simulator-64_13.6", appBundlePath: "apps/System.Foo.app"), + }; + + // Act + using var provider = collection.BuildServiceProvider(); + _task.InvokeExecute(provider).Should().BeTrue(); + + // Verify + _task.WorkItems.Length.Should().Be(2); + _fileSystem.RemovedFiles.Should().BeEmpty(); + + var workItem1 = _task.WorkItems.Last(); + workItem1.GetMetadata("Identity").Should().Be("item-2"); + + var payloadArchive = workItem1.GetMetadata("PayloadArchive"); + payloadArchive.Should().NotBeNullOrEmpty(); + _fileSystem.FileExists(payloadArchive).Should().BeTrue(); + + var command = workItem1.GetMetadata("Command"); + command.Should().Contain("--target \"ios-simulator-64_13.6\""); + + var workItem2 = _task.WorkItems.First(); + workItem2.GetMetadata("Identity").Should().Be("item-1"); + + payloadArchive = workItem2.GetMetadata("PayloadArchive"); + payloadArchive.Should().NotBeNullOrEmpty(); + _fileSystem.FileExists(payloadArchive).Should().BeTrue(); + + command = workItem2.GetMetadata("Command"); + command.Should().Contain("--target \"ios-simulator-64_13.5\""); + } + [Fact] public void AreDependenciesRegistered() { @@ -180,17 +220,18 @@ public void AreDependenciesRegistered() } private ITaskItem CreateAppBundle( - string path, + string itemSpec, string target, string? workItemTimeout = null, string? testTimeout = null, string? launchTimeout = null, int expectedExitCode = 0, bool includesTestRunner = true, - string? customCommands = null) + string? customCommands = null, + string? appBundlePath = null) { var mockBundle = new Mock(); - mockBundle.SetupGet(x => x.ItemSpec).Returns(path); + mockBundle.SetupGet(x => x.ItemSpec).Returns(itemSpec); mockBundle.Setup(x => x.GetMetadata(CreateXHarnessAppleWorkItems.MetadataNames.Target)).Returns(target); mockBundle.Setup(x => x.GetMetadata(CreateXHarnessAppleWorkItems.MetadataNames.IncludesTestRunner)).Returns(includesTestRunner.ToString()); @@ -219,7 +260,12 @@ private ITaskItem CreateAppBundle( mockBundle.Setup(x => x.GetMetadata(XHarnessTaskBase.MetadataName.CustomCommands)).Returns(customCommands); } - _fileSystem.CreateDirectory(path); + if (appBundlePath != null) + { + mockBundle.Setup(x => x.GetMetadata(CreateXHarnessAppleWorkItems.MetadataNames.AppBundlePath)).Returns(appBundlePath); + } + + _fileSystem.CreateDirectory(appBundlePath ?? itemSpec); return mockBundle.Object; } diff --git a/src/Microsoft.DotNet.Helix/Sdk/CreateXHarnessAndroidWorkItems.cs b/src/Microsoft.DotNet.Helix/Sdk/CreateXHarnessAndroidWorkItems.cs index d2910307991..bcec332732f 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/CreateXHarnessAndroidWorkItems.cs +++ b/src/Microsoft.DotNet.Helix/Sdk/CreateXHarnessAndroidWorkItems.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading.Tasks; @@ -15,13 +14,17 @@ namespace Microsoft.DotNet.Helix.Sdk /// public class CreateXHarnessAndroidWorkItems : XHarnessTaskBase { - private const string ArgumentsPropName = "Arguments"; - private const string AndroidInstrumentationNamePropName = "AndroidInstrumentationName"; - private const string DeviceOutputPathPropName = "DeviceOutputPath"; - private const string AndroidPackageNamePropName = "AndroidPackageName"; + public static class MetadataNames + { + public const string Arguments = "Arguments"; + public const string AndroidInstrumentationName = "AndroidInstrumentationName"; + public const string DeviceOutputPath = "DeviceOutputPath"; + public const string AndroidPackageName = "AndroidPackageName"; + public const string ApkPath = "ApkPath"; + } private const string PosixAndroidWrapperScript = "xharness-helix-job.android.sh"; - private const string NonPosixAndroidWrapperScript = "xharness-helix-job.android.bat"; + private const string NonPosixAndroidWrapperScript = "xharness-helix-job.android.ps1"; /// /// Boolean true if this is a posix shell, false if not. @@ -64,34 +67,114 @@ public bool ExecuteTask(IZipArchiveManager zipArchiveManager, IFileSystem fileSy /// An ITaskItem instance representing the prepared HelixWorkItem. private async Task PrepareWorkItem(IZipArchiveManager zipArchiveManager, IFileSystem fileSystem, ITaskItem appPackage) { - string workItemName = fileSystem.GetFileNameWithoutExtension(appPackage.ItemSpec); - - var (testTimeout, workItemTimeout, expectedExitCode, customCommands) = ParseMetadata(appPackage); + // The user can re-use the same .apk for 2 work items so the name of the work item will come from ItemSpec and path from metadata + // This can be useful when we want to run the same app with different parameters or run the same app on different test targets, e.g. iOS 13 and 13.5 + string workItemName; + string apkPath; + if (appPackage.TryGetMetadata(MetadataNames.ApkPath, out string apkPathMetadata) && !string.IsNullOrEmpty(apkPathMetadata)) + { + workItemName = appPackage.ItemSpec; + apkPath = apkPathMetadata; + } + else + { + workItemName = fileSystem.GetFileNameWithoutExtension(appPackage.ItemSpec); + apkPath = appPackage.ItemSpec; + } - string command = ValidateMetadataAndGetXHarnessAndroidCommand(appPackage, testTimeout, expectedExitCode); + if (!fileSystem.FileExists(apkPath)) + { + Log.LogError($"App package not found in {apkPath}"); + return null; + } - if (!fileSystem.GetExtension(appPackage.ItemSpec).Equals(".apk", StringComparison.OrdinalIgnoreCase)) + if (!fileSystem.GetExtension(apkPath).Equals(".apk", StringComparison.OrdinalIgnoreCase)) { - Log.LogError($"Unsupported app package type: {fileSystem.GetFileName(appPackage.ItemSpec)}"); + Log.LogError($"Unsupported app package type: {fileSystem.GetFileName(apkPath)}"); return null; } - Log.LogMessage($"Creating work item with properties Identity: {workItemName}, Payload: {appPackage.ItemSpec}, Command: {command}"); + // Validation of any metadata specific to Android stuff goes here + if (!appPackage.GetRequiredMetadata(Log, MetadataNames.AndroidPackageName, out string androidPackageName)) + { + Log.LogError($"{MetadataNames.AndroidPackageName} metadata must be specified; this may match, but can vary from file name"); + return null; + } - string workItemZip = await CreateZipArchiveOfPackageAsync(zipArchiveManager, fileSystem, appPackage.ItemSpec); + var (testTimeout, workItemTimeout, expectedExitCode, customCommands) = ParseMetadata(appPackage); - return new Build.Utilities.TaskItem(workItemName, new Dictionary() + if (customCommands == null) { - { "Identity", workItemName }, - { "PayloadArchive", workItemZip }, - { "Command", command }, - { "Timeout", workItemTimeout.ToString() }, - }); + // When no user commands are specified, we add the default `android test ...` command + customCommands = GetDefaultCommand(appPackage, expectedExitCode); + } + + string command = GetHelixCommand(appPackage, apkPath, androidPackageName, testTimeout, expectedExitCode); + + Log.LogMessage($"Creating work item with properties Identity: {workItemName}, Payload: {apkPath}, Command: {command}"); + + string workItemZip = await CreateZipArchiveOfPackageAsync(zipArchiveManager, fileSystem, workItemName, apkPath, customCommands); + + return CreateTaskItem(workItemName, workItemZip, command, workItemTimeout); } - private async Task CreateZipArchiveOfPackageAsync(IZipArchiveManager zipArchiveManager, IFileSystem fileSystem, string fileToZip) + private string GetDefaultCommand(ITaskItem appPackage, int expectedExitCode) { - string fileName = $"xharness-apk-payload-{fileSystem.GetFileNameWithoutExtension(fileToZip).ToLowerInvariant()}.zip"; + appPackage.TryGetMetadata(MetadataNames.Arguments, out string extraArguments); + + var exitCodeArg = expectedExitCode != 0 ? $"--expected-exit-code $expected_exit_code" : string.Empty; + var passthroughArgs = !string.IsNullOrEmpty(AppArguments) ? $" -- {AppArguments}" : string.Empty; + + var instrumentationArg = appPackage.TryGetMetadata(MetadataNames.AndroidInstrumentationName, out string androidInstrumentationName) + ? $"--instrumentation \"{androidInstrumentationName}\"" + : string.Empty; + + var devOutArg = appPackage.TryGetMetadata(MetadataNames.DeviceOutputPath, out string deviceOutputPath) + ? $"--dev-out \"{deviceOutputPath}\"" + : string.Empty; + + // In case user didn't specify custom commands, we use our default one + return "xharness android test " + + "--app \"$app\" " + + "--output-directory \"$output_directory\" " + + "--timeout \"$timeout\" " + + "--package-name \"$package_name\" " + + " -v " + + $"{ devOutArg } { instrumentationArg } { exitCodeArg } { extraArguments } { passthroughArgs }"; + } + + private string GetHelixCommand(ITaskItem appPackage, string apkPath, string androidPackageName, TimeSpan xHarnessTimeout, int expectedExitCode) + { + appPackage.TryGetMetadata(MetadataNames.AndroidInstrumentationName, out string androidInstrumentationName); + appPackage.TryGetMetadata(MetadataNames.DeviceOutputPath, out string deviceOutputPath); + + string wrapperScriptName = IsPosixShell ? PosixAndroidWrapperScript : NonPosixAndroidWrapperScript; + string xharnessHelixWrapperScript = IsPosixShell ? $"chmod +x ./{wrapperScriptName} && ./{wrapperScriptName}" + : $"powershell -ExecutionPolicy ByPass -NoProfile -File \"{wrapperScriptName}\""; + + // We either call .ps1 or .sh so we need to format the arguments well (PS has -argument, bash has --argument) + string dash = IsPosixShell ? "--" : "-"; + string xharnessRunCommand = $"{xharnessHelixWrapperScript} " + + $"{dash}app \"{Path.GetFileName(apkPath)}\" " + + $"{dash}timeout \"{xHarnessTimeout}\" " + + $"{dash}package_name \"{androidPackageName}\" " + + (expectedExitCode != 0 ? $" {dash}expected_exit_code \"{expectedExitCode}\" " : string.Empty) + + (string.IsNullOrEmpty(deviceOutputPath) ? string.Empty : $"{dash}device_output_path \"{deviceOutputPath}\" ") + + (string.IsNullOrEmpty(androidInstrumentationName) ? string.Empty : $"{dash}instrumentation \"{androidInstrumentationName}\" "); + + Log.LogMessage(MessageImportance.Low, $"Generated XHarness command: {xharnessRunCommand}"); + + return xharnessRunCommand; + } + + private async Task CreateZipArchiveOfPackageAsync( + IZipArchiveManager zipArchiveManager, + IFileSystem fileSystem, + string workItemName, + string fileToZip, + string injectedCommands) + { + string fileName = $"xharness-apk-payload-{workItemName.ToLowerInvariant()}.zip"; string outputZipPath = fileSystem.PathCombine(fileSystem.GetDirectoryName(fileToZip), fileName); if (fileSystem.FileExists(outputZipPath)) @@ -106,48 +189,9 @@ private async Task CreateZipArchiveOfPackageAsync(IZipArchiveManager zip // so we'll always include both scripts (very small) await zipArchiveManager.AddResourceFileToArchive(outputZipPath, ScriptNamespace + PosixAndroidWrapperScript, PosixAndroidWrapperScript); await zipArchiveManager.AddResourceFileToArchive(outputZipPath, ScriptNamespace + NonPosixAndroidWrapperScript, NonPosixAndroidWrapperScript); + await zipArchiveManager.AddContentToArchive(outputZipPath, CustomCommandsScript + (IsPosixShell ? ".sh" : ".ps1"), injectedCommands); return outputZipPath; } - - private string ValidateMetadataAndGetXHarnessAndroidCommand(ITaskItem appPackage, TimeSpan xHarnessTimeout, int expectedExitCode) - { - // Validation of any metadata specific to Android stuff goes here - if (!appPackage.GetRequiredMetadata(Log, AndroidPackageNamePropName, out string androidPackageName)) - { - Log.LogError($"{AndroidPackageNamePropName} metadata must be specified; this may match, but can vary from file name"); - return null; - } - - appPackage.TryGetMetadata(ArgumentsPropName, out string arguments); - appPackage.TryGetMetadata(AndroidInstrumentationNamePropName, out string androidInstrumentationName); - appPackage.TryGetMetadata(DeviceOutputPathPropName, out string deviceOutputPath); - - string outputPathArg = string.IsNullOrEmpty(deviceOutputPath) ? string.Empty : $"--dev-out={deviceOutputPath} "; - string instrumentationArg = string.IsNullOrEmpty(androidInstrumentationName) ? string.Empty : $"-i={androidInstrumentationName} "; - - string outputDirectory = IsPosixShell ? "$HELIX_WORKITEM_UPLOAD_ROOT" : "%HELIX_WORKITEM_UPLOAD_ROOT%"; - string wrapperScriptName = IsPosixShell ? PosixAndroidWrapperScript : NonPosixAndroidWrapperScript; - - string xharnessHelixWrapperScript = IsPosixShell ? $"chmod +x ./{wrapperScriptName} && ./{wrapperScriptName}" - : $"call {wrapperScriptName}"; - - string xharnessRunCommand = $"{xharnessHelixWrapperScript} " + - $"dotnet exec \"{(IsPosixShell ? "$XHARNESS_CLI_PATH" : "%XHARNESS_CLI_PATH%")}\" android test " + - $"--app \"{Path.GetFileName(appPackage.ItemSpec)}\" " + - $"--output-directory \"{outputDirectory}\" " + - $"--timeout \"{xHarnessTimeout}\" " + - $"-p=\"{androidPackageName}\" " + - "-v " + - (expectedExitCode != 0 ? $" --expected-exit-code \"{expectedExitCode}\" " : string.Empty) + - outputPathArg + - instrumentationArg + - arguments + - (!string.IsNullOrEmpty(AppArguments) ? $" -- {AppArguments}" : string.Empty); - - Log.LogMessage(MessageImportance.Low, $"Generated XHarness command: {xharnessRunCommand}"); - - return xharnessRunCommand; - } } } diff --git a/src/Microsoft.DotNet.Helix/Sdk/CreateXHarnessAppleWorkItems.cs b/src/Microsoft.DotNet.Helix/Sdk/CreateXHarnessAppleWorkItems.cs index 1cd81e66846..a21be345429 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/CreateXHarnessAppleWorkItems.cs +++ b/src/Microsoft.DotNet.Helix/Sdk/CreateXHarnessAppleWorkItems.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading.Tasks; @@ -24,6 +23,7 @@ public static class MetadataNames public const string LaunchTimeout = "LaunchTimeout"; public const string IncludesTestRunner = "IncludesTestRunner"; public const string ResetSimulator = "ResetSimulator"; + public const string AppBundlePath = "AppBundlePath"; } private const string EntryPointScript = "xharness-helix-job.apple.sh"; @@ -91,9 +91,28 @@ private async Task PrepareWorkItem( IFileSystem fileSystem, ITaskItem appBundleItem) { - string appFolderPath = appBundleItem.ItemSpec.TrimEnd(Path.DirectorySeparatorChar); + // The user can re-use the same .apk for 2 work items so the name of the work item will come from ItemSpec and path from metadata + string workItemName; + string appFolderPath; + if (appBundleItem.TryGetMetadata(MetadataNames.AppBundlePath, out string appPathMetadata) && !string.IsNullOrEmpty(appPathMetadata)) + { + workItemName = appBundleItem.ItemSpec; + appFolderPath = appPathMetadata; + } + else + { + workItemName = fileSystem.GetFileName(appBundleItem.ItemSpec); + appFolderPath = appBundleItem.ItemSpec; + } + + appFolderPath = appFolderPath.TrimEnd(Path.DirectorySeparatorChar); + + if (!fileSystem.DirectoryExists(appFolderPath)) + { + Log.LogError($"App bundle not found in {appFolderPath}"); + return null; + } - string workItemName = fileSystem.GetFileName(appFolderPath); if (workItemName.EndsWith(".app")) { workItemName = workItemName.Substring(0, workItemName.Length - 4); @@ -147,37 +166,34 @@ private async Task PrepareWorkItem( if (customCommands == null) { - // In case user didn't specify custom commands, we use our default one - customCommands = $"xharness apple {(includesTestRunner ? "test" : "run")} " + - "--app \"$app\" " + - "--output-directory \"$output_directory\" " + - "--target \"$target\" " + - "--timeout \"$timeout\" " + - (includesTestRunner - ? $"--launch-timeout \"$launch_timeout\" " - : $"--expected-exit-code $expected_exit_code ") + - (resetSimulator ? $"--reset-simulator " : string.Empty) + - (target.Contains("device") ? $"--signal-app-end " : string.Empty) + // iOS/tvOS 14+ workaround - "--xcode \"$xcode_path\" " + - "-v " + - (!string.IsNullOrEmpty(AppArguments) ? "-- " + AppArguments : string.Empty); + // When no user commands are specified, we add the default `apple test ...` command + customCommands = GetDefaultCommand(target, includesTestRunner, resetSimulator); } - string appName = fileSystem.GetFileName(appBundleItem.ItemSpec); + string appName = fileSystem.GetFileName(appFolderPath); string helixCommand = GetHelixCommand(appName, target, testTimeout, launchTimeout, includesTestRunner, expectedExitCode, resetSimulator); - string payloadArchivePath = await CreateZipArchiveOfFolder(zipArchiveManager, fileSystem, appFolderPath, customCommands); + string payloadArchivePath = await CreateZipArchiveOfFolder(zipArchiveManager, fileSystem, workItemName, appFolderPath, customCommands); Log.LogMessage($"Creating work item with properties Identity: {workItemName}, Payload: {appFolderPath}, Command: {helixCommand}"); - return new Build.Utilities.TaskItem(workItemName, new Dictionary() - { - { "Identity", workItemName }, - { "PayloadArchive", payloadArchivePath }, - { "Command", helixCommand }, - { "Timeout", workItemTimeout.ToString() }, - }); + return CreateTaskItem(workItemName, payloadArchivePath, helixCommand, workItemTimeout); } + private string GetDefaultCommand(string target, bool includesTestRunner, bool resetSimulator) => + $"xharness apple {(includesTestRunner ? "test" : "run")} " + + "--app \"$app\" " + + "--output-directory \"$output_directory\" " + + "--target \"$target\" " + + "--timeout \"$timeout\" " + + "--xcode \"$xcode_path\" " + + "-v " + + (includesTestRunner + ? $"--launch-timeout \"$launch_timeout\" " + : $"--expected-exit-code $expected_exit_code ") + + (resetSimulator ? $"--reset-simulator " : string.Empty) + + (target.Contains("device") ? $"--signal-app-end " : string.Empty) + // iOS/tvOS 14+ workaround + (!string.IsNullOrEmpty(AppArguments) ? "-- " + AppArguments : string.Empty); + private string GetHelixCommand( string appName, string target, @@ -201,6 +217,7 @@ private string GetHelixCommand( private async Task CreateZipArchiveOfFolder( IZipArchiveManager zipArchiveManager, IFileSystem fileSystem, + string workItemName, string folderToZip, string injectedCommands) { @@ -211,7 +228,7 @@ private async Task CreateZipArchiveOfFolder( } string appFolderDirectory = fileSystem.GetDirectoryName(folderToZip); - string fileName = $"xharness-app-payload-{fileSystem.GetFileName(folderToZip).ToLowerInvariant()}.zip"; + string fileName = $"xharness-app-payload-{workItemName.ToLowerInvariant()}.zip"; string outputZipPath = fileSystem.PathCombine(appFolderDirectory, fileName); if (fileSystem.FileExists(outputZipPath)) diff --git a/src/Microsoft.DotNet.Helix/Sdk/Microsoft.DotNet.Helix.Sdk.csproj b/src/Microsoft.DotNet.Helix/Sdk/Microsoft.DotNet.Helix.Sdk.csproj index e06aeef2fbe..2e510066ae4 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/Microsoft.DotNet.Helix.Sdk.csproj +++ b/src/Microsoft.DotNet.Helix/Sdk/Microsoft.DotNet.Helix.Sdk.csproj @@ -40,7 +40,7 @@ Never - + Never diff --git a/src/Microsoft.DotNet.Helix/Sdk/XharnessTaskBase.cs b/src/Microsoft.DotNet.Helix/Sdk/XharnessTaskBase.cs index 0797b49f38e..205bf33a6dd 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/XharnessTaskBase.cs +++ b/src/Microsoft.DotNet.Helix/Sdk/XharnessTaskBase.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using Microsoft.Arcade.Common; using Microsoft.Build.Framework; @@ -93,5 +94,14 @@ public class MetadataName return (testTimeout, workItemTimeout, expectedExitCode, customCommands); } + + protected Build.Utilities.TaskItem CreateTaskItem(string workItemName, string payloadArchivePath, string command, TimeSpan timeout) => + new (workItemName, new Dictionary() + { + { "Identity", workItemName }, + { "PayloadArchive", payloadArchivePath }, + { "Command", command }, + { "Timeout", timeout.ToString() }, + }); } } diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/Readme.md b/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/Readme.md index c953bbbb06c..bebc13f69fa 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/Readme.md +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/Readme.md @@ -181,6 +181,9 @@ Example: ``` +Please note that Android can run on both Windows and Linux based on the target queue. +For that reason, make sure the `` script you supply is either **bash** for Linux queues or **PowerShell** for Windows. + When using `CustomCommands`, several variables will be defined for you for easier run. #### Variables defined for Apple scenarios @@ -190,4 +193,34 @@ When using `CustomCommands`, several variables will be defined for you for easie - `$target`, `$timeout`, `$launch_timeout`, `$expected_exit_code`, `$includes_test_runner` - parsed metadata defined on the original `XHarnessAppBundleToTest` MSBuild item #### Variables defined for Android scenarios -Android is currently not supported - coming soon! +- `$app` - path to the application +- `$package_name` - name of the Android package +- `$output_directory` - path under which all files will be uploaded to Helix at the end of the job + - If a file named `testResults.xml` is found containing xUnit results, it will be uploaded back to Azure DevOps +- `$timeout`, `$expected_exit_code`, `$device_output_path`, `$instrumentation` - parsed metadata defined on the original `XHarnessApkToTest` MSBuild item + + +### Reusing app bundles / apks + +In some scenarios, you might need to re-use one application for multiple work items, i.e. to supply each with a different custom command to run the application with different parameters or to run the application on different test targets (e.g. different versions of iOS). + +You can then name the item however you like and supply the path to the app as metadata: + +```xml + + path/to/System.Text.Json.Tests.apk + net.dot.System.Buffers.Tests + net.dot.MonoRunner + + + + path/to/System.Text.Json.Tests.apk + net.dot.System.Buffers.Tests + net.dot.MonoRunner + + xharness android test --app "$app" --package-name "net.dot.System.Buffers.Tests" --output-directory "$output_directory" --instrumentation=net.dot.MonoRunner + + +``` + +For Apple it is the same, just the metadata property name is ``. diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.android.bat b/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.android.bat deleted file mode 100644 index 5ffe3a911dc..00000000000 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.android.bat +++ /dev/null @@ -1,23 +0,0 @@ -@echo off - -REM This script is used as a payload of Helix jobs that execute Android workloads through XHarness on Windows systems. -REM This is used as the entrypoint of the work item so that XHarness failures can be detected and (when appropriate) -REM cause the work item to retry and reboot the Helix agent the work is running on. - -REM Currently no special functionality is needed beyond causing infrastructure retry and reboot if the emulators -REM or devices have trouble, but to add more Helix-specific Android XHarness behaviors, this is one extensibility point. - -set ADB_DEVICE_ENUMERATION_FAILURE=85 - -echo Xharness Helix Wrapper: Arguments: %* -%* -set EXIT_CODE=%ERRORLEVEL% - -if %EXIT_CODE%==%ADB_DEVICE_ENUMERATION_FAILURE% ( - echo Encountered ADB_DEVICE_ENUMERATION_FAILURE. This is typically not a failure of the work item. We will run it again and reboot this computer to help its devices - echo If this occurs repeatedly, please check for architectural mismatch, e.g. sending x86 or x86_64 APKs to an arm64_v8a-only queue. - %HELIX_PYTHONPATH% -c "from helix.workitemutil import request_infra_retry; request_infra_retry('Retrying because we could not enumerate all Android devices')" - %HELIX_PYTHONPATH% -c "from helix.workitemutil import request_reboot; request_reboot('Rebooting to allow Android emulator or device to restart')" -) - -exit /B %EXIT_CODE% diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.android.ps1 b/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.android.ps1 new file mode 100644 index 00000000000..99dba979c0a --- /dev/null +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.android.ps1 @@ -0,0 +1,49 @@ +<# +This script is used as a payload of Helix jobs that execute Android workloads through XHarness on Windows systems. +This is used as the entrypoint of the work item so that XHarness failures can be detected and (when appropriate) +cause the work item to retry and reboot the Helix agent the work is running on. + +Currently no special functionality is needed beyond causing infrastructure retry and reboot if the emulators +or devices have trouble, but to add more Helix-specific Android XHarness behaviors, this is one extensibility point. +#> + +param ( + [Parameter(Mandatory)] + [string]$app, + [Parameter(Mandatory)] + [string]$timeout, + [Parameter(Mandatory)] + [string]$package_name, + [Parameter()] + [int]$expected_exit_code = 0, + [Parameter()] + [string]$device_output_path = $null, + [Parameter()] + [string]$instrumentation = $null +) + +$ErrorActionPreference="Stop" + +[Diagnostics.CodeAnalysis.SuppressMessageAttribute("PSUseDeclaredVarsMoreThanAssignments", "")] # Variable used in sourced script +$output_directory=$Env:HELIX_WORKITEM_UPLOAD_ROOT + +# The xharness alias +function xharness() { + dotnet exec $Env:XHARNESS_CLI_PATH @args +} + +# Act out the actual commands +. "$PSScriptRoot\command.ps1" + +$exit_code=$LASTEXITCODE + +# ADB_DEVICE_ENUMERATION_FAILURE +if ($exit_code -eq 85) { + $ErrorActionPreference="Continue" + Write-Error "Encountered ADB_DEVICE_ENUMERATION_FAILURE. This is typically not a failure of the work item. We will run it again and reboot this computer to help its devices" + Write-Error "If this occurs repeatedly, please check for architectural mismatch, e.g. sending x86 or x86_64 APKs to an arm64_v8a-only queue." + & "$Env:HELIX_PYTHONPATH" -c "from helix.workitemutil import request_infra_retry; request_infra_retry('Retrying because we could not enumerate all Android devices')" + & "$Env:HELIX_PYTHONPATH" -c "from helix.workitemutil import request_reboot; request_reboot('Rebooting to allow Android emulator or device to restart')" +} + +exit $exit_code diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.android.sh b/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.android.sh index 3206267624c..f7e2b920b91 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.android.sh +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.android.sh @@ -3,13 +3,77 @@ ### This script is used as a payload of Helix jobs that execute Android workloads through XHarness on Linux systems. ### This is used as the entrypoint of the work item so that XHarness failures can be detected and (when appropriate) ### cause the work item to retry and reboot the Helix agent the work is running on. -### -### Currently no special functionality is needed beyond causing infrastructure retry and reboot if the emulators -### or devices have trouble, but to add more Helix-specific Android XHarness behaviors, this is one extensibility point. +### +### This scripts sets up the environment and then sources the actual XHarness commands that come either from the Helix +### SDK or from user via the property. -set -x echo "XHarness Helix Job Wrapper calling '$@'" -"$@" + +set -x + +app='' +timeout='' +package_name='' +expected_exit_code=0 +device_output_path='' +instrumentation='' +output_directory=$HELIX_WORKITEM_UPLOAD_ROOT + +while [[ $# -gt 0 ]]; do + opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")" + case "$opt" in + --app) + app="$2" + shift + ;; + --timeout) + timeout="$2" + shift + ;; + --package_name) + package_name="$2" + shift + ;; + --expected_exit_code) + expected_exit_code="$2" + shift + ;; + --device_output_path) + device_output_path="$2" + shift + ;; + --instrumentation) + instrumentation="$2" + shift + ;; + esac + shift +done + +if [ -z "$app" ]; then + die "App bundle path wasn't provided"; +fi + +if [ -z "$timeout" ]; then + die "No timeout was provided"; +fi + +if [ -z "$package_name" ]; then + die "Package name path wasn't provided"; +fi + +if [ -z "$output_directory" ]; then + die "No output directory provided"; +fi + +# The xharness alias +function xharness() { + dotnet exec $XHARNESS_CLI_PATH "$@" +} + +# Act out the actual commands +source command.sh + exit_code=$? # This handles issues where devices or emulators fail to start. @@ -17,7 +81,7 @@ exit_code=$? # Too see where these values come from, check out https://github.com/dotnet/xharness/blob/master/src/Microsoft.DotNet.XHarness.Common/CLI/ExitCode.cs # Avoid any helix-ism in the Xharness! -ADB_DEVICE_ENUMERATION_FAILURE=85 +ADB_DEVICE_ENUMERATION_FAILURE=85 if [ $exit_code -eq $ADB_DEVICE_ENUMERATION_FAILURE ]; then echo 'Encountered ADB_DEVICE_ENUMERATION_FAILURE. This is typically not a failure of the work item. We will run it again and reboot this computer to help its devices' echo 'If this occurs repeatedly, please check for architectural mismatch, e.g. sending arm64_v8a APKs to an x86_64 / x86 only queue.' diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-runner.apple.sh b/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-runner.apple.sh index 8cfdc978e99..6096d835d48 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-runner.apple.sh +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-runner.apple.sh @@ -78,7 +78,7 @@ if [ -z "$app" ]; then fi if [ -z "$target" ]; then - die "No target were provided"; + die "No target was provided"; fi if [ -z "$output_directory" ]; then @@ -139,7 +139,11 @@ fi export XHARNESS_DISABLE_COLORED_OUTPUT=true export XHARNESS_LOG_WITH_TIMESTAMPS=true -alias xharness="dotnet exec $xharness_cli_path" + +# The xharness alias +function xharness() { + dotnet exec $xharness_cli_path "$@" +} # Act out the actual commands source command.sh diff --git a/tests/UnitTests.XHarness.Android.Device.proj b/tests/UnitTests.XHarness.Android.Device.proj new file mode 100644 index 00000000000..c3aeeb59dff --- /dev/null +++ b/tests/UnitTests.XHarness.Android.Device.proj @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/tests/UnitTests.XHarness.Android.Simulator.proj b/tests/UnitTests.XHarness.Android.Simulator.proj new file mode 100644 index 00000000000..11dfbaba887 --- /dev/null +++ b/tests/UnitTests.XHarness.Android.Simulator.proj @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + diff --git a/tests/UnitTests.XHarness.Android.WindowsQueues.proj b/tests/UnitTests.XHarness.Android.WindowsQueues.proj deleted file mode 100644 index 1614ac47ea5..00000000000 --- a/tests/UnitTests.XHarness.Android.WindowsQueues.proj +++ /dev/null @@ -1,52 +0,0 @@ - - - - - - - - $(MSBuildThisFileDirectory)../artifacts/bin/Microsoft.DotNet.Helix.Sdk/$(Configuration)/netcoreapp3.1/publish/Microsoft.DotNet.Helix.Sdk.dll - $(MSBuildThisFileDirectory)../artifacts/bin/Microsoft.DotNet.Helix.Sdk/$(Configuration)/net472/publish/Microsoft.DotNet.Helix.Sdk.dll - - - - test/product/ - $(AGENT_JOBNAME) - true - true - true - https://helix.dot.net - - - - - - - - - - - - - - - - - true - $(BUILD_SOURCEVERSIONAUTHOR) - anon - - - - - msbuild - - - - - - diff --git a/tests/UnitTests.XHarness.Android.LinuxQueues.proj b/tests/UnitTests.XHarness.Common.props similarity index 59% rename from tests/UnitTests.XHarness.Android.LinuxQueues.proj rename to tests/UnitTests.XHarness.Common.props index a8d3dbcd51f..d657a6d9e5c 100644 --- a/tests/UnitTests.XHarness.Android.LinuxQueues.proj +++ b/tests/UnitTests.XHarness.Common.props @@ -1,11 +1,9 @@ - + @@ -22,18 +20,11 @@ https://helix.dot.net - - - - - - - - - - - - + + + false + false + true @@ -45,8 +36,5 @@ msbuild - - - diff --git a/tests/UnitTests.XHarness.iOS.Device.proj b/tests/UnitTests.XHarness.iOS.Device.proj index 0e78ad60d95..d828cca75c7 100644 --- a/tests/UnitTests.XHarness.iOS.Device.proj +++ b/tests/UnitTests.XHarness.iOS.Device.proj @@ -1,6 +1,5 @@ - - + - - $(MSBuildThisFileDirectory)../artifacts/bin/Microsoft.DotNet.Helix.Sdk/$(Configuration)/netcoreapp3.1/publish/Microsoft.DotNet.Helix.Sdk.dll - $(MSBuildThisFileDirectory)../artifacts/bin/Microsoft.DotNet.Helix.Sdk/$(Configuration)/net472/publish/Microsoft.DotNet.Helix.Sdk.dll - - - - test/product/ - $(AGENT_JOBNAME) - true - true - true - https://helix.dot.net - - - - - - - true - $(BUILD_SOURCEVERSIONAUTHOR) - anon - - - - - msbuild - - diff --git a/tests/UnitTests.XHarness.iOS.Simulator.proj b/tests/UnitTests.XHarness.iOS.Simulator.proj index 7c07d677773..e483cb6384a 100644 --- a/tests/UnitTests.XHarness.iOS.Simulator.proj +++ b/tests/UnitTests.XHarness.iOS.Simulator.proj @@ -1,6 +1,5 @@ - - + - - $(MSBuildThisFileDirectory)../artifacts/bin/Microsoft.DotNet.Helix.Sdk/$(Configuration)/netcoreapp3.1/publish/Microsoft.DotNet.Helix.Sdk.dll - $(MSBuildThisFileDirectory)../artifacts/bin/Microsoft.DotNet.Helix.Sdk/$(Configuration)/net472/publish/Microsoft.DotNet.Helix.Sdk.dll - - - - test/product/ - $(AGENT_JOBNAME) - true - true - true - https://helix.dot.net - - - - - - - - - - true - $(BUILD_SOURCEVERSIONAUTHOR) - anon - - - - - msbuild - - diff --git a/tests/XHarness/XHarness.TestApks.proj b/tests/XHarness/XHarness.TestApks.proj index b26e872c26a..f44c1c26b3b 100644 --- a/tests/XHarness/XHarness.TestApks.proj +++ b/tests/XHarness/XHarness.TestApks.proj @@ -8,26 +8,38 @@ https://netcorenativeassets.blob.core.windows.net/resource-packages/external/android/test-apk/arm64_v8a/System.Buffers.Tests-arm64-v8a.apk + - + - + - + + - - + + %(FullPath) + + + net.dot.System.Buffers.Tests + + + net.dot.MonoRunner + + + + %(FullPath) net.dot.System.Buffers.Tests @@ -35,6 +47,8 @@ net.dot.MonoRunner + + xharness android test --app "$app" --package-name "$package_name" --output-directory "$output_directory" --instrumentation "$instrumentation" --timeout "$timeout" -v