Add support for tvOS devices in XHarness Helix SDK

src/Common/Microsoft.Arcade.Common/Helpers.cs

@@ -53,16 +53,22 @@ public T MutexExec<T>(Func<T> function, string mutexName)
             }
         }
 
-        public T DirectoryMutexExec<T>(Func<T> function, string path) =>
-            MutexExec(
-                function,
-                $"Global\\{ComputeSha256Hash(path)}");
-
         public T MutexExec<T>(Func<Task<T>> function, string mutexName) =>
             MutexExec(() => function().GetAwaiter().GetResult(), mutexName); // Can't await because of mutex
 
+        // This overload is here so that async Actions don't get routed to MutexExec<T>(Func<T> function, string path)
+        public void MutexExec(Func<Task> function, string mutexName) =>
+            MutexExec(() => { function().GetAwaiter().GetResult(); return true; }, mutexName);
+
+        public T DirectoryMutexExec<T>(Func<T> function, string path) =>
+            MutexExec(function, $"Global\\{ComputeSha256Hash(path)}");
+
         public T DirectoryMutexExec<T>(Func<Task<T>> function, string path) =>
-            DirectoryMutexExec(() => function().GetAwaiter().GetResult(), path); // Can't await because of mutex        
+            DirectoryMutexExec(() => function().GetAwaiter().GetResult(), path); // Can't await because of mutex
+
+        // This overload is here so that async Actions don't get routed to DirectoryMutexExec<T>(Func<T> function, string path)
+        public void DirectoryMutexExec(Func<Task> function, string path) => 
+            DirectoryMutexExec(() => { function().GetAwaiter().GetResult(); return true; }, path);
     }
 
     public static class KeyValuePairExtensions

src/Common/Microsoft.Arcade.Common/IHelpers.cs

@@ -1,15 +1,20 @@
-using System;
+using System;
 using System.Threading.Tasks;
 
 namespace Microsoft.Arcade.Common
 {
     public interface IHelpers
     {
         string ComputeSha256Hash(string normalizedPath);
-        T DirectoryMutexExec<T>(Func<T> function, string path);
-        T DirectoryMutexExec<T>(Func<Task<T>> function, string path);
+
         T MutexExec<T>(Func<T> function, string mutexName);
         T MutexExec<T>(Func<Task<T>> function, string mutexName);
+        void MutexExec(Func<Task> function, string mutexName);
+
+        T DirectoryMutexExec<T>(Func<T> function, string path);
+        T DirectoryMutexExec<T>(Func<Task<T>> function, string path);
+        void DirectoryMutexExec(Func<Task> function, string path);
+
         string RemoveTrailingSlash(string directoryPath);
     }
-}
+}

src/Microsoft.DotNet.Helix/Sdk/CreateXHarnessAppleWorkItems.cs

@@ -3,7 +3,9 @@
 using System.IO;
 using System.IO.Compression;
 using System.Linq;
+using System.Net;
 using System.Threading.Tasks;
+using Microsoft.Arcade.Common;
 using Microsoft.Build.Framework;
 
 namespace Microsoft.DotNet.Helix.Sdk
@@ -15,10 +17,12 @@ public class CreateXHarnessAppleWorkItems : XHarnessTaskBase
     {
         private const string EntryPointScriptName = "xharness-helix-job.apple.sh";
         private const string RunnerScriptName = "xharness-runner.apple.sh";
-        private const int DefaultLaunchTimeoutInMinutes = 10;
         private const string LaunchTimeoutPropName = "LaunchTimeout";
-        private const string TargetsPropName = "Targets";
+        private const string TargetPropName = "Targets";
         private const string IncludesTestRunnerPropName = "IncludesTestRunner";
+        private const int DefaultLaunchTimeoutInMinutes = 10;
+
+        private readonly IHelpers _helpers = new Arcade.Common.Helpers();
 
         /// <summary>
         /// An array of one or more paths to iOS app bundles (folders ending with ".app" usually)
@@ -32,9 +36,26 @@ public class CreateXHarnessAppleWorkItems : XHarnessTaskBase
         public string XcodeVersion { get; set; }
 
         /// <summary>
-        /// Path to the provisioning profile that will be used to sign the app (in case of real device targets).
+        /// URL template to get the provisioning profile that will be used to sign the app from (in case of real device targets).
+        /// The URL is a template in the following format:
+        /// https://storage.azure.com/signing/NET_Apple_Development_{PLATFORM}.mobileprovision
+        /// </summary>
+        public string ProvisioningProfileUrl { get; set; }
+
+        /// <summary>
+        /// Path where we can store intermediate files.
         /// </summary>
-        public string ProvisioningProfilePath { get; set; }
+        public string TmpDir { get; set; }
+
+        private enum TargetPlatform
+        {
+            iOS,
+            tvOS,
+        }
+
+        private string GetProvisioningProfileFileName(TargetPlatform platform) => Path.GetFileName(GetProvisioningProfileUrl(platform));
+
+        private string GetProvisioningProfileUrl(TargetPlatform platform) => ProvisioningProfileUrl.Replace("{PLATFORM}", platform.ToString());
 
         /// <summary>
         /// The main method of this MSBuild task which calls the asynchronous execution method and
@@ -59,6 +80,7 @@ public override bool Execute()
         /// <returns></returns>
         private async Task ExecuteAsync()
         {
+            DownloadProvisioningProfiles();
             WorkItems = (await Task.WhenAll(AppBundles.Select(PrepareWorkItem))).Where(wi => wi != null).ToArray();
         }
 
@@ -83,20 +105,14 @@ private async Task<ITaskItem> PrepareWorkItem(ITaskItem appBundleItem)
             var (testTimeout, workItemTimeout, expectedExitCode) = ParseMetadata(appBundleItem);
 
             // Validation of any metadata specific to iOS stuff goes here
-            if (!appBundleItem.TryGetMetadata(TargetsPropName, out string targets))
+            if (!appBundleItem.TryGetMetadata(TargetPropName, out string target))
             {
                 Log.LogError("'Targets' metadata must be specified - " +
                     "expecting list of target device/simulator platforms to execute tests on (e.g. ios-simulator-64)");
                 return null;
             }
 
-            bool isDeviceTarget = targets.Contains("device");
-            string provisioningProfileDest = Path.Combine(appFolderPath, "embedded.mobileprovision");
-            if (isDeviceTarget && string.IsNullOrEmpty(ProvisioningProfilePath) && !File.Exists(provisioningProfileDest))
-            {
-                Log.LogError("ProvisioningProfilePath parameter not set but required for real device targets!");
-                return null;
-            }
+            target = target.ToLowerInvariant();
 
             // Optional timeout for the how long it takes for the app to be installed, booted and tests start executing
             TimeSpan launchTimeout = TimeSpan.FromMinutes(DefaultLaunchTimeoutInMinutes);
@@ -120,24 +136,55 @@ private async Task<ITaskItem> PrepareWorkItem(ITaskItem appBundleItem)
 
             if (includesTestRunner && expectedExitCode != 0)
             {
-                Log.LogWarning("The ExpectedExitCode property is ignored in the `ios test` scenario");
+                Log.LogWarning("The ExpectedExitCode property is ignored in the `apple test` scenario");
             }
 
+            bool isDeviceTarget = target.Contains("device");
+            string provisioningProfileDest = Path.Combine(appFolderPath, "embedded.mobileprovision");
+
+            // Handle files needed for signing
             if (isDeviceTarget)
             {
+                if (string.IsNullOrEmpty(TmpDir))
+                {
+                    Log.LogError(nameof(TmpDir) + " parameter not set but required for real device targets!");
+                    return null;
+                }
+
+                if (string.IsNullOrEmpty(ProvisioningProfileUrl) && !File.Exists(provisioningProfileDest))
+                {
+                    Log.LogError(nameof(ProvisioningProfileUrl) + " parameter not set but required for real device targets!");
+                    return null;
+                }
+
                 if (!File.Exists(provisioningProfileDest))
                 {
-                    Log.LogMessage("Adding provisioning profile into the app bundle");
-                    File.Copy(ProvisioningProfilePath, provisioningProfileDest);
+                    // StartsWith because suffix can be the target OS version
+                    TargetPlatform? platform = null;
+                    if (target.StartsWith("ios-device"))
+                    {
+                        platform = TargetPlatform.iOS;
+                    }
+                    else if (target.StartsWith("tvos-device"))
+                    {
+                        platform = TargetPlatform.tvOS;
+                    }
+
+                    if (platform.HasValue)
+                    {
+                        string profilePath = Path.Combine(TmpDir, GetProvisioningProfileFileName(platform.Value));
+                        Log.LogMessage($"Adding provisioning profile `{profilePath}` into the app bundle at `{provisioningProfileDest}`");
+                        File.Copy(profilePath, provisioningProfileDest);
+                    }
                 }
                 else
                 {
-                    Log.LogMessage("Bundle already contains a provisioning profile");
+                    Log.LogMessage($"Bundle already contains a provisioning profile at `{provisioningProfileDest}`");
                 }
             }
 
             string appName = Path.GetFileName(appBundleItem.ItemSpec);
-            string command = GetHelixCommand(appName, targets, testTimeout, launchTimeout, includesTestRunner, expectedExitCode);
+            string command = GetHelixCommand(appName, target, testTimeout, launchTimeout, includesTestRunner, expectedExitCode);
             string payloadArchivePath = await CreateZipArchiveOfFolder(appFolderPath);
 
             Log.LogMessage($"Creating work item with properties Identity: {workItemName}, Payload: {appFolderPath}, Command: {command}");
@@ -190,5 +237,49 @@ private async Task<string> CreateZipArchiveOfFolder(string folderToZip)
 
             return outputZipPath;
         }
+
+        private void DownloadProvisioningProfiles()
+        {
+            if (string.IsNullOrEmpty(ProvisioningProfileUrl))
+            {
+                return;
+            }
+
+            string[] targets = AppBundles
+                .Select(appBundle => appBundle.TryGetMetadata(TargetPropName, out string target) ? target : null)
+                .Where(t => t != null)
+                .ToArray();
+
+            bool hasiOSTargets = targets.Contains("ios-device");
+            bool hastvOSTargets = targets.Contains("tvos-device");
+
+            if (hasiOSTargets)
+            {
+                DownloadProvisioningProfile(TargetPlatform.iOS);
+            }
+
+            if (hastvOSTargets)
+            {
+                DownloadProvisioningProfile(TargetPlatform.tvOS);
+            }
+        }
+
+        private void DownloadProvisioningProfile(TargetPlatform platform)
+        {
+            var targetFile = Path.Combine(TmpDir, GetProvisioningProfileFileName(platform));
+
+            using var client = new WebClient();
+            _helpers.DirectoryMutexExec(async () => {
+                if (File.Exists(targetFile))
+                {
+                    Log.LogMessage($"Provisioning profile is already downloaded");
+                    return;
+                }
+
+                Log.LogMessage($"Downloading {platform} provisioning profile to {targetFile}");
+
+                await client.DownloadFileTaskAsync(new Uri(GetProvisioningProfileUrl(platform)), targetFile);
+            }, TmpDir);
+        }
     }
 }

src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.props

@@ -9,9 +9,11 @@
   </PropertyGroup>
 
   <PropertyGroup>
-     <NETCORE_ENGINEERING_TELEMETRY>Helix</NETCORE_ENGINEERING_TELEMETRY>
+    <NETCORE_ENGINEERING_TELEMETRY>Helix</NETCORE_ENGINEERING_TELEMETRY>
   </PropertyGroup>
 
+  <UsingTask TaskName="Microsoft.DotNet.Arcade.Sdk.InstallDotNetTool" AssemblyFile="$(ArcadeSdkBuildTasksAssembly)"/>
+
   <UsingTask TaskName="SendHelixJob" AssemblyFile="$(MicrosoftDotNetHelixSdkTasksAssembly)"/>
   <UsingTask TaskName="WaitForHelixJobCompletion" AssemblyFile="$(MicrosoftDotNetHelixSdkTasksAssembly)"/>
   <UsingTask TaskName="CheckHelixJobStatus" AssemblyFile="$(MicrosoftDotNetHelixSdkTasksAssembly)"/>

src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/XHarnessRunner.props

@@ -8,6 +8,6 @@
     <XHarnessPackageSource Condition=" '$(XHarnessPackageSource)' == '' ">https://dnceng.pkgs.visualstudio.com/public/_packaging/dotnet-eng/nuget/v3/index.json</XHarnessPackageSource>
 
     <!-- Needed for app signing and tied to certificates installed in Helix machines -->
-    <XHarnessAppleProvisioningProfileUrl>https://netcorenativeassets.blob.core.windows.net/resource-packages/external/macos/signing/NET_Apple_Development_iOS.mobileprovision</XHarnessAppleProvisioningProfileUrl>
+    <XHarnessAppleProvisioningProfileUrl>https://netcorenativeassets.blob.core.windows.net/resource-packages/external/macos/signing/NET_Apple_Development_{PLATFORM}.mobileprovision</XHarnessAppleProvisioningProfileUrl>
   </PropertyGroup>
 </Project>

src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/XHarnessRunner.targets

@@ -38,7 +38,7 @@
     <ItemGroup>
       <_XHarnessExtraToolFiles Include="$(_XHarnessCliPath)\*.*" />
     </ItemGroup>
-    <Delete Files="@(_XHarnessExtraToolFiles)" />
+    <Delete Files="@(_XHarnessExtraToolFiles)" TreatErrorsAsWarnings="true" ContinueOnError="WarnAndContinue" />
 
     <ItemGroup>
       <HelixCorrelationPayload Include="$(_XHarnessCliPath)">
@@ -103,18 +103,11 @@
   <Target Name="CreateAppleWorkItems"
           Condition=" '@(XHarnessAppBundleToTest)' != '' "
           BeforeTargets="CoreTest">
-    <DownloadFile SourceUrl="$(XHarnessAppleProvisioningProfileUrl)"
-                  Condition=" '$(XHarnessAppleProvisioningProfileUrl)' != '' "
-                  DestinationFolder="$(ArtifactsTmpDir)"
-                  SkipUnchangedFiles="True"
-                  Retries="5">
-      <Output TaskParameter="DownloadedFile" ItemName="_XHarnessProvisioningProfile" />
-    </DownloadFile>
-
     <CreateXHarnessAppleWorkItems AppBundles="@(XHarnessAppBundleToTest)"
                                   IsPosixShell="$(IsPosixShell)"
                                   XcodeVersion="$(XHarnessXcodeVersion)"
-                                  ProvisioningProfilePath="@(_XHarnessProvisioningProfile)">
+                                  ProvisioningProfileUrl="$(XHarnessAppleProvisioningProfileUrl)"
+                                  TmpDir="$(ArtifactsTmpDir)">
       <Output TaskParameter="WorkItems" ItemName="HelixWorkItem"/>
     </CreateXHarnessAppleWorkItems>
   </Target>