Add infra retry in case of installation failure (78) on Android

src/Microsoft.DotNet.Helix/JobSender/Readme.md

@@ -32,7 +32,7 @@ namespace Sample
 ```
 
 ### Hello World
-This will print out `'Hai Wurld!'` in the work item's console log.
+This will print out `'Hai Wurld!'` in the work item's console log. Please notice that the command for Windows is placed inside a bat script file.
 
 ```csharp
 var job = await api.Job.Define()

src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.android.ps1

@@ -40,10 +40,18 @@ $exit_code=$LASTEXITCODE
 # ADB_DEVICE_ENUMERATION_FAILURE
 if ($exit_code -eq 85) {
     $ErrorActionPreference="Continue"
-    Write-Error "Encountered ADB_DEVICE_ENUMERATION_FAILURE.  This is typically not a failure of the work item.  We will run it again and reboot this computer to help its devices"
+    Write-Error "Encountered ADB_DEVICE_ENUMERATION_FAILURE. This is typically not a failure of the work item. We will run it again and reboot this computer to help its devices"
     Write-Error "If this occurs repeatedly, please check for architectural mismatch, e.g. sending x86 or x86_64 APKs to an arm64_v8a-only queue."
     & "$Env:HELIX_PYTHONPATH" -c "from helix.workitemutil import request_infra_retry; request_infra_retry('Retrying because we could not enumerate all Android devices')"
     & "$Env:HELIX_PYTHONPATH" -c "from helix.workitemutil import request_reboot; request_reboot('Rebooting to allow Android emulator or device to restart')"
 }
 
+# PACKAGE_INSTALLATION_FAILURE
+if ($exit_code -eq 78) {
+    $ErrorActionPreference="Continue"
+    Write-Error "Encountered PACKAGE_INSTALLATION_FAILURE. This is typically not a failure of the work item. We will try it again on another device"
+    Write-Error "If this occurs repeatedly, please check for architectural mismatch, e.g. requesting installation on arm64_v8a-only queue for x86 or x86_64 APKs."
+    & "$Env:HELIX_PYTHONPATH" -c "from helix.workitemutil import request_infra_retry; request_infra_retry('Retrying because we could not enumerate all Android devices')"
+}
+
 exit $exit_code

src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.android.sh

@@ -83,11 +83,22 @@ exit_code=$?
 
 ADB_DEVICE_ENUMERATION_FAILURE=85
 if [ $exit_code -eq $ADB_DEVICE_ENUMERATION_FAILURE ]; then
-    echo 'Encountered ADB_DEVICE_ENUMERATION_FAILURE.  This is typically not a failure of the work item.  We will run it again and reboot this computer to help its devices'
+    echo 'Encountered ADB_DEVICE_ENUMERATION_FAILURE. This is typically not a failure of the work item. We will run it again and reboot this computer to help its devices'
     echo 'If this occurs repeatedly, please check for architectural mismatch, e.g. sending arm64_v8a APKs to an x86_64 / x86 only queue.'
     # Since we run the payload script using launchctl, env vars are not set there and we have to do this part here
     "$HELIX_PYTHONPATH" -c "from helix.workitemutil import request_infra_retry; request_infra_retry('Retrying because we could not enumerate all Android devices')"
     "$HELIX_PYTHONPATH" -c "from helix.workitemutil import request_reboot; request_reboot('Rebooting to allow Android emulator or device to restart.')"
 fi
 
+# This handles issues where APKs fail to install.
+# We already reboot a device inside XHarness and now request a work item retry when this happens
+
+PACKAGE_INSTALLATION_FAILURE=78
+if [ $exit_code -eq PACKAGE_INSTALLATION_FAILURE ]; then
+    echo 'Encountered PACKAGE_INSTALLATION_FAILURE. This is typically not a failure of the work item. We will try it again on another device'
+    echo 'If this occurs repeatedly, please check for architectural mismatch, e.g. requesting installation on arm64_v8a-only queue for x86 or x86_64 APKs.'
+    # Since we run the payload script using launchctl, env vars are not set there and we have to do this part here
+    "$HELIX_PYTHONPATH" -c "from helix.workitemutil import request_infra_retry; request_infra_retry('Retrying because we could not enumerate all Android devices')"
+fi
+
 exit $exit_code