diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.android.header.ps1 b/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.android.header.ps1 index a8033609966..e91764050f2 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.android.header.ps1 +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.android.header.ps1 @@ -33,7 +33,10 @@ function xharness() { function report_infrastructure_failure($message) { Write-Output "Infrastructural problem reported by the user, requesting retry+reboot: $message" - & "$Env:HELIX_PYTHONPATH" -c "from helix.workitemutil import request_infra_retry; request_infra_retry('Retrying because we could not enumerate all Android devices')" - & "$Env:HELIX_PYTHONPATH" -c "from helix.workitemutil import request_reboot; request_reboot('Rebooting to allow Android emulator or device to restart')" + New-Item -Path "$Env:HELIX_WORKITEM_ROOT" -Name ".retry" -ItemType "file" -Force + New-Item -Path "$Env:HELIX_WORKITEM_ROOT" -Name ".reboot" -ItemType "file" -Force + + $message -replace "['\\]" | Out-File -FilePath "$Env:HELIX_WORKITEM_ROOT\.retry" + $message -replace "['\\]" | Out-File -FilePath "$Env:HELIX_WORKITEM_ROOT\.reboot" } diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.android.ps1 b/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.android.ps1 index 8c0a4e21a9b..7b82076a871 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.android.ps1 +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.android.ps1 @@ -52,8 +52,8 @@ if ($ev) { Write-Output "User command ended with $exit_code" } -$retry=$false -$reboot=$false +$retry = $false +$reboot = $false switch ($exit_code) { @@ -61,8 +61,8 @@ switch ($exit_code) 85 { Write-Error "Encountered ADB_DEVICE_ENUMERATION_FAILURE. This is typically not a failure of the work item. We will run it again and reboot this computer to help its devices" Write-Error "If this occurs repeatedly, please check for architectural mismatch, e.g. sending x86 or x86_64 APKs to an arm64_v8a-only queue." - $retry=$true - $reboot=$true + $retry = $true + $reboot = $true Break } @@ -70,17 +70,35 @@ switch ($exit_code) 78 { Write-Error "Encountered PACKAGE_INSTALLATION_FAILURE. This is typically not a failure of the work item. We will try it again on another Helix agent" Write-Error "If this occurs repeatedly, please check for architectural mismatch, e.g. requesting installation on arm64_v8a-only queue for x86 or x86_64 APKs." - $retry=$true + $retry = $true Break } } +if (Test-Path -Path "$Env:HELIX_WORKITEM_ROOT\.retry" -PathType Leaf) { + $retry = $true; + $retry_message = Get-Content -Path "$Env:HELIX_WORKITEM_ROOT\.retry" +} + +if (Test-Path -Path "$Env:HELIX_WORKITEM_ROOT\.reboot" -PathType Leaf) { + $reboot = $true; + $reboot_message = Get-Content -Path "$Env:HELIX_WORKITEM_ROOT\.reboot" +} + if ($retry) { - & "$Env:HELIX_PYTHONPATH" -c "from helix.workitemutil import request_infra_retry; request_infra_retry('Retrying because we could not enumerate all Android devices')" + if ([string]::IsNullOrEmpty($retry_message)) { + $retry_message = 'Retrying because we could not enumerate all Android devices' + } + + & "$Env:HELIX_PYTHONPATH" -c "from helix.workitemutil import request_infra_retry; request_infra_retry('$retry_message')" } if ($reboot) { - & "$Env:HELIX_PYTHONPATH" -c "from helix.workitemutil import request_reboot; request_reboot('Rebooting to allow Android emulator or device to restart')" + if ([string]::IsNullOrEmpty($reboot_message)) { + $reboot_message = 'Rebooting to allow Android emulator to restart' + } + + & "$Env:HELIX_PYTHONPATH" -c "from helix.workitemutil import request_reboot; request_reboot('$reboot_message')" } exit $exit_code diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.android.sh b/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.android.sh index ce025ebc736..5754e39f42f 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.android.sh +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.android.sh @@ -72,6 +72,13 @@ function xharness() { dotnet exec $XHARNESS_CLI_PATH "$@" } +function report_infrastructure_failure() { + echo "Infrastructural problem reported by the user, requesting retry+reboot: $1" + + echo "$1" > "$HELIX_WORKITEM_ROOT/.retry" + echo "$1" > "$HELIX_WORKITEM_ROOT/.reboot" +} + # Act out the actual commands (and time constrain them to create buffer for the end of this script) source command.sh & PID=$! ; (sleep $command_timeout && kill $PID 2> /dev/null & ) ; wait $PID @@ -105,12 +112,30 @@ case "$exit_code" in ;; esac +if [ -f "$HELIX_WORKITEM_ROOT/.retry" ]; then + retry=true + retry_message=$(cat "$HELIX_WORKITEM_ROOT/.retry" | tr -d "'\\\\") +fi + +if [ -f "$HELIX_WORKITEM_ROOT/.reboot" ]; then + reboot=true + reboot_message=$(cat "$HELIX_WORKITEM_ROOT/.reboot" | tr -d "'\\\\") +fi + if [ "$retry" == true ]; then - "$HELIX_PYTHONPATH" -c "from helix.workitemutil import request_infra_retry; request_infra_retry('Retrying because we could not enumerate all Android devices')" + if [ -z "$retry_message" ]; then + retry_message='Retrying because we could not enumerate all Android devices' + fi + + "$HELIX_PYTHONPATH" -c "from helix.workitemutil import request_infra_retry; request_infra_retry('$retry_message')" fi if [ "$reboot" == true ]; then - "$HELIX_PYTHONPATH" -c "from helix.workitemutil import request_reboot; request_reboot('Rebooting to allow Android emulator to restart')" + if [ -z "$reboot_message" ]; then + reboot_message='Rebooting to allow Android emulator to restart' + fi + + "$HELIX_PYTHONPATH" -c "from helix.workitemutil import request_reboot; request_reboot('$reboot_message')" fi exit $exit_code diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.apple.sh b/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.apple.sh index c87d0375930..7ed61784d5e 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.apple.sh +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.apple.sh @@ -44,12 +44,24 @@ exit_code=$? # We usually also ask the work item to be re-tried on a different machine # Since we run the payload script using launchctl, env vars such as PYTHON_PATH are not set there and we have to do this part here # We signal this by creating files -if [ -f './.retry' ]; then - "$HELIX_PYTHONPATH" -c "from helix.workitemutil import request_infra_retry; request_infra_retry('Retrying work item because XHarness workload requested it')" +if [ -f "$HELIX_WORKITEM_ROOT/.retry" ]; then + retry_message=$(cat "$HELIX_WORKITEM_ROOT/.retry" | tr -d "'\\\\") + + if [ -z "$retry_message" ]; then + retry_message='Retrying because we could not enumerate all Android devices' + fi + + "$HELIX_PYTHONPATH" -c "from helix.workitemutil import request_infra_retry; request_infra_retry('$retry_message')" fi -if [ -f './.reboot' ]; then - "$HELIX_PYTHONPATH" -c "from helix.workitemutil import request_reboot; request_reboot('Rebooting because XHarness workload requested it)" +if [ -f "$HELIX_WORKITEM_ROOT/.reboot" ]; then + reboot_message=$(cat "$HELIX_WORKITEM_ROOT/.reboot" | tr -d "'\\\\") + + if [ -z "$reboot_message" ]; then + reboot_message='Rebooting to allow Android emulator to restart' + fi + + "$HELIX_PYTHONPATH" -c "from helix.workitemutil import request_reboot; request_reboot('$reboot_message')" fi exit $exit_code diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-runner.apple.sh b/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-runner.apple.sh index 0f9188a88ab..836db2e74dd 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-runner.apple.sh +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-runner.apple.sh @@ -147,8 +147,8 @@ function xharness() { function report_infrastructure_failure() { echo "Infrastructural problem reported by the user, requesting retry+reboot: $1" - touch './.retry' - touch './.reboot' + echo "$1" > "$HELIX_WORKITEM_ROOT/.retry" + echo "$1" > "$HELIX_WORKITEM_ROOT/.reboot" } # Act out the actual commands (and time constrain them to create buffer for the end of this script)