diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index fa877beeab1..02ecda2f7a1 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -39,9 +39,9 @@ https://github.com/dotnet/arcade-services cac955fe259cb611f6a29d09209bd717deb69037 - + https://github.com/dotnet/xharness - 2e03d2d1081b606ad3c5f79709297a9c5627854d + c8c452d7746015847956618cf0c56f931ed8647a https://github.com/dotnet/roslyn diff --git a/eng/Versions.props b/eng/Versions.props index 00420b59f4f..d69092513f6 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -80,7 +80,7 @@ 6.0.0-beta.21451.3 1.0.0-beta.21431.1 1.1.0-beta.21378.2 - 1.0.0-prerelease.21427.1 + 1.0.0-prerelease.21456.1 1.1.156602 1.1.156602 6.0.100-preview.5.21254.11 diff --git a/src/Microsoft.DotNet.Helix/Sdk/Microsoft.DotNet.Helix.Sdk.csproj b/src/Microsoft.DotNet.Helix/Sdk/Microsoft.DotNet.Helix.Sdk.csproj index 2e510066ae4..e1e4d2c0175 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/Microsoft.DotNet.Helix.Sdk.csproj +++ b/src/Microsoft.DotNet.Helix/Sdk/Microsoft.DotNet.Helix.Sdk.csproj @@ -31,6 +31,9 @@ + + Never + Never diff --git a/src/Microsoft.DotNet.Helix/Sdk/XharnessTaskBase.cs b/src/Microsoft.DotNet.Helix/Sdk/XharnessTaskBase.cs index af666f56325..a1de1cc2428 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/XharnessTaskBase.cs +++ b/src/Microsoft.DotNet.Helix/Sdk/XharnessTaskBase.cs @@ -25,6 +25,7 @@ public class MetadataName private const string ScriptNamespace = "tools.xharness_runner."; private const string CustomCommandsScript = "command"; + private const string DiagnosticsScript = "xharness-event-reporter.py"; /// /// Extra arguments that will be passed to the iOS/Android/... app that is being run @@ -173,6 +174,11 @@ await zipArchiveManager.AddResourceFileToArchive( payloadScript); } + await zipArchiveManager.AddResourceFileToArchive( + outputZipPath, + ScriptNamespace + DiagnosticsScript, + DiagnosticsScript); + await zipArchiveManager.AddContentToArchive( outputZipPath, CustomCommandsScript + (isPosix ? ".sh" : ".ps1"), diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/XHarnessRunner.targets b/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/XHarnessRunner.targets index 91573291c8b..644b3bc2751 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/XHarnessRunner.targets +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/XHarnessRunner.targets @@ -60,6 +60,9 @@ $(HelixPreCommands);export XHARNESS_DISABLE_COLORED_OUTPUT=true $(HelixPreCommands);export XHARNESS_LOG_WITH_TIMESTAMPS=true $(HelixPreCommands);export XHARNESS_CLI_PATH=$HELIX_CORRELATION_PAYLOAD/microsoft.dotnet.xharness.cli/$(_XHarnessPackageVersion)/tools/net6.0/any/Microsoft.DotNet.XHarness.CLI.dll + + $(HelixPreCommands);export XHARNESS_DIAGNOSTICS_PATH=$HELIX_WORKITEM_ROOT/diagnostics.json + "$HELIX_PYTHONPATH" "$HELIX_WORKITEM_PAYLOAD/xharness-event-reporter.py";$(HelixPostCommands) @@ -68,6 +71,9 @@ $(HelixPreCommands);set XHARNESS_LOG_WITH_TIMESTAMPS=true $(HelixPreCommands);set XHARNESS_CLI_PATH=%HELIX_CORRELATION_PAYLOAD%\microsoft.dotnet.xharness.cli\$(_XHarnessPackageVersion)\tools\net6.0\any\Microsoft.DotNet.XHarness.CLI.dll $(HelixPreCommands);doskey xharness="dotnet exec %XHARNESS_CLI_PATH%" + + $(HelixPreCommands);set XHARNESS_DIAGNOSTICS_PATH=%HELIX_WORKITEM_ROOT%\diagnostics.json + "%HELIX_PYTHONPATH%" "%HELIX_WORKITEM_PAYLOAD%\xharness-event-reporter.py";$(HelixPostCommands) diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-event-reporter.py b/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-event-reporter.py new file mode 100644 index 00000000000..dde82f0e391 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-event-reporter.py @@ -0,0 +1,44 @@ +import getopt +import json +import os +import sys + +from helix.appinsights import app_insights + +opts, args = getopt.gnu_getopt(sys.argv[1:], 'd:', ['diagnostics-data=']) +opt_dict = dict(opts) + +diagnostics_file = None + +if '--data' in opt_dict: + diagnostics_file = opt_dict['--data'] +elif '-d' in opt_dict: + diagnostics_file = opt_dict['-d'] +else: + diagnostics_file = os.getenv('XHARNESS_DIAGNOSTICS_PATH') + +if not diagnostics_file: + print('ERROR: Expected path to the diagnostics JSON file generated by XHarness') + exit(1) + +if not os.path.isfile(diagnostics_file): + print(f"WARNING: Diagnostics file not found at `{diagnostics_file}`") + exit(2) + +# The JSON should be an array of objects (one per each executed XHarness command) +operations = json.load(open(diagnostics_file)) + +for operation in operations: + custom_dimensions = dict() + + custom_dimensions['command'] = operation['command'] + custom_dimensions['platform'] = operation['platform'] + + if 'target' in operation: + if 'targetOS' in operation: + custom_dimensions['target'] = operation['target'] + '_' + operation['targetOS'] + else: + custom_dimensions['target'] = operation['target'] + + app_insights.send_metric('XHarnessOperation', operation['exitCode'], properties=custom_dimensions) + app_insights.send_metric('XHarnessOperationDuration', operation['duration'], properties=custom_dimensions) diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.android.ps1 b/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.android.ps1 index f67b706f538..f9e0066068b 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.android.ps1 +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.android.ps1 @@ -32,9 +32,13 @@ function xharness() { dotnet exec $Env:XHARNESS_CLI_PATH @args } +$ErrorActionPreference="Continue" + # Act out the actual commands . "$PSScriptRoot\command.ps1" +$ErrorActionPreference="Continue" + $exit_code=$LASTEXITCODE $retry=$false @@ -44,7 +48,6 @@ switch ($exit_code) { # ADB_DEVICE_ENUMERATION_FAILURE 85 { - $ErrorActionPreference="Continue" Write-Error "Encountered ADB_DEVICE_ENUMERATION_FAILURE. This is typically not a failure of the work item. We will run it again and reboot this computer to help its devices" Write-Error "If this occurs repeatedly, please check for architectural mismatch, e.g. sending x86 or x86_64 APKs to an arm64_v8a-only queue." $retry=$true @@ -54,7 +57,6 @@ switch ($exit_code) # PACKAGE_INSTALLATION_FAILURE 78 { - $ErrorActionPreference="Continue" Write-Error "Encountered PACKAGE_INSTALLATION_FAILURE. This is typically not a failure of the work item. We will try it again on another Helix agent" Write-Error "If this occurs repeatedly, please check for architectural mismatch, e.g. requesting installation on arm64_v8a-only queue for x86 or x86_64 APKs." $retry=$true diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.apple.sh b/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.apple.sh index a6e535f629f..15e6af638be 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.apple.sh +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.apple.sh @@ -27,6 +27,8 @@ set -x # It is important we call the script via `launchctl asuser` in order to be able to spawn # the simulator which needs to run in a user session with GUI rendering capabilities. +# The problem with launchctl is that the spawned process won't share environment variables +# so we have to pass all of them as parameters are set them again. chmod +x xharness-runner.apple.sh helix_runner_uid=$(id -u) sudo launchctl asuser "$helix_runner_uid" sh ./xharness-runner.apple.sh \ @@ -34,6 +36,7 @@ sudo launchctl asuser "$helix_runner_uid" sh ./xharness-runner.apple.sh \ --app "$HELIX_WORKITEM_ROOT/$app" \ --xharness-cli-path "$XHARNESS_CLI_PATH" \ --output-directory "$HELIX_WORKITEM_UPLOAD_ROOT" \ + --diagnostics-path "$XHARNESS_DIAGNOSTICS_PATH" \ exit_code=$? diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-runner.apple.sh b/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-runner.apple.sh index 6096d835d48..90d21b98cba 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-runner.apple.sh +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-runner.apple.sh @@ -29,6 +29,10 @@ while [[ $# -gt 0 ]]; do output_directory="$2" shift ;; + --diagnostics-path) + export XHARNESS_DIAGNOSTICS_PATH="$2" + shift + ;; --target) target="$2" shift diff --git a/tests/UnitTests.XHarness.Common.props b/tests/UnitTests.XHarness.Common.props index d657a6d9e5c..cf6fd72ed5b 100644 --- a/tests/UnitTests.XHarness.Common.props +++ b/tests/UnitTests.XHarness.Common.props @@ -18,6 +18,7 @@ true true https://helix.dot.net + true