From 8f91458c759197853dc7ec8890fb957dba2c78c9 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 23 Jan 2023 13:28:37 -0800 Subject: [PATCH 01/11] Backport of the release/7.0 changes made to add maui android mobile to those runs: https://github.com/dotnet/performance/commit/f910fe509c80ee5acf7485b78149f43f569872f4. --- azure-pipelines.yml | 15 ++++ eng/performance/maui_scenarios_android.proj | 82 +++++++++++++++++++ scripts/dotnet.py | 31 +++---- src/scenarios/mauiandroid/pre.py | 69 +++++++--------- src/scenarios/mauiandroid/test.py | 8 +- src/scenarios/mauiandroidpodcast/post.py | 9 ++ src/scenarios/mauiandroidpodcast/pre.py | 35 ++++++++ src/scenarios/mauiandroidpodcast/test.py | 16 ++++ src/scenarios/mauiblazorandroid/post.py | 7 ++ src/scenarios/mauiblazorandroid/pre.py | 75 +++++++++++++++++ src/scenarios/mauiblazorandroid/test.py | 16 ++++ src/scenarios/shared/mauisharedpython.py | 24 ++++++ src/scenarios/shared/runner.py | 18 ++-- src/scenarios/shared/versionmanager.py | 31 +++++++ .../EnvironmentProviderMock.cs | 8 +- .../Reporting.Tests/ReporterTests.cs | 1 + .../Reporting/EnvironmentProvider.cs | 1 + src/tools/Reporting/Reporting/IEnvironment.cs | 1 + src/tools/Reporting/Reporting/Reporter.cs | 26 ++++-- 19 files changed, 401 insertions(+), 72 deletions(-) create mode 100644 eng/performance/maui_scenarios_android.proj create mode 100644 src/scenarios/mauiandroidpodcast/post.py create mode 100644 src/scenarios/mauiandroidpodcast/pre.py create mode 100644 src/scenarios/mauiandroidpodcast/test.py create mode 100644 src/scenarios/mauiblazorandroid/post.py create mode 100644 src/scenarios/mauiblazorandroid/pre.py create mode 100644 src/scenarios/mauiblazorandroid/test.py create mode 100644 src/scenarios/shared/mauisharedpython.py create mode 100644 src/scenarios/shared/versionmanager.py diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3c85b206967..791fb4fef6e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -302,6 +302,21 @@ jobs: projectFile: scenarios.proj channels: - release/6.0 + + # Maui Android scenario benchmarks + - template: /eng/performance/scenarios.yml + parameters: + osName: windows + architecture: x64 + osVersion: 19H1 + pool: + vmImage: 'windows-2022' + queue: Windows.10.Amd64.Pixel.Perf + machinePool: Pixel + kind: maui_scenarios_android + projectFile: maui_scenarios_android.proj + channels: + - release/6.0 # # Windows x64 micro benchmarks # - template: /eng/performance/benchmark_jobs.yml diff --git a/eng/performance/maui_scenarios_android.proj b/eng/performance/maui_scenarios_android.proj new file mode 100644 index 00000000000..270288b148a --- /dev/null +++ b/eng/performance/maui_scenarios_android.proj @@ -0,0 +1,82 @@ + + + + + + true + 1.0.0-prerelease.21566.2 + %HELIX_CORRELATION_PAYLOAD%\microsoft.dotnet.xharness.cli\$(MicrosoftDotNetXHarnessCLIVersion)\tools\net6.0\any\Microsoft.DotNet.XHarness.CLI.dll + + + + $(Python) post.py + scenarios_out + $(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)\ + $(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)/ + + + + + + + + + + 00:30 + + + + + + mauiandroid + $(ScenariosDir)%(ScenarioDirectoryName) + com.companyname.mauiandroiddefault-Signed + com.companyname.mauiandroiddefault + + + mauiandroidpodcast + $(ScenariosDir)%(ScenarioDirectoryName) + com.Microsoft.NetConf2021.Maui-Signed + com.Microsoft.NetConf2021.Maui + + + mauiblazorandroid + $(ScenariosDir)%(ScenarioDirectoryName) + com.companyname.mauiblazorandroiddefault-Signed + com.companyname.mauiblazorandroiddefault + + + + + + + $(Python) pre.py publish -f $(PERFLAB_Framework)-android -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName) -r android-arm64 --self-contained + %(PreparePayloadWorkItem.PayloadDirectory) + + + + + + + + echo on; xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y + $(Python) test.py sod --scenario-name "%(Identity)" + + + echo on; xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y; ren %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).apk %(HelixWorkItem.ApkName).zip; powershell.exe -nologo -noprofile -command "& {Expand-Archive %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).zip -DestinationPath %HELIX_WORKITEM_ROOT%\pub\}"; del %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).zip + $(Python) test.py sod --scenario-name "%(Identity)" + + + echo on; set XHARNESSPATH=$(XharnessPath); xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y + $(Python) test.py devicestartup --device-type android --package-path pub\%(HelixWorkItem.ApkName).apk --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" + + + echo on; set XHARNESSPATH=$(XharnessPath); xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y + $(Python) test.py devicestartup --device-type android --package-path pub\%(HelixWorkItem.ApkName).apk --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" --disable-animations + + + + + + + diff --git a/scripts/dotnet.py b/scripts/dotnet.py index 59b58172c15..ad7904e0094 100755 --- a/scripts/dotnet.py +++ b/scripts/dotnet.py @@ -5,13 +5,14 @@ """ import ssl +import datetime from argparse import Action, ArgumentParser, ArgumentTypeError, ArgumentError from collections import namedtuple from glob import iglob from json import loads from logging import getLogger from os import chmod, environ, listdir, makedirs, path, pathsep, system -from re import search +from re import search, match, MULTILINE from shutil import rmtree from stat import S_IRWXU from subprocess import CalledProcessError, check_output @@ -589,38 +590,40 @@ def get_commit_date( if not commit_sha: raise ValueError('.NET Commit sha was not defined.') + # Example URL: https://github.com/dotnet/runtime/commit/2d76178d5faa97be86fc8d049c7dbcbdf66dc497.patch url = None - urlformat = 'https://api.github.com/repos/%s/%s/commits/%s' if repository is None: # The origin of the repo where the commit belongs to has changed # between release. Here we attempt to naively guess the repo. core_sdk_frameworks = ChannelMap.get_supported_frameworks() - core_sdk_frameworks.remove('netcoreapp2.1') repo = 'core-sdk' if framework in core_sdk_frameworks else 'cli' - url = urlformat % ('dotnet', repo, commit_sha) + url = f'https://github.com/dotnet/{repo}/commit/{commit_sha}.patch' else: owner, repo = get_repository(repository) - url = urlformat % (owner, repo, commit_sha) + url = f'https://github.com/{owner}/{repo}/commit/{commit_sha}.patch' build_timestamp = None - retrycount = 0 - success = 0 - while success == 0 and retrycount <= 3: + sleep_time = 10 # Start with 10 second sleep timer + for retrycount in range(5): try: with urlopen(url) as response: getLogger().info("Commit: %s", url) - item = loads(response.read().decode('utf-8')) - build_timestamp = item['commit']['committer']['date'] - success = 1 - except URLError: - retrycount += 1 + patch = response.read().decode('utf-8') + dateMatch = search(r'^Date: (.+)$', patch, MULTILINE) + if dateMatch: + build_timestamp = datetime.datetime.strptime(dateMatch.group(1), '%a, %d %b %Y %H:%M:%S %z').astimezone(datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') + getLogger().info(f"Got UTC timestamp {build_timestamp} from {dateMatch.group(1)}") + break + except URLError as error: + getLogger().warning(f"URL Error trying to get commit date from {url}; Reason: {error.reason}; Attempt {retrycount}") + sleep(sleep_time) + sleep_time = sleep_time * 2 if not build_timestamp: raise RuntimeError( 'Could not get timestamp for commit %s' % commit_sha) return build_timestamp - def get_build_directory( bin_directory: str, project_name: str, diff --git a/src/scenarios/mauiandroid/pre.py b/src/scenarios/mauiandroid/pre.py index aed0d76c750..ce508b17fe6 100644 --- a/src/scenarios/mauiandroid/pre.py +++ b/src/scenarios/mauiandroid/pre.py @@ -2,48 +2,37 @@ pre-command ''' import sys -import os -from zipfile import ZipFile from performance.logger import setup_loggers, getLogger -from shutil import copyfile +from shared import const +from shared.mauisharedpython import remove_aab_files, install_versioned_maui from shared.precommands import PreCommands -from shared.const import PUBDIR -from argparse import ArgumentParser +from shared.versionmanager import versions_write_json, get_version_from_dll_powershell +from test import EXENAME setup_loggers(True) -parser = ArgumentParser() -parser.add_argument('--unzip', help='Unzip APK and report extracted tree', action='store_true', default=False) -parser.add_argument( - '--apk-name', - dest='apk', - required=True, - type=str, - help='Name of the APK to setup') -args = parser.parse_args() - -if not os.path.exists(PUBDIR): - os.mkdir(PUBDIR) -apkname = args.apk -apknamezip = '%s.zip' % (apkname) -if not os.path.exists(apkname): - getLogger().error('Cannot find %s' % (apkname)) - exit(-1) -if args.unzip: - if not os.path.exists(apknamezip): - copyfile(apkname, apknamezip) - - with ZipFile(apknamezip) as zip: - zip.extractall(os.path.join('.', PUBDIR)) - - assets_dir = os.path.join(PUBDIR, 'assets') - assets_zip = os.path.join(assets_dir, 'assets.zip') - with ZipFile(assets_zip) as zip: - zip.extractall(assets_dir) - - os.remove(assets_zip) -else: - copyfile(apkname, os.path.join(PUBDIR, apkname)) - - - +precommands = PreCommands() +install_versioned_maui(precommands) + +# Setup the Maui folder +precommands.new(template='maui', + output_dir=const.APPDIR, + bin_dir=const.BINDIR, + exename=EXENAME, + working_directory=sys.path[0], + no_restore=False) + +# Build the APK +precommands.execute(['--no-restore', '--source', 'MauiNuGet.config']) + +# Remove the aab files as we don't need them, this saves space +output_dir = const.PUBDIR +if precommands.output: + output_dir = precommands.output +remove_aab_files(output_dir) + +# Copy the MauiVersion to a file so we have it on the machine +maui_version = get_version_from_dll_powershell(rf".\{const.APPDIR}\obj\Release\{precommands.framework}\{precommands.runtime_identifier}\linked\Microsoft.Maui.dll") +version_dict = { "mauiVersion": maui_version } +versions_write_json(version_dict, rf"{output_dir}\versions.json") +print(f"Versions: {version_dict}") \ No newline at end of file diff --git a/src/scenarios/mauiandroid/test.py b/src/scenarios/mauiandroid/test.py index 349ef2ebcc7..5976d343ee1 100644 --- a/src/scenarios/mauiandroid/test.py +++ b/src/scenarios/mauiandroid/test.py @@ -1,11 +1,15 @@ ''' -C# Console app +Mobile Maui App ''' +from shared.const import PUBDIR from shared.runner import TestTraits, Runner +from shared.versionmanager import versions_read_json_file_save_env EXENAME = 'MauiAndroidDefault' -if __name__ == "__main__": +if __name__ == "__main__": + versions_read_json_file_save_env(rf".\{PUBDIR}\versions.json") + traits = TestTraits(exename=EXENAME, guiapp='false', ) diff --git a/src/scenarios/mauiandroidpodcast/post.py b/src/scenarios/mauiandroidpodcast/post.py new file mode 100644 index 00000000000..abdb7d8db9d --- /dev/null +++ b/src/scenarios/mauiandroidpodcast/post.py @@ -0,0 +1,9 @@ +''' +post cleanup script +''' + +from shared.postcommands import clean_directories +from performance.common import remove_directory + +remove_directory("dotnet-podcasts") +clean_directories() diff --git a/src/scenarios/mauiandroidpodcast/pre.py b/src/scenarios/mauiandroidpodcast/pre.py new file mode 100644 index 00000000000..30892a686f7 --- /dev/null +++ b/src/scenarios/mauiandroidpodcast/pre.py @@ -0,0 +1,35 @@ +''' +pre-command +''' +import subprocess +from performance.logger import setup_loggers, getLogger +from shared.precommands import PreCommands +from shared.mauisharedpython import remove_aab_files, install_versioned_maui +from shared.versionmanager import versions_write_json, get_version_from_dll_powershell +from shared import const + +setup_loggers(True) +precommands = PreCommands() +install_versioned_maui(precommands) + +branch = f'{precommands.framework[:6]}' +subprocess.run(['git', 'clone', 'https://github.com/microsoft/dotnet-podcasts.git', '-b', branch, '--single-branch', '--depth', '1']) +subprocess.run(['powershell', '-Command', r'Remove-Item -Path .\\dotnet-podcasts\\.git -Recurse -Force']) # Git files have permission issues, do their deletion separately + +precommands.existing(projectdir='./dotnet-podcasts', projectfile='./src/Mobile/Microsoft.NetConf2021.Maui.csproj') + +# Build the APK +precommands._restore() +precommands.execute(['--no-restore']) + +# Remove the aab files as we don't need them, this saves space +output_dir = const.PUBDIR +if precommands.output: + output_dir = precommands.output +remove_aab_files(output_dir) + +# Copy the MauiVersion to a file so we have it on the machine +maui_version = get_version_from_dll_powershell(rf".\{const.APPDIR}\obj\Release\{precommands.framework}\{precommands.runtime_identifier}\linked\Microsoft.Maui.dll") +version_dict = { "mauiVersion": maui_version } +versions_write_json(version_dict, rf"{output_dir}\versions.json") +print(f"Versions: {version_dict}") \ No newline at end of file diff --git a/src/scenarios/mauiandroidpodcast/test.py b/src/scenarios/mauiandroidpodcast/test.py new file mode 100644 index 00000000000..d715ae7eb3a --- /dev/null +++ b/src/scenarios/mauiandroidpodcast/test.py @@ -0,0 +1,16 @@ +''' +Mobile Maui App +''' +from shared.const import PUBDIR +from shared.runner import TestTraits, Runner +from shared.versionmanager import versions_read_json_file_save_env + +EXENAME = 'MauiAndroidPodcast' + +if __name__ == "__main__": + versions_read_json_file_save_env(rf".\{PUBDIR}\versions.json") + + traits = TestTraits(exename=EXENAME, + guiapp='false', + ) + Runner(traits).run() \ No newline at end of file diff --git a/src/scenarios/mauiblazorandroid/post.py b/src/scenarios/mauiblazorandroid/post.py new file mode 100644 index 00000000000..c87a49b4b7d --- /dev/null +++ b/src/scenarios/mauiblazorandroid/post.py @@ -0,0 +1,7 @@ +''' +post cleanup script +''' + +from shared.postcommands import clean_directories + +clean_directories() diff --git a/src/scenarios/mauiblazorandroid/pre.py b/src/scenarios/mauiblazorandroid/pre.py new file mode 100644 index 00000000000..c2c9f71f41d --- /dev/null +++ b/src/scenarios/mauiblazorandroid/pre.py @@ -0,0 +1,75 @@ +''' +pre-command +''' +import sys +from performance.logger import setup_loggers, getLogger +from shared import const +from shared.mauisharedpython import remove_aab_files, install_versioned_maui +from shared.precommands import PreCommands +from shared.versionmanager import versions_write_json, get_version_from_dll_powershell +from test import EXENAME + +setup_loggers(True) +precommands = PreCommands() +install_versioned_maui(precommands) + +# Setup the Maui folder +precommands.new(template='maui-blazor', + output_dir=const.APPDIR, + bin_dir=const.BINDIR, + exename=EXENAME, + working_directory=sys.path[0], + no_restore=False) + +# Add the index.razor.cs file +with open(f"{const.APPDIR}/Pages/Index.razor.cs", "w") as indexCSFile: + indexCSFile.write(''' + using Microsoft.AspNetCore.Components; + #if ANDROID + using Android.App; + #endif\n\n''' + + f" namespace {EXENAME}.Pages" + +''' + { + public partial class Index + { + protected override void OnAfterRender(bool firstRender) + { + if (firstRender) + { + #if ANDROID + var activity = MainActivity.Context as Activity; + activity.ReportFullyDrawn(); + #else + System.Console.WriteLine(\"__MAUI_Blazor_WebView_OnAfterRender__\"); + #endif + } + } + } + } +''') + +# Replace line in the Android MainActivity.cs file +with open(f"{const.APPDIR}/Platforms/Android/MainActivity.cs", "r") as mainActivityFile: + mainActivityFileLines = mainActivityFile.readlines() + +with open(f"{const.APPDIR}/Platforms/Android/MainActivity.cs", "w") as mainActivityFile: + for line in mainActivityFileLines: + if line.startswith("{"): + mainActivityFile.write("{\npublic static Android.Content.Context Context { get; private set; }\npublic MainActivity() { Context = this; }") + else: + mainActivityFile.write(line) + +# Build the APK +precommands.execute(['--no-restore', '--source', 'MauiNuGet.config']) + +output_dir = const.PUBDIR +if precommands.output: + output_dir = precommands.output +remove_aab_files(output_dir) + +# Copy the MauiVersion to a file so we have it on the machine +maui_version = get_version_from_dll_powershell(rf".\{const.APPDIR}\obj\Release\{precommands.framework}\{precommands.runtime_identifier}\linked\Microsoft.Maui.dll") +version_dict = { "mauiVersion": maui_version } +versions_write_json(version_dict, rf"{output_dir}\versions.json") +print(f"Versions: {version_dict}") \ No newline at end of file diff --git a/src/scenarios/mauiblazorandroid/test.py b/src/scenarios/mauiblazorandroid/test.py new file mode 100644 index 00000000000..9363fdfeb5b --- /dev/null +++ b/src/scenarios/mauiblazorandroid/test.py @@ -0,0 +1,16 @@ +''' +Mobile Maui App +''' +from shared.const import PUBDIR +from shared.runner import TestTraits, Runner +from shared.versionmanager import versions_read_json_file_save_env + +EXENAME = 'MauiBlazorAndroidDefault' + +if __name__ == "__main__": + versions_read_json_file_save_env(rf".\{PUBDIR}\versions.json") + + traits = TestTraits(exename=EXENAME, + guiapp='false', + ) + Runner(traits).run() \ No newline at end of file diff --git a/src/scenarios/shared/mauisharedpython.py b/src/scenarios/shared/mauisharedpython.py new file mode 100644 index 00000000000..a61be2cfa65 --- /dev/null +++ b/src/scenarios/shared/mauisharedpython.py @@ -0,0 +1,24 @@ +import subprocess +import os +import requests +from shared.precommands import PreCommands + +# Remove the aab files as we don't need them, this saves space in the correlation payload +def remove_aab_files(output_dir="."): + file_list = os.listdir(output_dir) + for file in file_list: + if file.endswith(".aab"): + os.remove(os.path.join(output_dir, file)) + +def install_versioned_maui(precommands): + target_framework_wo_platform = precommands.framework.split('-')[0] + + # Download what we need + with open ("MauiNuGet.config", "wb") as f: + f.write(requests.get(f'https://raw.githubusercontent.com/dotnet/maui/{target_framework_wo_platform}/NuGet.config', allow_redirects=True).content) + + workload_install_args = ['--configfile', 'MauiNuGet.config'] + if int(target_framework_wo_platform.split('.')[0][3:]) > 7: # Use the rollback file for versions greater than 7 + workload_install_args += ['--from-rollback-file', f'https://aka.ms/dotnet/maui/{target_framework_wo_platform}.json'] + + precommands.install_workload('maui', workload_install_args) \ No newline at end of file diff --git a/src/scenarios/shared/runner.py b/src/scenarios/shared/runner.py index 5e11bf230f7..c6c2503d96d 100644 --- a/src/scenarios/shared/runner.py +++ b/src/scenarios/shared/runner.py @@ -338,6 +338,13 @@ def run(self): 'size' ] RunCommand(cmdline, verbose=True).run() + stopAppCmd = [ + adb.stdout.strip(), + 'shell', + 'am', + 'force-stop', + self.packagename + ] installCmd = xharnesscommand() + [ self.devicetype, @@ -391,7 +398,7 @@ def run(self): # Actual testing some run stuff getLogger().info("Test run to check if permissions are needed") - activityname = getActivity.stdout + activityname = getActivity.stdout.strip() startAppCmd = [ adb.stdout.strip(), @@ -406,14 +413,7 @@ def run(self): testRun.run() testRunStats = re.findall(runSplitRegex, testRun.stdout) # Split results saving value (List: Starting, Status, LaunchState, Activity, TotalTime, WaitTime) getLogger().info(f"Test run activity: {testRunStats[3]}") - - stopAppCmd = [ - adb.stdout.strip(), - 'shell', - 'am', - 'force-stop', - self.packagename - ] + time.sleep(10) RunCommand(stopAppCmd, verbose=True).run() if "com.google.android.permissioncontroller" in testRunStats[3]: diff --git a/src/scenarios/shared/versionmanager.py b/src/scenarios/shared/versionmanager.py new file mode 100644 index 00000000000..3eeaf795a5a --- /dev/null +++ b/src/scenarios/shared/versionmanager.py @@ -0,0 +1,31 @@ +''' +Version File Manager +''' +import json +import os +import subprocess + +def versions_write_json(versiondict: dict, outputfile = 'versions.json'): + with open(outputfile, 'w') as file: + json.dump(versiondict, file) + +def versions_read_json(inputfile = 'versions.json'): + with open(inputfile, 'r') as file: + return json.load(file) + +def versions_write_env(versiondict: dict): + for key, value in versiondict.items(): + os.environ[key] = value + +def versions_read_json_file_save_env(inputfile = 'versions.json'): + versions = versions_read_json(inputfile) + print(f"Versions: {versions}") + versions_write_env(versions) + + # Remove the versions.json file if we are in the lab to ensure SOD doesn't pick it up + if "PERFLAB_INLAB" in os.environ and os.environ["PERFLAB_INLAB"] == "1": + os.remove(inputfile) + +def get_version_from_dll_powershell(dll_path: str): + result = subprocess.run(['powershell', '-Command', rf'Get-ChildItem {dll_path} | Select-Object -ExpandProperty VersionInfo | Select-Object -ExpandProperty ProductVersion'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) + return result.stdout.decode('utf-8').strip() \ No newline at end of file diff --git a/src/tools/Reporting/Reporting.Tests/EnvironmentProviderMock.cs b/src/tools/Reporting/Reporting.Tests/EnvironmentProviderMock.cs index cf33c408fe3..1939998e176 100644 --- a/src/tools/Reporting/Reporting.Tests/EnvironmentProviderMock.cs +++ b/src/tools/Reporting/Reporting.Tests/EnvironmentProviderMock.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using System; +using System.Collections; using System.Collections.Generic; namespace Reporting.Tests @@ -14,6 +15,10 @@ public string GetEnvironmentVariable(string variable) { return Vars?[variable]; } + public IDictionary GetEnvironmentVariables() + { + return Vars; + } } internal class PerfLabEnvironmentProviderMock : EnvironmentProviderMockBase { @@ -23,6 +28,7 @@ public PerfLabEnvironmentProviderMock() { {"PERFLAB_INLAB", "1" }, {"HELIX_CORRELATION_ID","testCorrelationId" }, + {"HELIX_WORKITEM_FRIENDLYNAME","Test Friendly Name" }, {"PERFLAB_PERFHASH","testPerfHash" }, {"PERFLAB_QUEUE","testQueue" }, {"PERFLAB_REPO","testRepo" }, @@ -34,7 +40,7 @@ public PerfLabEnvironmentProviderMock() {"PERFLAB_HIDDEN", "false" }, {"PERFLAB_RUNNAME", "testRunName" }, {"PERFLAB_CONFIGS", "KEY1=VALUE1;KEY2=VALUE2" }, - {"PERFLAB_BUILDTIMESTAMP", new DateTime(1970,1,1).ToString("o") }, + {"PERFLAB_BUILDTIMESTAMP", new DateTime(1970,1,1).ToString("o") } }; } diff --git a/src/tools/Reporting/Reporting.Tests/ReporterTests.cs b/src/tools/Reporting/Reporting.Tests/ReporterTests.cs index c354d23b90f..dca49fe5314 100644 --- a/src/tools/Reporting/Reporting.Tests/ReporterTests.cs +++ b/src/tools/Reporting/Reporting.Tests/ReporterTests.cs @@ -106,6 +106,7 @@ public void JsonCanBeGenerated() var jsonObj = JsonConvert.DeserializeAnonymousType(jsonString, jsonType); Assert.Equal(environment.GetEnvironmentVariable("HELIX_CORRELATION_ID"), jsonObj.Run.CorrelationId); + Assert.Equal(environment.GetEnvironmentVariable("HELIX_WORKITEM_FRIENDLYNAME"), jsonObj.Run.WorkItemName); Assert.Equal(environment.GetEnvironmentVariable("PERFLAB_PERFHASH"), jsonObj.Run.PerfRepoHash); Assert.Equal(environment.GetEnvironmentVariable("PERFLAB_QUEUE"), jsonObj.Run.Queue); Assert.Equal(environment.GetEnvironmentVariable("PERFLAB_REPO"), jsonObj.Build.Repo); diff --git a/src/tools/Reporting/Reporting/EnvironmentProvider.cs b/src/tools/Reporting/Reporting/EnvironmentProvider.cs index 90d28729284..11336a24fce 100644 --- a/src/tools/Reporting/Reporting/EnvironmentProvider.cs +++ b/src/tools/Reporting/Reporting/EnvironmentProvider.cs @@ -11,5 +11,6 @@ namespace Reporting public class EnvironmentProvider : IEnvironment { public string GetEnvironmentVariable(string variable) => Environment.GetEnvironmentVariable(variable); + public System.Collections.IDictionary GetEnvironmentVariables() => Environment.GetEnvironmentVariables(); } } diff --git a/src/tools/Reporting/Reporting/IEnvironment.cs b/src/tools/Reporting/Reporting/IEnvironment.cs index c7dbfb9b002..53adab94c63 100644 --- a/src/tools/Reporting/Reporting/IEnvironment.cs +++ b/src/tools/Reporting/Reporting/IEnvironment.cs @@ -11,5 +11,6 @@ namespace Reporting public interface IEnvironment { string GetEnvironmentVariable(string variable); + System.Collections.IDictionary GetEnvironmentVariables(); } } diff --git a/src/tools/Reporting/Reporting/Reporter.cs b/src/tools/Reporting/Reporting/Reporter.cs index 115914cee96..fb988f91321 100644 --- a/src/tools/Reporting/Reporting/Reporter.cs +++ b/src/tools/Reporting/Reporting/Reporter.cs @@ -5,6 +5,7 @@ using Newtonsoft.Json; using Newtonsoft.Json.Serialization; using System; +using System.Collections; using System.Collections.Generic; using System.Globalization; using System.Linq; @@ -88,14 +89,27 @@ private void Init() BuildName = environment.GetEnvironmentVariable("PERFLAB_BUILDNUM"), TimeStamp = DateTime.Parse(environment.GetEnvironmentVariable("PERFLAB_BUILDTIMESTAMP")), }; - build.AdditionalData["productVersion"] = environment.GetEnvironmentVariable("DOTNET_VERSION"); - - // Additional Data we only want populated if it is available - if (environment.GetEnvironmentVariable("MAUI_VERSION") != null) + + foreach (DictionaryEntry entry in environment.GetEnvironmentVariables()) { - build.AdditionalData["mauiVersion"] = environment.GetEnvironmentVariable("MAUI_VERSION"); - } + if (entry.Key.ToString().Equals("PERFLAB_TARGET_FRAMEWORKS", StringComparison.InvariantCultureIgnoreCase)) + { + build.AdditionalData["targetFrameworks"] = entry.Value.ToString(); + } + else if(entry.Key.ToString().EndsWith("version", true, CultureInfo.InvariantCulture)) + { + // Special case the original two special cases, MAUI_VERSION is only needed because runtime based runs use MAUI_VERSION + if(entry.Key.ToString().Equals("DOTNET_VERSION", StringComparison.InvariantCultureIgnoreCase)){ + build.AdditionalData["productVersion"] = entry.Value.ToString(); + } else if(entry.Key.ToString().Equals("MAUI_VERSION", StringComparison.InvariantCultureIgnoreCase)){ + build.AdditionalData["mauiVersion"] = entry.Value.ToString(); + } else { + build.AdditionalData[entry.Key.ToString()] = entry.Value.ToString(); + } + } + } } + public string GetJson() { if (!InLab) From 8c54d557d96a024d431b5994e96622c6ba83e293 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 23 Jan 2023 13:34:51 -0800 Subject: [PATCH 02/11] Add testing run. --- azure-pipelines.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 791fb4fef6e..3b8414947be 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -604,3 +604,18 @@ jobs: projectFile: sdk_scenarios.proj channels: - $(channel) + + # Maui Android scenario benchmarks + - template: /eng/performance/scenarios.yml + parameters: + osName: windows + architecture: x64 + osVersion: 19H1 + pool: + vmImage: 'windows-2022' + queue: Windows.10.Amd64.Pixel.Perf + machinePool: Pixel + kind: maui_scenarios_android + projectFile: maui_scenarios_android.proj + channels: + - $(channel) From 7dd69c76d8d4b2aa7f3a53ddae3fb313996493ed Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 23 Jan 2023 13:44:46 -0800 Subject: [PATCH 03/11] Copy necessary target and property files to support pre scenario all in one building. --- azure-pipelines.yml | 86 +-- .../PreparePayloadWorkItems.targets | 7 + eng/performance/Scenarios.Common.props | 30 + eng/performance/blazor_scenarios.proj | 2 + eng/performance/maui_scenarios_android.proj | 2 +- eng/performance/scenarios.proj | 1 + eng/performance/scenarios.yml | 26 + eng/performance/sdk_scenarios.proj | 1 + src/scenarios/shared/precommands.py | 80 ++- src/scenarios/shared/runner.py | 614 +++++++++++++++--- 10 files changed, 680 insertions(+), 169 deletions(-) create mode 100644 eng/performance/PreparePayloadWorkItems.targets create mode 100644 eng/performance/Scenarios.Common.props diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3b8414947be..c9acdaea990 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -559,51 +559,51 @@ jobs: - ${{ if and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Manual')) }}: - # Windows x64 SDK scenario benchmarks - - template: /eng/performance/scenarios.yml - parameters: - osName: windows - osVersion: RS5 - architecture: x64 - pool: - vmImage: windows-2019 - kind: sdk_scenarios - machinePool: Tiger - queue: Windows.10.Amd64.19H1.Tiger.Perf - projectFile: sdk_scenarios.proj - channels: - - $(channel) # for manual runs we can specify channel variable + # # Windows x64 SDK scenario benchmarks + # - template: /eng/performance/scenarios.yml + # parameters: + # osName: windows + # osVersion: RS5 + # architecture: x64 + # pool: + # vmImage: windows-2019 + # kind: sdk_scenarios + # machinePool: Tiger + # queue: Windows.10.Amd64.19H1.Tiger.Perf + # projectFile: sdk_scenarios.proj + # channels: + # - $(channel) # for manual runs we can specify channel variable - # Windows x86 SDK scenario benchmarks - - template: /eng/performance/scenarios.yml - parameters: - osName: windows - osVersion: RS5 - architecture: x86 - pool: - vmImage: windows-2019 - kind: sdk_scenarios - machinePool: Tiger - queue: Windows.10.Amd64.19H1.Tiger.Perf - projectFile: sdk_scenarios.proj - channels: - - $(channel) # for manual runs we can specify channel variable + # # Windows x86 SDK scenario benchmarks + # - template: /eng/performance/scenarios.yml + # parameters: + # osName: windows + # osVersion: RS5 + # architecture: x86 + # pool: + # vmImage: windows-2019 + # kind: sdk_scenarios + # machinePool: Tiger + # queue: Windows.10.Amd64.19H1.Tiger.Perf + # projectFile: sdk_scenarios.proj + # channels: + # - $(channel) # for manual runs we can specify channel variable - # Ubuntu 1804 x64 SDK scenario benchmarks - - template: /eng/performance/scenarios.yml - parameters: - osName: ubuntu - osVersion: 1804 - architecture: x64 - pool: - vmImage: ubuntu-latest - kind: sdk_scenarios - machinePool: Tiger - queue: Ubuntu.1804.Amd64.Tiger.Perf - container: ubuntu_x64_build_container - projectFile: sdk_scenarios.proj - channels: - - $(channel) + # # Ubuntu 1804 x64 SDK scenario benchmarks + # - template: /eng/performance/scenarios.yml + # parameters: + # osName: ubuntu + # osVersion: 1804 + # architecture: x64 + # pool: + # vmImage: ubuntu-latest + # kind: sdk_scenarios + # machinePool: Tiger + # queue: Ubuntu.1804.Amd64.Tiger.Perf + # container: ubuntu_x64_build_container + # projectFile: sdk_scenarios.proj + # channels: + # - $(channel) # Maui Android scenario benchmarks - template: /eng/performance/scenarios.yml diff --git a/eng/performance/PreparePayloadWorkItems.targets b/eng/performance/PreparePayloadWorkItems.targets new file mode 100644 index 00000000000..2615b211813 --- /dev/null +++ b/eng/performance/PreparePayloadWorkItems.targets @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/eng/performance/Scenarios.Common.props b/eng/performance/Scenarios.Common.props new file mode 100644 index 00000000000..2ae44748780 --- /dev/null +++ b/eng/performance/Scenarios.Common.props @@ -0,0 +1,30 @@ + + + + %(Identity) + + + + + + 4:00 + + + + + $(_Framework.Substring($([MSBuild]::Subtract($(_Framework.Length), 3)))) + + + + $(WorkItemDirectory)\src\scenarios\ + $(HelixPreCommands);set PYTHONPATH=%HELIX_CORRELATION_PAYLOAD%\scripts%3B%HELIX_CORRELATION_PAYLOAD% + win-$(Architecture) + + + + $(WorkItemDirectory)/src/scenarios/ + $(HelixPreCommands);sudo apt-get update;chmod +x $HELIX_CORRELATION_PAYLOAD/startup/perfcollect + $(HelixPreCommands);export PYTHONPATH=$HELIX_CORRELATION_PAYLOAD/scripts:$HELIX_CORRELATION_PAYLOAD + linux-$(Architecture) + + \ No newline at end of file diff --git a/eng/performance/blazor_scenarios.proj b/eng/performance/blazor_scenarios.proj index 3937e7fee99..f85d90c6ac9 100644 --- a/eng/performance/blazor_scenarios.proj +++ b/eng/performance/blazor_scenarios.proj @@ -79,4 +79,6 @@ + + diff --git a/eng/performance/maui_scenarios_android.proj b/eng/performance/maui_scenarios_android.proj index 270288b148a..c1e8a86e6fa 100644 --- a/eng/performance/maui_scenarios_android.proj +++ b/eng/performance/maui_scenarios_android.proj @@ -50,7 +50,7 @@ - $(Python) pre.py publish -f $(PERFLAB_Framework)-android -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName) -r android-arm64 --self-contained + $(Python) pre.py publish -f $(_Framework)-android -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName) -r android-arm64 --self-contained %(PreparePayloadWorkItem.PayloadDirectory) diff --git a/eng/performance/scenarios.proj b/eng/performance/scenarios.proj index 7a70390047e..7f933a111f0 100644 --- a/eng/performance/scenarios.proj +++ b/eng/performance/scenarios.proj @@ -121,5 +121,6 @@ post: %(HelixWorkItem.PostCommands) timeout: %(HelixWorkItem.Timeout) '"/> --> + diff --git a/eng/performance/scenarios.yml b/eng/performance/scenarios.yml index b170821c0ed..b73541f5014 100644 --- a/eng/performance/scenarios.yml +++ b/eng/performance/scenarios.yml @@ -104,6 +104,19 @@ jobs: displayName: Build FailureReporter tool env: PERFLAB_TARGET_FRAMEWORKS: $(_Framework) + - powershell: | + $(Python) -m pip install --upgrade pip + $(Python) -m pip install requests + .\src\scenarios\init.ps1 -DotnetDirectory $(CorrelationStaging)dotnet + dotnet --info + dotnet msbuild .\eng\performance\${{ parameters.projectFile }} /restore /t:PreparePayloadWorkItems /bl:.\artifacts\log\$(_BuildConfig)\PrepareWorkItemPayloads.binlog + displayName: Prepare scenarios + env: + CorrelationPayloadDirectory: $(CorrelationStaging) + Architecture: ${{ parameters.architecture }} + TargetsWindows: 'true' + WorkItemDirectory: $(Build.SourcesDirectory) + HelixTargetQueues: ${{ parameters.queue }} - ${{ if ne(parameters.osName, 'windows') }}: - script: cp ./NuGet.config $(CorrelationStaging);cp -r ./scripts $(CorrelationStaging)scripts;cp -r ./src/scenarios/shared $(CorrelationStaging)shared;cp -r ./src/scenarios/staticdeps $(CorrelationStaging)staticdeps displayName: Copy python libraries and NuGet.config @@ -119,6 +132,19 @@ jobs: displayName: Build FailureReporter tool env: PERFLAB_TARGET_FRAMEWORKS: $(_Framework) + - script: | + $(Python) -m pip install --upgrade pip + $(Python) -m pip install requests + . ./src/scenarios/init.sh -dotnetdir $(CorrelationStaging)dotnet + dotnet --info + dotnet msbuild ./eng/performance/${{ parameters.projectFile }} /restore /t:PreparePayloadWorkItems /bl:./artifacts/log/$(_BuildConfig)/PrepareWorkItemPayloads.binlog + displayName: Prepare scenarios + env: + CorrelationPayloadDirectory: $(CorrelationStaging) + Architecture: ${{ parameters.architecture }} + TargetsWindows: 'false' + WorkItemDirectory: $(Build.SourcesDirectory) + HelixTargetQueues: ${{ parameters.queue }} - template: /eng/performance/send-to-helix.yml parameters: HelixSource: '$(HelixSourcePrefix)/dotnet/performance/$(Build.SourceBranch)' # sources must start with pr/, official/, prodcon/, or agent/ diff --git a/eng/performance/sdk_scenarios.proj b/eng/performance/sdk_scenarios.proj index d40a8635c9f..d412f3ed443 100644 --- a/eng/performance/sdk_scenarios.proj +++ b/eng/performance/sdk_scenarios.proj @@ -189,5 +189,6 @@ 4:00 + diff --git a/src/scenarios/shared/precommands.py b/src/scenarios/shared/precommands.py index 43feff3dc62..7fe274a490b 100644 --- a/src/scenarios/shared/precommands.py +++ b/src/scenarios/shared/precommands.py @@ -19,6 +19,7 @@ PUBLISH = 'publish' CROSSGEN = 'crossgen' CROSSGEN2 = 'crossgen2' +EXTRACT = 'extract' DEBUG = 'Debug' RELEASE = 'Release' @@ -26,7 +27,8 @@ BUILD, PUBLISH, CROSSGEN, - CROSSGEN2 + CROSSGEN2, + EXTRACT ) class PreCommands: @@ -46,12 +48,30 @@ def __init__(self): default_parser = subparsers.add_parser(DEFAULT, help='Default operation (placeholder command and no specific operation will be executed)' ) self.add_common_arguments(default_parser) + + extract_parser = subparsers.add_parser(EXTRACT, help='Used for local runs that extract the binaries to be run from a zip file. Requires a path to the zip' ) + self.add_common_arguments(extract_parser) + extract_parser.add_argument('-p', '--pathtozip', + dest='pathtozip', + metavar='pathtozip', + help='Path to the zip file to extract', + required=True) build_parser = subparsers.add_parser(BUILD, help='Builds the project') self.add_common_arguments(build_parser) publish_parser = subparsers.add_parser(PUBLISH, help='Publishes the project') self.add_common_arguments(publish_parser) + publish_parser.add_argument('--self-contained', + dest='self_contained', + default=False, + action='store_true', + help='Publish SCD') + publish_parser.add_argument('--no-self-contained', + dest='no_self_contained', + default=False, + action='store_true', + help='Publish FDD') crossgen_parser = subparsers.add_parser(CROSSGEN, help='Runs crossgen on a particular file') self.add_common_arguments(crossgen_parser) @@ -77,11 +97,18 @@ def __init__(self): self.binlog = args.binlog self.has_workload = args.has_workload self.readonly_dotnet = args.readonly_dotnet - + self.windows = args.windows + self.output = args.output + + if self.operation == PUBLISH: + self.self_contained = args.self_contained + self.no_self_contained = args.no_self_contained if self.operation == CROSSGEN: self.crossgen_arguments.parse_crossgen_args(args) if self.operation == CROSSGEN2: self.crossgen_arguments.parse_crossgen2_args(args) + if self.operation == EXTRACT: + self.pathtozip = args.pathtozip def new(self, @@ -145,6 +172,14 @@ def add_common_arguments(self, parser: ArgumentParser): default=False, action='store_true', help='Indicates that the dotnet being used should not be modified (for example, when it is ahared with other builds)') + parser.add_argument('--windowsui', + dest='windows', + action='store_true', + help='must be set for UI tests so the proper rid is used') + parser.add_argument('-o', '--output', + dest='output', + metavar='output', + help='output directory') parser.set_defaults(configuration=RELEASE) def existing(self, projectdir: str, projectfile: str): @@ -160,12 +195,14 @@ def execute(self, build_args: list = []): pass if self.operation == BUILD: self._restore() - self._build(configuration=self.configuration, framework=self.framework, build_args=build_args) + self._build(configuration=self.configuration, framework=self.framework, output=self.output, build_args=build_args) if self.operation == PUBLISH: self._restore() - self._publish(configuration=self.configuration, - runtime_identifier=self.runtime_identifier, - framework=self.framework, build_args=build_args) + if self.self_contained: + build_args.append('--self-contained') + elif self.no_self_contained: + build_args.append('--no-self-contained') + self._publish(configuration=self.configuration, runtime_identifier=self.runtime_identifier, framework=self.framework, output=self.output, build_args=build_args) if self.operation == CROSSGEN: startup_args = [ os.path.join(self.crossgen_arguments.coreroot, 'crossgen%s' % extension()), @@ -230,30 +267,33 @@ def _parsemsbuildproperties(self) -> list: def _updateframework(self, projectfile: str): 'Update the property so we can re-use the template' if self.framework: - replace_line(projectfile, r'.*?', f'{self.framework}') + if self.windows: + replace_line(projectfile, r'.*?', f'{self.framework}-windows') + else: + replace_line(projectfile, r'.*?', f'{self.framework}') - def _publish(self, configuration: str, framework: str = None, runtime_identifier: str = None, build_args: list = []): + def _publish(self, configuration: str, framework: str = None, runtime_identifier: str = None, output: str = None, build_args: list = []): self.project.publish(configuration, - const.PUBDIR, + output or const.PUBDIR, True, os.path.join(get_packages_directory(), ''), # blazor publish targets require the trailing slash for joining the paths - framework, + framework if not self.windows else f'{framework}-windows', runtime_identifier, self._parsemsbuildproperties(), - '-bl:%s' % self.binlog if self.binlog else "", - *build_args - ) + *['-bl:%s' % self.binlog] if self.binlog else [], + *build_args) def _restore(self): self.project.restore(packages_path=get_packages_directory(), verbose=True) - def _build(self, configuration: str, framework: str = None, build_args: list = []): - self.project.build(configuration=configuration, - verbose=True, - packages_path=get_packages_directory(), - target_framework_monikers=[framework], - output_to_bindir=True, - *build_args) + def _build(self, configuration: str, framework: str = None, output: str = None, build_args: list = []): + self.project.build(configuration, + True, + get_packages_directory(), + [framework], + output is None, + None, + (['--output', output] if output else []) + build_args) def _backup(self, projectdir:str): 'Copy from projectdir to appdir so we do not modify the source code' diff --git a/src/scenarios/shared/runner.py b/src/scenarios/shared/runner.py index c6c2503d96d..3226827acaf 100644 --- a/src/scenarios/shared/runner.py +++ b/src/scenarios/shared/runner.py @@ -57,11 +57,15 @@ def parseargs(self): # parse only command parseonlyparser = subparsers.add_parser(const.DEVICESTARTUP, - description='measure time to main for Android apps') + description='measure time to startup for Android/iOS apps') parseonlyparser.add_argument('--device-type', choices=['android','ios'],type=str.lower,help='Device type for testing', dest='devicetype') parseonlyparser.add_argument('--package-path', help='Location of test application', dest='packagepath') - parseonlyparser.add_argument('--package-name', help='Classname of application', dest='packagename') - parseonlyparser.add_argument('--startup-iterations', help='Startups to run (1+)', type=int, default=5, dest='startupiterations') + parseonlyparser.add_argument('--package-name', help='Classname (Android) or Bundle ID (iOS) of application', dest='packagename') + parseonlyparser.add_argument('--startup-iterations', help='Startups to run (1+)', type=int, default=10, dest='startupiterations') + parseonlyparser.add_argument('--disable-animations', help='Disable Android device animations, does nothing on iOS.', action='store_true', dest='animationsdisabled') + parseonlyparser.add_argument('--use-fully-drawn-time', help='Use the startup time from reportFullyDrawn for android, the equivalent for iOS is handled via logging a magic string and passing it to --fully-drawn-magic-string', action='store_true', dest='usefullydrawntime') + parseonlyparser.add_argument('--fully-drawn-extra-delay', help='Set an additional delay time for an Android app to reportFullyDrawn (seconds), not on iOS. This should be greater than the greatest amount of extra time expected between first frame draw and reportFullyDrawn being called. Default = 3 seconds', type=int, default=3, dest='fullyDrawnDelaySecMax') + parseonlyparser.add_argument('--fully-drawn-magic-string', help='Set the magic string that is logged by the app to indicate when the app is fully drawn. Required when using --use-fully-drawn-time on iOS.', type=str, dest='fullyDrawnMagicString') self.add_common_arguments(parseonlyparser) # inner loop command @@ -144,6 +148,10 @@ def parseargs(self): self.packagename = args.packagename self.devicetype = args.devicetype self.startupiterations = args.startupiterations + self.animationsdisabled = args.animationsdisabled + self.usefullydrawntime = args.usefullydrawntime + self.fullyDrawnDelaySecMax = args.fullyDrawnDelaySecMax + self.fullyDrawnMagicString = args.fullyDrawnMagicString if args.scenarioname: self.scenarioname = args.scenarioname @@ -304,7 +312,7 @@ def run(self): startup.runtests(self.traits) - elif self.testtype == const.DEVICESTARTUP: + elif self.testtype == const.DEVICESTARTUP and self.devicetype == 'android': # ADB Key Event corresponding numbers: https://gist.github.com/arjunv/2bbcca9a1a1c127749f8dcb6d36fb0bc # Regex used to split the response from starting the activity and saving each value #Example: @@ -325,7 +333,7 @@ def run(self): getLogger().info("Removed: " + os.path.join(const.TRACEDIR, file)) os.remove(file) - cmdline = xharnesscommand() + [self.devicetype, 'state', '--adb'] + cmdline = xharnesscommand() + ['android', 'state', '--adb'] adb = RunCommand(cmdline, verbose=True) adb.run() @@ -338,130 +346,526 @@ def run(self): 'size' ] RunCommand(cmdline, verbose=True).run() - stopAppCmd = [ + + # Get animation values + getLogger().info("Getting Values we will need set specifically") + cmdline = [ adb.stdout.strip(), - 'shell', - 'am', - 'force-stop', - self.packagename + 'shell', 'settings', 'get', 'global', 'window_animation_scale' ] - - installCmd = xharnesscommand() + [ - self.devicetype, - 'install', - '--app', self.packagepath, - '--package-name', - self.packagename, - '-o', - const.TRACEDIR, - '-v' + window_animation_scale_cmd = RunCommand(cmdline, verbose=True) + window_animation_scale_cmd.run() + cmdline = [ + adb.stdout.strip(), + 'shell', 'settings', 'get', 'global', 'transition_animation_scale' ] - RunCommand(installCmd, verbose=True).run() - - getLogger().info("Completed install, running shell.") - cmdline = [ + transition_animation_scale_cmd = RunCommand(cmdline, verbose=True) + transition_animation_scale_cmd.run() + cmdline = [ adb.stdout.strip(), - 'shell', - f'cmd package resolve-activity --brief {self.packagename} | tail -n 1' + 'shell', 'settings', 'get', 'global', 'animator_duration_scale' ] - getActivity = RunCommand(cmdline, verbose=True) - getActivity.run() - getLogger().info(f"Target Activity {getActivity.stdout}") - - # More setup stuff - checkScreenOnCmd = [ + animator_duration_scale_cmd = RunCommand(cmdline, verbose=True) + animator_duration_scale_cmd.run() + cmdline = [ adb.stdout.strip(), - 'shell', - f'dumpsys input_method | grep mInteractive' + 'shell', 'settings', 'get', 'system', 'screen_off_timeout' ] - checkScreenOn = RunCommand(checkScreenOnCmd, verbose=True) - checkScreenOn.run() - - keyInputCmd = [ + screen_off_timeout_cmd = RunCommand(cmdline, verbose=True) + screen_off_timeout_cmd.run() + getLogger().info(f"Retrieved values window {window_animation_scale_cmd.stdout.strip()}, transition {transition_animation_scale_cmd.stdout.strip()}, animator {animator_duration_scale_cmd.stdout.strip()}, screen timeout {screen_off_timeout_cmd.stdout.strip()}") + + # Make sure animations are set to 1 or disabled + getLogger().info("Setting needed values") + if self.animationsdisabled: + animationValue = 0 + else: + animationValue = 1 + minimumTimeoutValue = 2 * 60 * 1000 # milliseconds + cmdline = [ adb.stdout.strip(), - 'shell', - 'input', - 'keyevent' + 'shell', 'settings', 'put', 'global', 'window_animation_scale', str(animationValue) ] + RunCommand(cmdline, verbose=True).run() + cmdline = [ + adb.stdout.strip(), + 'shell', 'settings', 'put', 'global', 'transition_animation_scale', str(animationValue) + ] + RunCommand(cmdline, verbose=True).run() + cmdline = [ + adb.stdout.strip(), + 'shell', 'settings', 'put', 'global', 'animator_duration_scale', str(animationValue) + ] + RunCommand(cmdline, verbose=True).run() + cmdline = [ + adb.stdout.strip(), + 'shell', 'settings', 'put', 'system', 'screen_off_timeout', str(minimumTimeoutValue) + ] + if minimumTimeoutValue > int(screen_off_timeout_cmd.stdout.strip()): + getLogger().info("Screen off value is lower than minimum time, setting to higher time") + RunCommand(cmdline, verbose=True).run() - if("mInteractive=false" in checkScreenOn.stdout): - # Turn on the screen to make interactive and see if it worked - getLogger().info("Screen was off, turning on.") - screenWasOff = True - RunCommand(keyInputCmd + ['26'], verbose=True).run() # Press the power key - RunCommand(keyInputCmd + ['82'], verbose=True).run() # Unlock the screen with menu key (only works if it is not a password lock) - + # Check for success + getLogger().info("Getting animation values to verify it worked") + cmdline = [ + adb.stdout.strip(), + 'shell', 'settings', 'get', 'global', 'window_animation_scale' + ] + windowSetValue = RunCommand(cmdline, verbose=True) + windowSetValue.run() + cmdline = [ + adb.stdout.strip(), + 'shell', 'settings', 'get', 'global', 'transition_animation_scale' + ] + transitionSetValue = RunCommand(cmdline, verbose=True) + transitionSetValue.run() + cmdline = [ + adb.stdout.strip(), + 'shell', 'settings', 'get', 'global', 'animator_duration_scale' + ] + animatorSetValue = RunCommand(cmdline, verbose=True) + animatorSetValue.run() + if int(windowSetValue.stdout.strip()) != animationValue or int(transitionSetValue.stdout.strip()) != animationValue or int(animatorSetValue.stdout.strip()) != animationValue: + # Setting the values didn't work, error out + getLogger().exception(f"Failed to set animation values to {animationValue}.") + raise Exception(f"Failed to set animation values to {animationValue}.") + else: + getLogger().info(f"Animation values successfully set to {animationValue}.") + + try: + stopAppCmd = [ + adb.stdout.strip(), + 'shell', + 'am', + 'force-stop', + self.packagename + ] + + installCmd = xharnesscommand() + [ + 'android', + 'install', + '--app', self.packagepath, + '--package-name', + self.packagename, + '-o', + const.TRACEDIR, + '-v' + ] + RunCommand(installCmd, verbose=True).run() + + getLogger().info("Completed install, running shell.") + cmdline = [ + adb.stdout.strip(), + 'shell', + f'cmd package resolve-activity --brief {self.packagename} | tail -n 1' + ] + getActivity = RunCommand(cmdline, verbose=True) + getActivity.run() + getLogger().info(f"Target Activity {getActivity.stdout}") + + # More setup stuff + checkScreenOnCmd = [ + adb.stdout.strip(), + 'shell', + f'dumpsys input_method | grep mInteractive' + ] checkScreenOn = RunCommand(checkScreenOnCmd, verbose=True) checkScreenOn.run() - if("mInteractive=false" in checkScreenOn.stdout): - getLogger().exception("Failed to make screen interactive.") - # Actual testing some run stuff - getLogger().info("Test run to check if permissions are needed") - activityname = getActivity.stdout.strip() - - startAppCmd = [ - adb.stdout.strip(), - 'shell', - 'am', - 'start-activity', - '-W', - '-n', - activityname - ] - testRun = RunCommand(startAppCmd, verbose=True) - testRun.run() - testRunStats = re.findall(runSplitRegex, testRun.stdout) # Split results saving value (List: Starting, Status, LaunchState, Activity, TotalTime, WaitTime) - getLogger().info(f"Test run activity: {testRunStats[3]}") - time.sleep(10) - RunCommand(stopAppCmd, verbose=True).run() - - if "com.google.android.permissioncontroller" in testRunStats[3]: - # On perm screen, use the buttons to close it. it will stay away until the app is reinstalled - RunCommand(keyInputCmd + ['22'], verbose=True).run() # Select next button - time.sleep(1) - RunCommand(keyInputCmd + ['22'], verbose=True).run() # Select next button - time.sleep(1) - RunCommand(keyInputCmd + ['66'], verbose=True).run() # Press enter to close main perm screen - time.sleep(1) - RunCommand(keyInputCmd + ['22'], verbose=True).run() # Select next button - time.sleep(1) - RunCommand(keyInputCmd + ['66'], verbose=True).run() # Press enter to close out of second screen - time.sleep(1) - - # Check to make sure it worked + keyInputCmd = [ + adb.stdout.strip(), + 'shell', + 'input', + 'keyevent' + ] + + if "mInteractive=false" in checkScreenOn.stdout: + # Turn on the screen to make interactive and see if it worked + getLogger().info("Screen was off, turning on.") + screenWasOff = True + RunCommand(keyInputCmd + ['26'], verbose=True).run() # Press the power key + RunCommand(keyInputCmd + ['82'], verbose=True).run() # Unlock the screen with menu key (only works if it is not a password lock) + + checkScreenOn = RunCommand(checkScreenOnCmd, verbose=True) + checkScreenOn.run() + if "mInteractive=false" in checkScreenOn.stdout: + getLogger().exception("Failed to make screen interactive.") + raise Exception("Failed to make screen interactive.") + + # Actual testing some run stuff + getLogger().info("Test run to check if permissions are needed") + activityname = getActivity.stdout.strip() + + # -W in the start command waits for the app to finish initial draw. + startAppCmd = [ + adb.stdout.strip(), + 'shell', + 'am', + 'start-activity', + '-W', + '-n', + activityname + ] testRun = RunCommand(startAppCmd, verbose=True) testRun.run() - testRunStats = re.findall(runSplitRegex, testRun.stdout) + testRunStats = re.findall(runSplitRegex, testRun.stdout) # Split results saving value (List: Starting, Status, LaunchState, Activity, TotalTime, WaitTime) getLogger().info(f"Test run activity: {testRunStats[3]}") - RunCommand(stopAppCmd, verbose=True).run() - - if "com.google.android.permissioncontroller" in testRunStats[3]: - getLogger().exception("Failed to get past permission screen, run locally to see if enough next button presses were used.") + time.sleep(10) # Add delay to ensure app is fully installed and give it some time to settle - allResults = [] - for i in range(self.startupiterations): - startStats = RunCommand(startAppCmd, verbose=True) - startStats.run() RunCommand(stopAppCmd, verbose=True).run() - allResults.append(startStats.stdout) # Save results (List is Intent, Status, LaunchState Activity, TotalTime, WaitTime) - time.sleep(3) # Delay in seconds for ensuring a cold start - getLogger().info("Stopping App for uninstall") - RunCommand(stopAppCmd, verbose=True).run() + if "com.google.android.permissioncontroller" in testRunStats[3]: + # On perm screen, use the buttons to close it. it will stay away until the app is reinstalled + RunCommand(keyInputCmd + ['22'], verbose=True).run() # Select next button + time.sleep(1) + RunCommand(keyInputCmd + ['22'], verbose=True).run() # Select next button + time.sleep(1) + RunCommand(keyInputCmd + ['66'], verbose=True).run() # Press enter to close main perm screen + time.sleep(1) + RunCommand(keyInputCmd + ['22'], verbose=True).run() # Select next button + time.sleep(1) + RunCommand(keyInputCmd + ['66'], verbose=True).run() # Press enter to close out of second screen + time.sleep(1) + + # Check to make sure it worked + testRun = RunCommand(startAppCmd, verbose=True) + testRun.run() + testRunStats = re.findall(runSplitRegex, testRun.stdout) + getLogger().info(f"Test run activity: {testRunStats[3]}") + RunCommand(stopAppCmd, verbose=True).run() + if "com.google.android.permissioncontroller" in testRunStats[3]: + getLogger().exception("Failed to get past permission screen, run locally to see if enough next button presses were used.") + raise Exception("Failed to get past permission screen, run locally to see if enough next button presses were used.") + + # Create the fullydrawn command + fullyDrawnRetrieveCmd = [ + adb.stdout.strip(), + 'shell', + f"logcat -d | grep 'ActivityTaskManager: Fully drawn {self.packagename}'" + ] + + basicStartupRetrieveCmd = [ + adb.stdout.strip(), + 'shell', + f"logcat -d | grep 'ActivityTaskManager: Displayed {activityname}'" + ] + + clearLogsCmd = [ + adb.stdout.strip(), + 'logcat', + '-c' + ] + + allResults = [] + for i in range(self.startupiterations): + # Clear logs + RunCommand(clearLogsCmd, verbose=True).run() + startStats = RunCommand(startAppCmd, verbose=True) + startStats.run() + # Make sure we cold started (TODO Add other starts) + if "LaunchState: COLD" not in startStats.stdout: + getLogger().error("App Start not COLD!") + + # Save the results and get them from the log + if self.usefullydrawntime: time.sleep(self.fullyDrawnDelaySecMax) # Start command doesn't wait for fully drawn report, force a wait for it. -W in the start command waits for the app to finish initial draw. + RunCommand(stopAppCmd, verbose=True).run() + if self.usefullydrawntime: + retrieveTimeCmd = RunCommand(fullyDrawnRetrieveCmd, verbose=True) + else: + retrieveTimeCmd = RunCommand(basicStartupRetrieveCmd, verbose=True) + retrieveTimeCmd.run() + dirtyCapture = re.search("\+(\d*s?\d+)ms", retrieveTimeCmd.stdout) + if not dirtyCapture: + raise Exception("Failed to capture the reported start time!") + captureList = dirtyCapture.group(1).split('s') + if len(captureList) == 1: # Only have the ms, everything should be good + formattedTime = f"TotalTime: {captureList[0]}\n" + elif len(captureList) == 2: # Have s and ms, but maybe not padded ms, pad and combine (zfill left pads with 0) + formattedTime = f"TotalTime: {captureList[0]}{captureList[1].zfill(3)}\n" + else: + getLogger().error("Time capture failed, found {len(captureList)}") + raise Exception("Android Time Capture Failed! Incorrect number of captures found.") + allResults.append(formattedTime) # append TotalTime: (TIME) + time.sleep(3) # Delay in seconds for ensuring a cold start + + finally: + getLogger().info("Stopping App for uninstall") + RunCommand(stopAppCmd, verbose=True).run() + + getLogger().info("Uninstalling app") + uninstallAppCmd = xharnesscommand() + [ + 'android', + 'uninstall', + '--package-name', + self.packagename + ] + RunCommand(uninstallAppCmd, verbose=True).run() + + # Reset animation values + getLogger().info("Resetting animation values to pretest values") + cmdline = [ + adb.stdout.strip(), + 'shell', 'settings', 'put', 'global', 'window_animation_scale', window_animation_scale_cmd.stdout.strip() + ] + RunCommand(cmdline, verbose=True).run() + cmdline = [ + adb.stdout.strip(), + 'shell', 'settings', 'put', 'global', 'transition_animation_scale', transition_animation_scale_cmd.stdout.strip() + ] + RunCommand(cmdline, verbose=True).run() + cmdline = [ + adb.stdout.strip(), + 'shell', 'settings', 'put', 'global', 'animator_duration_scale', animator_duration_scale_cmd.stdout.strip() + ] + RunCommand(cmdline, verbose=True).run() + cmdline = [ + adb.stdout.strip(), + 'shell', 'settings', 'put', 'system', 'screen_off_timeout', screen_off_timeout_cmd.stdout.strip() + ] + RunCommand(cmdline, verbose=True).run() + + if screenWasOff: + RunCommand(keyInputCmd + ['26'], verbose=True).run() # Turn the screen back off + + # Create traces to store the data so we can keep the current general parse trace flow + getLogger().info(f"Logs: \n{allResults}") + os.makedirs(f"{const.TRACEDIR}/PerfTest", exist_ok=True) + traceFile = open(f"{const.TRACEDIR}/PerfTest/runoutput.trace", "w") + for result in allResults: + traceFile.write(result) + traceFile.close() + + startup = StartupWrapper() + self.traits.add_traits(overwrite=True, apptorun="app", startupmetric=const.STARTUP_DEVICETIMETOMAIN, tracefolder='PerfTest/', tracename='runoutput.trace', scenarioname=self.scenarioname) + startup.parsetraces(self.traits) + + elif self.testtype == const.DEVICESTARTUP and self.devicetype == 'ios': + + getLogger().info("Clearing potential previous run nettraces") + for file in glob.glob(os.path.join(const.TRACEDIR, 'PerfTest', 'runoutput.trace')): + if exists(file): + getLogger().info("Removed: " + os.path.join(const.TRACEDIR, file)) + os.remove(file) + + if not exists(const.TMPDIR): + os.mkdir(const.TMPDIR) + + getLogger().info("Clearing potential previous run *.logarchive") + for logarchive in glob.glob(os.path.join(const.TMPDIR, '*.logarchive')): + if exists(logarchive): + getLogger().info("Removed: " + os.path.join(const.TMPDIR, logarchive)) + rmtree(logarchive) + + getLogger().info("Checking device state.") + cmdline = xharnesscommand() + ['apple', 'state'] + adb = RunCommand(cmdline, verbose=True) + adb.run() + + getLogger().info("Installing app on device.") + installCmd = xharnesscommand() + [ + 'apple', + 'install', + '--app', self.packagepath, + '--target', 'ios-device', + '-o', + const.TRACEDIR, + '-v' + ] + RunCommand(installCmd, verbose=True).run() + getLogger().info("Completed install.") + + allResults = [] + for i in range(self.startupiterations + 1): # adding one iteration to account for the warmup iteration + getLogger().info("Waiting 10 secs to ensure we're not getting confused with previous app run.") + time.sleep(10) + + getLogger().info(f"Collect startup data for iteration {i}.") + runCmdTimestamp = datetime.now() + runCmd = xharnesscommand() + [ + 'apple', + 'mlaunch', + '--', + f'--launchdevbundleid={self.packagename}', + ] + runCmdCommand = RunCommand(runCmd, verbose=True) + + try: + runCmdCommand.run() + except CalledProcessError as ex: + if ex.returncode == 70: + # Exit code 70 from xharness means time out, this can happen sometimes when the device is in a screwed state + # and doesn't correctly launch apps anymore. In that case we reboot the device + getLogger().error("Device is in a broken state, rebooting.") + rebootCmd = xharnesscommand() + [ + 'apple', + 'mlaunch', + '--', + '--rebootdev', + ] + rebootCmdCommand = RunCommand(rebootCmd, verbose=True) + rebootCmdCommand.run() + + getLogger().info("Waiting 30 secs for the device to boot.") + time.sleep(30) + + # if we're in Helix, schedule the work item for retry by writing a special file to the workitem root + if helixworkitemroot(): + getLogger().info("Requesting retry from Helix.") + with open(f"{helixworkitemroot()}/.retry", "w") as retryFile: + retryFile.write("Device was in a broken state, rebooted the device and retrying work item.") + + # rethrow exception so we end the process + getLogger().error("App launch failed, please rerun the script to start a new measurement.") + raise + + app_pid_search = re.search("Launched application.*with pid (?P\d+)", runCmdCommand.stdout) + app_pid = int(app_pid_search.group('app_pid')) + + logarchive_filename = os.path.join(const.TMPDIR, f'iteration{i}.logarchive') + getLogger().info(f"Waiting 5 secs to ensure app with PID {app_pid} is fully started.") + time.sleep(5) + collectCmd = [ + 'sudo', + 'log', + 'collect', + '--device', + '--start', runCmdTimestamp.strftime("%Y-%m-%d %H:%M:%S"), + '--output', logarchive_filename, + ] + RunCommand(collectCmd, verbose=True).run() + + getLogger().info(f"Kill app with PID {app_pid}.") + killCmd = xharnesscommand() + [ + 'apple', + 'mlaunch', + '--', + f'--killdev={app_pid}', + ] + killCmdCommand = RunCommand(killCmd, verbose=True) + killCmdCommand.run() + + # Process Data + + # There are four watchdog events from SpringBoard during an application startup: + # + # [application:770] [realTime] Now monitoring resource allowance of 20.00s (at refreshInterval -1.00s) + # [application:770] [realTime] Stopped monitoring. + # [application:770] [realTime] Now monitoring resource allowance of 19.28s (at refreshInterval -1.00s) + # [application:770] [realTime] Stopped monitoring. + # + # The first two are monitoring the time it takes the OS to create the process, load .dylibs and call into the app's main() + # The second two are monitoring the time it takes the app to draw the first frame of UI from main() + # + # An app has 20 seconds to complete this sequence or the OS will kill the app. + # We collect these log events to do our measurements. + + logShowCmd = [ + 'log', + 'show', + '--predicate', '(process == "SpringBoard") && (category == "Watchdog")', + '--info', + '--style', 'ndjson', + logarchive_filename, + ] + logShowCmdCommand = RunCommand(logShowCmd, verbose=True) + logShowCmdCommand.run() + + events = [] + for line in logShowCmdCommand.stdout.splitlines(): + try: + lineData = json.loads(line) + if 'Now monitoring resource allowance' in lineData['eventMessage'] or 'Stopped monitoring' in lineData['eventMessage']: + events.append (lineData) + except: + break + + # the startup measurement relies on the date/time of the device to be pretty much in sync with the host + # since we use the timestamps from the host to decide which parts of the device log to get and + # we then use that to calculate the time delta from watchdog events + if len(events) != 4: + raise Exception("Didn't get the right amount of watchdog events, this could mean the app crashed or the device clock is not in sync with the host.") + + timeToMainEventStart = events[0] + timeToMainEventStop = events[1] + timeToFirstDrawEventStart = events[2] + timeToFirstDrawEventStop = events[3] + + # validate log messages + if f'application<{self.packagename}>:{app_pid}' not in timeToMainEventStart['eventMessage'] or 'Now monitoring resource allowance of 20.00s' not in timeToMainEventStart['eventMessage']: + raise Exception(f"Invalid timeToMainEventStart: {timeToMainEventStart['eventMessage']}") + + if f'application<{self.packagename}>:{app_pid}' not in timeToMainEventStop['eventMessage'] or 'Stopped monitoring' not in timeToMainEventStop['eventMessage']: + raise Exception(f"Invalid timeToMainEventStop: {timeToMainEventStop['eventMessage']}") + + if f'application<{self.packagename}>:{app_pid}' not in timeToFirstDrawEventStart['eventMessage'] or 'Now monitoring resource allowance of' not in timeToFirstDrawEventStart['eventMessage']: + raise Exception(f"Invalid timeToFirstDrawEventStart: {timeToFirstDrawEventStart['eventMessage']}") + + if f'application<{self.packagename}>:{app_pid}' not in timeToFirstDrawEventStop['eventMessage'] or 'Stopped monitoring' not in timeToFirstDrawEventStop['eventMessage']: + raise Exception(f"Invalid timeToFirstDrawEventStop: {timeToFirstDrawEventStop['eventMessage']}") + + timeToMainEventStartDateTime = datetime.strptime(timeToMainEventStart['timestamp'], '%Y-%m-%d %H:%M:%S.%f%z') + timeToMainEventEndDateTime = datetime.strptime(timeToMainEventStop['timestamp'], '%Y-%m-%d %H:%M:%S.%f%z') + timeToMainMilliseconds = (timeToMainEventEndDateTime - timeToMainEventStartDateTime).total_seconds() * 1000 + + timeToFirstDrawEventStartDateTime = datetime.strptime(timeToFirstDrawEventStart['timestamp'], '%Y-%m-%d %H:%M:%S.%f%z') + timeToFirstDrawEventEndDateTime = datetime.strptime(timeToFirstDrawEventStop['timestamp'], '%Y-%m-%d %H:%M:%S.%f%z') + timeToFirstDrawMilliseconds = (timeToFirstDrawEventEndDateTime - timeToFirstDrawEventStartDateTime).total_seconds() * 1000 + + if self.usefullydrawntime: + # grab log event with the magic string in it + logShowMagicStringCmd = [ + 'log', + 'show', + '--predicate', f'(processIdentifier == {app_pid}) && (composedMessage contains "{self.fullyDrawnMagicString}")', + '--info', + '--style', 'ndjson', + logarchive_filename, + ] + logShowMagicStringCmd = RunCommand(logShowMagicStringCmd, verbose=True) + logShowMagicStringCmd.run() + + magicStringEvent = '' + for line in logShowMagicStringCmd.stdout.splitlines(): + try: + lineData = json.loads(line) + if self.fullyDrawnMagicString in lineData['eventMessage']: + magicStringEvent = lineData + except: + break + + if magicStringEvent == '': + raise Exception("Didn't get the fully-drawn magic string event.") + + timeToMagicStringEventDateTime = datetime.strptime(magicStringEvent['timestamp'], '%Y-%m-%d %H:%M:%S.%f%z') + + # startup time is time to the magic string event + totalTimeMilliseconds = (timeToMagicStringEventDateTime - timeToMainEventStartDateTime).total_seconds() * 1000 + else: + # startup time is time to first draw + totalTimeMilliseconds = timeToMainMilliseconds + timeToFirstDrawMilliseconds + + if i == 0: + # ignore the warmup iteration + getLogger().info(f'Warmup iteration took {totalTimeMilliseconds}') + else: + # TODO: this isn't really a COLD run, we should have separate measurements for starting the app right after install + launchState = 'COLD' + allResults.append(f'LaunchState: {launchState}\nTotalTime: {int(totalTimeMilliseconds)}\nTimeToMain: {int(timeToMainMilliseconds)}\n\n') + + # Done with testing, uninstall the app getLogger().info("Uninstalling app") uninstallAppCmd = xharnesscommand() + [ - 'android', + 'apple', 'uninstall', - '--package-name', - self.packagename + '--app', self.packagename, + '--target', 'ios-device', + '-o', + const.TRACEDIR, + '-v' ] RunCommand(uninstallAppCmd, verbose=True).run() - if screenWasOff: - RunCommand(keyInputCmd + ['26'], verbose=True).run() # Turn the screen back off - # Create traces to store the data so we can keep the current general parse trace flow getLogger().info(f"Logs: \n{allResults}") os.makedirs(f"{const.TRACEDIR}/PerfTest", exist_ok=True) @@ -471,9 +875,9 @@ def run(self): traceFile.close() startup = StartupWrapper() - self.traits.add_traits(overwrite=True, apptorun="app", startupmetric=const.STARTUP_DEVICETIMETOMAIN, tracefolder='PerfTest/', tracename='runoutput.trace', scenarioname='Device Startup - Android %s' % (self.packagename)) + self.traits.add_traits(overwrite=True, apptorun="app", startupmetric=const.STARTUP_DEVICETIMETOMAIN, tracefolder='PerfTest/', tracename='runoutput.trace', scenarioname=self.scenarioname) startup.parsetraces(self.traits) - + elif self.testtype == const.SOD: sod = SODWrapper() builtdir = const.PUBDIR if os.path.exists(const.PUBDIR) else None From bcd5e3c62d8d20655c4ac6035a8d1c15e797a301 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 23 Jan 2023 16:27:55 -0800 Subject: [PATCH 04/11] Remove testing run and reenable sdk_scenarios that where disabled for testing. --- azure-pipelines.yml | 89 +++++++++++++++++++-------------------------- 1 file changed, 37 insertions(+), 52 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c9acdaea990..791fb4fef6e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -559,63 +559,48 @@ jobs: - ${{ if and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Manual')) }}: - # # Windows x64 SDK scenario benchmarks - # - template: /eng/performance/scenarios.yml - # parameters: - # osName: windows - # osVersion: RS5 - # architecture: x64 - # pool: - # vmImage: windows-2019 - # kind: sdk_scenarios - # machinePool: Tiger - # queue: Windows.10.Amd64.19H1.Tiger.Perf - # projectFile: sdk_scenarios.proj - # channels: - # - $(channel) # for manual runs we can specify channel variable - - # # Windows x86 SDK scenario benchmarks - # - template: /eng/performance/scenarios.yml - # parameters: - # osName: windows - # osVersion: RS5 - # architecture: x86 - # pool: - # vmImage: windows-2019 - # kind: sdk_scenarios - # machinePool: Tiger - # queue: Windows.10.Amd64.19H1.Tiger.Perf - # projectFile: sdk_scenarios.proj - # channels: - # - $(channel) # for manual runs we can specify channel variable - - # # Ubuntu 1804 x64 SDK scenario benchmarks - # - template: /eng/performance/scenarios.yml - # parameters: - # osName: ubuntu - # osVersion: 1804 - # architecture: x64 - # pool: - # vmImage: ubuntu-latest - # kind: sdk_scenarios - # machinePool: Tiger - # queue: Ubuntu.1804.Amd64.Tiger.Perf - # container: ubuntu_x64_build_container - # projectFile: sdk_scenarios.proj - # channels: - # - $(channel) + # Windows x64 SDK scenario benchmarks + - template: /eng/performance/scenarios.yml + parameters: + osName: windows + osVersion: RS5 + architecture: x64 + pool: + vmImage: windows-2019 + kind: sdk_scenarios + machinePool: Tiger + queue: Windows.10.Amd64.19H1.Tiger.Perf + projectFile: sdk_scenarios.proj + channels: + - $(channel) # for manual runs we can specify channel variable - # Maui Android scenario benchmarks + # Windows x86 SDK scenario benchmarks - template: /eng/performance/scenarios.yml parameters: osName: windows + osVersion: RS5 + architecture: x86 + pool: + vmImage: windows-2019 + kind: sdk_scenarios + machinePool: Tiger + queue: Windows.10.Amd64.19H1.Tiger.Perf + projectFile: sdk_scenarios.proj + channels: + - $(channel) # for manual runs we can specify channel variable + + # Ubuntu 1804 x64 SDK scenario benchmarks + - template: /eng/performance/scenarios.yml + parameters: + osName: ubuntu + osVersion: 1804 architecture: x64 - osVersion: 19H1 pool: - vmImage: 'windows-2022' - queue: Windows.10.Amd64.Pixel.Perf - machinePool: Pixel - kind: maui_scenarios_android - projectFile: maui_scenarios_android.proj + vmImage: ubuntu-latest + kind: sdk_scenarios + machinePool: Tiger + queue: Ubuntu.1804.Amd64.Tiger.Perf + container: ubuntu_x64_build_container + projectFile: sdk_scenarios.proj channels: - $(channel) From 0ad00dc2ec94542275b64c6345a3d4f443fee30d Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Tue, 24 Jan 2023 10:50:00 -0800 Subject: [PATCH 05/11] Update scenarios.proj to use the prepare scenarios properly. --- eng/performance/scenarios.proj | 99 ++++++++++++++++++++++------------ scripts/dotnet.py | 6 +-- 2 files changed, 68 insertions(+), 37 deletions(-) diff --git a/eng/performance/scenarios.proj b/eng/performance/scenarios.proj index 7f933a111f0..812f290b38f 100644 --- a/eng/performance/scenarios.proj +++ b/eng/performance/scenarios.proj @@ -20,34 +20,73 @@ - + 4:00 - + + + $(Python) post.py + scenarios_out + $(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)\ + $(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)/ + + - $(ScenariosDir)staticconsoletemplate + staticconsoletemplate + $(ScenariosDir)%(ScenarioDirectoryName) - $(ScenariosDir)staticvbconsoletemplate + staticvbconsoletemplate + $(ScenariosDir)%(ScenarioDirectoryName) - - $(ScenariosDir)emptyconsoletemplate + emptyconsoletemplate + $(ScenariosDir)%(ScenarioDirectoryName) - $(ScenariosDir)emptyvbconsoletemplate + emptyvbconsoletemplate + $(ScenariosDir)%(ScenarioDirectoryName) + + + mauidesktop + $(ScenariosDir)%(ScenarioDirectoryName) + + + mauiblazordesktop + $(ScenariosDir)%(ScenarioDirectoryName) + + + + + + + $(Python) pre.py publish -f $(_Framework) -c Release -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName)_fdd + %(PreparePayloadWorkItem.PayloadDirectory) + + + $(Python) pre.py publish -f $(_Framework) -c Release -r $(RID) -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName)_scd + %(PreparePayloadWorkItem.PayloadDirectory) + + + $(Python) pre.py build -c Release -f $(_Framework) -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName)_build + %(PreparePayloadWorkItem.PayloadDirectory) + + + $(Python) pre.py publish -c Release -f $(_Framework)-windows10.0.19041.0 -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName)_fdd + %(PreparePayloadWorkItem.PayloadDirectory) + + - $(Python) pre.py publish -f $(_Framework) -c Release + xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)_fdd %HELIX_WORKITEM_ROOT%\pub /E /I /Y + cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)_fdd $HELIX_WORKITEM_ROOT/pub $(Python) test.py startup --scenario-name "%(Identity)" @@ -55,7 +94,8 @@ - $(Python) pre.py publish -f $(_Framework) -c Release + xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)_fdd %HELIX_WORKITEM_ROOT%\pub /E /I /Y + cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)_fdd $HELIX_WORKITEM_ROOT/pub $(Python) test.py sod --scenario-name "%(Identity)" @@ -63,7 +103,8 @@ - $(Python) pre.py publish -f $(_Framework) -c Release -r $(RID) + xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)_scd %HELIX_WORKITEM_ROOT%\pub /E /I /Y + cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)_scd $HELIX_WORKITEM_ROOT/pub $(Python) test.py sod --scenario-name "%(Identity)" @@ -71,38 +112,29 @@ - $(Python) pre.py build -c Release -f $(_Framework) + xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)_build %HELIX_WORKITEM_ROOT%\bin /E /I /Y + cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)_build $HELIX_WORKITEM_ROOT/bin $(Python) test.py sod --scenario-name "%(Identity)" - + - - $(ScenariosDir)mauidesktop - $(Python) pre.py publish -c Release -f $(_Framework)-windows10.0.19041.0 + + xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)_fdd %HELIX_WORKITEM_ROOT%\pub /E /I /Y + cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)_fdd $HELIX_WORKITEM_ROOT/pub $(Python) test.py sod --scenario-name "%(Identity)" - - $(ScenariosDir)mauidesktop - $(Python) pre.py publish -c Release -f $(_Framework)-windows10.0.19041.0 + + xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)_fdd %HELIX_WORKITEM_ROOT%\pub /E /I /Y + cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)_fdd $HELIX_WORKITEM_ROOT/pub $(Python) test.py startup --scenario-name "%(Identity)" - - - - $(ScenariosDir)mauiblazordesktop - $(Python) pre.py publish -c Release -f $(_Framework)-windows10.0.19041.0 - $(Python) test.py sod --scenario-name "%(Identity)" - - - $(ScenariosDir)mauiblazordesktop - $(Python) pre.py publish -c Release -f $(_Framework)-windows10.0.19041.0 - $(Python) test.py startup --scenario-name "%(Identity)" - - + + + - diff --git a/scripts/dotnet.py b/scripts/dotnet.py index ad7904e0094..823f7b81e5a 100755 --- a/scripts/dotnet.py +++ b/scripts/dotnet.py @@ -308,7 +308,7 @@ def build(self, target_framework_monikers: list = None, output_to_bindir: bool = False, runtime_identifier: str = None, - *args) -> None: + args: list = None) -> None: '''Calls dotnet to build the specified project.''' if not target_framework_monikers: # Build all supported frameworks. cmdline = [ @@ -327,7 +327,7 @@ def build(self, cmdline = cmdline + ['--runtime', runtime_identifier] if args: - cmdline = cmdline + list(args) + cmdline = cmdline + args RunCommand(cmdline, verbose=verbose).run( self.working_directory) @@ -351,7 +351,7 @@ def build(self, cmdline = cmdline + ['--runtime', runtime_identifier] if args: - cmdline = cmdline + list(args) + cmdline = cmdline + args RunCommand(cmdline, verbose=verbose).run( self.working_directory) From 45e43356a598a1eed1f41105b0c34b0130025034 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Tue, 24 Jan 2023 14:00:44 -0800 Subject: [PATCH 06/11] Fix maui desktop scenario runs and make it so that they do not uninstall maui since we prepare the runs in the cloud. --- src/scenarios/mauiblazordesktop/post.py | 4 +- src/scenarios/mauiblazordesktop/pre.py | 53 ----------------------- src/scenarios/mauiblazordesktop/test.py | 56 +++++++++++++++++++++++++ src/scenarios/mauidesktop/post.py | 4 +- 4 files changed, 58 insertions(+), 59 deletions(-) diff --git a/src/scenarios/mauiblazordesktop/post.py b/src/scenarios/mauiblazordesktop/post.py index 35392ac63cc..c87a49b4b7d 100644 --- a/src/scenarios/mauiblazordesktop/post.py +++ b/src/scenarios/mauiblazordesktop/post.py @@ -2,8 +2,6 @@ post cleanup script ''' -from shared.postcommands import PostCommands, clean_directories +from shared.postcommands import clean_directories -postcommands = PostCommands() clean_directories() -postcommands.uninstall_workload('maui') \ No newline at end of file diff --git a/src/scenarios/mauiblazordesktop/pre.py b/src/scenarios/mauiblazordesktop/pre.py index 6639a9f563d..f64002adde2 100644 --- a/src/scenarios/mauiblazordesktop/pre.py +++ b/src/scenarios/mauiblazordesktop/pre.py @@ -15,62 +15,9 @@ setup_loggers(True) NugetURL = 'https://raw.githubusercontent.com/dotnet/maui/net6.0/NuGet.config' -WebViewURL = 'https://go.microsoft.com/fwlink/p/?LinkId=2124703' # Obtained from https://developer.microsoft.com/en-us/microsoft-edge/webview2/#download-section -WebViewInstalled = False NugetFile = requests.get(NugetURL) open('./Nuget.config', 'wb').write(NugetFile.content) -lmkey = r"SOFTWARE\WOW6432Node\Microsoft\EdgeUpdate\Clients\{F3017226-FE2A-4295-8BDF-00C3A9A7E4C5}" -cukey = r"Software\Microsoft\EdgeUpdate\Clients\{F3017226-FE2A-4295-8BDF-00C3A9A7E4C5}" -with winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) as hklm_hive: - try: - with winreg.OpenKey(hklm_hive, lmkey) as openkey: - pvvalue = winreg.QueryValueEx(openkey, 'pv')[0] - if pvvalue and pvvalue != '' and pvvalue != '0.0.0.0': - WebViewInstalled = True - getLogger().info(f"WebView Found; pvvalue(version) {pvvalue}") - except: - getLogger().warning("WebView not verified in Local_Machine Registry") -if not WebViewInstalled: - try: - with winreg.ConnectRegistry(None, winreg.HKEY_CURRENT_USER) as hkcu_hive: - with winreg.OpenKey(hkcu_hive, cukey) as openkey: - pvvalue = winreg.QueryValueEx(openkey, 'pv')[0] - if pvvalue and pvvalue != '' and pvvalue != '0.0.0.0': - WebViewInstalled = True - getLogger().info(f"WebView Found; pvvalue(version) {pvvalue}") - except: - getLogger().warning("WebView not verified in Current_Machine Registry") -if not WebViewInstalled: - getLogger().info("Installing WebView2") - WebViewInstallFile = requests.get(WebViewURL) - open('./MicrosoftEdgeWebview2Setup.exe', 'wb').write(WebViewInstallFile.content) - subprocess.run(['powershell', '-Command', r'Start-Process "./MicrosoftEdgeWebview2Setup.exe" -Wait'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) - with winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) as hklm_hive: - try: - with winreg.OpenKey(hklm_hive, lmkey) as openkey: - pvvalue = winreg.QueryValueEx(openkey, 'pv')[0] - if pvvalue and pvvalue != '' and pvvalue != '0.0.0.0': - WebViewInstalled = True - getLogger().info(f"WebView Found; pvvalue(version) {pvvalue}") - except: - getLogger().warning("WebView not verified in Local_Machine Registry") - if not WebViewInstalled: - try: - with winreg.ConnectRegistry(None, winreg.HKEY_CURRENT_USER) as hkcu_hive: - with winreg.OpenKey(hkcu_hive, cukey) as openkey: - pvvalue = winreg.QueryValueEx(openkey, 'pv')[0] - if pvvalue and pvvalue != '' and pvvalue != '0.0.0.0': - WebViewInstalled = True - getLogger().info(f"WebView Found; pvvalue(version) {pvvalue}") - except: - getLogger().warning("WebView not verified in Current_Machine Registry.") - getLogger().error("Blazor cannot run without WebView installed, exiting execution.") - sys.exit(-1) -else: - getLogger().info("WebViewAlreadyInstalled") - - precommands = PreCommands() precommands.install_workload('maui', ['--from-rollback-file', 'https://aka.ms/dotnet/maui/net6.0.json', '--configfile', './Nuget.config']) precommands.new(template='maui-blazor', diff --git a/src/scenarios/mauiblazordesktop/test.py b/src/scenarios/mauiblazordesktop/test.py index 7fe688b7567..b6fe3798077 100644 --- a/src/scenarios/mauiblazordesktop/test.py +++ b/src/scenarios/mauiblazordesktop/test.py @@ -5,6 +5,8 @@ EXENAME = 'MauiBlazorDesktopTesting' def main(): + setup_loggers(True) + install_webview() traits = TestTraits(exename=EXENAME, guiapp='true', startupmetric='WinUIBlazor', @@ -16,6 +18,60 @@ def main(): runner = Runner(traits) runner.run() +def install_webview(): + WebViewURL = 'https://go.microsoft.com/fwlink/p/?LinkId=2124703' # Obtained from https://developer.microsoft.com/en-us/microsoft-edge/webview2/#download-section + WebViewInstalled = False + lmkey = r"SOFTWARE\WOW6432Node\Microsoft\EdgeUpdate\Clients\{F3017226-FE2A-4295-8BDF-00C3A9A7E4C5}" + cukey = r"Software\Microsoft\EdgeUpdate\Clients\{F3017226-FE2A-4295-8BDF-00C3A9A7E4C5}" + with winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) as hklm_hive: + try: + with winreg.OpenKey(hklm_hive, lmkey) as openkey: + pvvalue = winreg.QueryValueEx(openkey, 'pv')[0] + if pvvalue and pvvalue != '' and pvvalue != '0.0.0.0': + WebViewInstalled = True + getLogger().info(f"WebView Found; pvvalue(version) {pvvalue}") + except: + getLogger().warning("WebView not verified in Local_Machine Registry") + if not WebViewInstalled: + try: + with winreg.ConnectRegistry(None, winreg.HKEY_CURRENT_USER) as hkcu_hive: + with winreg.OpenKey(hkcu_hive, cukey) as openkey: + pvvalue = winreg.QueryValueEx(openkey, 'pv')[0] + if pvvalue and pvvalue != '' and pvvalue != '0.0.0.0': + WebViewInstalled = True + getLogger().info(f"WebView Found; pvvalue(version) {pvvalue}") + except: + getLogger().warning("WebView not verified in Current_Machine Registry") + if not WebViewInstalled: + getLogger().info("Installing WebView2") + WebViewInstallFile = requests.get(WebViewURL) + open('./MicrosoftEdgeWebview2Setup.exe', 'wb').write(WebViewInstallFile.content) + subprocess.run(['powershell', '-Command', r'Start-Process "./MicrosoftEdgeWebview2Setup.exe" -Wait'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) + with winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) as hklm_hive: + try: + with winreg.OpenKey(hklm_hive, lmkey) as openkey: + pvvalue = winreg.QueryValueEx(openkey, 'pv')[0] + if pvvalue and pvvalue != '' and pvvalue != '0.0.0.0': + WebViewInstalled = True + getLogger().info(f"WebView Found; pvvalue(version) {pvvalue}") + except: + getLogger().warning("WebView not verified in Local_Machine Registry") + if not WebViewInstalled: + try: + with winreg.ConnectRegistry(None, winreg.HKEY_CURRENT_USER) as hkcu_hive: + with winreg.OpenKey(hkcu_hive, cukey) as openkey: + pvvalue = winreg.QueryValueEx(openkey, 'pv')[0] + if pvvalue and pvvalue != '' and pvvalue != '0.0.0.0': + WebViewInstalled = True + getLogger().info(f"WebView Found; pvvalue(version) {pvvalue}") + except: + getLogger().warning("WebView not verified in Current_Machine Registry.") + getLogger().error("Blazor cannot run without WebView installed, exiting execution.") + sys.exit(-1) + else: + getLogger().info("WebViewAlreadyInstalled") + + if __name__ == "__main__": result = subprocess.run(['powershell', '-Command', r'Get-ChildItem .\pub\Microsoft.Maui.dll | Select-Object -ExpandProperty VersionInfo | Select-Object ProductVersion | Select-Object -ExpandProperty ProductVersion'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) diff --git a/src/scenarios/mauidesktop/post.py b/src/scenarios/mauidesktop/post.py index 35392ac63cc..c87a49b4b7d 100644 --- a/src/scenarios/mauidesktop/post.py +++ b/src/scenarios/mauidesktop/post.py @@ -2,8 +2,6 @@ post cleanup script ''' -from shared.postcommands import PostCommands, clean_directories +from shared.postcommands import clean_directories -postcommands = PostCommands() clean_directories() -postcommands.uninstall_workload('maui') \ No newline at end of file From cd877e21bb0476a00f94c02223877c1f6786639a Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Tue, 24 Jan 2023 15:51:05 -0800 Subject: [PATCH 07/11] Remove dotnet packs before sending payloads to helix. --- eng/performance/scenarios.proj | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/eng/performance/scenarios.proj b/eng/performance/scenarios.proj index 812f290b38f..f8b357a5d0e 100644 --- a/eng/performance/scenarios.proj +++ b/eng/performance/scenarios.proj @@ -32,6 +32,11 @@ $(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)/ + + + + + From f9546afa434aba16ed24fa57b0f99d6d7f8c2b67 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Wed, 25 Jan 2023 09:43:42 -0800 Subject: [PATCH 08/11] Force Nuget.Config overwrite so xcopy doesn't prompt for overwrite. --- eng/performance/scenarios.yml | 2 +- src/scenarios/mauiblazordesktop/pre.py | 1 - src/scenarios/mauiblazordesktop/test.py | 5 ++++- src/scenarios/mauidesktop/test.py | 1 + 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/eng/performance/scenarios.yml b/eng/performance/scenarios.yml index b73541f5014..521fa389622 100644 --- a/eng/performance/scenarios.yml +++ b/eng/performance/scenarios.yml @@ -27,7 +27,7 @@ jobs: - name: CorrelationStaging value: $(Build.SourcesDirectory)\CorrelationStaging\ - name: AdditionalHelixPreCommands - value: 'call %HELIX_CORRELATION_PAYLOAD%\machine-setup$(ScriptExtension);xcopy %HELIX_CORRELATION_PAYLOAD%\NuGet.config %HELIX_WORKITEM_ROOT%' + value: 'call %HELIX_CORRELATION_PAYLOAD%\machine-setup$(ScriptExtension);xcopy %HELIX_CORRELATION_PAYLOAD%\NuGet.config %HELIX_WORKITEM_ROOT% /Y' - name: PreservePythonPath value: 'set ORIGPYPATH=%PYTHONPATH%' - name: HelixPostCommand diff --git a/src/scenarios/mauiblazordesktop/pre.py b/src/scenarios/mauiblazordesktop/pre.py index f64002adde2..f3370a0d0b5 100644 --- a/src/scenarios/mauiblazordesktop/pre.py +++ b/src/scenarios/mauiblazordesktop/pre.py @@ -11,7 +11,6 @@ from shared import const from test import EXENAME import requests -import winreg setup_loggers(True) NugetURL = 'https://raw.githubusercontent.com/dotnet/maui/net6.0/NuGet.config' diff --git a/src/scenarios/mauiblazordesktop/test.py b/src/scenarios/mauiblazordesktop/test.py index b6fe3798077..4004521a265 100644 --- a/src/scenarios/mauiblazordesktop/test.py +++ b/src/scenarios/mauiblazordesktop/test.py @@ -1,11 +1,14 @@ import os import subprocess +import sys from shared.runner import TestTraits, Runner +from performance.logger import setup_loggers, getLogger +import winreg +import requests EXENAME = 'MauiBlazorDesktopTesting' def main(): - setup_loggers(True) install_webview() traits = TestTraits(exename=EXENAME, guiapp='true', diff --git a/src/scenarios/mauidesktop/test.py b/src/scenarios/mauidesktop/test.py index 80e7f513478..c0adf823d85 100644 --- a/src/scenarios/mauidesktop/test.py +++ b/src/scenarios/mauidesktop/test.py @@ -1,6 +1,7 @@ import os import subprocess from shared.runner import TestTraits, Runner +from performance.logger import setup_loggers EXENAME = 'MauiDesktopTesting' From 58c0e2182c8500fc317c6ae216b1eb386591df1f Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Wed, 25 Jan 2023 14:25:30 -0800 Subject: [PATCH 09/11] Add logger setup for pre-runner.py setup information. --- src/scenarios/mauiblazordesktop/test.py | 1 + src/scenarios/mauidesktop/test.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/scenarios/mauiblazordesktop/test.py b/src/scenarios/mauiblazordesktop/test.py index 4004521a265..df157b21ead 100644 --- a/src/scenarios/mauiblazordesktop/test.py +++ b/src/scenarios/mauiblazordesktop/test.py @@ -9,6 +9,7 @@ EXENAME = 'MauiBlazorDesktopTesting' def main(): + setup_loggers(True) install_webview() traits = TestTraits(exename=EXENAME, guiapp='true', diff --git a/src/scenarios/mauidesktop/test.py b/src/scenarios/mauidesktop/test.py index c0adf823d85..1ff702d3988 100644 --- a/src/scenarios/mauidesktop/test.py +++ b/src/scenarios/mauidesktop/test.py @@ -6,6 +6,7 @@ EXENAME = 'MauiDesktopTesting' def main(): + setup_loggers(True) traits = TestTraits(exename=EXENAME, guiapp='true', startupmetric='WinUI', From 439256b9444717985dc21847d62db8e0126e3564 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Wed, 25 Jan 2023 14:28:34 -0800 Subject: [PATCH 10/11] Setup better testing. --- azure-pipelines.yml | 392 ++++++++++++++++----------------- eng/performance/scenarios.proj | 8 +- 2 files changed, 200 insertions(+), 200 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 791fb4fef6e..3b83ababf3c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -58,213 +58,213 @@ jobs: channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks - release/6.0 - # Ubuntu 1804 x64 scenario benchmarks - - template: /eng/performance/scenarios.yml - parameters: - osName: ubuntu - osVersion: 1804 - kind: scenarios - architecture: x64 - pool: - vmImage: ubuntu-latest - machinePool: Open - queue: Ubuntu.1804.Amd64.Open - container: ubuntu_x64_build_container - projectFile: scenarios.proj - channels: - - release/6.0 + # # Ubuntu 1804 x64 scenario benchmarks + # - template: /eng/performance/scenarios.yml + # parameters: + # osName: ubuntu + # osVersion: 1804 + # kind: scenarios + # architecture: x64 + # pool: + # vmImage: ubuntu-latest + # machinePool: Open + # queue: Ubuntu.1804.Amd64.Open + # container: ubuntu_x64_build_container + # projectFile: scenarios.proj + # channels: + # - release/6.0 - # Windows x64 Blazor scenario benchmarks - - template: /eng/performance/scenarios.yml - parameters: - osName: windows - osVersion: 22H2 - architecture: x64 - pool: - vmImage: windows-2019 - kind: blazor_scenarios - machinePool: Open - queue: Windows.10.Amd64.Client.Open - projectFile: blazor_scenarios.proj - channels: - - release/6.0 - - # Windows x64 SDK scenario benchmarks - - template: /eng/performance/scenarios.yml - parameters: - osName: windows - osVersion: 22H2 - architecture: x64 - pool: - vmImage: windows-2019 - kind: sdk_scenarios - machinePool: Open - queue: Windows.10.Amd64.Client.Open - projectFile: sdk_scenarios.proj - channels: - - release/6.0 + # # Windows x64 Blazor scenario benchmarks + # - template: /eng/performance/scenarios.yml + # parameters: + # osName: windows + # osVersion: 22H2 + # architecture: x64 + # pool: + # vmImage: windows-2019 + # kind: blazor_scenarios + # machinePool: Open + # queue: Windows.10.Amd64.Client.Open + # projectFile: blazor_scenarios.proj + # channels: + # - release/6.0 + + # # Windows x64 SDK scenario benchmarks + # - template: /eng/performance/scenarios.yml + # parameters: + # osName: windows + # osVersion: 22H2 + # architecture: x64 + # pool: + # vmImage: windows-2019 + # kind: sdk_scenarios + # machinePool: Open + # queue: Windows.10.Amd64.Client.Open + # projectFile: sdk_scenarios.proj + # channels: + # - release/6.0 - # Ubuntu 1804 x64 SDK scenario benchmarks - - template: /eng/performance/scenarios.yml - parameters: - osName: ubuntu - osVersion: 1804 - kind: sdk_scenarios - architecture: x64 - pool: - vmImage: ubuntu-latest - machinePool: Open - queue: Ubuntu.1804.Amd64.Open - container: ubuntu_x64_build_container - projectFile: sdk_scenarios.proj - channels: - - release/6.0 + # # Ubuntu 1804 x64 SDK scenario benchmarks + # - template: /eng/performance/scenarios.yml + # parameters: + # osName: ubuntu + # osVersion: 1804 + # kind: sdk_scenarios + # architecture: x64 + # pool: + # vmImage: ubuntu-latest + # machinePool: Open + # queue: Ubuntu.1804.Amd64.Open + # container: ubuntu_x64_build_container + # projectFile: sdk_scenarios.proj + # channels: + # - release/6.0 - # Windows x86 SDK scenario benchmarks - - template: /eng/performance/scenarios.yml - parameters: - osName: windows - osVersion: 22H2 - architecture: x86 - pool: - vmImage: windows-2019 - kind: sdk_scenarios - machinePool: Open - queue: Windows.10.Amd64.Client.Open - projectFile: sdk_scenarios.proj - channels: - - release/6.0 + # # Windows x86 SDK scenario benchmarks + # - template: /eng/performance/scenarios.yml + # parameters: + # osName: windows + # osVersion: 22H2 + # architecture: x86 + # pool: + # vmImage: windows-2019 + # kind: sdk_scenarios + # machinePool: Open + # queue: Windows.10.Amd64.Client.Open + # projectFile: sdk_scenarios.proj + # channels: + # - release/6.0 - # Windows x64 micro benchmarks - - template: /eng/performance/benchmark_jobs.yml - parameters: - osName: windows - osVersion: 22H2 - kind: micro - architecture: x64 - pool: - vmImage: windows-2019 - machinePool: Open - queue: Windows.10.Amd64.Client.Open - csproj: src\benchmarks\micro\MicroBenchmarks.csproj - runCategories: 'runtime libraries' - channels: - - release/6.0 + # # Windows x64 micro benchmarks + # - template: /eng/performance/benchmark_jobs.yml + # parameters: + # osName: windows + # osVersion: 22H2 + # kind: micro + # architecture: x64 + # pool: + # vmImage: windows-2019 + # machinePool: Open + # queue: Windows.10.Amd64.Client.Open + # csproj: src\benchmarks\micro\MicroBenchmarks.csproj + # runCategories: 'runtime libraries' + # channels: + # - release/6.0 - # Windows x64 net461 micro benchmarks - - template: /eng/performance/benchmark_jobs.yml - parameters: - osName: windows - osVersion: RS4 - kind: micro_net461 - architecture: x64 - pool: - vmImage: windows-2019 - machinePool: Open - queue: Windows.10.Amd64.ClientRS5.Open - csproj: src\benchmarks\micro\MicroBenchmarks.csproj - runCategories: 'runtime libraries' - channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks - - LTS # use LTS channel for net461 framework + # # Windows x64 net461 micro benchmarks + # - template: /eng/performance/benchmark_jobs.yml + # parameters: + # osName: windows + # osVersion: RS4 + # kind: micro_net461 + # architecture: x64 + # pool: + # vmImage: windows-2019 + # machinePool: Open + # queue: Windows.10.Amd64.ClientRS5.Open + # csproj: src\benchmarks\micro\MicroBenchmarks.csproj + # runCategories: 'runtime libraries' + # channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks + # - LTS # use LTS channel for net461 framework - # Windows x86 micro benchmarks - - template: /eng/performance/benchmark_jobs.yml - parameters: - osName: windows - osVersion: 22H2 - kind: micro - architecture: x86 - pool: - vmImage: windows-2019 - machinePool: Open - queue: Windows.10.Amd64.Client.Open - csproj: src\benchmarks\micro\MicroBenchmarks.csproj - runCategories: 'runtime libraries' - channels: # for public jobs we want to make sure that the PRs don't break x86 - - release/6.0 + # # Windows x86 micro benchmarks + # - template: /eng/performance/benchmark_jobs.yml + # parameters: + # osName: windows + # osVersion: 22H2 + # kind: micro + # architecture: x86 + # pool: + # vmImage: windows-2019 + # machinePool: Open + # queue: Windows.10.Amd64.Client.Open + # csproj: src\benchmarks\micro\MicroBenchmarks.csproj + # runCategories: 'runtime libraries' + # channels: # for public jobs we want to make sure that the PRs don't break x86 + # - release/6.0 - # Windows x64 ML.NET benchmarks - - template: /eng/performance/benchmark_jobs.yml - parameters: - osName: windows - osVersion: 22H2 - kind: mlnet - architecture: x64 - pool: - vmImage: windows-2019 - machinePool: Open - queue: Windows.10.Amd64.Client.Open - csproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj - runCategories: 'mldotnet' - channels: # for ML.NET jobs we want to check .NET Core 3.1 and 5.0 only - - release/6.0 + # # Windows x64 ML.NET benchmarks + # - template: /eng/performance/benchmark_jobs.yml + # parameters: + # osName: windows + # osVersion: 22H2 + # kind: mlnet + # architecture: x64 + # pool: + # vmImage: windows-2019 + # machinePool: Open + # queue: Windows.10.Amd64.Client.Open + # csproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj + # runCategories: 'mldotnet' + # channels: # for ML.NET jobs we want to check .NET Core 3.1 and 5.0 only + # - release/6.0 - # Windows x64 Roslyn benchmarks - - template: /eng/performance/benchmark_jobs.yml - parameters: - osName: windows - osVersion: 22H2 - kind: roslyn - architecture: x64 - pool: - vmImage: windows-2019 - machinePool: Open - queue: Windows.10.Amd64.Client.Open - csproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj - runCategories: 'roslyn' - channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only - - release/6.0 + # # Windows x64 Roslyn benchmarks + # - template: /eng/performance/benchmark_jobs.yml + # parameters: + # osName: windows + # osVersion: 22H2 + # kind: roslyn + # architecture: x64 + # pool: + # vmImage: windows-2019 + # machinePool: Open + # queue: Windows.10.Amd64.Client.Open + # csproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj + # runCategories: 'roslyn' + # channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only + # - release/6.0 - # Ubuntu 1804 x64 micro benchmarks - - template: /eng/performance/benchmark_jobs.yml - parameters: - osName: ubuntu - osVersion: 1804 - kind: micro - architecture: x64 - pool: - vmImage: ubuntu-latest - machinePool: Open - queue: Ubuntu.1804.Amd64.Open - container: ubuntu_x64_build_container - csproj: src/benchmarks/micro/MicroBenchmarks.csproj - runCategories: 'runtime libraries' - channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks - - release/6.0 + # # Ubuntu 1804 x64 micro benchmarks + # - template: /eng/performance/benchmark_jobs.yml + # parameters: + # osName: ubuntu + # osVersion: 1804 + # kind: micro + # architecture: x64 + # pool: + # vmImage: ubuntu-latest + # machinePool: Open + # queue: Ubuntu.1804.Amd64.Open + # container: ubuntu_x64_build_container + # csproj: src/benchmarks/micro/MicroBenchmarks.csproj + # runCategories: 'runtime libraries' + # channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks + # - release/6.0 - # Ubuntu 1804 x64 ML.NET benchmarks - - template: /eng/performance/benchmark_jobs.yml - parameters: - osName: ubuntu - osVersion: 1804 - kind: mlnet - architecture: x64 - pool: - vmImage: ubuntu-latest - machinePool: Open - queue: Ubuntu.1804.Amd64.Open - container: ubuntu_x64_build_container - runCategories: 'mldotnet' - csproj: src/benchmarks/real-world/Microsoft.ML.Benchmarks/Microsoft.ML.Benchmarks.csproj - channels: # for ML.NET jobs we want to check .NET Core 3.1 and 5.0 only - - release/6.0 + # # Ubuntu 1804 x64 ML.NET benchmarks + # - template: /eng/performance/benchmark_jobs.yml + # parameters: + # osName: ubuntu + # osVersion: 1804 + # kind: mlnet + # architecture: x64 + # pool: + # vmImage: ubuntu-latest + # machinePool: Open + # queue: Ubuntu.1804.Amd64.Open + # container: ubuntu_x64_build_container + # runCategories: 'mldotnet' + # csproj: src/benchmarks/real-world/Microsoft.ML.Benchmarks/Microsoft.ML.Benchmarks.csproj + # channels: # for ML.NET jobs we want to check .NET Core 3.1 and 5.0 only + # - release/6.0 - # Ubuntu 1804 x64 Roslyn benchmarks - - template: /eng/performance/benchmark_jobs.yml - parameters: - osName: ubuntu - osVersion: 1804 - kind: roslyn - architecture: x64 - pool: - vmImage: ubuntu-latest - machinePool: Open - queue: Ubuntu.1804.Amd64.Open - container: ubuntu_x64_build_container - runCategories: 'roslyn' - csproj: src/benchmarks/real-world/Roslyn/CompilerBenchmarks.csproj - channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only - - release/6.0 + # # Ubuntu 1804 x64 Roslyn benchmarks + # - template: /eng/performance/benchmark_jobs.yml + # parameters: + # osName: ubuntu + # osVersion: 1804 + # kind: roslyn + # architecture: x64 + # pool: + # vmImage: ubuntu-latest + # machinePool: Open + # queue: Ubuntu.1804.Amd64.Open + # container: ubuntu_x64_build_container + # runCategories: 'roslyn' + # csproj: src/benchmarks/real-world/Roslyn/CompilerBenchmarks.csproj + # channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only + # - release/6.0 ########################################### # Private Jobs diff --git a/eng/performance/scenarios.proj b/eng/performance/scenarios.proj index f8b357a5d0e..fb4b6bfcae6 100644 --- a/eng/performance/scenarios.proj +++ b/eng/performance/scenarios.proj @@ -39,7 +39,7 @@ - + mauidesktop @@ -68,7 +68,7 @@ - + $(Python) pre.py publish -c Release -f $(_Framework)-windows10.0.19041.0 -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName)_fdd %(PreparePayloadWorkItem.PayloadDirectory) From 30621be9c458766973f8888304f5785b62d64e64 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Wed, 25 Jan 2023 15:47:28 -0800 Subject: [PATCH 11/11] Revert "Setup better testing." This reverts commit 439256b9444717985dc21847d62db8e0126e3564. --- azure-pipelines.yml | 392 ++++++++++++++++----------------- eng/performance/scenarios.proj | 8 +- 2 files changed, 200 insertions(+), 200 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3b83ababf3c..791fb4fef6e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -58,213 +58,213 @@ jobs: channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks - release/6.0 - # # Ubuntu 1804 x64 scenario benchmarks - # - template: /eng/performance/scenarios.yml - # parameters: - # osName: ubuntu - # osVersion: 1804 - # kind: scenarios - # architecture: x64 - # pool: - # vmImage: ubuntu-latest - # machinePool: Open - # queue: Ubuntu.1804.Amd64.Open - # container: ubuntu_x64_build_container - # projectFile: scenarios.proj - # channels: - # - release/6.0 + # Ubuntu 1804 x64 scenario benchmarks + - template: /eng/performance/scenarios.yml + parameters: + osName: ubuntu + osVersion: 1804 + kind: scenarios + architecture: x64 + pool: + vmImage: ubuntu-latest + machinePool: Open + queue: Ubuntu.1804.Amd64.Open + container: ubuntu_x64_build_container + projectFile: scenarios.proj + channels: + - release/6.0 - # # Windows x64 Blazor scenario benchmarks - # - template: /eng/performance/scenarios.yml - # parameters: - # osName: windows - # osVersion: 22H2 - # architecture: x64 - # pool: - # vmImage: windows-2019 - # kind: blazor_scenarios - # machinePool: Open - # queue: Windows.10.Amd64.Client.Open - # projectFile: blazor_scenarios.proj - # channels: - # - release/6.0 - - # # Windows x64 SDK scenario benchmarks - # - template: /eng/performance/scenarios.yml - # parameters: - # osName: windows - # osVersion: 22H2 - # architecture: x64 - # pool: - # vmImage: windows-2019 - # kind: sdk_scenarios - # machinePool: Open - # queue: Windows.10.Amd64.Client.Open - # projectFile: sdk_scenarios.proj - # channels: - # - release/6.0 + # Windows x64 Blazor scenario benchmarks + - template: /eng/performance/scenarios.yml + parameters: + osName: windows + osVersion: 22H2 + architecture: x64 + pool: + vmImage: windows-2019 + kind: blazor_scenarios + machinePool: Open + queue: Windows.10.Amd64.Client.Open + projectFile: blazor_scenarios.proj + channels: + - release/6.0 + + # Windows x64 SDK scenario benchmarks + - template: /eng/performance/scenarios.yml + parameters: + osName: windows + osVersion: 22H2 + architecture: x64 + pool: + vmImage: windows-2019 + kind: sdk_scenarios + machinePool: Open + queue: Windows.10.Amd64.Client.Open + projectFile: sdk_scenarios.proj + channels: + - release/6.0 - # # Ubuntu 1804 x64 SDK scenario benchmarks - # - template: /eng/performance/scenarios.yml - # parameters: - # osName: ubuntu - # osVersion: 1804 - # kind: sdk_scenarios - # architecture: x64 - # pool: - # vmImage: ubuntu-latest - # machinePool: Open - # queue: Ubuntu.1804.Amd64.Open - # container: ubuntu_x64_build_container - # projectFile: sdk_scenarios.proj - # channels: - # - release/6.0 + # Ubuntu 1804 x64 SDK scenario benchmarks + - template: /eng/performance/scenarios.yml + parameters: + osName: ubuntu + osVersion: 1804 + kind: sdk_scenarios + architecture: x64 + pool: + vmImage: ubuntu-latest + machinePool: Open + queue: Ubuntu.1804.Amd64.Open + container: ubuntu_x64_build_container + projectFile: sdk_scenarios.proj + channels: + - release/6.0 - # # Windows x86 SDK scenario benchmarks - # - template: /eng/performance/scenarios.yml - # parameters: - # osName: windows - # osVersion: 22H2 - # architecture: x86 - # pool: - # vmImage: windows-2019 - # kind: sdk_scenarios - # machinePool: Open - # queue: Windows.10.Amd64.Client.Open - # projectFile: sdk_scenarios.proj - # channels: - # - release/6.0 + # Windows x86 SDK scenario benchmarks + - template: /eng/performance/scenarios.yml + parameters: + osName: windows + osVersion: 22H2 + architecture: x86 + pool: + vmImage: windows-2019 + kind: sdk_scenarios + machinePool: Open + queue: Windows.10.Amd64.Client.Open + projectFile: sdk_scenarios.proj + channels: + - release/6.0 - # # Windows x64 micro benchmarks - # - template: /eng/performance/benchmark_jobs.yml - # parameters: - # osName: windows - # osVersion: 22H2 - # kind: micro - # architecture: x64 - # pool: - # vmImage: windows-2019 - # machinePool: Open - # queue: Windows.10.Amd64.Client.Open - # csproj: src\benchmarks\micro\MicroBenchmarks.csproj - # runCategories: 'runtime libraries' - # channels: - # - release/6.0 + # Windows x64 micro benchmarks + - template: /eng/performance/benchmark_jobs.yml + parameters: + osName: windows + osVersion: 22H2 + kind: micro + architecture: x64 + pool: + vmImage: windows-2019 + machinePool: Open + queue: Windows.10.Amd64.Client.Open + csproj: src\benchmarks\micro\MicroBenchmarks.csproj + runCategories: 'runtime libraries' + channels: + - release/6.0 - # # Windows x64 net461 micro benchmarks - # - template: /eng/performance/benchmark_jobs.yml - # parameters: - # osName: windows - # osVersion: RS4 - # kind: micro_net461 - # architecture: x64 - # pool: - # vmImage: windows-2019 - # machinePool: Open - # queue: Windows.10.Amd64.ClientRS5.Open - # csproj: src\benchmarks\micro\MicroBenchmarks.csproj - # runCategories: 'runtime libraries' - # channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks - # - LTS # use LTS channel for net461 framework + # Windows x64 net461 micro benchmarks + - template: /eng/performance/benchmark_jobs.yml + parameters: + osName: windows + osVersion: RS4 + kind: micro_net461 + architecture: x64 + pool: + vmImage: windows-2019 + machinePool: Open + queue: Windows.10.Amd64.ClientRS5.Open + csproj: src\benchmarks\micro\MicroBenchmarks.csproj + runCategories: 'runtime libraries' + channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks + - LTS # use LTS channel for net461 framework - # # Windows x86 micro benchmarks - # - template: /eng/performance/benchmark_jobs.yml - # parameters: - # osName: windows - # osVersion: 22H2 - # kind: micro - # architecture: x86 - # pool: - # vmImage: windows-2019 - # machinePool: Open - # queue: Windows.10.Amd64.Client.Open - # csproj: src\benchmarks\micro\MicroBenchmarks.csproj - # runCategories: 'runtime libraries' - # channels: # for public jobs we want to make sure that the PRs don't break x86 - # - release/6.0 + # Windows x86 micro benchmarks + - template: /eng/performance/benchmark_jobs.yml + parameters: + osName: windows + osVersion: 22H2 + kind: micro + architecture: x86 + pool: + vmImage: windows-2019 + machinePool: Open + queue: Windows.10.Amd64.Client.Open + csproj: src\benchmarks\micro\MicroBenchmarks.csproj + runCategories: 'runtime libraries' + channels: # for public jobs we want to make sure that the PRs don't break x86 + - release/6.0 - # # Windows x64 ML.NET benchmarks - # - template: /eng/performance/benchmark_jobs.yml - # parameters: - # osName: windows - # osVersion: 22H2 - # kind: mlnet - # architecture: x64 - # pool: - # vmImage: windows-2019 - # machinePool: Open - # queue: Windows.10.Amd64.Client.Open - # csproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj - # runCategories: 'mldotnet' - # channels: # for ML.NET jobs we want to check .NET Core 3.1 and 5.0 only - # - release/6.0 + # Windows x64 ML.NET benchmarks + - template: /eng/performance/benchmark_jobs.yml + parameters: + osName: windows + osVersion: 22H2 + kind: mlnet + architecture: x64 + pool: + vmImage: windows-2019 + machinePool: Open + queue: Windows.10.Amd64.Client.Open + csproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj + runCategories: 'mldotnet' + channels: # for ML.NET jobs we want to check .NET Core 3.1 and 5.0 only + - release/6.0 - # # Windows x64 Roslyn benchmarks - # - template: /eng/performance/benchmark_jobs.yml - # parameters: - # osName: windows - # osVersion: 22H2 - # kind: roslyn - # architecture: x64 - # pool: - # vmImage: windows-2019 - # machinePool: Open - # queue: Windows.10.Amd64.Client.Open - # csproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj - # runCategories: 'roslyn' - # channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only - # - release/6.0 + # Windows x64 Roslyn benchmarks + - template: /eng/performance/benchmark_jobs.yml + parameters: + osName: windows + osVersion: 22H2 + kind: roslyn + architecture: x64 + pool: + vmImage: windows-2019 + machinePool: Open + queue: Windows.10.Amd64.Client.Open + csproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj + runCategories: 'roslyn' + channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only + - release/6.0 - # # Ubuntu 1804 x64 micro benchmarks - # - template: /eng/performance/benchmark_jobs.yml - # parameters: - # osName: ubuntu - # osVersion: 1804 - # kind: micro - # architecture: x64 - # pool: - # vmImage: ubuntu-latest - # machinePool: Open - # queue: Ubuntu.1804.Amd64.Open - # container: ubuntu_x64_build_container - # csproj: src/benchmarks/micro/MicroBenchmarks.csproj - # runCategories: 'runtime libraries' - # channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks - # - release/6.0 + # Ubuntu 1804 x64 micro benchmarks + - template: /eng/performance/benchmark_jobs.yml + parameters: + osName: ubuntu + osVersion: 1804 + kind: micro + architecture: x64 + pool: + vmImage: ubuntu-latest + machinePool: Open + queue: Ubuntu.1804.Amd64.Open + container: ubuntu_x64_build_container + csproj: src/benchmarks/micro/MicroBenchmarks.csproj + runCategories: 'runtime libraries' + channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks + - release/6.0 - # # Ubuntu 1804 x64 ML.NET benchmarks - # - template: /eng/performance/benchmark_jobs.yml - # parameters: - # osName: ubuntu - # osVersion: 1804 - # kind: mlnet - # architecture: x64 - # pool: - # vmImage: ubuntu-latest - # machinePool: Open - # queue: Ubuntu.1804.Amd64.Open - # container: ubuntu_x64_build_container - # runCategories: 'mldotnet' - # csproj: src/benchmarks/real-world/Microsoft.ML.Benchmarks/Microsoft.ML.Benchmarks.csproj - # channels: # for ML.NET jobs we want to check .NET Core 3.1 and 5.0 only - # - release/6.0 + # Ubuntu 1804 x64 ML.NET benchmarks + - template: /eng/performance/benchmark_jobs.yml + parameters: + osName: ubuntu + osVersion: 1804 + kind: mlnet + architecture: x64 + pool: + vmImage: ubuntu-latest + machinePool: Open + queue: Ubuntu.1804.Amd64.Open + container: ubuntu_x64_build_container + runCategories: 'mldotnet' + csproj: src/benchmarks/real-world/Microsoft.ML.Benchmarks/Microsoft.ML.Benchmarks.csproj + channels: # for ML.NET jobs we want to check .NET Core 3.1 and 5.0 only + - release/6.0 - # # Ubuntu 1804 x64 Roslyn benchmarks - # - template: /eng/performance/benchmark_jobs.yml - # parameters: - # osName: ubuntu - # osVersion: 1804 - # kind: roslyn - # architecture: x64 - # pool: - # vmImage: ubuntu-latest - # machinePool: Open - # queue: Ubuntu.1804.Amd64.Open - # container: ubuntu_x64_build_container - # runCategories: 'roslyn' - # csproj: src/benchmarks/real-world/Roslyn/CompilerBenchmarks.csproj - # channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only - # - release/6.0 + # Ubuntu 1804 x64 Roslyn benchmarks + - template: /eng/performance/benchmark_jobs.yml + parameters: + osName: ubuntu + osVersion: 1804 + kind: roslyn + architecture: x64 + pool: + vmImage: ubuntu-latest + machinePool: Open + queue: Ubuntu.1804.Amd64.Open + container: ubuntu_x64_build_container + runCategories: 'roslyn' + csproj: src/benchmarks/real-world/Roslyn/CompilerBenchmarks.csproj + channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only + - release/6.0 ########################################### # Private Jobs diff --git a/eng/performance/scenarios.proj b/eng/performance/scenarios.proj index fb4b6bfcae6..f8b357a5d0e 100644 --- a/eng/performance/scenarios.proj +++ b/eng/performance/scenarios.proj @@ -39,7 +39,7 @@ - + mauidesktop @@ -68,7 +68,7 @@ - + $(Python) pre.py publish -c Release -f $(_Framework)-windows10.0.19041.0 -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName)_fdd %(PreparePayloadWorkItem.PayloadDirectory)