Skip to content

Commit f910fe5

Browse files
LoopedBard3LoopedBard3
authored
[Release/7.0] Add Maui Mobile Android Benchmarks (#2789)
Add Maui Mobile Android Benchmarks to the dotnet performance runs, rather than having them in the dotnet runtime runs. This included making new scenario folders for each and moving the logic for running and building the apps, along with changes for keeping track of versions and commit date time gathering.
1 parent 591d69d commit f910fe5

File tree

20 files changed

+386
-111
lines changed

20 files changed

+386
-111
lines changed

‎azure-pipelines.yml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,19 @@ jobs:
247247
projectFile: maui_scenarios.proj
248248
channels:
249249
- release/7.0
250+
251+
# Maui Android scenario benchmarks
252+
- template: /eng/performance/build_machine_matrix.yml
253+
parameters:
254+
jobTemplate: /eng/performance/scenarios.yml
255+
buildMachines:
256+
- win-x64-android-arm64
257+
isPublic: false
258+
jobParameters:
259+
kind: maui_scenarios_android
260+
projectFile: maui_scenarios_android.proj
261+
channels:
262+
- release/7.0
250263

251264
################################################
252265
# Scheduled Private jobs

‎eng/performance/build_machine_matrix.yml‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,4 +91,16 @@ jobs:
9191
vmImage: windows-2019
9292
machinePool: Tiger
9393
queue: Windows.10.Arm64.Perf.Surf
94-
${{ insert }}: ${{ parameters.jobParameters }}
94+
${{ insert }}: ${{ parameters.jobParameters }}
95+
96+
- ${{ if and(containsValue(parameters.buildMachines, 'win-x64-android-arm64'), not(eq(parameters.isPublic, true))) }}: # Windows ARM64 Pixel only used in private builds currently
97+
- template: ${{ parameters.jobTemplate }}
98+
parameters:
99+
osName: windows
100+
architecture: x64
101+
osVersion: 19H1
102+
pool:
103+
vmImage: 'windows-2022'
104+
queue: Windows.10.Amd64.Pixel.Perf
105+
machinePool: Pixel
106+
${{ insert }}: ${{ parameters.jobParameters }}

‎eng/performance/maui_scenarios_android.proj‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
<Project Sdk="Microsoft.DotNet.Helix.Sdk" DefaultTargets="Test">
2+
3+
<Import Project="Scenarios.Common.props" />
4+
5+
<PropertyGroup>
6+
<IncludeXHarnessCli>true</IncludeXHarnessCli>
7+
<MicrosoftDotNetXHarnessCLIVersion>1.0.0-prerelease.21566.2</MicrosoftDotNetXHarnessCLIVersion>
8+
<XharnessPath>%HELIX_CORRELATION_PAYLOAD%\microsoft.dotnet.xharness.cli\$(MicrosoftDotNetXHarnessCLIVersion)\tools\net6.0\any\Microsoft.DotNet.XHarness.CLI.dll</XharnessPath>
9+
</PropertyGroup>
10+
11+
<PropertyGroup>
12+
<AfterPreparePayloadWorkItemCommand>$(Python) post.py</AfterPreparePayloadWorkItemCommand>
13+
<PreparePayloadOutDirectoryName>scenarios_out</PreparePayloadOutDirectoryName>
14+
<PreparePayloadWorkItemBaseDirectory Condition="'$(TargetsWindows)' == 'true'">$(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)\</PreparePayloadWorkItemBaseDirectory>
15+
<PreparePayloadWorkItemBaseDirectory Condition="'$(TargetsWindows)' != 'true'">$(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)/</PreparePayloadWorkItemBaseDirectory>
16+
</PropertyGroup>
17+
18+
<Target Name="RemoveDotnetFromCorrelationStaging" BeforeTargets="BeforeTest">
19+
<Message Text="Removing Dotnet from Correlation Staging" Importance="high" />
20+
<RemoveDir Directories="$(CorrelationPayloadDirectory)dotnet\packs" />
21+
</Target>
22+
23+
<ItemDefinitionGroup>
24+
<HelixWorkItem>
25+
<Timeout>00:30</Timeout>
26+
</HelixWorkItem>
27+
</ItemDefinitionGroup>
28+
29+
<ItemGroup>
30+
<MAUIAndroidScenario Include="Maui Android Default">
31+
<ScenarioDirectoryName>mauiandroid</ScenarioDirectoryName>
32+
<PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
33+
<ApkName>com.companyname.mauiandroiddefault-Signed</ApkName>
34+
<PackageName>com.companyname.mauiandroiddefault</PackageName>
35+
</MAUIAndroidScenario>
36+
<MAUIAndroidScenario Include="Maui Android Podcast">
37+
<ScenarioDirectoryName>mauiandroidpodcast</ScenarioDirectoryName>
38+
<PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
39+
<ApkName>com.Microsoft.NetConf2021.Maui-Signed</ApkName>
40+
<PackageName>com.Microsoft.NetConf2021.Maui</PackageName>
41+
</MAUIAndroidScenario>
42+
<MAUIAndroidScenario Include="Maui Blazor Android Default">
43+
<ScenarioDirectoryName>mauiblazorandroid</ScenarioDirectoryName>
44+
<PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
45+
<ApkName>com.companyname.mauiblazorandroiddefault-Signed</ApkName>
46+
<PackageName>com.companyname.mauiblazorandroiddefault</PackageName>
47+
</MAUIAndroidScenario>
48+
</ItemGroup>
49+
50+
51+
<ItemGroup>
52+
<PreparePayloadWorkItem Include="@(MAUIAndroidScenario)">
53+
<Command>$(Python) pre.py publish -f $(PERFLAB_Framework)-android -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName) -r android-arm64 --self-contained</Command>
54+
<WorkingDirectory>%(PreparePayloadWorkItem.PayloadDirectory)</WorkingDirectory>
55+
</PreparePayloadWorkItem>
56+
</ItemGroup>
57+
58+
59+
<!-- We only run the android tests from Windows machines (at least for now) -->
60+
<ItemGroup>
61+
<HelixWorkItem Include="@(MAUIAndroidScenario -> 'SOD - %(Identity) APK Size')">
62+
<PreCommands>echo on; xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y</PreCommands>
63+
<Command>$(Python) test.py sod --scenario-name &quot;%(Identity)&quot;</Command>
64+
</HelixWorkItem>
65+
<HelixWorkItem Include="@(MAUIAndroidScenario -> 'SOD - %(Identity) Extracted Size')">
66+
<PreCommands>echo on; xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y; ren %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).apk %(HelixWorkItem.ApkName).zip; powershell.exe -nologo -noprofile -command "&amp; {Expand-Archive %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).zip -DestinationPath %HELIX_WORKITEM_ROOT%\pub\}"; del %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).zip</PreCommands>
67+
<Command>$(Python) test.py sod --scenario-name &quot;%(Identity)&quot;</Command>
68+
</HelixWorkItem>
69+
<HelixWorkItem Include="@(MAUIAndroidScenario -> 'Device Startup - %(Identity)')">
70+
<PreCommands>echo on; set XHARNESSPATH=$(XharnessPath); xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y</PreCommands>
71+
<Command>$(Python) test.py devicestartup --device-type android --package-path pub\%(HelixWorkItem.ApkName).apk --package-name %(HelixWorkItem.PackageName) --scenario-name &quot;%(Identity)&quot;</Command>
72+
</HelixWorkItem>
73+
<HelixWorkItem Include="@(MAUIAndroidScenario -> 'Device Startup - %(Identity) NoAnimation')">
74+
<PreCommands>echo on; set XHARNESSPATH=$(XharnessPath); xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y</PreCommands>
75+
<Command>$(Python) test.py devicestartup --device-type android --package-path pub\%(HelixWorkItem.ApkName).apk --package-name %(HelixWorkItem.PackageName) --scenario-name &quot;%(Identity)&quot; --disable-animations</Command>
76+
</HelixWorkItem>
77+
</ItemGroup>
78+
79+
80+
<Import Project="PreparePayloadWorkItems.targets" />
81+
82+
</Project>

‎scripts/dotnet.py‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,14 @@
55
"""
66

77
import ssl
8+
import datetime
89
from argparse import Action, ArgumentParser, ArgumentTypeError, ArgumentError
910
from collections import namedtuple
1011
from glob import iglob
1112
from json import loads
1213
from logging import getLogger
1314
from os import chmod, environ, listdir, makedirs, path, pathsep, system
14-
from re import search
15+
from re import search, match, MULTILINE
1516
from shutil import rmtree
1617
from stat import S_IRWXU
1718
from subprocess import CalledProcessError, check_output
@@ -611,27 +612,30 @@ def get_commit_date(
611612
if not commit_sha:
612613
raise ValueError('.NET Commit sha was not defined.')
613614

615+
# Example URL: https://github.com/dotnet/runtime/commit/2d76178d5faa97be86fc8d049c7dbcbdf66dc497.patch
614616
url = None
615-
urlformat = 'https://api.github.com/repos/%s/%s/commits/%s'
616617
if repository is None:
617618
# The origin of the repo where the commit belongs to has changed
618619
# between release. Here we attempt to naively guess the repo.
619620
core_sdk_frameworks = ChannelMap.get_supported_frameworks()
620621
repo = 'core-sdk' if framework in core_sdk_frameworks else 'cli'
621-
url = urlformat % ('dotnet', repo, commit_sha)
622+
url = f'https://github.com/dotnet/{repo}/commit/{commit_sha}.patch'
622623
else:
623624
owner, repo = get_repository(repository)
624-
url = urlformat % (owner, repo, commit_sha)
625+
url = f'https://github.com/{owner}/{repo}/commit/{commit_sha}.patch'
625626

626627
build_timestamp = None
627-
sleep_time = 10 # Start with 10 second sleep timer
628+
sleep_time = 10 # Start with 10 second sleep timer
628629
for retrycount in range(5):
629630
try:
630631
with urlopen(url) as response:
631632
getLogger().info("Commit: %s", url)
632-
item = loads(response.read().decode('utf-8'))
633-
build_timestamp = item['commit']['committer']['date']
634-
break
633+
patch = response.read().decode('utf-8')
634+
dateMatch = search(r'^Date: (.+)$', patch, MULTILINE)
635+
if dateMatch:
636+
build_timestamp = datetime.datetime.strptime(dateMatch.group(1), '%a, %d %b %Y %H:%M:%S %z').astimezone(datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
637+
getLogger().info(f"Got UTC timestamp {build_timestamp} from {dateMatch.group(1)}")
638+
break
635639
except URLError as error:
636640
getLogger().warning(f"URL Error trying to get commit date from {url}; Reason: {error.reason}; Attempt {retrycount}")
637641
sleep(sleep_time)

‎src/scenarios/mauiandroid/pre.py‎

Lines changed: 29 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -2,48 +2,37 @@
22
pre-command
33
'''
44
import sys
5-
import os
6-
from zipfile import ZipFile
75
from performance.logger import setup_loggers, getLogger
8-
from shutil import copyfile
6+
from shared import const
7+
from shared.mauisharedpython import remove_aab_files, install_versioned_maui
98
from shared.precommands import PreCommands
10-
from shared.const import PUBDIR
11-
from argparse import ArgumentParser
9+
from shared.versionmanager import versions_write_json, get_version_from_dll_powershell
10+
from test import EXENAME
1211

1312
setup_loggers(True)
1413

15-
parser = ArgumentParser()
16-
parser.add_argument('--unzip', help='Unzip APK and report extracted tree', action='store_true', default=False)
17-
parser.add_argument(
18-
'--apk-name',
19-
dest='apk',
20-
required=True,
21-
type=str,
22-
help='Name of the APK to setup')
23-
args = parser.parse_args()
24-
25-
if not os.path.exists(PUBDIR):
26-
os.mkdir(PUBDIR)
27-
apkname = args.apk
28-
apknamezip = '%s.zip' % (apkname)
29-
if not os.path.exists(apkname):
30-
getLogger().error('Cannot find %s' % (apkname))
31-
exit(-1)
32-
if args.unzip:
33-
if not os.path.exists(apknamezip):
34-
copyfile(apkname, apknamezip)
35-
36-
with ZipFile(apknamezip) as zip:
37-
zip.extractall(os.path.join('.', PUBDIR))
38-
39-
assets_dir = os.path.join(PUBDIR, 'assets')
40-
assets_zip = os.path.join(assets_dir, 'assets.zip')
41-
with ZipFile(assets_zip) as zip:
42-
zip.extractall(assets_dir)
43-
44-
os.remove(assets_zip)
45-
else:
46-
copyfile(apkname, os.path.join(PUBDIR, apkname))
47-
48-
49-
14+
precommands = PreCommands()
15+
install_versioned_maui(precommands)
16+
17+
# Setup the Maui folder
18+
precommands.new(template='maui',
19+
output_dir=const.APPDIR,
20+
bin_dir=const.BINDIR,
21+
exename=EXENAME,
22+
working_directory=sys.path[0],
23+
no_restore=False)
24+
25+
# Build the APK
26+
precommands.execute(['--no-restore', '--source', 'MauiNuGet.config'])
27+
28+
# Remove the aab files as we don't need them, this saves space
29+
output_dir = const.PUBDIR
30+
if precommands.output:
31+
output_dir = precommands.output
32+
remove_aab_files(output_dir)
33+
34+
# Copy the MauiVersion to a file so we have it on the machine
35+
maui_version = get_version_from_dll_powershell(rf".\{const.APPDIR}\obj\Release\{precommands.framework}\{precommands.runtime_identifier}\linked\Microsoft.Maui.dll")
36+
version_dict = { "mauiVersion": maui_version }
37+
versions_write_json(version_dict, rf"{output_dir}\versions.json")
38+
print(f"Versions: {version_dict}")

‎src/scenarios/mauiandroid/test.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,15 @@
11
'''
2-
C# Console app
2+
Mobile Maui App
33
'''
4+
from shared.const import PUBDIR
45
from shared.runner import TestTraits, Runner
6+
from shared.versionmanager import versions_read_json_file_save_env
57

68
EXENAME = 'MauiAndroidDefault'
79

8-
if __name__ == "__main__":
10+
if __name__ == "__main__":
11+
versions_read_json_file_save_env(rf".\{PUBDIR}\versions.json")
12+
913
traits = TestTraits(exename=EXENAME,
1014
guiapp='false',
1115
)

‎src/scenarios/mauiandroidpodcast/post.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
'''
2+
post cleanup script
3+
'''
4+
5+
from shared.postcommands import clean_directories
6+
from performance.common import remove_directory
7+
8+
remove_directory("dotnet-podcasts")
9+
clean_directories()

‎src/scenarios/mauiandroidpodcast/pre.py‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
'''
2+
pre-command
3+
'''
4+
import subprocess
5+
from performance.logger import setup_loggers, getLogger
6+
from shared.precommands import PreCommands
7+
from shared.mauisharedpython import remove_aab_files, install_versioned_maui
8+
from shared.versionmanager import versions_write_json, get_version_from_dll_powershell
9+
from shared import const
10+
11+
setup_loggers(True)
12+
precommands = PreCommands()
13+
install_versioned_maui(precommands)
14+
15+
branch = f'{precommands.framework[:6]}'
16+
subprocess.run(['git', 'clone', 'https://github.com/microsoft/dotnet-podcasts.git', '-b', branch, '--single-branch', '--depth', '1'])
17+
subprocess.run(['powershell', '-Command', r'Remove-Item -Path .\\dotnet-podcasts\\.git -Recurse -Force']) # Git files have permission issues, do their deletion separately
18+
19+
precommands.existing(projectdir='./dotnet-podcasts', projectfile='./src/Mobile/Microsoft.NetConf2021.Maui.csproj')
20+
21+
# Build the APK
22+
precommands._restore()
23+
precommands.execute(['--no-restore'])
24+
25+
# Remove the aab files as we don't need them, this saves space
26+
output_dir = const.PUBDIR
27+
if precommands.output:
28+
output_dir = precommands.output
29+
remove_aab_files(output_dir)
30+
31+
# Copy the MauiVersion to a file so we have it on the machine
32+
maui_version = get_version_from_dll_powershell(rf".\{const.APPDIR}\obj\Release\{precommands.framework}\{precommands.runtime_identifier}\linked\Microsoft.Maui.dll")
33+
version_dict = { "mauiVersion": maui_version }
34+
versions_write_json(version_dict, rf"{output_dir}\versions.json")
35+
print(f"Versions: {version_dict}")

‎src/scenarios/mauiandroidpodcast/test.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
'''
2+
Mobile Maui App
3+
'''
4+
from shared.const import PUBDIR
5+
from shared.runner import TestTraits, Runner
6+
from shared.versionmanager import versions_read_json_file_save_env
7+
8+
EXENAME = 'MauiAndroidPodcast'
9+
10+
if __name__ == "__main__":
11+
versions_read_json_file_save_env(rf".\{PUBDIR}\versions.json")
12+
13+
traits = TestTraits(exename=EXENAME,
14+
guiapp='false',
15+
)
16+
Runner(traits).run()

0 commit comments

Comments
 (0)