[Release/6.0] Add Mobile Android Maui Scenarios

azure-pipelines.yml

@@ -302,6 +302,21 @@ jobs:
       projectFile: scenarios.proj
       channels:
         - release/6.0
+
+    # Maui Android scenario benchmarks
+  - template: /eng/performance/scenarios.yml
+    parameters:
+      osName: windows
+      architecture: x64
+      osVersion: 19H1
+      pool:
+        vmImage: 'windows-2022'
+      queue: Windows.10.Amd64.Pixel.Perf
+      machinePool: Pixel
+      kind: maui_scenarios_android
+      projectFile: maui_scenarios_android.proj
+      channels:
+        - release/6.0
 
   # # Windows x64 micro benchmarks
   # - template: /eng/performance/benchmark_jobs.yml

eng/performance/PreparePayloadWorkItems.targets

@@ -0,0 +1,7 @@
+<Project>
+  <Target Name="PreparePayloadWorkItems" Inputs="@(PreparePayloadWorkItem)" Outputs="%(PreparePayloadWorkItem.Identity)">
+    <Message Text="Command: %(PreparePayloadWorkItem.Command)" />
+    <Exec Command="%(PreparePayloadWorkItem.Command)" WorkingDirectory="%(PreparePayloadWorkItem.WorkingDirectory)" />
+    <Exec Command="$(AfterPreparePayloadWorkItemCommand)" WorkingDirectory="%(PreparePayloadWorkItem.WorkingDirectory)" Condition="'$(AfterPreparePayloadWorkItemCommand)' != ''" />
+  </Target>
+</Project>

eng/performance/Scenarios.Common.props

@@ -0,0 +1,30 @@
+<Project>
+  <ItemGroup>
+    <HelixCorrelationPayload Include="$(CorrelationPayloadDirectory)">
+      <PayloadDirectory>%(Identity)</PayloadDirectory>
+    </HelixCorrelationPayload>
+  </ItemGroup>
+
+  <ItemDefinitionGroup>
+    <HelixWorkItem>
+      <Timeout>4:00</Timeout>
+    </HelixWorkItem>
+  </ItemDefinitionGroup>
+
+  <PropertyGroup>
+    <FrameworkVersion>$(_Framework.Substring($([MSBuild]::Subtract($(_Framework.Length), 3))))</FrameworkVersion>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(TargetsWindows)' == 'true'">
+    <ScenariosDir>$(WorkItemDirectory)\src\scenarios\</ScenariosDir>
+    <HelixPreCommands>$(HelixPreCommands);set PYTHONPATH=%HELIX_CORRELATION_PAYLOAD%\scripts%3B%HELIX_CORRELATION_PAYLOAD%</HelixPreCommands>
+    <RID>win-$(Architecture)</RID>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(TargetsWindows)' != 'true'">
+    <ScenariosDir>$(WorkItemDirectory)/src/scenarios/</ScenariosDir>
+    <HelixPreCommands>$(HelixPreCommands);sudo apt-get update;chmod +x $HELIX_CORRELATION_PAYLOAD/startup/perfcollect</HelixPreCommands>
+    <HelixPreCommands>$(HelixPreCommands);export PYTHONPATH=$HELIX_CORRELATION_PAYLOAD/scripts:$HELIX_CORRELATION_PAYLOAD</HelixPreCommands>
+    <RID>linux-$(Architecture)</RID>
+  </PropertyGroup>
+</Project>

eng/performance/blazor_scenarios.proj

@@ -79,4 +79,6 @@
     </HelixWorkItem>
   </ItemGroup>
 
+  <Import Project="PreparePayloadWorkItems.targets" />
+
 </Project>

eng/performance/maui_scenarios_android.proj

@@ -0,0 +1,82 @@
+<Project Sdk="Microsoft.DotNet.Helix.Sdk" DefaultTargets="Test">
+
+  <Import Project="Scenarios.Common.props" />
+
+  <PropertyGroup>
+    <IncludeXHarnessCli>true</IncludeXHarnessCli>
+    <MicrosoftDotNetXHarnessCLIVersion>1.0.0-prerelease.21566.2</MicrosoftDotNetXHarnessCLIVersion>
+    <XharnessPath>%HELIX_CORRELATION_PAYLOAD%\microsoft.dotnet.xharness.cli\$(MicrosoftDotNetXHarnessCLIVersion)\tools\net6.0\any\Microsoft.DotNet.XHarness.CLI.dll</XharnessPath>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <AfterPreparePayloadWorkItemCommand>$(Python) post.py</AfterPreparePayloadWorkItemCommand>
+    <PreparePayloadOutDirectoryName>scenarios_out</PreparePayloadOutDirectoryName>
+    <PreparePayloadWorkItemBaseDirectory Condition="'$(TargetsWindows)' == 'true'">$(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)\</PreparePayloadWorkItemBaseDirectory>
+    <PreparePayloadWorkItemBaseDirectory Condition="'$(TargetsWindows)' != 'true'">$(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)/</PreparePayloadWorkItemBaseDirectory>
+  </PropertyGroup>
+
+  <Target Name="RemoveDotnetFromCorrelationStaging" BeforeTargets="BeforeTest">
+    <Message Text="Removing Dotnet from Correlation Staging" Importance="high" />
+    <RemoveDir Directories="$(CorrelationPayloadDirectory)dotnet\packs" />
+  </Target>
+
+  <ItemDefinitionGroup>
+    <HelixWorkItem>
+      <Timeout>00:30</Timeout>
+    </HelixWorkItem>
+  </ItemDefinitionGroup>
+
+  <ItemGroup>
+    <MAUIAndroidScenario Include="Maui Android Default">
+      <ScenarioDirectoryName>mauiandroid</ScenarioDirectoryName>
+      <PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
+      <ApkName>com.companyname.mauiandroiddefault-Signed</ApkName>
+      <PackageName>com.companyname.mauiandroiddefault</PackageName>
+    </MAUIAndroidScenario>
+    <MAUIAndroidScenario Include="Maui Android Podcast">
+      <ScenarioDirectoryName>mauiandroidpodcast</ScenarioDirectoryName>
+      <PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
+      <ApkName>com.Microsoft.NetConf2021.Maui-Signed</ApkName>
+      <PackageName>com.Microsoft.NetConf2021.Maui</PackageName>
+    </MAUIAndroidScenario>
+    <MAUIAndroidScenario Include="Maui Blazor Android Default">
+      <ScenarioDirectoryName>mauiblazorandroid</ScenarioDirectoryName>
+      <PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
+      <ApkName>com.companyname.mauiblazorandroiddefault-Signed</ApkName>
+      <PackageName>com.companyname.mauiblazorandroiddefault</PackageName>
+    </MAUIAndroidScenario>
+  </ItemGroup>
+
+
+  <ItemGroup>
+    <PreparePayloadWorkItem Include="@(MAUIAndroidScenario)">
+      <Command>$(Python) pre.py publish -f $(_Framework)-android -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName) -r android-arm64 --self-contained</Command>
+      <WorkingDirectory>%(PreparePayloadWorkItem.PayloadDirectory)</WorkingDirectory>
+    </PreparePayloadWorkItem>
+  </ItemGroup>
+
+
+<!-- We only run the android tests from Windows machines (at least for now) -->
+  <ItemGroup>
+    <HelixWorkItem Include="@(MAUIAndroidScenario -> 'SOD - %(Identity) APK Size')">
+      <PreCommands>echo on; xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y</PreCommands>
+      <Command>$(Python) test.py sod --scenario-name &quot;%(Identity)&quot;</Command>
+    </HelixWorkItem>
+    <HelixWorkItem Include="@(MAUIAndroidScenario -> 'SOD - %(Identity) Extracted Size')">
+      <PreCommands>echo on; xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y; ren %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).apk %(HelixWorkItem.ApkName).zip; powershell.exe -nologo -noprofile -command "&amp; {Expand-Archive %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).zip -DestinationPath %HELIX_WORKITEM_ROOT%\pub\}"; del %HELIX_WORKITEM_ROOT%\pub\%(HelixWorkItem.ApkName).zip</PreCommands>
+      <Command>$(Python) test.py sod --scenario-name &quot;%(Identity)&quot;</Command>
+    </HelixWorkItem>
+    <HelixWorkItem Include="@(MAUIAndroidScenario -> 'Device Startup - %(Identity)')">
+      <PreCommands>echo on; set XHARNESSPATH=$(XharnessPath); xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y</PreCommands>
+      <Command>$(Python) test.py devicestartup --device-type android --package-path pub\%(HelixWorkItem.ApkName).apk --package-name %(HelixWorkItem.PackageName) --scenario-name &quot;%(Identity)&quot;</Command>
+    </HelixWorkItem>
+    <HelixWorkItem Include="@(MAUIAndroidScenario -> 'Device Startup - %(Identity) NoAnimation')">
+      <PreCommands>echo on; set XHARNESSPATH=$(XharnessPath); xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y</PreCommands>
+      <Command>$(Python) test.py devicestartup --device-type android --package-path pub\%(HelixWorkItem.ApkName).apk --package-name %(HelixWorkItem.PackageName) --scenario-name &quot;%(Identity)&quot; --disable-animations</Command>
+    </HelixWorkItem>
+  </ItemGroup>
+
+
+  <Import Project="PreparePayloadWorkItems.targets" />
+
+</Project>

eng/performance/scenarios.proj

@@ -121,5 +121,6 @@
      post: %(HelixWorkItem.PostCommands)
      timeout: %(HelixWorkItem.Timeout)  '"/>
   </Target> -->
+  <Import Project="PreparePayloadWorkItems.targets" />
 
 </Project>

eng/performance/scenarios.yml

@@ -104,6 +104,19 @@ jobs:
             displayName: Build FailureReporter tool
             env:
               PERFLAB_TARGET_FRAMEWORKS: $(_Framework)
+          - powershell: |
+              $(Python) -m pip install --upgrade pip
+              $(Python) -m pip install requests
+              .\src\scenarios\init.ps1 -DotnetDirectory $(CorrelationStaging)dotnet
+              dotnet --info
+              dotnet msbuild .\eng\performance\${{ parameters.projectFile }} /restore /t:PreparePayloadWorkItems /bl:.\artifacts\log\$(_BuildConfig)\PrepareWorkItemPayloads.binlog
+            displayName: Prepare scenarios
+            env:
+              CorrelationPayloadDirectory: $(CorrelationStaging)
+              Architecture: ${{ parameters.architecture }}
+              TargetsWindows: 'true'
+              WorkItemDirectory: $(Build.SourcesDirectory)
+              HelixTargetQueues: ${{ parameters.queue }}
         - ${{ if ne(parameters.osName, 'windows') }}:
           - script: cp ./NuGet.config $(CorrelationStaging);cp -r ./scripts $(CorrelationStaging)scripts;cp -r ./src/scenarios/shared $(CorrelationStaging)shared;cp -r ./src/scenarios/staticdeps $(CorrelationStaging)staticdeps
             displayName: Copy python libraries and NuGet.config
@@ -119,6 +132,19 @@ jobs:
             displayName: Build FailureReporter tool
             env:
               PERFLAB_TARGET_FRAMEWORKS: $(_Framework)
+          - script: |
+              $(Python) -m pip install --upgrade pip
+              $(Python) -m pip install requests
+              . ./src/scenarios/init.sh -dotnetdir $(CorrelationStaging)dotnet
+              dotnet --info
+              dotnet msbuild ./eng/performance/${{ parameters.projectFile }} /restore /t:PreparePayloadWorkItems /bl:./artifacts/log/$(_BuildConfig)/PrepareWorkItemPayloads.binlog
+            displayName: Prepare scenarios
+            env:
+              CorrelationPayloadDirectory: $(CorrelationStaging)
+              Architecture: ${{ parameters.architecture }}
+              TargetsWindows: 'false'
+              WorkItemDirectory: $(Build.SourcesDirectory)
+              HelixTargetQueues: ${{ parameters.queue }}
         - template: /eng/performance/send-to-helix.yml
           parameters:
             HelixSource: '$(HelixSourcePrefix)/dotnet/performance/$(Build.SourceBranch)' # sources must start with pr/, official/, prodcon/, or agent/

eng/performance/sdk_scenarios.proj

@@ -189,5 +189,6 @@
       <Timeout>4:00</Timeout>
     </HelixWorkItem>
   </ItemGroup>
+  <Import Project="PreparePayloadWorkItems.targets" />
 </Project>
 

scripts/dotnet.py

@@ -5,13 +5,14 @@
 """
 
 import ssl
+import datetime
 from argparse import Action, ArgumentParser, ArgumentTypeError, ArgumentError
 from collections import namedtuple
 from glob import iglob
 from json import loads
 from logging import getLogger
 from os import chmod, environ, listdir, makedirs, path, pathsep, system
-from re import search
+from re import search, match, MULTILINE
 from shutil import rmtree
 from stat import S_IRWXU
 from subprocess import CalledProcessError, check_output
@@ -589,38 +590,40 @@ def get_commit_date(
     if not commit_sha:
         raise ValueError('.NET Commit sha was not defined.')
 
+    # Example URL: https://github.com/dotnet/runtime/commit/2d76178d5faa97be86fc8d049c7dbcbdf66dc497.patch
     url = None
-    urlformat = 'https://api.github.com/repos/%s/%s/commits/%s'
     if repository is None:
         # The origin of the repo where the commit belongs to has changed
         # between release. Here we attempt to naively guess the repo.
         core_sdk_frameworks = ChannelMap.get_supported_frameworks()
-        core_sdk_frameworks.remove('netcoreapp2.1')
         repo = 'core-sdk' if framework  in core_sdk_frameworks else 'cli'
-        url = urlformat % ('dotnet', repo, commit_sha)
+        url = f'https://github.com/dotnet/{repo}/commit/{commit_sha}.patch'
     else:
         owner, repo = get_repository(repository)
-        url = urlformat % (owner, repo, commit_sha)
+        url = f'https://github.com/{owner}/{repo}/commit/{commit_sha}.patch'
 
     build_timestamp = None
-    retrycount = 0
-    success = 0
-    while success == 0 and retrycount <= 3:
+    sleep_time = 10 # Start with 10 second sleep timer        
+    for retrycount in range(5):
         try:
             with urlopen(url) as response:
                 getLogger().info("Commit: %s", url)
-                item = loads(response.read().decode('utf-8'))
-                build_timestamp = item['commit']['committer']['date']
-                success = 1
-        except URLError:
-            retrycount += 1
+                patch = response.read().decode('utf-8')
+                dateMatch = search(r'^Date: (.+)$', patch, MULTILINE)
+                if dateMatch:
+                    build_timestamp = datetime.datetime.strptime(dateMatch.group(1), '%a, %d %b %Y %H:%M:%S %z').astimezone(datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
+                    getLogger().info(f"Got UTC timestamp {build_timestamp} from {dateMatch.group(1)}")
+                    break
+        except URLError as error:
+            getLogger().warning(f"URL Error trying to get commit date from {url}; Reason: {error.reason}; Attempt {retrycount}")
+            sleep(sleep_time)
+            sleep_time = sleep_time * 2
 
     if not build_timestamp:
         raise RuntimeError(
             'Could not get timestamp for commit %s' % commit_sha)
     return build_timestamp
 
-
 def get_build_directory(
         bin_directory: str,
         project_name: str,

src/scenarios/mauiandroid/pre.py

@@ -2,48 +2,37 @@
 pre-command
 '''
 import sys
-import os
-from zipfile import ZipFile
 from performance.logger import setup_loggers, getLogger
-from shutil import copyfile
+from shared import const
+from shared.mauisharedpython import remove_aab_files, install_versioned_maui
 from shared.precommands import PreCommands
-from shared.const import PUBDIR
-from argparse import ArgumentParser
+from shared.versionmanager import versions_write_json, get_version_from_dll_powershell
+from test import EXENAME
 
 setup_loggers(True)
 
-parser = ArgumentParser()
-parser.add_argument('--unzip', help='Unzip APK and report extracted tree', action='store_true', default=False)
-parser.add_argument(
-        '--apk-name',
-        dest='apk',
-        required=True,
-        type=str,
-        help='Name of the APK to setup')
-args = parser.parse_args()
-
-if not os.path.exists(PUBDIR):
-    os.mkdir(PUBDIR)
-apkname = args.apk
-apknamezip = '%s.zip' % (apkname)
-if not os.path.exists(apkname):
-    getLogger().error('Cannot find %s' % (apkname))
-    exit(-1)
-if args.unzip:
-    if not os.path.exists(apknamezip):
-        copyfile(apkname, apknamezip)
-
-    with ZipFile(apknamezip) as zip:
-        zip.extractall(os.path.join('.', PUBDIR))
-
-    assets_dir = os.path.join(PUBDIR, 'assets')
-    assets_zip = os.path.join(assets_dir, 'assets.zip')
-    with ZipFile(assets_zip) as zip:
-        zip.extractall(assets_dir)
-
-    os.remove(assets_zip)
-else:
-    copyfile(apkname, os.path.join(PUBDIR, apkname))
-
-
-
+precommands = PreCommands()
+install_versioned_maui(precommands)
+
+# Setup the Maui folder
+precommands.new(template='maui',
+                output_dir=const.APPDIR,
+                bin_dir=const.BINDIR,
+                exename=EXENAME,
+                working_directory=sys.path[0],
+                no_restore=False)
+
+# Build the APK
+precommands.execute(['--no-restore', '--source', 'MauiNuGet.config'])
+
+# Remove the aab files as we don't need them, this saves space
+output_dir = const.PUBDIR
+if precommands.output:
+    output_dir = precommands.output
+remove_aab_files(output_dir)
+
+# Copy the MauiVersion to a file so we have it on the machine
+maui_version = get_version_from_dll_powershell(rf".\{const.APPDIR}\obj\Release\{precommands.framework}\{precommands.runtime_identifier}\linked\Microsoft.Maui.dll")
+version_dict = { "mauiVersion": maui_version }
+versions_write_json(version_dict, rf"{output_dir}\versions.json")
+print(f"Versions: {version_dict}")

src/scenarios/mauiandroid/test.py

@@ -1,11 +1,15 @@
 '''
-C# Console app
+Mobile Maui App
 '''
+from shared.const import PUBDIR
 from shared.runner import TestTraits, Runner
+from shared.versionmanager import versions_read_json_file_save_env
 
 EXENAME = 'MauiAndroidDefault'
 
-if __name__ == "__main__":
+if __name__ == "__main__":    
+    versions_read_json_file_save_env(rf".\{PUBDIR}\versions.json")
+
     traits = TestTraits(exename=EXENAME, 
                         guiapp='false',
                         )

src/scenarios/mauiandroidpodcast/post.py

@@ -0,0 +1,9 @@
+'''
+post cleanup script
+'''
+
+from shared.postcommands import clean_directories
+from performance.common import remove_directory
+
+remove_directory("dotnet-podcasts")
+clean_directories()