From 9ff0e621124672bcb31b14234a3dd4eb7fc777a6 Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Mon, 19 May 2025 15:10:30 +0200 Subject: [PATCH 01/12] Add build time measurements for MAUI Android and iOS scenarios --- eng/performance/maui_scenarios_android.proj | 4 ++++ eng/performance/maui_scenarios_ios.proj | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/eng/performance/maui_scenarios_android.proj b/eng/performance/maui_scenarios_android.proj index 6a0c11404a6..721b0408cef 100644 --- a/eng/performance/maui_scenarios_android.proj +++ b/eng/performance/maui_scenarios_android.proj @@ -101,6 +101,10 @@ echo on; xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y $(Python) test.py devicememoryconsumption --device-type android --package-path pub\%(HelixWorkItem.ApkName).apk --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" --runtime 30 --test-iteration 2 $(ScenarioArgs) + + mkdir %HELIX_WORKITEM_ROOT%\pub\traces && copy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName).$(RunConfigsString).binlog %HELIX_WORKITEM_ROOT%\pub\traces\ + $(Python) test.py buildtime --scenario-name "%(Identity)" $(ScenarioArgs) --binlog-path .\%(HelixWorkItem.ScenarioDirectoryName).$(RunConfigsString).binlog + diff --git a/eng/performance/maui_scenarios_ios.proj b/eng/performance/maui_scenarios_ios.proj index 991e7bd33ab..c30bdf433b6 100644 --- a/eng/performance/maui_scenarios_ios.proj +++ b/eng/performance/maui_scenarios_ios.proj @@ -48,7 +48,7 @@ - sudo xcode-select -s /Applications/Xcode_16.3.app; $(Python) pre.py publish -f $(PERFLAB_Framework)-ios --self-contained -c Release -r ios-arm64 $(NativeAOTCommandProps) -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName); cd ../; zip -r %(PreparePayloadWorkItem.ScenarioDirectoryName).zip %(PreparePayloadWorkItem.ScenarioDirectoryName) + sudo xcode-select -s /Applications/Xcode_16.3.app; $(Python) pre.py publish -f $(PERFLAB_Framework)-ios --self-contained -c Release -r ios-arm64 $(NativeAOTCommandProps) --binlog %(PreparePayloadWorkItem.ScenarioDirectoryName).$(RunConfigsString).binlog -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName); cd ../; zip -r %(PreparePayloadWorkItem.ScenarioDirectoryName).zip %(PreparePayloadWorkItem.ScenarioDirectoryName) %(PreparePayloadWorkItem.PayloadDirectory) @@ -62,6 +62,10 @@ cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName) $HELIX_WORKITEM_ROOT/pub; mv $HELIX_WORKITEM_ROOT/pub/%(HelixWorkItem.IPAName).ipa $HELIX_WORKITEM_ROOT/pub/%(HelixWorkItem.IPAName).zip; unzip -d $HELIX_WORKITEM_ROOT/pub $HELIX_WORKITEM_ROOT/pub/%(HelixWorkItem.IPAName).zip; rm $HELIX_WORKITEM_ROOT/pub/%(HelixWorkItem.IPAName).zip $(Python) test.py sod --scenario-name "%(Identity)" $(ScenarioArgs) + + mkdir $HELIX_WORKITEM_ROOT/pub/traces;cp -v $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName).$(RunConfigsString).binlog $HELIX_WORKITEM_ROOT/pub/traces + $(Python) test.py buildtime --scenario-name "%(Identity)" $(ScenarioArgs) --binlog-path ./%(HelixWorkItem.ScenarioDirectoryName).$(RunConfigsString).binlog + $(ScenariosDir)netios.zip 00:15:00 From 58426db566077a824be2c9860035ee0d3c251487 Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Wed, 28 May 2025 13:37:42 +0200 Subject: [PATCH 02/12] Add AndroidCoreCLR JIT Static Linking scenarios --- eng/pipelines/runtime-perf-jobs.yml | 21 ++++++++++++++++++++ eng/pipelines/templates/runtime-perf-job.yml | 8 ++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/runtime-perf-jobs.yml b/eng/pipelines/runtime-perf-jobs.yml index fff1a958db7..9f9d045487a 100644 --- a/eng/pipelines/runtime-perf-jobs.yml +++ b/eng/pipelines/runtime-perf-jobs.yml @@ -146,6 +146,27 @@ jobs: ${{ each parameter in parameters.jobParameters }}: ${{ parameter.key }}: ${{ parameter.value }} + # run android scenarios - CoreCLR JIT Static Linking + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + buildConfig: release + runtimeFlavor: coreclr + platforms: + - windows_x64 + jobParameters: + runtimeType: AndroidCoreCLR + codeGenType: JIT + linkingType: static + projectFile: $(Build.SourcesDirectory)/eng/testing/performance/android_scenarios.proj + runKind: android_scenarios + isScenario: true + logicalMachine: 'perfpixel4a' + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + # run android scenarios - CoreCLR R2R - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: diff --git a/eng/pipelines/templates/runtime-perf-job.yml b/eng/pipelines/templates/runtime-perf-job.yml index b4541f72109..21b02a32066 100644 --- a/eng/pipelines/templates/runtime-perf-job.yml +++ b/eng/pipelines/templates/runtime-perf-job.yml @@ -10,6 +10,7 @@ parameters: liveLibrariesBuildConfig: '' runtimeType: 'coreclr' codeGenType: 'JIT' + linkingType: 'dynamic' iOSLlvmBuild: 'False' iOSStripSymbols: 'False' hybridGlobalization: 'False' @@ -171,10 +172,13 @@ jobs: # AndroidCoreCLR ${{ if eq(parameters.runtimeType, 'AndroidCoreCLR')}}: - ${{ if eq(parameters.codeGenType, 'JIT') }}: + ${{ if and(eq(parameters.codeGenType, 'JIT'), eq(parameters.linkingType, 'dynamic')) }}: artifactFileName: 'AndroidHelloWorldArm64CoreCLR.tar.gz' artifactName: 'AndroidHelloWorldArm64CoreCLR' - ${{ if eq(parameters.codeGenType, 'R2R') }}: + ${{ if and(eq(parameters.codeGenType, 'JIT'), eq(parameters.linkingType, 'static')) }}: + artifactFileName: 'AndroidHelloWorldArm64CoreCLRStaticLinking.tar.gz' + artifactName: 'AndroidHelloWorldArm64CoreCLRStaticLinking' + ${{ if and(eq(parameters.codeGenType, 'R2R'), eq(parameters.linkingType, 'dynamic')) }}: artifactFileName: 'AndroidHelloWorldArm64CoreCLRR2R.tar.gz' artifactName: 'AndroidHelloWorldArm64CoreCLRR2R' From 61319228b43a2f93af326b1403b34c604f91313d Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Wed, 28 May 2025 13:49:03 +0200 Subject: [PATCH 03/12] Add linkingType parameter to performance job configuration --- eng/pipelines/templates/run-performance-job.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/run-performance-job.yml b/eng/pipelines/templates/run-performance-job.yml index 7d716fa1442..c16c4ceb9eb 100644 --- a/eng/pipelines/templates/run-performance-job.yml +++ b/eng/pipelines/templates/run-performance-job.yml @@ -44,6 +44,7 @@ parameters: projectFile: '' # optional -- Full path to project file framework: '' # optional -- Framework to run against (use frameworks). Ignored if frameworks is non-empty. codeGenType: '' # optional -- CodeGen type (e.g. JIT, AOT) + linkingType: '' # optional -- Linking type (e.g. static, dynamic) runtimeType: '' # optional -- Runtime type (e.g. mono, wasm, coreclr) useLocalCommitTime: false # optional -- Whether to use the local commit time or fetch it from the remote repository isCompare: false # optional -- If the job is comparing two configurations @@ -60,8 +61,8 @@ jobs: - template: ${{ parameters.jobTemplate }} parameters: enableTelemetry: ${{ parameters.enableTelemetry }} - name: run_performance_test_${{ replace(format('{0}_{1}_{2}_{3}_{4}{5}_{6}_{7}_{8}', parameters.runKind, coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(replace(parameters.additionalJobIdentifier, ' ', '_'), 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine)), '_NULL', '') }} - displayName: Performance ${{ replace(format('{0} {1} {2} {3} {4}{5} {6} {7} {8}', parameters.runKind, coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(parameters.additionalJobIdentifier, 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine)), ' NULL', '') }} + name: run_performance_test_${{ replace(format('{0}_{1}_{2}_{3}_{4}{5}_{6}_{7}_{8}_{9}', parameters.runKind, coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(replace(parameters.additionalJobIdentifier, ' ', '_'), 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine), coalesce(parameters.linkingType, 'NULL')), '_NULL', '') }} + displayName: Performance ${{ replace(format('{0} {1} {2} {3} {4}{5} {6} {7} {8} {9}', parameters.runKind, coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(parameters.additionalJobIdentifier, 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine), coalesce(parameters.linkingType, 'NULL')), ' NULL', '') }} timeoutInMinutes: ${{ parameters.timeoutInMinutes }} workspace: clean: all From 111f6bde641b57eb51855b79a5876e3b85c12efa Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Wed, 28 May 2025 13:53:47 +0200 Subject: [PATCH 04/12] Dynamic linking is default --- eng/pipelines/templates/run-performance-job.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/run-performance-job.yml b/eng/pipelines/templates/run-performance-job.yml index c16c4ceb9eb..56f596b94e7 100644 --- a/eng/pipelines/templates/run-performance-job.yml +++ b/eng/pipelines/templates/run-performance-job.yml @@ -61,8 +61,8 @@ jobs: - template: ${{ parameters.jobTemplate }} parameters: enableTelemetry: ${{ parameters.enableTelemetry }} - name: run_performance_test_${{ replace(format('{0}_{1}_{2}_{3}_{4}{5}_{6}_{7}_{8}_{9}', parameters.runKind, coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(replace(parameters.additionalJobIdentifier, ' ', '_'), 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine), coalesce(parameters.linkingType, 'NULL')), '_NULL', '') }} - displayName: Performance ${{ replace(format('{0} {1} {2} {3} {4}{5} {6} {7} {8} {9}', parameters.runKind, coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(parameters.additionalJobIdentifier, 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine), coalesce(parameters.linkingType, 'NULL')), ' NULL', '') }} + name: run_performance_test_${{ replace(format('{0}_{1}_{2}_{3}_{4}{5}_{6}_{7}_{8}_{9}', parameters.runKind, coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(replace(parameters.additionalJobIdentifier, ' ', '_'), 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine), coalesce(eq(parameters.linkingType, 'dynamic'), '', parameters.linkingType, 'NULL')), '_NULL', '') }} + displayName: Performance ${{ replace(format('{0} {1} {2} {3} {4}{5} {6} {7} {8} {9}', parameters.runKind, coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(parameters.additionalJobIdentifier, 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine), coalesce(eq(parameters.linkingType, 'dynamic'), '', parameters.linkingType, 'NULL')), ' NULL', '') }} timeoutInMinutes: ${{ parameters.timeoutInMinutes }} workspace: clean: all From 961f6b0259a34d3451979385285cc831f69ef992 Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Wed, 28 May 2025 13:56:01 +0200 Subject: [PATCH 05/12] Handle dynamic linking type correctly --- eng/pipelines/templates/run-performance-job.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/run-performance-job.yml b/eng/pipelines/templates/run-performance-job.yml index 56f596b94e7..eb3d0d20ad7 100644 --- a/eng/pipelines/templates/run-performance-job.yml +++ b/eng/pipelines/templates/run-performance-job.yml @@ -61,8 +61,8 @@ jobs: - template: ${{ parameters.jobTemplate }} parameters: enableTelemetry: ${{ parameters.enableTelemetry }} - name: run_performance_test_${{ replace(format('{0}_{1}_{2}_{3}_{4}{5}_{6}_{7}_{8}_{9}', parameters.runKind, coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(replace(parameters.additionalJobIdentifier, ' ', '_'), 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine), coalesce(eq(parameters.linkingType, 'dynamic'), '', parameters.linkingType, 'NULL')), '_NULL', '') }} - displayName: Performance ${{ replace(format('{0} {1} {2} {3} {4}{5} {6} {7} {8} {9}', parameters.runKind, coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(parameters.additionalJobIdentifier, 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine), coalesce(eq(parameters.linkingType, 'dynamic'), '', parameters.linkingType, 'NULL')), ' NULL', '') }} + name: run_performance_test_${{ replace(format('{0}_{1}_{2}_{3}_{4}{5}_{6}_{7}_{8}_{9}', parameters.runKind, coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(replace(parameters.additionalJobIdentifier, ' ', '_'), 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine), if(eq(parameters.linkingType, 'dynamic'), '', coalesce(parameters.linkingType, 'NULL'))), '_NULL', '') }} + displayName: Performance ${{ replace(format('{0} {1} {2} {3} {4}{5} {6} {7} {8} {9}', parameters.runKind, coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(parameters.additionalJobIdentifier, 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine), if(eq(parameters.linkingType, 'dynamic'), '', coalesce(parameters.linkingType, 'NULL'))), ' NULL', '') }} timeoutInMinutes: ${{ parameters.timeoutInMinutes }} workspace: clean: all From 418331a82b98121f4f5623a2d9303b95abe2a07c Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Wed, 28 May 2025 14:04:01 +0200 Subject: [PATCH 06/12] Handle dynamic linking type correctly --- eng/pipelines/templates/run-performance-job.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/run-performance-job.yml b/eng/pipelines/templates/run-performance-job.yml index eb3d0d20ad7..26ea6a439c3 100644 --- a/eng/pipelines/templates/run-performance-job.yml +++ b/eng/pipelines/templates/run-performance-job.yml @@ -57,12 +57,19 @@ parameters: iOSStripSymbols: false # optional -- Whether to strip symbols from the iOS build additionalSetupParameters: '' # optional -- Additional arguments to pass to the script +variables: +- name: linkingTypeString + ${{ if eq(parameters.linkingType, 'dynamic') }}: + value: '' + ${{ else }}: + value: ${{ coalesce(parameters.linkingType, 'NULL') }} + jobs: - template: ${{ parameters.jobTemplate }} parameters: enableTelemetry: ${{ parameters.enableTelemetry }} - name: run_performance_test_${{ replace(format('{0}_{1}_{2}_{3}_{4}{5}_{6}_{7}_{8}_{9}', parameters.runKind, coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(replace(parameters.additionalJobIdentifier, ' ', '_'), 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine), if(eq(parameters.linkingType, 'dynamic'), '', coalesce(parameters.linkingType, 'NULL'))), '_NULL', '') }} - displayName: Performance ${{ replace(format('{0} {1} {2} {3} {4}{5} {6} {7} {8} {9}', parameters.runKind, coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(parameters.additionalJobIdentifier, 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine), if(eq(parameters.linkingType, 'dynamic'), '', coalesce(parameters.linkingType, 'NULL'))), ' NULL', '') }} + name: run_performance_test_${{ replace(format('{0}_{1}_{2}_{3}_{4}{5}_{6}_{7}_{8}_{9}', parameters.runKind, coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(replace(parameters.additionalJobIdentifier, ' ', '_'), 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine), coalesce(parameters.linkingTypeString, 'NULL')), '_NULL', '') }} + displayName: Performance ${{ replace(format('{0} {1} {2} {3} {4}{5} {6} {7} {8} {9}', parameters.runKind, coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(parameters.additionalJobIdentifier, 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine), coalesce(parameters.linkingTypeString, 'NULL')), ' NULL', '') }} timeoutInMinutes: ${{ parameters.timeoutInMinutes }} workspace: clean: all From bbff369e6488e8d965d9fd3d2865bae7ca7218a2 Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Wed, 28 May 2025 14:07:32 +0200 Subject: [PATCH 07/12] Refactor linkingType handling to default to empty string --- eng/pipelines/templates/run-performance-job.yml | 11 ++--------- eng/pipelines/templates/runtime-perf-job.yml | 6 +++--- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/eng/pipelines/templates/run-performance-job.yml b/eng/pipelines/templates/run-performance-job.yml index 26ea6a439c3..c16c4ceb9eb 100644 --- a/eng/pipelines/templates/run-performance-job.yml +++ b/eng/pipelines/templates/run-performance-job.yml @@ -57,19 +57,12 @@ parameters: iOSStripSymbols: false # optional -- Whether to strip symbols from the iOS build additionalSetupParameters: '' # optional -- Additional arguments to pass to the script -variables: -- name: linkingTypeString - ${{ if eq(parameters.linkingType, 'dynamic') }}: - value: '' - ${{ else }}: - value: ${{ coalesce(parameters.linkingType, 'NULL') }} - jobs: - template: ${{ parameters.jobTemplate }} parameters: enableTelemetry: ${{ parameters.enableTelemetry }} - name: run_performance_test_${{ replace(format('{0}_{1}_{2}_{3}_{4}{5}_{6}_{7}_{8}_{9}', parameters.runKind, coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(replace(parameters.additionalJobIdentifier, ' ', '_'), 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine), coalesce(parameters.linkingTypeString, 'NULL')), '_NULL', '') }} - displayName: Performance ${{ replace(format('{0} {1} {2} {3} {4}{5} {6} {7} {8} {9}', parameters.runKind, coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(parameters.additionalJobIdentifier, 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine), coalesce(parameters.linkingTypeString, 'NULL')), ' NULL', '') }} + name: run_performance_test_${{ replace(format('{0}_{1}_{2}_{3}_{4}{5}_{6}_{7}_{8}_{9}', parameters.runKind, coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(replace(parameters.additionalJobIdentifier, ' ', '_'), 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine), coalesce(parameters.linkingType, 'NULL')), '_NULL', '') }} + displayName: Performance ${{ replace(format('{0} {1} {2} {3} {4}{5} {6} {7} {8} {9}', parameters.runKind, coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(parameters.additionalJobIdentifier, 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine), coalesce(parameters.linkingType, 'NULL')), ' NULL', '') }} timeoutInMinutes: ${{ parameters.timeoutInMinutes }} workspace: clean: all diff --git a/eng/pipelines/templates/runtime-perf-job.yml b/eng/pipelines/templates/runtime-perf-job.yml index 21b02a32066..c0b4336be3d 100644 --- a/eng/pipelines/templates/runtime-perf-job.yml +++ b/eng/pipelines/templates/runtime-perf-job.yml @@ -10,7 +10,7 @@ parameters: liveLibrariesBuildConfig: '' runtimeType: 'coreclr' codeGenType: 'JIT' - linkingType: 'dynamic' + linkingType: '' # dynamic is default iOSLlvmBuild: 'False' iOSStripSymbols: 'False' hybridGlobalization: 'False' @@ -172,13 +172,13 @@ jobs: # AndroidCoreCLR ${{ if eq(parameters.runtimeType, 'AndroidCoreCLR')}}: - ${{ if and(eq(parameters.codeGenType, 'JIT'), eq(parameters.linkingType, 'dynamic')) }}: + ${{ if and(eq(parameters.codeGenType, 'JIT'), eq(parameters.linkingType, '')) }}: artifactFileName: 'AndroidHelloWorldArm64CoreCLR.tar.gz' artifactName: 'AndroidHelloWorldArm64CoreCLR' ${{ if and(eq(parameters.codeGenType, 'JIT'), eq(parameters.linkingType, 'static')) }}: artifactFileName: 'AndroidHelloWorldArm64CoreCLRStaticLinking.tar.gz' artifactName: 'AndroidHelloWorldArm64CoreCLRStaticLinking' - ${{ if and(eq(parameters.codeGenType, 'R2R'), eq(parameters.linkingType, 'dynamic')) }}: + ${{ if and(eq(parameters.codeGenType, 'R2R'), eq(parameters.linkingType, '')) }}: artifactFileName: 'AndroidHelloWorldArm64CoreCLRR2R.tar.gz' artifactName: 'AndroidHelloWorldArm64CoreCLRR2R' From b5f022c07ee8b8dd9c512eeba5d5d09099ca3c88 Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Wed, 28 May 2025 14:09:22 +0200 Subject: [PATCH 08/12] Fix linkingType handling --- eng/pipelines/templates/run-performance-job.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/run-performance-job.yml b/eng/pipelines/templates/run-performance-job.yml index c16c4ceb9eb..1cd08f443db 100644 --- a/eng/pipelines/templates/run-performance-job.yml +++ b/eng/pipelines/templates/run-performance-job.yml @@ -61,8 +61,8 @@ jobs: - template: ${{ parameters.jobTemplate }} parameters: enableTelemetry: ${{ parameters.enableTelemetry }} - name: run_performance_test_${{ replace(format('{0}_{1}_{2}_{3}_{4}{5}_{6}_{7}_{8}_{9}', parameters.runKind, coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(replace(parameters.additionalJobIdentifier, ' ', '_'), 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine), coalesce(parameters.linkingType, 'NULL')), '_NULL', '') }} - displayName: Performance ${{ replace(format('{0} {1} {2} {3} {4}{5} {6} {7} {8} {9}', parameters.runKind, coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(parameters.additionalJobIdentifier, 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine), coalesce(parameters.linkingType, 'NULL')), ' NULL', '') }} + name: run_performance_test_${{ replace(format('{0}_{1}_{2}_{3}_{4}{5}_{6}_{7}_{8}_{9}', parameters.runKind, coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(parameters.linkingType, 'NULL'), coalesce(replace(parameters.additionalJobIdentifier, ' ', '_'), 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine)), '_NULL', '') }} + displayName: Performance ${{ replace(format('{0} {1} {2} {3} {4} {5} {6} {7} {8} {9}', parameters.runKind, coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(parameters.linkingType, 'NULL'), coalesce(parameters.additionalJobIdentifier, 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine)), ' NULL', '') }} timeoutInMinutes: ${{ parameters.timeoutInMinutes }} workspace: clean: all From 896a445e2eea1788a8be96e06530bcc62d087f5f Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Thu, 29 May 2025 09:28:50 +0200 Subject: [PATCH 09/12] Update binlog path --- eng/performance/maui_scenarios_android.proj | 2 +- eng/performance/maui_scenarios_ios.proj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/performance/maui_scenarios_android.proj b/eng/performance/maui_scenarios_android.proj index 721b0408cef..8fb02ba37c2 100644 --- a/eng/performance/maui_scenarios_android.proj +++ b/eng/performance/maui_scenarios_android.proj @@ -69,7 +69,7 @@ - $(Python) pre.py publish -f $(PERFLAB_Framework)-android -r android-arm64 --self-contained --msbuild="$(_MSBuildArgs)" --binlog %(PreparePayloadWorkItem.ScenarioDirectoryName).$(RunConfigsString).binlog -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName) + $(Python) pre.py publish -f $(PERFLAB_Framework)-android -r android-arm64 --self-contained --msbuild="$(_MSBuildArgs)" --binlog $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName)\%(PreparePayloadWorkItem.ScenarioDirectoryName).$(RunConfigsString).binlog -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName) %(PreparePayloadWorkItem.PayloadDirectory) diff --git a/eng/performance/maui_scenarios_ios.proj b/eng/performance/maui_scenarios_ios.proj index c30bdf433b6..950ef91ad72 100644 --- a/eng/performance/maui_scenarios_ios.proj +++ b/eng/performance/maui_scenarios_ios.proj @@ -48,7 +48,7 @@ - sudo xcode-select -s /Applications/Xcode_16.3.app; $(Python) pre.py publish -f $(PERFLAB_Framework)-ios --self-contained -c Release -r ios-arm64 $(NativeAOTCommandProps) --binlog %(PreparePayloadWorkItem.ScenarioDirectoryName).$(RunConfigsString).binlog -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName); cd ../; zip -r %(PreparePayloadWorkItem.ScenarioDirectoryName).zip %(PreparePayloadWorkItem.ScenarioDirectoryName) + sudo xcode-select -s /Applications/Xcode_16.3.app; $(Python) pre.py publish -f $(PERFLAB_Framework)-ios --self-contained -c Release -r ios-arm64 $(NativeAOTCommandProps) --binlog $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName)/%(PreparePayloadWorkItem.ScenarioDirectoryName).$(RunConfigsString).binlog -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName); cd ../; zip -r %(PreparePayloadWorkItem.ScenarioDirectoryName).zip %(PreparePayloadWorkItem.ScenarioDirectoryName) %(PreparePayloadWorkItem.PayloadDirectory) From 11352944465ca0d3571b007f33d3836d1c55456b Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Thu, 29 May 2025 11:34:03 +0200 Subject: [PATCH 10/12] Update binlog path --- eng/performance/maui_scenarios_android.proj | 2 +- eng/performance/maui_scenarios_ios.proj | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/performance/maui_scenarios_android.proj b/eng/performance/maui_scenarios_android.proj index 8fb02ba37c2..a1660758f62 100644 --- a/eng/performance/maui_scenarios_android.proj +++ b/eng/performance/maui_scenarios_android.proj @@ -102,7 +102,7 @@ $(Python) test.py devicememoryconsumption --device-type android --package-path pub\%(HelixWorkItem.ApkName).apk --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" --runtime 30 --test-iteration 2 $(ScenarioArgs) - mkdir %HELIX_WORKITEM_ROOT%\pub\traces && copy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName).$(RunConfigsString).binlog %HELIX_WORKITEM_ROOT%\pub\traces\ + echo on; xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y && mkdir %HELIX_WORKITEM_ROOT%\pub\traces && copy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName).$(RunConfigsString).binlog %HELIX_WORKITEM_ROOT%\pub\traces\ $(Python) test.py buildtime --scenario-name "%(Identity)" $(ScenarioArgs) --binlog-path .\%(HelixWorkItem.ScenarioDirectoryName).$(RunConfigsString).binlog diff --git a/eng/performance/maui_scenarios_ios.proj b/eng/performance/maui_scenarios_ios.proj index 950ef91ad72..d16f57b0e08 100644 --- a/eng/performance/maui_scenarios_ios.proj +++ b/eng/performance/maui_scenarios_ios.proj @@ -48,7 +48,7 @@ - sudo xcode-select -s /Applications/Xcode_16.3.app; $(Python) pre.py publish -f $(PERFLAB_Framework)-ios --self-contained -c Release -r ios-arm64 $(NativeAOTCommandProps) --binlog $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName)/%(PreparePayloadWorkItem.ScenarioDirectoryName).$(RunConfigsString).binlog -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName); cd ../; zip -r %(PreparePayloadWorkItem.ScenarioDirectoryName).zip %(PreparePayloadWorkItem.ScenarioDirectoryName) + sudo xcode-select -s /Applications/Xcode_16.3.app; $(Python) pre.py publish -f $(PERFLAB_Framework)-ios --self-contained -c Release -r ios-arm64 $(NativeAOTCommandProps) --binlog $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName)/%(PreparePayloadWorkItem.ScenarioDirectoryName).binlog -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName); cd ../; zip -r %(PreparePayloadWorkItem.ScenarioDirectoryName).zip %(PreparePayloadWorkItem.ScenarioDirectoryName) %(PreparePayloadWorkItem.PayloadDirectory) @@ -63,8 +63,8 @@ $(Python) test.py sod --scenario-name "%(Identity)" $(ScenarioArgs) - mkdir $HELIX_WORKITEM_ROOT/pub/traces;cp -v $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName).$(RunConfigsString).binlog $HELIX_WORKITEM_ROOT/pub/traces - $(Python) test.py buildtime --scenario-name "%(Identity)" $(ScenarioArgs) --binlog-path ./%(HelixWorkItem.ScenarioDirectoryName).$(RunConfigsString).binlog + cp -v $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)/versions.json $HELIX_WORKITEM_ROOT/pub/versions.json; mkdir -p $HELIX_WORKITEM_ROOT/pub/traces; cp -v $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName).binlog $HELIX_WORKITEM_ROOT/pub/traces + $(Python) test.py buildtime --scenario-name "%(Identity)" $(ScenarioArgs) --binlog-path ./%(HelixWorkItem.ScenarioDirectoryName).binlog $(ScenariosDir)netios.zip From 5523bc416a204ed7dce09a9c794db568eb55e699 Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Thu, 29 May 2025 13:13:16 +0200 Subject: [PATCH 11/12] Fix traces path --- eng/performance/maui_scenarios_android.proj | 2 +- eng/performance/maui_scenarios_ios.proj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/performance/maui_scenarios_android.proj b/eng/performance/maui_scenarios_android.proj index a1660758f62..f04d0d90c98 100644 --- a/eng/performance/maui_scenarios_android.proj +++ b/eng/performance/maui_scenarios_android.proj @@ -102,7 +102,7 @@ $(Python) test.py devicememoryconsumption --device-type android --package-path pub\%(HelixWorkItem.ApkName).apk --package-name %(HelixWorkItem.PackageName) --scenario-name "%(Identity)" --runtime 30 --test-iteration 2 $(ScenarioArgs) - echo on; xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y && mkdir %HELIX_WORKITEM_ROOT%\pub\traces && copy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName).$(RunConfigsString).binlog %HELIX_WORKITEM_ROOT%\pub\traces\ + echo on; xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y && mkdir %HELIX_WORKITEM_ROOT%\traces && copy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName).$(RunConfigsString).binlog %HELIX_WORKITEM_ROOT%\traces\ $(Python) test.py buildtime --scenario-name "%(Identity)" $(ScenarioArgs) --binlog-path .\%(HelixWorkItem.ScenarioDirectoryName).$(RunConfigsString).binlog diff --git a/eng/performance/maui_scenarios_ios.proj b/eng/performance/maui_scenarios_ios.proj index d16f57b0e08..a2ac531a5b6 100644 --- a/eng/performance/maui_scenarios_ios.proj +++ b/eng/performance/maui_scenarios_ios.proj @@ -63,7 +63,7 @@ $(Python) test.py sod --scenario-name "%(Identity)" $(ScenarioArgs) - cp -v $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)/versions.json $HELIX_WORKITEM_ROOT/pub/versions.json; mkdir -p $HELIX_WORKITEM_ROOT/pub/traces; cp -v $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName).binlog $HELIX_WORKITEM_ROOT/pub/traces + cp -v $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)/versions.json $HELIX_WORKITEM_ROOT/pub/versions.json; mkdir -p $HELIX_WORKITEM_ROOT/traces; cp -v $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName).binlog $HELIX_WORKITEM_ROOT/traces $(Python) test.py buildtime --scenario-name "%(Identity)" $(ScenarioArgs) --binlog-path ./%(HelixWorkItem.ScenarioDirectoryName).binlog From b349a00cde62d3dfc742829f65abfe231f376a90 Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Thu, 29 May 2025 16:20:28 +0200 Subject: [PATCH 12/12] Fix copy command --- eng/performance/maui_scenarios_ios.proj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/performance/maui_scenarios_ios.proj b/eng/performance/maui_scenarios_ios.proj index a2ac531a5b6..02765520a04 100644 --- a/eng/performance/maui_scenarios_ios.proj +++ b/eng/performance/maui_scenarios_ios.proj @@ -63,7 +63,7 @@ $(Python) test.py sod --scenario-name "%(Identity)" $(ScenarioArgs) - cp -v $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)/versions.json $HELIX_WORKITEM_ROOT/pub/versions.json; mkdir -p $HELIX_WORKITEM_ROOT/traces; cp -v $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName).binlog $HELIX_WORKITEM_ROOT/traces + cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName) $HELIX_WORKITEM_ROOT/pub; cp -v $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)/versions.json $HELIX_WORKITEM_ROOT/pub/versions.json; mkdir -p $HELIX_WORKITEM_ROOT/traces; cp -v $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName).binlog $HELIX_WORKITEM_ROOT/traces $(Python) test.py buildtime --scenario-name "%(Identity)" $(ScenarioArgs) --binlog-path ./%(HelixWorkItem.ScenarioDirectoryName).binlog