Run libraries package testing on build agent (#53905)

* Run libraries package testing on build agent instead of on Helix as with recent changes the entire package testing doesn't take more than 2 minutes. Helix created a work item per package test even though it only took seconds and the average wait time for a client was 10-15min. Also cleaning up how the generated package test projects are restored and incorporating a fix from Eric St John to not hard code the package feeds. * Expose useHelix argument
dotnet · Jun 9, 2021 · eb9a5ab · eb9a5ab
1 parent 197cfb6
commit eb9a5ab
Show file tree

Hide file tree

Showing 19 changed files with 97 additions and 217 deletions.
diff --git a/eng/Tools.props b/eng/Tools.props
@@ -7,7 +7,6 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.DotNet.Build.Tasks.Packaging" Version="$(MicrosoftDotNetBuildTasksPackagingVersion)" />
-    <PackageReference Include="Microsoft.DotNet.PackageTesting" Version="$(MicrosoftDotNetPackageTestingVersion)" />
     <!-- enable source link in pkgproj -->
     <PackageReference Include="Microsoft.SourceLink.GitHub" Version="$(MicrosoftSourceLinkVersion)" PrivateAssets="all" IsImplicitlyDefined="true" />
     <PackageReference Include="Microsoft.SourceLink.AzureRepos.Git" Version="$(MicrosoftSourceLinkVersion)" PrivateAssets="all" IsImplicitlyDefined="true" />

diff --git a/eng/pipelines/libraries/base-job.yml b/eng/pipelines/libraries/base-job.yml
@@ -80,7 +80,6 @@ jobs:
         - ${{ if eq(parameters.framework, 'allConfigurations') }}:
           - _finalFrameworkArg: -allConfigurations
           - _testModeArg: /p:TestAssemblies=false /p:TestPackages=true
-          - _extraHelixArguments: /p:TestPackages=true
 
         - ${{ if eq(parameters.isOfficialAllConfigurations, true) }}:
           - librariesBuildArtifactName: 'libraries_bin_official_allconfigurations'

diff --git a/eng/pipelines/libraries/build-job.yml b/eng/pipelines/libraries/build-job.yml
@@ -29,6 +29,7 @@ parameters:
   variables: {}
   pool: ''
   runTests: false
+  useHelix: true
   testScope: ''
   testBuildPlatforms: []
 
@@ -66,14 +67,18 @@ jobs:
       variables:
         - librariesTestsArtifactName: ${{ format('libraries_test_assets_{0}_{1}_{2}', parameters.osGroup, parameters.archType, parameters.buildConfig) }}
         - _subset: libs
+        - _buildAction: ''
         - _additionalBuildArguments: ''
         - ${{ parameters.variables }}
 
         # Tests only run for 'allConfiguration' and 'net48' build-jobs
         # If platform is in testBuildPlatforms we build tests as well.
         - ${{ if or(eq(parameters.runTests, true), containsValue(parameters.testBuildPlatforms, parameters.platform)) }}:
           - _subset: libs+libs.tests
-          - _additionalBuildArguments: /p:ArchiveTests=true
+          - ${{ if eq(parameters.useHelix, false) }}:
+            - _buildAction: -restore -build -test
+          - ${{ if eq(parameters.useHelix, true) }}:
+            - _additionalBuildArguments: /p:ArchiveTests=true
 
         - ${{ parameters.variables }}
 
@@ -92,6 +97,7 @@ jobs:
 
         - script: $(_buildScript)
                 -subset $(_subset)
+                $(_buildAction)
                 $(_buildArguments)
                 $(_additionalBuildArguments)
           displayName: Restore and Build Product
@@ -136,7 +142,7 @@ jobs:
               parameters:
                 name: Libraries_AllConfigurations
 
-        - ${{ if eq(parameters.runTests, true) }}:
+        - ${{ if and(eq(parameters.runTests, true), eq(parameters.useHelix, true)) }}:
           - template: /eng/pipelines/libraries/helix.yml
             parameters:
               osGroup: ${{ parameters.osGroup }}

diff --git a/eng/pipelines/libraries/helix-queues-setup.yml b/eng/pipelines/libraries/helix-queues-setup.yml
@@ -128,7 +128,7 @@ jobs:
     # windows x64
     - ${{ if eq(parameters.platform, 'windows_x64') }}:
       # netcoreapp
-      - ${{ if notIn(parameters.jobParameters.framework, 'allConfigurations', 'net48') }}:
+      - ${{ if notIn(parameters.jobParameters.framework, 'net48') }}:
         - ${{ if and(eq(parameters.jobParameters.testScope, 'outerloop'), eq(parameters.jobParameters.runtimeFlavor, 'mono')) }}:
           - Windows.81.Amd64.Open
           - Windows.10.Amd64.Server19H1.Open
@@ -152,14 +152,10 @@ jobs:
       - ${{ if eq(parameters.jobParameters.framework, 'net48') }}:
         - Windows.10.Amd64.Client19H1.Open
 
-      # AllConfigurations
-      - ${{ if eq(parameters.jobParameters.framework, 'allConfigurations') }}:
-        - Windows.10.Amd64.Server19H1.Open
-
     # windows x86
     - ${{ if eq(parameters.platform, 'windows_x86') }}:
       # netcoreapp
-      - ${{ if notIn(parameters.jobParameters.framework, 'allConfigurations', 'net48') }}:
+      - ${{ if notIn(parameters.jobParameters.framework, 'net48') }}:
         - ${{ if and(eq(parameters.jobParameters.testScope, 'outerloop'), eq(parameters.jobParameters.runtimeFlavor, 'mono')) }}:
           - Windows.7.Amd64.Open
           - Windows.10.Amd64.ServerRS5.Open

diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml
@@ -758,11 +758,11 @@ jobs:
     buildConfig: ${{ variables.debugOnPrReleaseOnRolling }}
     platforms:
     - windows_x64
-    helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml
     jobParameters:
       isFullMatrix: ${{ variables.isFullMatrix }}
       framework: allConfigurations
       runTests: true
+      useHelix: false
       condition: >-
         or(
           eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true),

diff --git a/src/libraries/pkg/Directory.Build.props b/src/libraries/pkg/Directory.Build.props
@@ -1,6 +1,6 @@
 <Project>
   <PropertyGroup>
-    <DisableProjectRestore>true</DisableProjectRestore>
+    <DisableProjectRestore Condition="'$(MSBuildProjectName)' != 'testPackages'">true</DisableProjectRestore>
   </PropertyGroup>
 
   <Import Project="..\Directory.Build.props" />

diff --git a/src/libraries/pkg/test/Directory.Build.props b/src/libraries/pkg/test/Directory.Build.props
diff --git a/src/libraries/pkg/test/Directory.Build.targets b/src/libraries/pkg/test/Directory.Build.targets
diff --git a/src/libraries/pkg/test/build/Directory.Build.props b/src/libraries/pkg/test/build/Directory.Build.props
@@ -0,0 +1,9 @@
+<Project>
+  <PropertyGroup>
+    <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
+    <RestoreUseStaticGraphEvaluation>true</RestoreUseStaticGraphEvaluation>
+  </PropertyGroup>
+
+  <Import Project="Versions.props" />
+  <Import Project="netfxreference.props" />
+</Project>
diff --git a/src/libraries/pkg/test/build/Directory.Build.targets b/src/libraries/pkg/test/build/Directory.Build.targets
@@ -0,0 +1,4 @@
+<Project>
+  <Import Project="targetingpacks.targets" />
+  <Import Project="packageTest.targets" />
+</Project>
diff --git a/src/libraries/pkg/test/build/dirs.proj b/src/libraries/pkg/test/build/dirs.proj
@@ -0,0 +1,16 @@
+<Project>
+  <PropertyGroup>
+    <ImportDirectoryBuildProps>false</ImportDirectoryBuildProps>
+    <ImportDirectoryBuildTargets>false</ImportDirectoryBuildTargets>
+    <RestoreUseStaticGraphEvaluation>true</RestoreUseStaticGraphEvaluation>
+    <RestorePackagesPath>$(MSBuildThisFileDirectory)cache\</RestorePackagesPath>
+  </PropertyGroup>
+
+  <Import Project="Sdk.props" Sdk="Microsoft.Build.Traversal" />
+
+  <ItemGroup>
+    <ProjectReference Include="projects\**\*.csproj" />
+  </ItemGroup>
+
+  <Import Project="Sdk.targets" Sdk="Microsoft.Build.Traversal" />
+</Project>
diff --git a/src/libraries/pkg/test/packageTest.targets → ...raries/pkg/test/build/packageTest.targets b/src/libraries/pkg/test/packageTest.targets → ...raries/pkg/test/build/packageTest.targets
@@ -4,14 +4,6 @@
   </PropertyGroup>
 
   <PropertyGroup>
-    <RestoreSources>
-      https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json;
-      https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json;
-      https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6/nuget/v3/index.json;
-      https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6-transport/nuget/v3/index.json;
-      $(LocalPackagesPath)
-    </RestoreSources>
-
     <!-- Make sure the SDK raises the runtime items so that they are passed to conflict resolution -->
     <CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>
     <!-- Suppress any SYSLIB9000 errors, as in these cases restore/build would succeed, the failure would be at run-time -->
@@ -38,7 +30,8 @@
   </ItemGroup>
 
 
-  <Target Name="LogBeginTest">
+  <Target Name="LogBeginTest"
+          Condition="'$(TargetFramework)' != ''">
     <Message Importance="High" Text="Testing $(TestPackageID) TFM=$(TargetFramework)" />
   </Target>
 

diff --git a/src/libraries/pkg/test/frameworkSettings/netcoreapp/settings.targets b/src/libraries/pkg/test/frameworkSettings/netcoreapp/settings.targets
@@ -1,6 +1,4 @@
 <Project>
-  <Import Project="$(SupportFilesDir)netfxreference.props" />
-
   <PropertyGroup>
     <ShouldVerifyClosure>true</ShouldVerifyClosure>
     <ShouldVerifyTypes>true</ShouldVerifyTypes>

diff --git a/src/libraries/pkg/test/frameworkSettings/netstandard/settings.targets b/src/libraries/pkg/test/frameworkSettings/netstandard/settings.targets
@@ -1,6 +1,4 @@
 <Project>
-  <Import Project="$(SupportFilesDir)netfxreference.props" />
-
   <PropertyGroup>
     <ShouldVerifyClosure>true</ShouldVerifyClosure>
     <ShouldVerifyTypes>true</ShouldVerifyTypes>

diff --git a/src/libraries/pkg/test/project.csproj.template b/src/libraries/pkg/test/project.csproj.template
@@ -1,19 +1,18 @@
-<Project>
-  <PropertyGroup>
-    <DirectoryBuildPropsPath>$(SupportFilesDir)Directory.Build.props</DirectoryBuildPropsPath>
-    <DirectoryBuildTargetsPath>$(SupportFilesDir)Directory.Build.targets</DirectoryBuildTargetsPath>
-  </PropertyGroup>
-
-  <Import Project="Sdk.props" Sdk="Microsoft.NET.Sdk" />
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFrameworks>{TargetFrameworks}</TargetFrameworks>
     <TestPackageId>{PackageId}</TestPackageId>
 
+    <!-- Restore settings -->
+    <RestorePackagesPath>{RestorePackagesPath}</RestorePackagesPath>
+    <RestoreAdditionalProjectSources>{RestoreAdditionalProjectSources}</RestoreAdditionalProjectSources>
+
     <!-- Used in targetingpacks.targets -->
     <NetCoreAppCurrentVersion>{NetCoreAppCurrentVersion}</NetCoreAppCurrentVersion>
     <NetCoreAppCurrent>{NetCoreAppCurrent}</NetCoreAppCurrent>
     <MicrosoftNetCoreAppFrameworkName>{MicrosoftNetCoreAppFrameworkName}</MicrosoftNetCoreAppFrameworkName>
+    <MicrosoftNetCoreAppRefPackDir>{MicrosoftNetCoreAppRefPackDir}</MicrosoftNetCoreAppRefPackDir>
 
     <!-- Turn off end of life target framework checks as we intentionally build older .NETCoreApp configurations. -->
     <CheckEolTargetFramework>false</CheckEolTargetFramework>
@@ -24,5 +23,4 @@
     <PackageReference Include="{PackageId}" Version="{PackageVersion}" />
   </ItemGroup>
 
-  <Import Project="Sdk.targets" Sdk="Microsoft.NET.Sdk" />
 </Project>
diff --git a/src/libraries/pkg/test/test.msbuild b/src/libraries/pkg/test/test.msbuild