Enable installer and SDK repos in VMR build (#18632)

Co-authored-by: Viktor Hofer <[email protected]>

Directory.Build.props

@@ -8,7 +8,7 @@
   </PropertyGroup>
 
   <PropertyGroup>
-    <BuildArchitecture>$([System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString().ToLowerInvariant())</BuildArchitecture>
+    <BuildArchitecture Condition="'$(BuildArchitecture)' == ''">$([System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString().ToLowerInvariant())</BuildArchitecture>
     <Architecture Condition="'$(Architecture)' == '' AND ('$(BuildArchitecture)' == 'arm64' OR '$(BuildArchitecture)' == 'arm')">$(BuildArchitecture)</Architecture>
     <Architecture Condition="'$(Architecture)' == '' AND '$(BuildArchitecture)' == 's390x'">$(BuildArchitecture)</Architecture>
     <Architecture Condition="'$(Architecture)' == '' AND '$(BuildArchitecture)' == 'ppc64le'">$(BuildArchitecture)</Architecture>

eng/SourceBuild.props

@@ -8,12 +8,25 @@
   </PropertyGroup>
 
   <Target Name="ConfigureInnerBuildArg" BeforeTargets="GetSourceBuildCommandConfiguration">
-    <PropertyGroup>
+    <PropertyGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
       <InnerBuildArgs>$(InnerBuildArgs) /p:SkipBuildingInstallers=true</InnerBuildArgs>
       <InnerBuildArgs>$(InnerBuildArgs) /p:IncludeAdditionalSharedFrameworks=false</InnerBuildArgs>
       <InnerBuildArgs Condition="'$(SourceBuildUseMonoRuntime)' == 'true'">$(InnerBuildArgs) /p:DISABLE_CROSSGEN=true</InnerBuildArgs>
       <InnerBuildArgs Condition="'$(PgoInstrument)' == 'true'">$(InnerBuildArgs) /p:PgoInstrument=true</InnerBuildArgs>
     </PropertyGroup>
   </Target>
 
+  <!-- This should be resolved/removed with https://github.com/dotnet/source-build/issues/4101 -->
+  <Target Name="AddInstallers"
+          BeforeTargets="GetCategorizedIntermediateNupkgContents">
+    <ItemGroup Condition="'$(DotNetBuildOrchestrator)' == 'true'">
+      <!-- Include installers when in product VMR builds. These are not necessary when building the repo-only build as we don't
+           need them in downstream source-only PR legs. We could include them, but it may bump us over the package size limit. -->
+      <IntermediateNupkgArtifactFile Include="$(CurrentRepoSourceBuildArtifactsPackagesDir)Shipping\*.msi" />
+      <IntermediateNupkgArtifactFile Include="$(CurrentRepoSourceBuildArtifactsPackagesDir)Shipping\*.deb" />
+      <IntermediateNupkgArtifactFile Include="$(CurrentRepoSourceBuildArtifactsPackagesDir)Shipping\*.rpm" />
+      <IntermediateNupkgArtifactFile Include="$(CurrentRepoSourceBuildArtifactsPackagesDir)Shipping\*.pkg" />
+    </ItemGroup>
+  </Target>
+
 </Project>

eng/restore-toolset.ps1

@@ -5,7 +5,7 @@ function InitializeCustomSDKToolset {
 
   # The following frameworks and tools are used only for testing.
   # Do not attempt to install them in source build.
-  if ($env:DotNetBuildFromSource -eq "true") {
+  if ($productBuild -or $properties -like "*DotNetBuildRepo=true*") {
     return
   }
 

eng/restore-toolset.sh

@@ -5,7 +5,7 @@ function InitializeCustomSDKToolset {
 
   # The following frameworks and tools are used only for testing.
   # Do not attempt to install them in source build.
-  if [[ "${DotNetBuildFromSource:-}" == "true" ]]; then
+  if [[ $product_build == true || $properties == *"DotNetBuildRepo=true"* ]]; then
     return
   fi
 

src/SourceBuild/content/Directory.Build.props

@@ -34,12 +34,15 @@
     <RootRepo Condition="'$(ShortStack)' == 'true'">runtime</RootRepo>
   </PropertyGroup>
 
+  <!-- See https://github.com/dotnet/arcade/blob/main/Documentation/UnifiedBuild/Unified-Build-Controls.md#output-controls for 
+       control set definition. -->
   <PropertyGroup Label="CalculateArch">
+    <!-- Build architecture is what we are building on. -->
     <BuildArchitecture>$([System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString().ToLowerInvariant)</BuildArchitecture>
-    <HostArchitecture Condition="'$(HostArchitecture)' == ''">$(BuildArchitecture)</HostArchitecture>
-
-    <!-- When building on non-x64 architectures, there may be no cross-compilation available, select a target architecture that is the same as the build. -->
+    <!-- The target architecture is the what the customer is targeting their outputs to run on. -->
     <TargetArchitecture Condition="'$(TargetArchitecture)' == ''">$(BuildArchitecture)</TargetArchitecture>
+    <!-- The host architecture is the what the customer will build on. Much of the time, Host==Target. -->
+    <HostArchitecture Condition="'$(HostArchitecture)' == ''">$(TargetArchitecture)</HostArchitecture>
 
     <Platform Condition="'$(Platform)' == ''">AnyCPU</Platform>
   </PropertyGroup>

src/SourceBuild/content/repo-projects/Directory.Build.props

@@ -240,20 +240,15 @@
     <ExtraPackageVersionPropsPackageInfo Include="MicrosoftNETCoreAppRuntimePackageVersion" Version="%24(MicrosoftNETCoreAppRefPackageVersion)" />
     <ExtraPackageVersionPropsPackageInfo Include="MicrosoftNETCoreAppRuntimeVersion" Version="%24(MicrosoftNETCoreAppRefPackageVersion)" />
     <ExtraPackageVersionPropsPackageInfo Include="MicrosoftNETCoreAppHostPackageVersion" Version="%24(MicrosoftNETCoreAppRefPackageVersion)" />
-    <!-- core-sdk uses this property for ASP.NET blob directory -->
-    <ExtraPackageVersionPropsPackageInfo Include="VSRedistCommonAspNetCoreTargetingPackx6430PackageVersion" Version="$(aspnetcoreOutputPackageVersion)" />
+
+    <ExtraPackageVersionPropsPackageInfo Include="MicrosoftWindowsDesktopAppRuntimewinx64PackageVersion" Version="$(windowsdesktopOutputPackageVersion)" />
 
     <!-- Used by installer to determine sdk version -->
     <ExtraPackageVersionPropsPackageInfo Include="MicrosoftDotnetToolsetInternalPackageVersion" Version="%24(MicrosoftNETSdkPackageVersion)" />
 
     <!-- Used by sdk to determine msbuild version for fsharp -->
     <ExtraPackageVersionPropsPackageInfo Include="FSharpBuildVersion" Version="%24(MicrosoftBuildPackageVersion)" />
 
-    <!-- property used by Arcade to determine what version of SourceLink to use -->
-    <!-- if MicrosoftSourceLinkCommonPackageVersion is non-empty, then we've already built SourceLink, regardless of whether
-         this is the production or offline build, so we should use that version.  -->
-    <ExtraPackageVersionPropsPackageInfo Include="MicrosoftSourceLinkVersion" Version="%24(MicrosoftSourceLinkCommonPackageVersion)" />
-
     <!-- non-rid-specific versions of RID-specific version variables to use for bootstrapping -->
     <ExtraPackageVersionPropsPackageInfo Include="MicrosoftAspNetCoreAppRuntimeVersion" Version="%24(MicrosoftAspNetCoreAppRefPackageVersion)" />
     <ExtraPackageVersionPropsPackageInfo Include="MicrosoftNETCoreAppCrossgen2Version" Version="%24(MicrosoftNETCoreAppRefPackageVersion)" />

src/SourceBuild/content/repo-projects/Directory.Build.targets

@@ -742,7 +742,6 @@
       <TransitiveRepositoryReference Remove="nuget-client" />
       <TransitiveRepositoryReference Remove="fsharp" />
       <TransitiveRepositoryReference Remove="vstest" />
-      <TransitiveRepositoryReference Remove="installer" />
     </ItemGroup>
   </Target>
 

src/SourceBuild/content/repo-projects/installer.proj

@@ -4,24 +4,21 @@
     <!-- Need to set to false to calculate RepositoryCommit. -->
     <EnableSourceControlManagerQueries>false</EnableSourceControlManagerQueries>
 
-    <!-- We need to extract the non-portable OS name from the non-portable RID and pass that to installer build script -->
-    <OSNameOverride>$(TargetOS)</OSNameOverride>
-    <OSNameOverride Condition="'$(PortableBuild)' != 'true'">$(TargetRid.Substring(0, $(TargetRid.IndexOf("-"))))</OSNameOverride>
-
     <!-- Use the repo root build script -->
     <BuildScript>$(ProjectDirectory)build$(ShellExtension)</BuildScript>
 
     <!-- Restore and Build actions are already passed in by the root script. -->
     <BuildActions>$(FlagParameterPrefix)pack $(FlagParameterPrefix)publish</BuildActions>
 
-    <BuildArgs>$(BuildArgs) $(FlagParameterPrefix)runtime-id $(TargetRid)</BuildArgs>
-
     <!--
       Setting NETCoreAppMaximumVersion to a high version so that the sdk doesn't complain if we're restoring/publishing for a higher version than the sdk.
       See https://github.com/dotnet/sdk/issues/1512#issuecomment-377082883
     -->
     <BuildArgs>$(BuildArgs) /p:NETCoreAppMaximumVersion=99.9</BuildArgs>
-    <BuildArgs>$(BuildArgs) /p:OSName=$(OSNameOverride)</BuildArgs>
+    <!-- We need to extract the non-portable OS name from the non-portable RID and pass that to installer build script.
+         This should not happen except when building non-portable. installer generally extracts the OSName from the host OS,
+         or from the Rid if supplied. -->
+    <BuildArgs Condition="$(PortableBuild) != 'true'">$(BuildArgs) /p:OSName=$(TargetRid.Substring(0, $(TargetRid.IndexOf("-"))))</BuildArgs>
     <BuildArgs>$(BuildArgs) /p:PortableOSName=$(__PortableTargetOS)</BuildArgs>
     <BuildArgs>$(BuildArgs) /p:Rid=$(TargetRid)</BuildArgs>
     <BuildArgs>$(BuildArgs) /p:Architecture=$(TargetArchitecture)</BuildArgs>
@@ -30,10 +27,13 @@
     <BuildArgs Condition="'$(TargetOS)' != 'windows'">$(BuildArgs) /p:AspNetCoreInstallerRid=$(TargetRid)</BuildArgs>
     <!-- installer always wants to build portable on FreeBSD -->
     <BuildArgs Condition="'$(TargetOS)' == 'freebsd' and '$(DotNetBuildSourceOnly)' == 'true'">$(BuildArgs) /p:PortableBuild=true</BuildArgs>
-    <BuildArgs Condition="'$(TargetOS)' != 'windows'">$(BuildArgs) /p:CoreSetupRid=$(TargetRid)</BuildArgs>
+    <BuildArgs Condition="'$(TargetOS)' != 'windows'">$(BuildArgs) /p:NetRuntimeRid=$(TargetRid)</BuildArgs>
+    <!-- https://github.com/dotnet/source-build/issues/4138 -->
+    <BuildArgs Condition="'$(TargetOS)' != 'windows' and '$(TargetOS)' != 'osx'">$(BuildArgs) /p:SkipBuildingInstallers=true</BuildArgs>
 
     <BuildArgs>$(BuildArgs) /p:PublicBaseURL=file:%2F%2F$(ArtifactsAssetsDir)</BuildArgs>
-
+    <!-- In non-source-only scenarios, currently consume aspnetcore from the normal public base url -->
+    <BuildArgs>$(BuildArgs) /p:FallbackPublicBaseURL=https://dotnetbuilds.blob.core.windows.net/public/</BuildArgs>
     <BuildArgs>$(BuildArgs) /p:UsePortableLinuxSharedFramework=false</BuildArgs>
 
     <BuildArgs Condition="'$(PgoInstrument)' == 'true'">$(BuildArgs) /p:PgoInstrument=true</BuildArgs>
@@ -74,15 +74,8 @@
 
   <ItemGroup>
     <EnvironmentVariables Include="CLIBUILD_SKIP_TESTS=true" />
-
-    <!-- Disable bundled tools until we can figure out:
-      Unable to find package dotnet-dev-certs.
-      Unable to find package dotnet-ef.
-      Unable to find package dotnet-sql-cache.
-      Unable to find package dotnet-user-secrets.
-      Unable to find package dotnet-user-jwts.
-      Unable to find package dotnet-watch.  -->
-    <EnvironmentVariables Include="CLIBUILD_SKIP_BUNDLEDDOTNETTOOLS=true" />
+    <!-- https://github.com/dotnet/source-build/issues/4115. -->
+    <EnvironmentVariables Include="PublishWindowsPdb=false" />
   </ItemGroup>
 
   <Target Name="GetInputsOutputForCreatePrivateSourceBuiltArtifactsArchive"

src/core-sdk-tasks/core-sdk-tasks.csproj

@@ -4,7 +4,7 @@
     <TargetFrameworks Condition="'$(OS)' != 'Windows_NT'">$(CoreSdkTargetFramework)</TargetFrameworks>
     <CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>
     <RootNamespace>Microsoft.DotNet.Cli.Build</RootNamespace>
-    <DefineConstants Condition="'$(DotNetBuildFromSource)' == 'true'">$(DefineConstants);SOURCE_BUILD</DefineConstants>
+    <DefineConstants Condition="'$(DotNetBuildSourceOnly)' == 'true'">$(DefineConstants);SOURCE_BUILD</DefineConstants>
     <DisableImplicitNamespaceImports>true</DisableImplicitNamespaceImports>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
   </PropertyGroup>
@@ -16,7 +16,7 @@
     <PackageReference Include="NuGet.Versioning" Version="$(NuGetBuildTasksPackageVersion)" />
     <PackageReference Include="NuGet.Packaging" Version="$(NuGetBuildTasksPackageVersion)" />
     <PackageReference Include="System.Reflection.Metadata" Version="1.4.2" />
-    <PackageReference Include="WindowsAzure.Storage" Version="9.3.3" Condition="'$(DotNetBuildFromSource)' != 'true'" />
+    <PackageReference Include="WindowsAzure.Storage" Version="9.3.3" Condition="'$(DotNetBuildSourceOnly)' != 'true'" />
     <PackageReference Include="System.Net.Http" Version="4.3.4" />
   </ItemGroup>
 

src/finalizer/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.15.5)
+cmake_minimum_required(VERSION 3.20)
 
 # Create project named finalizer, this will
 # will generate Finalizer.vcxproj

src/redist/redist.csproj

@@ -6,7 +6,7 @@
     <ProduceReferenceAssembly>false</ProduceReferenceAssembly>
     <ResolveAssemblyReferencesSilent>true</ResolveAssemblyReferencesSilent>
     <ResolveAssemblyWarnOrErrorOnTargetArchitectureMismatch>none</ResolveAssemblyWarnOrErrorOnTargetArchitectureMismatch>
-    <BundleRuntimePacks Condition="'$(BundleRuntimePacks)' == '' And '$(DotNetBuildFromSource)' == 'true'">true</BundleRuntimePacks>
+    <BundleRuntimePacks Condition="'$(BundleRuntimePacks)' == '' and '$(DotNetBuildSourceOnly)' == 'true'">true</BundleRuntimePacks>
   </PropertyGroup>
 
   <ItemGroup>
@@ -15,8 +15,8 @@
                         RuntimeFrameworkVersion="$(MicrosoftNETCoreAppRuntimePackageVersion)"
                         />
 
-    <ProjectReference Include="..\SdkResolver\SdkResolver.csproj" ReferenceOutputAssembly="false" Condition="'$(DotNetBuildFromSource)' != 'true'" />
-    <ProjectReference Include="..\VSTemplateLocator\VSTemplateLocator.csproj" ReferenceOutputAssembly="false" Condition="'$(DotNetBuildFromSource)' != 'true'" />
+    <ProjectReference Include="..\SdkResolver\SdkResolver.csproj" ReferenceOutputAssembly="false" Condition="'$(DotNetBuildSourceOnly)' != 'true'" />
+    <ProjectReference Include="..\VSTemplateLocator\VSTemplateLocator.csproj" ReferenceOutputAssembly="false" Condition="'$(DotNetBuildSourceOnly)' != 'true'" />
   </ItemGroup>
 
   <ItemGroup>

src/redist/targets/BundledManifests.targets

@@ -3,12 +3,12 @@
     <NetFeatureBand>8.0.100</NetFeatureBand>
   </PropertyGroup>
   <ItemGroup>
-    <BundledManifests Include="Microsoft.NET.Sdk.Android" FeatureBand="$(MauiFeatureBand)" Version="$(XamarinAndroidWorkloadManifestVersion)" Condition="'$(DotNetBuildFromSource)' != 'true'" />
-    <BundledManifests Include="Microsoft.NET.Sdk.iOS" FeatureBand="$(MauiFeatureBand)" Version="$(XamarinIOSWorkloadManifestVersion)" Condition="'$(DotNetBuildFromSource)' != 'true'" />
-    <BundledManifests Include="Microsoft.NET.Sdk.MacCatalyst" FeatureBand="$(MauiFeatureBand)" Version="$(XamarinMacCatalystWorkloadManifestVersion)" Condition="'$(DotNetBuildFromSource)' != 'true'" />
-    <BundledManifests Include="Microsoft.NET.Sdk.macOS" FeatureBand="$(MauiFeatureBand)" Version="$(XamarinMacOSWorkloadManifestVersion)" Condition="'$(DotNetBuildFromSource)' != 'true'" />
-    <BundledManifests Include="Microsoft.NET.Sdk.Maui" FeatureBand="$(MauiFeatureBand)" Version="$(MauiWorkloadManifestVersion)" Condition="'$(DotNetBuildFromSource)' != 'true'" />
-    <BundledManifests Include="Microsoft.NET.Sdk.tvOS" FeatureBand="$(MauiFeatureBand)" Version="$(XamarinTvOSWorkloadManifestVersion)" Condition="'$(DotNetBuildFromSource)' != 'true'" />
+    <BundledManifests Include="Microsoft.NET.Sdk.Android" FeatureBand="$(MauiFeatureBand)" Version="$(XamarinAndroidWorkloadManifestVersion)" Condition="'$(DotNetBuildSourceOnly)' != 'true'" />
+    <BundledManifests Include="Microsoft.NET.Sdk.iOS" FeatureBand="$(MauiFeatureBand)" Version="$(XamarinIOSWorkloadManifestVersion)" Condition="'$(DotNetBuildSourceOnly)' != 'true'" />
+    <BundledManifests Include="Microsoft.NET.Sdk.MacCatalyst" FeatureBand="$(MauiFeatureBand)" Version="$(XamarinMacCatalystWorkloadManifestVersion)" Condition="'$(DotNetBuildSourceOnly)' != 'true'" />
+    <BundledManifests Include="Microsoft.NET.Sdk.macOS" FeatureBand="$(MauiFeatureBand)" Version="$(XamarinMacOSWorkloadManifestVersion)" Condition="'$(DotNetBuildSourceOnly)' != 'true'" />
+    <BundledManifests Include="Microsoft.NET.Sdk.Maui" FeatureBand="$(MauiFeatureBand)" Version="$(MauiWorkloadManifestVersion)" Condition="'$(DotNetBuildSourceOnly)' != 'true'" />
+    <BundledManifests Include="Microsoft.NET.Sdk.tvOS" FeatureBand="$(MauiFeatureBand)" Version="$(XamarinTvOSWorkloadManifestVersion)" Condition="'$(DotNetBuildSourceOnly)' != 'true'" />
     <!-- Bundled Manifests are ordered by the reference here - verify before altering -->
     <BundledManifests Include="Microsoft.NET.Workload.Mono.ToolChain.Current" FeatureBand="$(MonoWorkloadFeatureBand)" Version="$(MonoWorkloadManifestVersion)" />
     <BundledManifests Include="Microsoft.NET.Workload.Emscripten.Current" FeatureBand="$(EmscriptenWorkloadFeatureBand)" Version="$(EmscriptenWorkloadManifestVersion)" />