ArPow stage 1: local source-build infrastructure

azure-pipelines.yml

@@ -84,6 +84,7 @@ stages:
         enablePublishTestResults: false
         enablePublishBuildAssets: true
         enablePublishUsingPipelines: $(_PublishUsingPipelines)
+        enableSourceBuild: true
         enableTelemetry: true
         helixRepo: dotnet/fsharp
         jobs:
@@ -184,6 +185,7 @@ stages:
         enablePublishTestResults: false
         enablePublishBuildAssets: true
         enablePublishUsingPipelines: $(_PublishUsingPipelines)
+        enableSourceBuild: true
         enableTelemetry: true
         helixRepo: dotnet/fsharp
         jobs:
@@ -331,32 +333,6 @@ stages:
           - script: .\tests\EndToEndBuildTests\EndToEndBuildTests.cmd -c Release
             displayName: End to end build tests
 
-        # Source Build Windows
-        - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
-          - job: SourceBuild_Windows
-            pool:
-              vmImage: windows-latest
-            steps:
-            - checkout: self
-              clean: true
-            - script: eng\CIBuild.cmd -configuration Release -noSign -prepareMachine -sourceBuild
-              displayName: Build
-            - task: PublishPipelineArtifact@1
-              displayName: Publish Logs
-              inputs:
-                targetPath: '$(Build.SourcesDirectory)/artifacts/log/Release'
-                artifactName: 'SourceBuild_Windows Logs'
-              continueOnError: true
-              condition: not(succeeded())
-            - task: PublishBuildArtifacts@1
-              displayName: Publish Logs
-              inputs:
-                PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/Release'
-                ArtifactName: 'SourceBuild_Windows_Test'
-                publishLocation: Container
-              continueOnError: true
-              condition: not(succeeded())
-
         # Up-to-date - disabled due to it being flaky
         #- job: UpToDate_Windows
         #  pool:
@@ -370,35 +346,6 @@ stages:
         #      filePath: eng\tests\UpToDate.ps1
         #      arguments: -configuration $(_BuildConfig) -ci -binaryLog
 
-        # Source Build Semi-Official
-        - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
-          - job: SourceBuild_Official
-            # used until https://github.com/dotnet/source-build/issues/1795 is fixed
-            pool:
-              name: NetCorePublic-Pool
-              queue: BuildPool.Ubuntu.1604.amd64.Open
-            timeoutInMinutes: 90
-            steps:
-            - checkout: self
-              clean: true
-            - script: ./eng/cibuild.sh --configuration Release --prepareMachine --docker --sourceBuild
-              displayName: Build
-            - task: PublishPipelineArtifact@1
-              displayName: Publish Logs
-              inputs:
-                targetPath: '$(Build.SourcesDirectory)/artifacts/log/Release'
-                artifactName: 'SourceBuild_Official Logs'
-              continueOnError: true
-              condition: not(succeeded())
-            - task: PublishBuildArtifacts@1
-              displayName: Publish Logs
-              inputs:
-                PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/Release'
-                ArtifactName: 'SourceBuild_Official_Test'
-                publishLocation: Container
-              continueOnError: true
-              condition: not(succeeded())
-
         # Plain build Windows
         # Disabled until the Windows Proto compiler is coreclr
         # - job: Plain_Build_Windows

eng/SourceBuild.props

@@ -0,0 +1,47 @@
+<Project>
+
+  <PropertyGroup>
+    <GitHubRepositoryName>fsharp</GitHubRepositoryName>
+    <SourceBuildManagedOnly>true</SourceBuildManagedOnly>
+  </PropertyGroup>
+
+  <Target Name="ApplySourceBuildPatchFiles"
+          Condition="
+            '$(ArcadeBuildFromSource)' == 'true' and
+            '$(ArcadeInnerBuildFromSource)' == 'true'"
+          BeforeTargets="Execute">
+    <ItemGroup>
+      <SourceBuildPatchFile Include="$(RepositoryEngineeringDir)source-build-patches\*.patch" />
+    </ItemGroup>
+
+    <Exec
+      Command="git apply --ignore-whitespace --whitespace=nowarn &quot;%(SourceBuildPatchFile.FullPath)&quot;"
+      WorkingDirectory="$(RepoRoot)"
+      Condition="'@(SourceBuildPatchFile)' != ''" />
+  </Target>
+
+  <!--
+    The build script passes in the full path of the sln to build.  This must be overridden in order to build
+    the cloned source in the inner build.
+  -->
+  <Target Name="ConfigureInnerBuildArg" BeforeTargets="GetSourceBuildCommandConfiguration">
+    <PropertyGroup>
+      <InnerBuildArgs>$(InnerBuildArgs) /p:Projects="$(InnerSourceBuildRepoRoot)\FSharp.sln"</InnerBuildArgs>
+    </PropertyGroup>
+  </Target>
+
+  <!--
+    The build script bootstraps some tooling for the build.  Since the inner build is triggerred via msbuild,
+    trigger the bootstrapping for the inner build.
+  -->
+  <Target Name="CopyBootstrap"
+          DependsOnTargets="PrepareInnerSourceBuildRepoRoot"
+          BeforeTargets="RunInnerSourceBuildCommand">
+
+    <Exec
+      Command="./build.sh --bootstrap --skipBuild"
+      WorkingDirectory="$(InnerSourceBuildRepoRoot)"
+      EnvironmentVariables="@(InnerBuildEnv)" />
+  </Target>
+
+</Project>

eng/SourceBuildPrebuiltBaseline.xml

@@ -0,0 +1,5 @@
+<UsageData>
+  <IgnorePatterns>
+    <UsagePattern IdentityGlob="*/*" />
+  </IgnorePatterns>
+</UsageData>

eng/Version.Details.xml

@@ -1,11 +1,17 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Dependencies>
   <ProductDependencies>
+    <Dependency Name="Microsoft.SourceBuild.Intermediate.xliff-tasks" Version="1.0.0-beta.21126.1">
+      <Uri>https://github.com/dotnet/xliff-tasks</Uri>
+      <Sha>7e80445ee82adbf9a8e6ae601ac5e239d982afaa</Sha>
+      <SourceBuild RepoName="xliff-tasks" ManagedOnly="true" />
+    </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
     <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.21125.5">
       <Uri>https://github.com/dotnet/arcade</Uri>
       <Sha>15246f4af00a1cb2e580783d32ec2937b1878a64</Sha>
+      <SourceBuild RepoName="arcade" ManagedOnly="true"/>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>

eng/build.sh

@@ -29,6 +29,7 @@ usage()
   echo "  --ci                       Building in CI"
   echo "  --docker                   Run in a docker container if applicable"
   echo "  --skipAnalyzers            Do not run analyzers during build operations"
+  echo "  --skipBuild                Do not run the build"
   echo "  --prepareMachine           Prepare machine for CI run, clean up processes after build"
   echo "  --sourceBuild              Simulate building for source-build"
   echo ""
@@ -60,6 +61,7 @@ binary_log=false
 force_bootstrap=false
 ci=false
 skip_analyzers=false
+skip_build=false
 prepare_machine=false
 source_build=false
 properties=""
@@ -126,6 +128,9 @@ while [[ $# > 0 ]]; do
     --skipanalyzers)
       skip_analyzers=true
       ;;
+    --skipbuild)
+      skip_build=true
+      ;;
     --preparemachine)
       prepare_machine=true
       ;;
@@ -260,25 +265,27 @@ function BuildSolution {
     cp -pr $artifacts_dir/bin/fsc/$bootstrap_config/net5.0 $bootstrap_dir/fsc
   fi
 
-  # do real build
-  BuildMessage="Error building solution"
-  MSBuild $toolset_build_proj \
-    $bl \
-    /v:$verbosity \
-    /p:Configuration=$configuration \
-    /p:Projects="$projects" \
-    /p:RepoRoot="$repo_root" \
-    /p:Restore=$restore \
-    /p:Build=$build \
-    /p:Rebuild=$rebuild \
-    /p:Pack=$pack \
-    /p:Publish=$publish \
-    /p:UseRoslynAnalyzers=$enable_analyzers \
-    /p:ContinuousIntegrationBuild=$ci \
-    /p:QuietRestore=$quiet_restore \
-    /p:QuietRestoreBinaryLog="$binary_log" \
-    /p:DotNetBuildFromSource=$source_build \
-    $properties
+  if [[ "$skip_build" != true ]]; then
+    # do real build
+    BuildMessage="Error building solution"
+    MSBuild $toolset_build_proj \
+      $bl \
+      /v:$verbosity \
+      /p:Configuration=$configuration \
+      /p:Projects="$projects" \
+      /p:RepoRoot="$repo_root" \
+      /p:Restore=$restore \
+      /p:Build=$build \
+      /p:Rebuild=$rebuild \
+      /p:Pack=$pack \
+      /p:Publish=$publish \
+      /p:UseRoslynAnalyzers=$enable_analyzers \
+      /p:ContinuousIntegrationBuild=$ci \
+      /p:QuietRestore=$quiet_restore \
+      /p:QuietRestoreBinaryLog="$binary_log" \
+      /p:ArcadeBuildFromSource=$source_build \
+      $properties
+  fi
 }
 
 function TrapAndReportError {

eng/common/templates/steps/source-build.yml

@@ -36,7 +36,7 @@ steps:
 
     ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \
       --configuration $buildConfig \
-      --restore --build --pack --publish \
+      --restore --build --pack --publish -bl\
       $officialBuildArgs \
       $targetRidArgs \
       /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \

eng/source-build-patches/0005-Fix-package-downgrade-warning.patch

@@ -0,0 +1,56 @@
+From 7f0d25bfaa9a38da2097c9420461232a08951165 Mon Sep 17 00:00:00 2001
+From: Chris Rummel <crummel@microsoft.com>
+Date: Thu, 3 Sep 2020 19:02:07 -0500
+Subject: [PATCH 5/5] Fix package downgrade warning
+
+---
+ .../FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj    | 4 ++--
+ .../Microsoft.FSharp.Compiler.nuspec                          | 4 ++--
+ src/fsharp/fsc/fsc.fsproj                                     | 1 +
+ 3 files changed, 7 insertions(+), 6 deletions(-)
+
+diff --git a/src/fsharp/FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj b/src/fsharp/FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj
+index aced6e1b4..fcdacc30b 100644
+--- a/src/fsharp/FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj
++++ b/src/fsharp/FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj
+@@ -770,8 +770,8 @@
+     <PackageReference Include="System.Threading.Tasks.Parallel" Version="$(SystemThreadingTasksParallelVersion)" />
+     <PackageReference Include="System.Threading.Thread" Version="$(SystemThreadingThreadVersion)" />
+     <PackageReference Include="System.Threading.ThreadPool" Version="$(SystemThreadingThreadPoolVersion)" />
+-    <PackageReference Include="System.Buffers" Version="$(SystemBuffersVersion)" />
+-    <PackageReference Include="System.Memory" Version="$(SystemMemoryVersion)" />
++    <PackageReference Include="System.Buffers" Version="4.5.1" />
++    <PackageReference Include="System.Memory" Version="4.5.4" />
+     <PackageReference Include="Microsoft.Build.Framework" Version="$(MicrosoftBuildVersion)" />
+     <PackageReference Include="Microsoft.Build.Tasks.Core" Version="$(MicrosoftBuildVersion)" />
+     <PackageReference Include="Microsoft.Build.Utilities.Core" Version="$(MicrosoftBuildVersion)" />
+diff --git a/src/fsharp/Microsoft.FSharp.Compiler/Microsoft.FSharp.Compiler.nuspec b/src/fsharp/Microsoft.FSharp.Compiler/Microsoft.FSharp.Compiler.nuspec
+index b96f90d2a..366488b47 100644
+--- a/src/fsharp/Microsoft.FSharp.Compiler/Microsoft.FSharp.Compiler.nuspec
++++ b/src/fsharp/Microsoft.FSharp.Compiler/Microsoft.FSharp.Compiler.nuspec
+@@ -22,8 +22,8 @@
+                 <dependency id="System.Threading.Thread" version="4.3.0" />
+                 <dependency id="System.Threading.ThreadPool" version="4.3.0" />
+                 <dependency id="System.ValueTuple" version="4.4.0" />
+-                <dependency id="System.Buffers" version="4.5.0" />
+-                <dependency id="System.Memory" version="4.5.3" />
++                <dependency id="System.Buffers" version="4.5.1" />
++                <dependency id="System.Memory" version="4.5.4" />
+             </group>
+         </dependencies>
+         <contentFiles>
+diff --git a/src/fsharp/fsc/fsc.fsproj b/src/fsharp/fsc/fsc.fsproj
+index 776cddc78..df16c1554 100644
+--- a/src/fsharp/fsc/fsc.fsproj
++++ b/src/fsharp/fsc/fsc.fsproj
+@@ -45,6 +45,7 @@
+     <PackageReference Include="System.Reflection.Metadata" Version="$(SystemReflectionMetadataVersion)" />
+     <PackageReference Include="System.Runtime.Loader" Version="$(SystemRuntimeLoaderVersion)" />
+     <PackageReference Include="System.Security.Principal" Version="$(SystemSecurityPrincipalVersion)" />
++    <PackageReference Include="System.Memory" Version="4.5.4" />
+   </ItemGroup>
+
+ </Project>
+--
+2.18.0
+