Copy symbols to intermediates

src/SourceBuild/content/eng/build.sourcebuild.targets

@@ -15,13 +15,28 @@
     <MSBuild Projects="$(RepoProjectsDir)$(RootRepo).proj" Targets="WritePrebuiltUsageData;ReportPrebuiltUsage" />
   </Target>
 
-  <!-- After building, repackage symbols into a single tarball. -->
-  <Target Name="RepackageSymbols"
+  <!--
+    Determine symbols tarball names and discover all intermediate symbols,
+    to be used as inputs and outputs of symbols repackaging targets.
+  -->
+  <Target Name="DetermineSymbolsTargetsInputsAndOutputs"
           AfterTargets="Build"
           DependsOnTargets="DetermineSourceBuiltSdkVersion">
     <PropertyGroup>
       <UnifiedSymbolsTarball>$(ArtifactsAssetsDir)dotnet-symbols-all-$(SourceBuiltSdkVersion)-$(TargetRid)$(ArchiveExtension)</UnifiedSymbolsTarball>
+      <SdkSymbolsTarball>$(ArtifactsAssetsDir)dotnet-symbols-sdk-$(SourceBuiltSdkVersion)-$(TargetRid)$(ArchiveExtension)</SdkSymbolsTarball>
     </PropertyGroup>
+    <ItemGroup>
+      <IntermediateSymbol Include="$(IntermediateSymbolsRootDir)**/*" />
+    </ItemGroup>
+  </Target>
+
+  <!-- After building, repackage symbols into a single tarball. -->
+  <Target Name="RepackageSymbols"
+          AfterTargets="Build"
+          DependsOnTargets="DetermineSymbolsTargetsInputsAndOutputs"
+          Inputs="@(IntermediateSymbol)"
+          Outputs="$(UnifiedSymbolsTarball)">
 
     <MakeDir Directories="$([System.IO.Path]::GetDirectoryName('$(UnifiedSymbolsTarball)'))" />
     <Exec Command="tar --numeric-owner -czf $(UnifiedSymbolsTarball) *"
@@ -34,10 +49,10 @@
   <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.CreateSdkSymbolsLayout" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" />
   <Target Name="CreateSdkSymbolsTarball"
           AfterTargets="Build"
-          DependsOnTargets="RepackageSymbols">
+          DependsOnTargets="RepackageSymbols"
+          Inputs="@(IntermediateSymbol);$(SdkTarballPath)"
+          Outputs="$(SdkSymbolsTarball)">
     <PropertyGroup>
-      <SdkSymbolsTarball>$(ArtifactsAssetsDir)dotnet-symbols-sdk-$(SourceBuiltSdkVersion)-$(TargetRid)$(ArchiveExtension)</SdkSymbolsTarball>
-
       <IntermediateSdkSymbolsLayout>$(BaseIntermediateOutputPath)SdkSymbols</IntermediateSdkSymbolsLayout>
       <IntermediateSdkLayout>$(BaseIntermediateOutputPath)Sdk</IntermediateSdkLayout>
     </PropertyGroup>
@@ -55,7 +70,6 @@
 
     <Message Importance="High" Text="Packaged sdk symbols in '$(SdkSymbolsTarball)'" />
 
-    <RemoveDir Directories="$(IntermediateSymbolsRootDir)" />
     <RemoveDir Directories="$(IntermediateSdkSymbolsLayout)" />
     <RemoveDir Directories="$(IntermediateSdkLayout)" />
   </Target>

src/SourceBuild/content/repo-projects/Directory.Build.props

@@ -39,6 +39,7 @@
     <NuGetConfigFile Condition="'$(OriginalNuGetConfigFile)' != ''">$(BaseIntermediateOutputPath)$([System.IO.Path]::GetFileName('$(OriginalNuGetConfigFile)'))</NuGetConfigFile>
 
     <RepoAssetManifestsDir>$([MSBuild]::NormalizeDirectory('$(AssetManifestsIntermediateDir)', '$(RepositoryName)'))</RepoAssetManifestsDir>
+    <IntermediateSymbolsRepoDir>$([MSBuild]::NormalizeDirectory('$(IntermediateSymbolsRootDir)', '$(RepositoryName)'))</IntermediateSymbolsRepoDir>
 
     <SourceBuiltSdksDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'source-built-sdks'))</SourceBuiltSdksDir>
 
@@ -92,6 +93,7 @@
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
+    <BuildArgs>$(BuildArgs) /p:SourceBuiltSymbolsDir=$(IntermediateSymbolsRepoDir)</BuildArgs>
     <BuildArgs>$(BuildArgs) /p:ArcadeBuildFromSource=true</BuildArgs>
     <BuildArgs>$(BuildArgs) /p:DotNetBuildSourceOnly=true</BuildArgs>
     <BuildArgs>$(BuildArgs) /p:PreviouslySourceBuiltNupkgCacheDir="$(PrebuiltSourceBuiltPackagesPath)"</BuildArgs>

src/SourceBuild/content/repo-projects/Directory.Build.targets

@@ -24,8 +24,6 @@
     <PackageReportDataFile>$(PackageReportDir)prebuilt-usage.xml</PackageReportDataFile>
     <ProjectAssetsJsonArchiveFile>$(PackageReportDir)all-project-assets-json-files.zip</ProjectAssetsJsonArchiveFile>
     <ProdConManifestFile>$(PackageReportDir)prodcon-build.xml</ProdConManifestFile>
-
-    <IntermediateSymbolsRepoDir>$(IntermediateSymbolsRootDir)$(RepositoryName)</IntermediateSymbolsRepoDir>
   </PropertyGroup>
 
   <!-- Returns the repository references of this project and all the projects this project references, recursively -->
@@ -538,43 +536,13 @@
     </Touch>
   </Target>
 
-  <Target Name="DiscoverSymbolsArchiveFile"
-          Condition="'$(DotNetBuildSourceOnly)' == 'true' and
-                     '$(IsUtilityProject)' != 'true'">
-    <PropertyGroup>
-      <RepoAssetsSymbolsArchive>$(ArtifactsAssetsSymbolsDir)Symbols.$(RepositoryName)$(ArchiveExtension)</RepoAssetsSymbolsArchive>
-    </PropertyGroup>
-
-    <ItemGroup>
-      <RepoSymbolsArchive Include="$(RepoAssetsSymbolsArchive)" Condition="Exists('$(RepoAssetsSymbolsArchive)')" />
-    </ItemGroup>
-  </Target>
-
-  <Target Name="ExtractSymbolsArchiveFile"
-          DependsOnTargets="RepoBuild;DiscoverSymbolsArchiveFile"
-          Condition="'$(DotNetBuildSourceOnly)' == 'true' and
-                     '$(IsUtilityProject)' != 'true' and
-                     '@(RepoSymbolsArchive)' != ''"
-          Inputs="@(RepoSymbolsArchive)"
-          Outputs="$(IntermediateSymbolsRepoDir)">
-
-    <Error Text="There must only be a single repository symbols archive file. Found: @(RepoSymbolsArchive)"
-           Condition="@(RepoSymbolsArchive->Count()) &gt; 1" />
-
-    <!-- Extract repo symbols tarball -->
-    <MakeDir Directories="$(IntermediateSymbolsRepoDir)" />
-    <Exec Command="tar -xzf %(RepoSymbolsArchive.Identity) -C $(IntermediateSymbolsRepoDir)" />
-  </Target>
-
   <Target Name="Build"
           DependsOnTargets="
     BuildRepoReferences;
     RepoBuild;
     LogRepoArtifacts;
     CopyInnerBuildRestoredPackages;
     ExtractToolPackage;
-    DiscoverSymbolsArchiveFile;
-    ExtractSymbolsArchiveFile;
     CleanupRepo" />
 
   <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.UsageReport.WritePackageUsageData" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" />

src/SourceBuild/patches/arcade/0001-Copy-repo-symbols-to-intermediates.patch

@@ -0,0 +1,111 @@
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: Nikola Milosavljevic <[email protected]>
+Date: Wed, 3 Apr 2024 21:23:27 +0000
+Subject: [PATCH] Copy repo symbols to intermediates
+
+Backport: https://github.com/dotnet/arcade/pull/14681
+---
+ .../tools/Publish.proj                        |  8 -----
+ .../tools/SourceBuild/AfterSourceBuild.proj   |  2 +-
+ .../SourceBuild/SourceBuildArcade.targets     | 35 +++++++++++++++----
+ 3 files changed, 29 insertions(+), 16 deletions(-)
+
+diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/Publish.proj b/src/Microsoft.DotNet.Arcade.Sdk/tools/Publish.proj
+index 43fadc322..bec1a4963 100644
+--- a/src/Microsoft.DotNet.Arcade.Sdk/tools/Publish.proj
++++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/Publish.proj
+@@ -118,14 +118,6 @@
+       <ItemsToSignPostBuild Include="@(SymbolPackagesToGenerate->'%(Filename)%(Extension)')" Condition="'$(PostBuildSign)' == 'true'" />
+     </ItemGroup>
+
+-    <!-- Include Symbols.<repo>.tar.gz, if running in inner source-only build -->
+-    <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true' and '$(DotNetBuildInnerRepo)' == 'true'">
+-      <UnifiedSymbolsPackage Include="$(ArtifactsNonShippingPackagesDir)Symbols.*.tar.gz" />
+-      <Artifact Include="@(UnifiedSymbolsPackage)"
+-                RelativeBlobPath="Symbols/%(Filename)%(Extension)"
+-                IsShipping="false" />
+-    </ItemGroup>
+-
+     <!--
+       If a symbol package doesn't exist yet we assume that the regular package contains Portable PDBs.
+       Such packages can act as symbol packages since they have the same structure.
+diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/SourceBuild/AfterSourceBuild.proj b/src/Microsoft.DotNet.Arcade.Sdk/tools/SourceBuild/AfterSourceBuild.proj
+index c2ba0644b..6a1fd449d 100644
+--- a/src/Microsoft.DotNet.Arcade.Sdk/tools/SourceBuild/AfterSourceBuild.proj
++++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/SourceBuild/AfterSourceBuild.proj
+@@ -14,7 +14,7 @@
+   <Target Name="AfterSourceBuildInnerBuild"
+           Condition="('$(ArcadeInnerBuildFromSource)' == 'true' or '$(DotNetBuildInnerRepo)' == 'true') and
+                      '$(DotNetBuildOrchestrator)' == 'true'"
+-          DependsOnTargets="CreateRepoSymbolsArchive;ClearPreviousSBRP" />
++          DependsOnTargets="CopyRepoSymbolsToIntermediates;ClearPreviousSBRP" />
+
+   <Target Name="ClearPreviousSBRP"
+           Condition="'$(GitHubRepositoryName)' == 'source-build-reference-packages'">
+diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/SourceBuild/SourceBuildArcade.targets b/src/Microsoft.DotNet.Arcade.Sdk/tools/SourceBuild/SourceBuildArcade.targets
+index 41817c3a9..f06d33095 100644
+--- a/src/Microsoft.DotNet.Arcade.Sdk/tools/SourceBuild/SourceBuildArcade.targets
++++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/SourceBuild/SourceBuildArcade.targets
+@@ -159,20 +159,20 @@
+
+     Conditioning out for Windows as the tar execution below doesn't work cross-plat.
+   -->
++
+   <Target Name="CreateRepoSymbolsArchive"
++          Condition="'$(OS)' != 'Windows_NT' and ('$(DotNetBuildFromSourceFlavor)' == 'Product' or '$(DotNetBuildOrchestrator)' == 'true' or '$(SupplementalIntermediateNupkgCategory)' == '$(SymbolsIntermediateNupkgCategory)')"
++          DependsOnTargets="$(CreateRepoSymbolsArchiveDependsOn);
++                            DiscoverRepoSymbols;
++                            PackageRepoSymbols"/>
++
++  <Target Name="DiscoverRepoSymbols"
+           Condition="'$(OS)' != 'Windows_NT' and ('$(DotNetBuildFromSourceFlavor)' == 'Product' or '$(DotNetBuildOrchestrator)' == 'true' or '$(SupplementalIntermediateNupkgCategory)' == '$(SymbolsIntermediateNupkgCategory)')"
+           DependsOnTargets="$(CreateRepoSymbolsArchiveDependsOn)">
+     <PropertyGroup>
+       <SymbolsRoot>$(CurrentRepoSourceBuildArtifactsDir)</SymbolsRoot>
+       <!-- Fall back to repo root for source-build-externals or repos that don't have the regular SymbolsRoot as defined above -->
+       <SymbolsRoot Condition="!Exists('$(SymbolsRoot)') or '$(GitHubRepositoryName)' == 'source-build-externals'">$(RepoRoot)</SymbolsRoot>
+-      <SymbolsArchiveLocation>$(CurrentRepoSourceBuildArtifactsNonShippingPackagesDir)</SymbolsArchiveLocation>
+-      <SymbolsArchiveLocation Condition="'$(GitHubRepositoryName)' == 'nuget-client' and '$(PackageOutputPath)' != ''">$([MSBuild]::EnsureTrailingSlash('$(PackageOutputPath)'))</SymbolsArchiveLocation>
+-      <SymbolsList>$([MSBuild]::NormalizePath('$(SymbolsArchiveLocation)', 'symbols.lst'))</SymbolsList>
+-      <SymbolsArchivePrefix>Symbols.</SymbolsArchivePrefix>
+-      <!-- $(Version) and $(TargetRid) are only available when target is executed as part of intermediate package creation. -->
+-      <SymbolsArchiveSuffix Condition="'$(CreateIntermediatePackage)' == 'true'">.$(Version).$(TargetRid)</SymbolsArchiveSuffix>
+-      <SymbolsArchiveFile>$(SymbolsArchiveLocation)$(SymbolsArchivePrefix)$(GitHubRepositoryName)$(SymbolsArchiveSuffix)$(ArchiveExtension)</SymbolsArchiveFile>
+     </PropertyGroup>
+
+     <ItemGroup>
+@@ -181,6 +181,20 @@
+         <RelativePath>$([MSBuild]::MakeRelative($(SymbolsRoot), %(FullPath)))</RelativePath>
+       </AbsoluteSymbolPath>
+     </ItemGroup>
++  </Target>
++
++  <Target Name="PackageRepoSymbols"
++          Condition="'$(OS)' != 'Windows_NT' and ('$(DotNetBuildFromSourceFlavor)' == 'Product' or '$(DotNetBuildOrchestrator)' == 'true' or '$(SupplementalIntermediateNupkgCategory)' == '$(SymbolsIntermediateNupkgCategory)')"
++          DependsOnTargets="$(CreateRepoSymbolsArchiveDependsOn);DiscoverRepoSymbols">
++    <PropertyGroup>
++      <SymbolsArchiveLocation>$(CurrentRepoSourceBuildArtifactsNonShippingPackagesDir)</SymbolsArchiveLocation>
++      <SymbolsArchiveLocation Condition="'$(GitHubRepositoryName)' == 'nuget-client' and '$(PackageOutputPath)' != ''">$([MSBuild]::EnsureTrailingSlash('$(PackageOutputPath)'))</SymbolsArchiveLocation>
++      <SymbolsList>$([MSBuild]::NormalizePath('$(SymbolsArchiveLocation)', 'symbols.lst'))</SymbolsList>
++      <SymbolsArchivePrefix>Symbols.</SymbolsArchivePrefix>
++      <!-- $(Version) and $(TargetRid) are only available when target is executed as part of intermediate package creation. -->
++      <SymbolsArchiveSuffix Condition="'$(CreateIntermediatePackage)' == 'true'">.$(Version).$(TargetRid)</SymbolsArchiveSuffix>
++      <SymbolsArchiveFile>$(SymbolsArchiveLocation)$(SymbolsArchivePrefix)$(GitHubRepositoryName)$(SymbolsArchiveSuffix)$(ArchiveExtension)</SymbolsArchiveFile>
++    </PropertyGroup>
+
+     <WriteLinesToFile
+       File="$(SymbolsList)"
+@@ -203,6 +217,13 @@
+     <Delete Files="$(SymbolsList)" Condition="Exists($(SymbolsList))" />
+   </Target>
+
++  <Target Name="CopyRepoSymbolsToIntermediates"
++          DependsOnTargets="DiscoverRepoSymbols">
++    <MakeDir Directories="$(SourceBuiltSymbolsDir)" />
++    <Copy
++      SourceFiles="@(AbsoluteSymbolPath)"
++      DestinationFolder="$(SourceBuiltSymbolsDir)%(RecursiveDir)" />
++  </Target>
+   <!--
+     This target can be removed once we enable standard repo assets manifests and SB orchestrator
+     starts using it - https://github.com/dotnet/source-build/issues/3970