Update workaround-publishing branch with current main

Arcade.sln

@@ -129,6 +129,12 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Arcade.Common", "
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Arcade.Test.Common", "src\Common\Microsoft.Arcade.Test.Common\Microsoft.Arcade.Test.Common.csproj", "{6CA09DC9-E654-4906-A977-1279F6EDC109}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.DotNet.PackageValidation", "src\Microsoft.DotNet.PackageValidation\Microsoft.DotNet.PackageValidation.csproj", "{B691A17B-B577-431C-AF4D-199BBAC8EC97}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.DotNet.PackageValidation.Tests", "src\Microsoft.DotNet.PackageValidation.Tests\Microsoft.DotNet.PackageValidation.Tests.csproj", "{8BBF14AC-48F0-4282-910E-48E816021660}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Arcade.Common.Tests", "src\Common\Microsoft.Arcade.Common.Tests\Microsoft.Arcade.Common.Tests.csproj", "{B5E9D9D8-59E0-49F8-9C3C-75138A2D452C}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -579,18 +585,6 @@ Global
 		{6E19C6B6-4ADF-4DD6-86CC-6C1624BCDB71}.Release|x64.Build.0 = Release|Any CPU
 		{6E19C6B6-4ADF-4DD6-86CC-6C1624BCDB71}.Release|x86.ActiveCfg = Release|Any CPU
 		{6E19C6B6-4ADF-4DD6-86CC-6C1624BCDB71}.Release|x86.Build.0 = Release|Any CPU
-		{62B929C4-3D15-4D43-AEFC-2D0BD3CFC20D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{62B929C4-3D15-4D43-AEFC-2D0BD3CFC20D}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{62B929C4-3D15-4D43-AEFC-2D0BD3CFC20D}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{62B929C4-3D15-4D43-AEFC-2D0BD3CFC20D}.Debug|x64.Build.0 = Debug|Any CPU
-		{62B929C4-3D15-4D43-AEFC-2D0BD3CFC20D}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{62B929C4-3D15-4D43-AEFC-2D0BD3CFC20D}.Debug|x86.Build.0 = Debug|Any CPU
-		{62B929C4-3D15-4D43-AEFC-2D0BD3CFC20D}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{62B929C4-3D15-4D43-AEFC-2D0BD3CFC20D}.Release|Any CPU.Build.0 = Release|Any CPU
-		{62B929C4-3D15-4D43-AEFC-2D0BD3CFC20D}.Release|x64.ActiveCfg = Release|Any CPU
-		{62B929C4-3D15-4D43-AEFC-2D0BD3CFC20D}.Release|x64.Build.0 = Release|Any CPU
-		{62B929C4-3D15-4D43-AEFC-2D0BD3CFC20D}.Release|x86.ActiveCfg = Release|Any CPU
-		{62B929C4-3D15-4D43-AEFC-2D0BD3CFC20D}.Release|x86.Build.0 = Release|Any CPU
 		{3376C769-211F-4537-A156-5F841FF7840B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{3376C769-211F-4537-A156-5F841FF7840B}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{3376C769-211F-4537-A156-5F841FF7840B}.Debug|x64.ActiveCfg = Debug|Any CPU
@@ -819,6 +813,42 @@ Global
 		{6CA09DC9-E654-4906-A977-1279F6EDC109}.Release|x64.Build.0 = Release|Any CPU
 		{6CA09DC9-E654-4906-A977-1279F6EDC109}.Release|x86.ActiveCfg = Release|Any CPU
 		{6CA09DC9-E654-4906-A977-1279F6EDC109}.Release|x86.Build.0 = Release|Any CPU
+		{B691A17B-B577-431C-AF4D-199BBAC8EC97}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B691A17B-B577-431C-AF4D-199BBAC8EC97}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B691A17B-B577-431C-AF4D-199BBAC8EC97}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{B691A17B-B577-431C-AF4D-199BBAC8EC97}.Debug|x64.Build.0 = Debug|Any CPU
+		{B691A17B-B577-431C-AF4D-199BBAC8EC97}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{B691A17B-B577-431C-AF4D-199BBAC8EC97}.Debug|x86.Build.0 = Debug|Any CPU
+		{B691A17B-B577-431C-AF4D-199BBAC8EC97}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B691A17B-B577-431C-AF4D-199BBAC8EC97}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B691A17B-B577-431C-AF4D-199BBAC8EC97}.Release|x64.ActiveCfg = Release|Any CPU
+		{B691A17B-B577-431C-AF4D-199BBAC8EC97}.Release|x64.Build.0 = Release|Any CPU
+		{B691A17B-B577-431C-AF4D-199BBAC8EC97}.Release|x86.ActiveCfg = Release|Any CPU
+		{B691A17B-B577-431C-AF4D-199BBAC8EC97}.Release|x86.Build.0 = Release|Any CPU
+		{8BBF14AC-48F0-4282-910E-48E816021660}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{8BBF14AC-48F0-4282-910E-48E816021660}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{8BBF14AC-48F0-4282-910E-48E816021660}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{8BBF14AC-48F0-4282-910E-48E816021660}.Debug|x64.Build.0 = Debug|Any CPU
+		{8BBF14AC-48F0-4282-910E-48E816021660}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{8BBF14AC-48F0-4282-910E-48E816021660}.Debug|x86.Build.0 = Debug|Any CPU
+		{8BBF14AC-48F0-4282-910E-48E816021660}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{8BBF14AC-48F0-4282-910E-48E816021660}.Release|Any CPU.Build.0 = Release|Any CPU
+		{8BBF14AC-48F0-4282-910E-48E816021660}.Release|x64.ActiveCfg = Release|Any CPU
+		{8BBF14AC-48F0-4282-910E-48E816021660}.Release|x64.Build.0 = Release|Any CPU
+		{8BBF14AC-48F0-4282-910E-48E816021660}.Release|x86.ActiveCfg = Release|Any CPU
+		{8BBF14AC-48F0-4282-910E-48E816021660}.Release|x86.Build.0 = Release|Any CPU
+		{B5E9D9D8-59E0-49F8-9C3C-75138A2D452C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B5E9D9D8-59E0-49F8-9C3C-75138A2D452C}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B5E9D9D8-59E0-49F8-9C3C-75138A2D452C}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{B5E9D9D8-59E0-49F8-9C3C-75138A2D452C}.Debug|x64.Build.0 = Debug|Any CPU
+		{B5E9D9D8-59E0-49F8-9C3C-75138A2D452C}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{B5E9D9D8-59E0-49F8-9C3C-75138A2D452C}.Debug|x86.Build.0 = Debug|Any CPU
+		{B5E9D9D8-59E0-49F8-9C3C-75138A2D452C}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B5E9D9D8-59E0-49F8-9C3C-75138A2D452C}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B5E9D9D8-59E0-49F8-9C3C-75138A2D452C}.Release|x64.ActiveCfg = Release|Any CPU
+		{B5E9D9D8-59E0-49F8-9C3C-75138A2D452C}.Release|x64.Build.0 = Release|Any CPU
+		{B5E9D9D8-59E0-49F8-9C3C-75138A2D452C}.Release|x86.ActiveCfg = Release|Any CPU
+		{B5E9D9D8-59E0-49F8-9C3C-75138A2D452C}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -851,6 +881,7 @@ Global
 		{CE5278A3-2442-4309-A543-5BA5C1C76A2A} = {C53DD924-C212-49EA-9BC4-1827421361EF}
 		{E941EDE6-3FFB-4776-A4CE-750755D57817} = {C53DD924-C212-49EA-9BC4-1827421361EF}
 		{6CA09DC9-E654-4906-A977-1279F6EDC109} = {C53DD924-C212-49EA-9BC4-1827421361EF}
+		{B5E9D9D8-59E0-49F8-9C3C-75138A2D452C} = {C53DD924-C212-49EA-9BC4-1827421361EF}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {32B9C883-432E-4FC8-A1BF-090EB033DD5B}

Documentation/CorePackages/PostBuildSigning.md

@@ -7,12 +7,43 @@ If PostBuildSign==true then no other steps are needed, post-build signing is ena
 
 Steps to ensure that post-build signing is correctly set up for a repo: A manifest file is created as part of the build process, it can be found in AssetManifests/Manifest.xml. This file lists everything that will be signed once the build is complete. If this file has been created and the contents of the ItemsToSign section match the expected set of signed files for the build then no further action is needed.
 
+## Post Build Signing Technical Details
+
+When performing normal in-build signing, the build process generates a list of files to sign (e.g. nupkg files, dlls, msi's, etc.) during the build process. These are combined with information about what certificates to use for each file type or specific file name (located in [Sign.props](https://github.com/dotnet/arcade/blob/master/src/Microsoft.DotNet.Arcade.Sdk/tools/Sign.props)) and fed to SignTool. SignTool processes the input files and performs signing via ESRP, replacing the input files in place with the signed versions. Depending on the build scenario, SignTool may be called multiple times within a build to iteratively build up some archives. For instance, it may be called on individual dlls which are then packaged into an msi, which is then passed to SignTool.
+
+Performing post build signing is more involved. The drop that is to be signed is comprised of the outputs of many repos. Some of those outputs may already be signed, some of those outputs may have been repackaged within the outputs of other builds, etc. Thus, most of the effort in post build signing is related to two aspects of the process:
+- **Deconstructing (and reconstructing) archives** - With in-build signing, SignTool knew how to deconstruct zip and nupkg archives. These could simply be unzipped, their contents replaced in-line, then zipped back up. Post build signing requires dealing with msi installers too. These are not as amenable to changes to contents (e.g. the msi database encodes file lengths, which change with signing).
+- **Tracking the source of all the files** - Let's say dotnet/runtime notes that apphost.exe should not be signed, then the installer repackages that binary in a zip file. When post-build signing runs, it needs to be able to identify that apphost.exe, which appears to be located within an installer produced zip file, actually came from runtime and thus should not be signed.
+
+Post build signing deals with this by recording sidecar information at build time, then gathering the closure of that info when post build signing runs. The additional information includes the following:
+- Archives that can be used to reconstruct installers produced by a build.
+- Manifest files which describe the signing data for a build (e.g. items to sign and file extension -> certificate mappings)
+
+The process works as follows:
+1. Gather the drop of a build that needs to be signed, including all un-released inputs builds.
+2. Locate all build manifests (containing signing information for each input build)
+3. Build a list of all files to sign (files to sign should be found in the drop), and the specific certificate info required for those files/extensions (and any files within containers). Each input item to sign is associated with the certificate info present in the same build.
+4. Pass this data to SignTool
+5. In SignTool:
+   1. Walk each item to sign. If the item is an archive, unpack it and begin to process the contained files.
+   2. After a file is unpacked (if necessary), determine what signing cert and/or strong name it should get (if any). Add info to a map of filename+hash of file -> info.
+   3. Continue until all files are unpacked.
+   4. Starting at the most nested set of files, create a Microbuild compatible project file and submit for signing.
+   5. Repack those files into their enclosing containers, if any.
+   6. Repeat, moving upwards in the tree of files to sign until all files have been repacked.
+6. Upload files to storage (feeds and blob storage).
+
+### Determining certificates
+
+The crux of the SignTool algorithm is determining the certificate for a specific file. *Side note: Certificates are associated with a specific file name or file extension, though they cannot be associated with a specific path. For instance, 'foo/bar/baz.zip' is only dealt with as 'baz.zip'*. When dealing with a single repo, SignTool just interprets the signing info as applying to all files passed to it. A baz.zip found anywhere in the input file set is treated identically. When dealing with post build signing, it's possible that a foo.dll found in an sdk asset may actually come from the runtime, which may have indicated that *.dll get a different certificate. *So SignTool needs to track where foo.dll originally came from*. Did it come from the build of dotnet/runtime, or from dotnet/sdk? To do this, SignTool associates each input top level item to sign and the signing info for a build with a 'collision priority ID' This collision priority ID serves as a way to disambiguate the source signing information for a nested artifact. As each top-level asset is unpacked, the hash of each nested file is determined and looked up in a map. If the map already contains a file with the same name and hash, the collision priority ID's are compared. If the current one in the map is lower, the signing info for the existing file is used. Otherwise, the certificate information is recomputed based on the lower collision priority ID.
+
+## Additional Information
 
 More information about the file types that are signed and the certificates used to sign them can be found here:
 https://github.com/dotnet/arcade/blob/master/src/Microsoft.DotNet.Arcade.Sdk/tools/Sign.props
 
 Additional information about the publishing process in general can be found here:
 https://github.com/dotnet/arcade/blob/master/Documentation/CorePackages/Publishing.md
 
-Additional information about the legacy signing process can be found here:
+Additional information about the in-build signing process can be found here:
 https://github.com/dotnet/arcade/blob/master/Documentation/CorePackages/Signing.md

Documentation/Projects/M2MRenaming/Master-to-Main-renaming-guide.md

@@ -95,7 +95,7 @@ All of the steps are easily revert-able, so it is not a problem to go back to `m
 3. [Update the build mirroring in `subscriptions.json`](#3-update-the-build-mirroring-in-subscriptionsjson)
 4. [Create the `main` branch in the internal mirrored AzDO repository](#4-create-the-main-branch-in-the-internal-mirrored-azdo-repository)
 5. [Change the default branch to `main` for your GitHub repository](#5-change-the-default-branch-to-main-for-your-github-repository)
-6. [Search your repository for any references to the `main` branch specific to your repo](#6-search-your-repository-for-any-references-to-the-main-branch-specific-to-your-repo)
+6. [Search your repository for any references to the `master` branch specific to your repo](#6-search-your-repository-for-any-references-to-the-master-branch-specific-to-your-repo)
 7. [Use a `darc` script to migrate channels and subscriptions](#7-use-a-darc-script-to-migrate-channels-and-subscriptions)
 8. [Change the default branch for AzDO builds for pipelines](#8-change-the-default-branch-for-azdo-pipelines)
 9. [Switch the default branch of the AzDO repository to `main`](#9-switch-the-default-branch-of-the-azdo-repository-to-main)
@@ -190,8 +190,8 @@ This will effectively disable code mirroring.
 ![AzDO mirrored](images/azdo-mirrored.png)
 
 1. Go to the [internally mirrored repository](https://dev.azure.com/dnceng/internal/_git) - repository should have the same name, only replace `/` with `-`, e.g. `dotnet/xharness` becomes `dotnet-xharness`
-2. Wait for the code-mirror build to propagate the change from the previous step to the internal mirrored repository
-    > Note: Go to the [code-mirror build](https://dev.azure.com/dnceng/internal/_build?definitionId=16&_a=summary) and filter the pipeline runs by Tags (select your repo).
+2. Wait for the code-mirror build to propagate changes made in **step 2** to the internal mirrored repository (meaning: internal repo has main triggers)
+    > Note: You can go to the [code-mirror build](https://dev.azure.com/dnceng/internal/_build?definitionId=16&_a=summary) and filter the pipeline runs by Tags (select your repo).
 3. Go to `Branches`
 4. Create a new branch called `main` off of the `master` branch
 5. Mirror policies from branch `master` to branch `main` using [M2MTool](https://devdiv.visualstudio.com/DefaultCollection/Engineering/_git/M2MTool?path=%2FREADME.md&_a=preview)
@@ -226,7 +226,7 @@ Example for `dotnet/xharness`:
 
 ![Changing the default branch in GitHub](images/github-branch-rename-tool.png)
 
-## 6. Search your repository for any references to the `main` branch specific to your repo
+## 6. Search your repository for any references to the `master` branch specific to your repo
 
 Search your repository for any references to the `master` branch specific to your repo, replace them to `main` and push them to `main`.