Document ROSLYN_TEST_USEDASSEMBLIES (#60478)

Author: jcouv

docs/contributing/Building, Debugging, and Testing on Windows.md

@@ -196,6 +196,20 @@ See more details in the [IOperation test hook](https://github.com/dotnet/roslyn/
 `C:\Source\roslyn> cd ..`
 `C:\Source> dotnet-format analyzers .\roslyn\Compilers.sln --diagnostics=RS0016 --no-restore --include-generated -v diag`
 
+## Replicating Failures in the Used Assemblies leg
+
+In order to replicate test failures in that leg, there are a few options:
+
+1. Uncomment `src/Compilers/Test/Core/Compilation/CompilationExtensions.cs:9`, which defines `ROSLYN_TEST_USEDASSEMBLIES`, and run your tests. Do _not_ check this in, as it
+will enable the test hook for every test in every project and significantly slow down regular test runs.
+2. Set the `ROSLYN_TEST_USEDASSEMBLIES` environment variable and restart VS with it set.
+3. Set a breakpoint at the start of `CSharpTestBase.VerifyUsedAssemblyReferences` in `src/Compilers/Test/Utilities/CSharp/CSharpTestBase.cs`. When it breaks, use VS's jump to location or
+drag the instruction pointer past the early check and return on `EnableVerifyUsedAssemblies`.
+
+When a test failure is isolated, please add a _dedicated_ test for this (ie. failing even when the Used Assemblies validation isn't enabled) to make it easier to avoid future regressions.  
+Preferrably, don't replicate the entire original test, just enough to hit the bug to ensure that it's protected against regressions.  
+Before pushing a relevant fix to CI, you can validate locally using the `-testUsedAssemblies` command-line option for `build.cmd`. For example: `build.cmd -testCoreClr -testCompilerOnly -testUsedAssemblies`.
+
 ## Contributing
 
 Please see [Contributing Code](https://github.com/dotnet/roslyn/blob/main/CONTRIBUTING.md) for details on contributing changes back to the code.

eng/build.ps1

@@ -98,7 +98,7 @@ function Print-Usage() {
   Write-Host "  -testCompilerOnly         Run only the compiler unit tests"
   Write-Host "  -testVsi                  Run all integration tests"
   Write-Host "  -testIOperation           Run extra checks to validate IOperations"
-  Write-Host "  -testUsedAssemblies       Run extra checks to validate used assemblies feature"
+  Write-Host "  -testUsedAssemblies       Run extra checks to validate used assemblies feature (see ROSLYN_TEST_USEDASSEMBLIES in codebase)"
   Write-Host ""
   Write-Host "Advanced settings:"
   Write-Host "  -ci                       Set when running on CI server"

src/Compilers/Test/Core/Compilation/CompilationExtensions.cs

@@ -6,27 +6,26 @@
 // Uncomment to enable the IOperation test hook on all test runs. Do not commit this uncommented.
 //#define ROSLYN_TEST_IOPERATION
 
+// Uncomment to enable the Used Assemblies test hook on all test runs. Do not commit this uncommented.
+//#define ROSLYN_TEST_USEDASSEMBLIES
+
 using System;
 using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
-using System.Reflection.Metadata;
 using System.Text;
-using System.Text.RegularExpressions;
 using System.Threading;
 using Microsoft.CodeAnalysis.CodeGen;
 using Microsoft.CodeAnalysis.CSharp;
 using Microsoft.CodeAnalysis.Emit;
 using Microsoft.CodeAnalysis.FlowAnalysis;
 using Microsoft.CodeAnalysis.Operations;
 using Microsoft.CodeAnalysis.PooledObjects;
-using Microsoft.CodeAnalysis.Symbols;
 using Roslyn.Test.Utilities;
 using Roslyn.Utilities;
 using Xunit;
-using Xunit.Sdk;
 
 namespace Microsoft.CodeAnalysis.Test.Utilities
 {
@@ -39,7 +38,12 @@ public static class CompilationExtensions
             !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("ROSLYN_TEST_IOPERATION"));
 #endif
 
-        internal static bool EnableVerifyUsedAssemblies { get; } = !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("ROSLYN_TEST_USEDASSEMBLIES"));
+        internal static bool EnableVerifyUsedAssemblies { get; } =
+#if ROSLYN_TEST_USEDASSEMBLIES
+            true;
+#else
+            !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("ROSLYN_TEST_USEDASSEMBLIES"));
+#endif
 
         internal static ImmutableArray<byte> EmitToArray(
             this Compilation compilation,

src/Compilers/Test/Utilities/CSharp/CSharpTestBase.cs

@@ -1210,6 +1210,7 @@ private static void ValidateCompilation(Func<CSharpCompilation> createCompilatio
 
         private static void VerifyUsedAssemblyReferences(Func<CSharpCompilation> createCompilationLambda)
         {
+            // To run the additional validation below, comment this out or define ROSLYN_TEST_USEDASSEMBLIES
             if (!CompilationExtensions.EnableVerifyUsedAssemblies)
             {
                 return;