Infra fixes for bugs uncovered by testing of the JIT/Methodical merged tests

eng/pipelines/common/templates/runtimes/run-test-job.yml

@@ -300,7 +300,7 @@ jobs:
     # Compose the Core_Root folder containing all artifacts needed for running
     # CoreCLR tests. This step also compiles the framework using Crossgen / Crossgen2
     # in ReadyToRun jobs.
-    - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) generatelayoutonly $(logRootNameArg)Layout $(runtimeFlavorArgs) $(crossgenArg) $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(librariesOverrideArg)
+    - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) generatelayoutonly $(logRootNameArg)Layout $(runtimeFlavorArgs) $(crossgenArg) $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(librariesOverrideArg) $(runtimeVariantArg)
       displayName: Generate CORE_ROOT
 
     # Build a Mono LLVM AOT cross-compiler for non-amd64 targets (in this case, just arm64)

src/tests/Common/CLRTest.Execute.Bash.targets

@@ -306,8 +306,6 @@ $__Command msbuild $CORE_ROOT/wasm-test-runner/WasmTestRunner.proj /p:NetCoreApp
     <![CDATA[
 $(BashLinkerTestLaunchCmds)
 
-export TestExclusionListPath=$CORE_ROOT/TestExclusionList.txt
-
 _DebuggerArgsSeparator=
 if [[ "$_DebuggerFullPath" == *lldb* ]];
 then
@@ -549,6 +547,7 @@ $(BashCLRTestExitCodePrep)
 $(IlasmRoundTripBashScript)
 # Allow precommands to override the ExePath
 ExePath=$(InputAssemblyName)
+export TestExclusionListPath=$CORE_ROOT/TestExclusionList.txt
 # PreCommands
 $(BashCLRTestPreCommands)
 # Launch
@@ -561,15 +560,23 @@ $(BashCLRTestExitCodeCheck)
 
     </PropertyGroup>
 
+    <PropertyGroup>
+      <ExecutionBashScriptPath>$(OutputPath)/$(MSBuildProjectName).sh</ExecutionBashScriptPath>
+    </PropertyGroup>
+
     <!-- Write the file.
          Note: under the hood, this will rely on Environment.NewLine for line
          endings. This means that if the scripts are being generated on Windows,
          the line endings will need to be changed from CR-LF to Unix (LF) line
-         endings before running the scripts on Unix platforms. -->
+         endings before running the scripts on Unix platforms. In our current lab
+         infra it shouldn't really matter as the execution scripts are regenerated
+         in the 'test run' phase before sending the items to Helix i.o.w. at
+         the point where we already know the exact targeting platform. -->
     <WriteLinesToFile
-      File="$(OutputPath)\$(MSBuildProjectName).sh"
+      File="$(ExecutionBashScriptPath)"
       Lines="$(_CLRTestExecutionScriptText)"
       Overwrite="true" />
+    <Exec Command="chmod +x $(ExecutionBashScriptPath)" EchoOff="true" />
   </Target>
 
 </Project>

src/tests/Common/CLRTest.Execute.Batch.targets

@@ -301,8 +301,6 @@ COPY /y %CORE_ROOT%\CoreShim.dll .
 $(BatchLinkerTestLaunchCmds)
 $(BatchCopyCoreShimLocalCmds)
 
-set TestExclusionListPath=%CORE_ROOT%\TestExclusionList.txt
-
 IF NOT "%CLRCustomTestLauncher%"=="" (
   set LAUNCHER=call %CLRCustomTestLauncher% %scriptPath%
 ) ELSE (
@@ -449,6 +447,7 @@ $(IlasmRoundTripBatchScript)
 
 REM Allow precommands to override the ExePath
 set ExePath=$(InputAssemblyName)
+set TestExclusionListPath=%CORE_ROOT%\TestExclusionList.txt
 
 REM Precommands
 $(CLRTestBatchPreCommands)

src/tests/Common/Directory.Build.targets

@@ -191,4 +191,7 @@
 
   </Target>
 
+  <!-- At this point Common test dependencies don't have any native components -->
+  <Target Name="CopyAllNativeProjectReferenceBinaries" />
+
 </Project>

src/tests/Common/XHarnessRunnerLibrary/GeneratedTestRunner.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
+using System.Text;
 using System.Threading.Tasks;
 using Microsoft.DotNet.XHarness.Common;
 using Microsoft.DotNet.XHarness.TestRunners.Common;
@@ -15,16 +16,20 @@ public sealed class GeneratedTestRunner : TestRunner
     TestFilter.ISearchClause? _filter;
     Func<TestFilter?, TestSummary> _runTestsCallback;
     HashSet<string> _testExclusionList;
+    private readonly Boolean _writeBase64TestResults;
+
     public GeneratedTestRunner(
         LogWriter logger, 
         Func<TestFilter?, TestSummary> runTestsCallback, 
         string assemblyName,
-        HashSet<string> testExclusionList)
+        HashSet<string> testExclusionList,
+        bool writeBase64TestResults)
         :base(logger)
     {
         _assemblyName = assemblyName;
         _runTestsCallback = runTestsCallback;
         _testExclusionList = testExclusionList;
+        _writeBase64TestResults = writeBase64TestResults;
         ResultsFileName = $"{_assemblyName}.testResults.xml";
     }
 
@@ -53,7 +58,17 @@ public override string WriteResultsToFile(XmlResultJargon xmlResultJargon)
     public override void WriteResultsToFile(TextWriter writer, XmlResultJargon jargon)
     {
         Debug.Assert(jargon == XmlResultJargon.xUnit);
-        writer.WriteLine(LastTestRun.GetTestResultOutput(_assemblyName));
+        string lastTestResults = LastTestRun.GetTestResultOutput(_assemblyName);
+        if (_writeBase64TestResults)
+        {
+            byte[] encodedBytes = Encoding.Unicode.GetBytes(lastTestResults);
+            string base64Results = Convert.ToBase64String(encodedBytes);
+            writer.WriteLine($"STARTRESULTXML {encodedBytes.Length} {base64Results} ENDRESULTXML");
+        }
+        else
+        {
+            writer.WriteLine(lastTestResults);
+        }
     }
 
     public override void SkipTests(IEnumerable<string> tests)

src/tests/Common/XHarnessRunnerLibrary/RunnerEntryPoint.cs

@@ -36,6 +36,12 @@ public static async Task<int> RunTests(
         bool anyFailedTests = false;
         entryPoint.TestsCompleted += (o, e) => anyFailedTests = e.FailedTests > 0;
         await entryPoint.RunAsync();
+
+        if (OperatingSystem.IsBrowser())
+        {
+            // Browser expects all xharness processes to exit with 0, even in case of failure
+            return 0;
+        }
         return anyFailedTests ? 1 : 0;
     }
 
@@ -65,7 +71,7 @@ public AppleEntryPoint(
         protected override bool IsXunit => true;
         protected override TestRunner GetTestRunner(LogWriter logWriter)
         {
-            var runner = new GeneratedTestRunner(logWriter, _runTestsCallback, _assemblyName, _testExclusionList);
+            var runner = new GeneratedTestRunner(logWriter, _runTestsCallback, _assemblyName, _testExclusionList, writeBase64TestResults: true);
             if (_methodNameToRun is not null)
             {
                 runner.SkipMethod(_methodNameToRun, isExcluded: false);
@@ -103,7 +109,7 @@ public AndroidEntryPoint(
         protected override bool IsXunit => true;
         protected override TestRunner GetTestRunner(LogWriter logWriter)
         {
-            var runner = new GeneratedTestRunner(logWriter, _runTestsCallback, _assemblyName, _testExclusionList);
+            var runner = new GeneratedTestRunner(logWriter, _runTestsCallback, _assemblyName, _testExclusionList, writeBase64TestResults: false);
             if (_methodNameToRun is not null)
             {
                 runner.SkipMethod(_methodNameToRun, isExcluded: false);
@@ -151,7 +157,7 @@ public WasmEntryPoint(
         protected override bool IsXunit => true;
         protected override TestRunner GetTestRunner(LogWriter logWriter)
         {
-            var runner = new GeneratedTestRunner(logWriter, _runTestsCallback, _assemblyName, _testExclusionList);
+            var runner = new GeneratedTestRunner(logWriter, _runTestsCallback, _assemblyName, _testExclusionList, writeBase64TestResults: true);
             if (_methodNameToRun is not null)
             {
                 runner.SkipMethod(_methodNameToRun, isExcluded: false);

src/tests/Common/XUnitWrapperLibrary/TestSummary.cs

@@ -70,17 +70,18 @@ public string GetTestResultOutput(string assemblyName)
         foreach (var test in _testResults)
         {
             resultsFile.Append($@"<test name=""{test.Name}"" type=""{test.ContainingTypeName}"" method=""{test.MethodName}"" time=""{test.Duration.TotalSeconds:F6}"" ");
+            string outputElement = !string.IsNullOrWhiteSpace(test.Output) ? $"<output><![CDATA[{test.Output}]]></output>" : string.Empty;
             if (test.Exception is not null)
             {
-                resultsFile.AppendLine($@"result=""Fail""><failure exception-type=""{test.Exception.GetType()}""><message><![CDATA[{test.Exception.Message}]]></message><stack-trace><![CDATA[{test.Exception.StackTrace}]]></stack-trace></failure><output><![CDATA[{test.Output}]]></output></test>");
+                resultsFile.AppendLine($@"result=""Fail""><failure exception-type=""{test.Exception.GetType()}""><message><![CDATA[{test.Exception.Message}]]></message><stack-trace><![CDATA[{test.Exception.StackTrace}]]></stack-trace></failure>{outputElement}</test>");
             }
             else if (test.SkipReason is not null)
             {
                 resultsFile.AppendLine($@"result=""Skip""><reason><![CDATA[{test.SkipReason}]]></reason></test>");
             }
             else
             {
-                resultsFile.AppendLine($@" result=""Pass""><output><![CDATA[{test.Output}]]></output></test>");
+                resultsFile.AppendLine($@" result=""Pass"">{outputElement}</test>");
             }
         }
 

src/tests/Common/helixpublishwitharcade.proj

@@ -282,7 +282,8 @@
 
   <Target Name="DiscoverMergedTestWrappers">
     <ItemGroup>
-      <_MergedWrapperMarker Include="$(TestBinDir)**\*.MergedTestAssembly" />
+      <!-- Exclude WASM support files. They can interfere with our discovery process and create extra work items that don't work. -->
+      <_MergedWrapperMarker Include="$(TestBinDir)**\*.MergedTestAssembly" Exclude="$(TestBinDir)**\supportFiles\*.MergedTestAssembly" />
     </ItemGroup>
   </Target>
 
@@ -294,14 +295,22 @@
       <_MergedWrapperRunScript Include="$([System.IO.Path]::ChangeExtension('%(_MergedWrapperMarker.Identity)', '.$(TestScriptExtension)'))" />
     </ItemGroup>
     <PropertyGroup>
-      <_MergedWrapperDirectory>%(_MergedWrapperRunScript.RootDir)%(Directory)</_MergedWrapperDirectory>
+      <_MergedWrapperDirectory>$([System.IO.Path]::GetDirectoryName('%(_MergedWrapperRunScript.Identity)'))</_MergedWrapperDirectory>
+      <_MergedWrapperParentDirectory>$([System.IO.Path]::GetDirectoryName('$(_MergedWrapperDirectory)'))</_MergedWrapperParentDirectory>
       <_MergedWrapperName>%(_MergedWrapperRunScript.FileName)</_MergedWrapperName>
       <_MergedWrapperRunScriptRelative Condition="'%(_MergedWrapperRunScript.Identity)' != ''">$([System.IO.Path]::GetRelativePath($(TestBinDir), %(_MergedWrapperRunScript.FullPath)))</_MergedWrapperRunScriptRelative>
     </PropertyGroup>
+    <ItemGroup>
+      <_MergedWrapperOutOfProcessTestMarkers Include="$(_MergedWrapperParentDirectory)/**/*.OutOfProcessTest" />
+      <_MergedWrapperOutOfProcessTestFiles
+        Include="%(_MergedWrapperOutOfProcessTestMarkers.RootDir)%(_MergedWrapperOutOfProcessTestMarkers.Directory)/**"
+        Condition="'@(_MergedWrapperOutOfProcessTestMarkers)' != ''" />
+    </ItemGroup>
 
     <ItemGroup>
       <_MergedPayloadGroups Include="$(_MergedWrapperName)" />
-      <_MergedPayloadFiles Include="$(_MergedWrapperDirectory)**" />
+      <_MergedPayloadFiles Include="$(_MergedWrapperDirectory)/**" />
+      <_MergedPayloadFiles Include="@(_MergedWrapperOutOfProcessTestFiles)" />
       <_MergedPayloadFiles Update="@(_MergedPayloadFiles)">
         <!-- Never use [MSBuild]::MakeRelative here! We have some files containing Unicode characters in their %(FullPath) and
              MakeRelative function calls Escape function internally that replaces all the Unicode characters with %<xx>. -->
@@ -312,7 +321,7 @@
     <ItemGroup>
       <!-- Remove the managed pdbs from our payloads.
            This is for performance reasons to reduce our helix payload size  -->
-      <ReducedMergedPayloadFilesFinal Include="@(_MergedPayloadFiles)" Condition=" '%(Extension)' != '.pdb' " />
+      <ReducedMergedPayloadFilesFinal Include="@(_MergedPayloadFiles)" Condition=" '%(Extension)' != '.pdb' and '%(Extension)' != '.OutOfProcessTest' " />
     </ItemGroup>
 
     <Copy SourceFiles="@(ReducedMergedPayloadFilesFinal)" DestinationFiles="@(ReducedMergedPayloadFilesFinal->'$(MergedPayloadsRootDirectory)\$(_MergedWrapperName)\%(FileRelativeToPayloadsRootDirectory)')" />

src/tests/Directory.Build.targets

@@ -114,21 +114,6 @@
     <SkipImportILTargets Condition="'$(CLRTestBuildAllTargets)' != '' And '$(CLRTestNeedTarget)' != '$(CLRTestBuildAllTargets)'">true</SkipImportILTargets>
   </PropertyGroup>
 
-  <Target Name="CopyMergedWrapperReferences"
-      Condition="'$(IsMergedTestRunnerAssembly)' == 'true'"
-      AfterTargets="Build"
-      BeforeTargets="CopyNativeProjectBinaries">
-    <ItemGroup>
-      <MergedWrapperReferenceFolders Include="@(ProjectReference->'$([System.IO.Path]::GetFullPath('%(ProjectReference.Identity)/..', '$(OutDir)/..'))/%(ProjectReference.FileName)')" />
-      <!-- For merged project wrappers, include native libraries in all project references -->
-      <MergedWrapperReferenceFiles Include="%(MergedWrapperReferenceFolders.Identity)/*$(LibSuffix)" />
-    </ItemGroup>
-    <Copy SourceFiles="@(MergedWrapperReferenceFiles)"
-        DestinationFiles="@(MergedWrapperReferenceFiles->'$(OutDir)/%(FileName)%(Extension)')"
-        SkipUnchangedFiles="true"
-        />
-  </Target>
-
   <Target Name="CopyNativeProjectBinaries" Condition="'$(_CopyNativeProjectBinaries)' == 'true'">
     <ItemGroup Condition="'$(UseVisualStudioNativeBinariesLayout)' == 'true'">
       <NativeProjectBinaries Include="$(NativeProjectOutputFolder)\*.*" />
@@ -225,9 +210,26 @@
 
     <MSBuild Projects="$(MSBuildProjectFile)" Targets="CopyNativeProjectBinaries" Properties="NativeProjectOutputFolder=%(NativeProjectOutputFoldersToCopy.Identity)" Condition="'@(NativeProjectReference)' != ''" />
 
+    <MSBuild Projects="@(ProjectReference)"
+        Targets="CopyAllNativeProjectReferenceBinaries"
+        Condition="'$(IsMergedTestRunnerAssembly)' == 'true'"
+        BuildInParallel="true" />
   </Target>
 
-  <Target Name="CopyAllNativeProjectReferenceBinaries" DependsOnTargets="ResolveCMakeNativeProjectReference;ConsolidateNativeProjectReference" />
+  <Target Name="CopyAllNativeProjectReferenceBinaries"
+          DependsOnTargets="ResolveCMakeNativeProjectReference;ConsolidateNativeProjectReference">
+
+    <ItemGroup Condition="'$(IsMergedTestRunnerAssembly)' == 'true'">
+      <MergedWrapperReferenceFolders Include="@(ProjectReference->'$([System.IO.Path]::GetFullPath('%(ProjectReference.Identity)/..', '$(OutDir)/..'))/%(ProjectReference.FileName)')" />
+      <!-- For merged project wrappers, include native libraries in all project references -->
+      <MergedWrapperReferenceFiles Include="%(MergedWrapperReferenceFolders.Identity)/*$(LibSuffix)" />
+    </ItemGroup>
+    <Copy SourceFiles="@(MergedWrapperReferenceFiles)"
+        DestinationFiles="@(MergedWrapperReferenceFiles->'$(OutDir)/%(FileName)%(Extension)')"
+        SkipUnchangedFiles="true"
+        Condition="'@(MergedWrapperReferenceFiles)' != ''"
+        />
+  </Target>
 
   <!-- Build shell or command scripts whenever we copy native binaries -->
   <Import Project="$(MSBuildThisFileDirectory)Common\CLRTest.Execute.targets" />
@@ -259,20 +261,40 @@
     </ItemGroup>
   </Target>
 
-  <Target Name="AfterBuild">
+  <Target Name="GenerateMarkerFiles" BeforeTargets="AssignTargetPaths">
+    <ItemGroup>
+      <Content Include="$(AssemblyName).reflect.xml" Condition="Exists('$(AssemblyName).reflect.xml')" CopyToOutputDirectory="PreserveNewest" />
+      <MarkerFile Include="$(IntermediateOutputPath)\$(MSBuildProjectName).MergedTestAssembly" Condition="'$(IsMergedTestRunnerAssembly)' == 'true'" />
+      <MarkerFile Include="$(IntermediateOutputPath)\$(AssemblyName).NoMonoAot" Condition="'$(MonoAotIncompatible)' == 'true'" />
+      <Content Include="@(MarkerFile)" CopyToOutputDirectory="PreserveNewest" />
+    </ItemGroup>
+
      <Copy SourceFiles="$(AssemblyName).reflect.xml"
            DestinationFolder="$(OutputPath)"
        Condition="Exists('$(AssemblyName).reflect.xml')"/>
 
     <WriteLinesToFile
         Condition="'$(IsMergedTestRunnerAssembly)' == 'true'"
-        File="$(OutputPath)\$(MSBuildProjectName).MergedTestAssembly"
-        Lines="MergedTestAssembly" />
+        File="$(IntermediateOutputPath)\$(MSBuildProjectName).MergedTestAssembly"
+        Lines="MergedTestAssembly"
+        Overwrite="true"
+        WriteOnlyWhenDifferent="true" />
+
 
+    <WriteLinesToFile
+        Condition="'$(MonoAotIncompatible)' == 'true'"
+        File="$(IntermediateOutputPath)\$(AssemblyName).NoMonoAot"
+        Lines="NoMonoAot"
+        Overwrite="true"
+        WriteOnlyWhenDifferent="true" />
+
+    <!-- We don't want the out-of-process test marker file to be included in referencing projects, so we don't include it as content. -->
     <WriteLinesToFile
         Condition="'$(RequiresProcessIsolation)' == 'true' and '$(BuildAsStandalone)' != 'true'"
         File="$(OutputPath)\$(MSBuildProjectName).OutOfProcessTest"
-        Lines="OutOfProcessTest" />
+        Lines="OutOfProcessTest"
+        Overwrite="true"
+        WriteOnlyWhenDifferent="true" />
   </Target>
 
   <PropertyGroup>

src/tests/Exceptions/ForeignThread/ForeignThreadExceptions.csproj

@@ -1,6 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <OutputType>Exe</OutputType>
+    <MonoAotIncompatible>true</MonoAotIncompatible>
   </PropertyGroup>
   <ItemGroup>
     <Compile Include="ForeignThreadExceptions.cs" />

src/tests/Interop/DisabledRuntimeMarshalling/DisabledRuntimeMarshalling_Disabled_NativeAssemblyEnabled.csproj

@@ -1,6 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+    <MonoAotIncompatible>true</MonoAotIncompatible>
   </PropertyGroup>
   <ItemGroup>
     <Compile Include="PInvokeAssemblyMarshallingEnabled/*.cs" />

src/tests/Interop/SuppressGCTransition/SuppressGCTransitionTest.csproj

@@ -2,6 +2,7 @@
   <PropertyGroup>
     <OutputType>Exe</OutputType>
     <AllowUnsafeBlocks>True</AllowUnsafeBlocks>
+    <MonoAotIncompatible>true</MonoAotIncompatible>
   </PropertyGroup>
   <ItemGroup>
     <Compile Include="$(InteropCommonDir)CheckGCMode.cs" />

src/tests/Interop/UnmanagedCallersOnly/UnmanagedCallersOnlyTest.csproj

@@ -2,6 +2,7 @@
   <PropertyGroup>
     <OutputType>Exe</OutputType>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+    <MonoAotIncompatible>true</MonoAotIncompatible>
   </PropertyGroup>
   <ItemGroup>
     <Compile Include="UnmanagedCallersOnlyTest.cs" />

src/tests/JIT/Directed/callconv/Directory.Build.props

@@ -0,0 +1,7 @@
+<Project>
+  <Import Project="$([MSBuild]::GetPathOfFileAbove(Directory.Build.props, $(MSBuildThisFileDirectory)..))" />
+
+  <PropertyGroup>
+    <MonoAotIncompatible>true</MonoAotIncompatible>
+  </PropertyGroup>
+</Project>

src/tests/JIT/Directed/pinning/object-pin/CMakeLists.txt

@@ -1,12 +1,4 @@
 project(object_pin_mirror)
 
-set(CMAKE_SHARED_LIBRARY_PREFIX "")
-
 add_library(mirror SHARED mirror.cpp)
 SET_TARGET_PROPERTIES(mirror PROPERTIES COMPILE_FLAGS "-c") 
-
-# add the install targets (this "installs" the native file on Windows systems)
-install(TARGETS mirror DESTINATION bin)
-
-# This "installs" the native file on System V systems
-set_target_properties(mirror PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/mirror)

src/tests/JIT/Directed/pinvoke/CMakeLists.txt

@@ -1,12 +1,4 @@
 project(PInvokeExampleNative)
 
-set(CMAKE_SHARED_LIBRARY_PREFIX "")
-
 add_library(PInvokeExampleNative SHARED pinvokeexamplenative.cpp)
 SET_TARGET_PROPERTIES(PInvokeExampleNative PROPERTIES COMPILE_FLAGS "-c")
-
-# add the install targets (this "installs" the native file on Windows systems)
-install(TARGETS PInvokeExampleNative DESTINATION bin)
-
-# This "installs" the native file on System V systems
-set_target_properties(PInvokeExampleNative PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/PInvokeExampleNative)