Convert internal framework to MTP

Directory.Build.props

@@ -13,6 +13,7 @@
     <LangVersion>preview</LangVersion>
     <ImplicitUsings>enable</ImplicitUsings>
     <GenerateDocumentationFile>true</GenerateDocumentationFile>
+    <Nullable>enable</Nullable>
   </PropertyGroup>
 
   <!-- The TFMs to build and test against. -->
@@ -58,6 +59,12 @@
     <StrongNameKeyId>Microsoft</StrongNameKeyId>
   </PropertyGroup>
 
+  <!-- Test config -->
+  <PropertyGroup>
+    <!-- Needs to be setup globally so that arcade doesn't bring xUnit to playground and other test projects -->
+    <TestRunnerName>TestingPlatformRunner</TestRunnerName>
+  </PropertyGroup>
+
   <!-- Polyfill config -->
   <PropertyGroup>
     <PolyGuard>true</PolyGuard>

eng/TestingPlatformRunner/TestingPlatformRunner.targets

@@ -1,8 +1,3 @@
 <Project>
-  <Import Project="$(NuGetPackageRoot)\microsoft.dotnet.arcade.sdk\$(ArcadeSdkVersion)\tools\VSTest.targets" Condition=" '$(UseVSTestRunner)' == 'true' "/>
-  <Import Project=".\TestingPlatform.Runner.targets" Condition=" '$(UseVSTestRunner)' != 'true' " />
-
-  <PropertyGroup>
-    <TestRunnerAdditionalArguments Condition=" '$(UseVSTestRunner)' == 'true' ">--blame-crash --blame-crash-dump-type full</TestRunnerAdditionalArguments>
-  </PropertyGroup>
+  <Import Project=".\TestingPlatform.Runner.targets" />
 </Project>

global.json

@@ -10,6 +10,8 @@
         "9.0.2"
       ],
       "dotnet/x86": [
+        "3.1.32",
+        "6.0.35",
         "9.0.2"
       ],
       "aspnetcore": [

samples/FxExtensibility/AssertEx.cs

@@ -11,7 +11,7 @@ namespace MSTest.Extensibility.Samples;
 [SuppressMessage("Naming", "CA1711:Identifiers should not have incorrect suffix", Justification = "This is only some sample type")]
 public static class AssertEx
 {
-    private static AssertIs s_assertIs;
+    private static AssertIs? s_assertIs;
 
     /// <summary>
     /// A simple assert extension to validate if an object is of a given type.
@@ -22,13 +22,14 @@ public static class AssertEx
     /// <returns>True if object is of the given type.</returns>
     /// <exception cref="AssertFailedException">If object is not of the given type.</exception>
 #pragma warning disable IDE0060 // Remove unused parameter
-    public static bool IsOfType<T>(this Assert assert, object obj) => obj is T
+    public static bool IsOfType<T>(this Assert assert, object obj)
+        => obj is T
             ? true
             : throw new AssertFailedException(string.Format(
                 CultureInfo.InvariantCulture,
                 "Expected object of type {0} but found object of type {1}",
                 typeof(T),
-                obj ?? obj.GetType()));
+                obj ?? obj?.GetType()));
 
     /// <summary>
     /// A chain/grouping of assert statements.

src/Adapter/MSTest.TestAdapter/Discovery/TestMethodValidator.cs

@@ -55,7 +55,7 @@ internal virtual bool IsValidTestMethod(MethodInfo testMethodInfo, Type type, IC
         // Todo: Decide whether parameter count matters.
         bool isValidTestMethod = isAccessible &&
                                  testMethodInfo is { IsAbstract: false, IsStatic: false } &&
-                                 testMethodInfo.IsValidReturnType();
+                                 testMethodInfo.IsValidReturnType(_reflectHelper);
 
         if (!isValidTestMethod)
         {

src/Adapter/MSTest.TestAdapter/Extensions/MethodInfoExtensions.cs

@@ -87,22 +87,23 @@ internal static bool HasCorrectTestMethodSignature(this MethodInfo method, bool
     /// Check is return type is void for non async and Task for async methods.
     /// </summary>
     /// <param name="method">The method to verify.</param>
+    /// <param name="reflectHelper">The reflection service to use.</param>
     /// <returns>True if the method has a void/task return type..</returns>
-    internal static bool IsValidReturnType(this MethodInfo method)
+    internal static bool IsValidReturnType(this MethodInfo method, ReflectHelper? reflectHelper = null)
         => ReflectHelper.MatchReturnType(method, typeof(Task))
         || ReflectHelper.MatchReturnType(method, typeof(ValueTask))
-        || (ReflectHelper.MatchReturnType(method, typeof(void)) && method.GetAsyncTypeName() == null);
+        || (ReflectHelper.MatchReturnType(method, typeof(void)) && method.GetAsyncTypeName(reflectHelper) == null);
 
     /// <summary>
     /// For async methods compiler generates different type and method.
     /// Gets the compiler generated type name for given async test method.
     /// </summary>
     /// <param name="method">The method to verify.</param>
+    /// <param name="reflectHelper">The reflection service to use.</param>
     /// <returns>Compiler generated type name for given async test method..</returns>
-    internal static string? GetAsyncTypeName(this MethodInfo method)
+    internal static string? GetAsyncTypeName(this MethodInfo method, ReflectHelper? reflectHelper = null)
     {
-        AsyncStateMachineAttribute? asyncStateMachineAttribute = ReflectHelper.Instance.GetFirstNonDerivedAttributeOrDefault<AsyncStateMachineAttribute>(method, inherit: false);
-
+        AsyncStateMachineAttribute? asyncStateMachineAttribute = (reflectHelper ?? ReflectHelper.Instance).GetFirstNonDerivedAttributeOrDefault<AsyncStateMachineAttribute>(method, inherit: false);
         return asyncStateMachineAttribute?.StateMachineType?.FullName;
     }
 

src/Adapter/MSTest.TestAdapter/Helpers/ReflectHelper.cs

@@ -107,8 +107,8 @@ public virtual bool IsNonDerivedAttributeDefined<TAttribute>(MemberInfo memberIn
     /// <param name="attributeProvider">The type, assembly or method.</param>
     /// <param name="inherit">If we should inspect parents of this type.</param>
     /// <returns>The attribute that is found or null.</returns>
-    public TAttribute? GetFirstNonDerivedAttributeOrDefault<TAttribute>(ICustomAttributeProvider attributeProvider, bool inherit)
-    where TAttribute : Attribute
+    public virtual /* for tests, for moq */ TAttribute? GetFirstNonDerivedAttributeOrDefault<TAttribute>(ICustomAttributeProvider attributeProvider, bool inherit)
+        where TAttribute : Attribute
     {
         Attribute[] cachedAttributes = GetCustomAttributesCached(attributeProvider, inherit);
 
@@ -134,7 +134,7 @@ public virtual bool IsNonDerivedAttributeDefined<TAttribute>(MemberInfo memberIn
     /// <returns>The attribute that is found or null.</returns>
     /// <exception cref="InvalidOperationException">Throws when multiple attributes are found (the attribute must allow multiple).</exception>
     public virtual /* for tests, for moq */ TAttribute? GetFirstDerivedAttributeOrDefault<TAttribute>(ICustomAttributeProvider attributeProvider, bool inherit)
-    where TAttribute : Attribute
+        where TAttribute : Attribute
     {
         Attribute[] cachedAttributes = GetCustomAttributesCached(attributeProvider, inherit);
 

src/Platform/Microsoft.Testing.Platform/Microsoft.Testing.Platform.csproj

@@ -57,6 +57,8 @@ This package provides the core platform and the .NET implementation of the proto
     <InternalsVisibleTo Include="MSTest.Acceptance.IntegrationTests" Key="$(VsPublicKey)" />
     <InternalsVisibleTo Include="MSTest.Engine" Key="$(VsPublicKey)" />
     <InternalsVisibleTo Include="MSTest.Engine.UnitTests" Key="$(VsPublicKey)" />
+    <InternalsVisibleTo Include="TestFramework.ForTestingMSTest" Key="$(VsPublicKey)" />
+    <InternalsVisibleTo Include="MSTest.IntegrationTests" Key="$(VsPublicKey)" />
   </ItemGroup>
 
   <ItemGroup>

test/Directory.Build.props

@@ -3,16 +3,11 @@
 
   <Import Project="../Directory.Build.props" />
 
-  <PropertyGroup>
-    <TestRunnerName>TestingPlatformRunner</TestRunnerName>
-    <!-- By default we run tests with VSTest runner and custom test framework. Platform projects will use a different runner and framework -->
-    <UseVSTestRunner Condition=" '$(UseVSTestRunner)' == '' " >true</UseVSTestRunner>
-    <Nullable>enable</Nullable>
-  </PropertyGroup>
-
   <!-- Build & pack tests -->
   <PropertyGroup>
     <IsPackable>false</IsPackable>
+    <!-- Prevent arcade from adding Microsoft.NET.Test.Sdk -->
+    <ExcludeMicrosoftNetTestSdk>true</ExcludeMicrosoftNetTestSdk>
     <PublishAot Condition=" '$(PublishAot)' == '' ">false</PublishAot>
     <!-- At the moment we don't run single file tests we should do it in integration/isolation -->
     <EnableSingleFileAnalyzer>false</EnableSingleFileAnalyzer>

test/Directory.Build.targets

@@ -3,10 +3,8 @@
 
   <Import Project="../Directory.Build.targets" />
 
-  <PropertyGroup Condition=" '$(UseVSTestRunner)' == 'false' ">
-    <Nullable>enable</Nullable>
-    <OutputType>Exe</OutputType>
-    <EnableMSTestRunner>true</EnableMSTestRunner>
+  <PropertyGroup Condition=" '$(EnableMSTestRunner)' == 'true' OR '$(UseInternalTestFramework)' == 'true' ">
+    <UseMSTest Condition=" '$(UseMSTest)' == '' ">true</UseMSTest>
     <GenerateProgramFile>false</GenerateProgramFile>
     <GenerateTestingPlatformEntryPoint>false</GenerateTestingPlatformEntryPoint>
     <Architecture Condition=" '$(Architecture)' == '' ">$(PlatformTarget)</Architecture>
@@ -21,29 +19,32 @@
     <!-- /Arcade runner settings -->
   </PropertyGroup>
 
-  <ItemGroup Condition=" '$(UseVSTestRunner)' == 'false' ">
-    <ProjectReference Include="$(RepoRoot)test\Utilities\Microsoft.Testing.TestInfrastructure\Microsoft.Testing.TestInfrastructure.csproj" />
+  <ItemGroup Condition=" '$(EnableMSTestRunner)' == 'true' OR '$(UseInternalTestFramework)' == 'true' ">
+    <ProjectReference Include="$(RepoRoot)test\Utilities\Microsoft.Testing.TestInfrastructure\Microsoft.Testing.TestInfrastructure.csproj"
+                      Condition="'$(UseInternalTestFramework)' != 'true'" />
     <ProjectReference Include="$(RepoRoot)src\Platform\Microsoft.Testing.Extensions.CrashDump\Microsoft.Testing.Extensions.CrashDump.csproj" />
     <ProjectReference Include="$(RepoRoot)src\Platform\Microsoft.Testing.Extensions.HangDump\Microsoft.Testing.Extensions.HangDump.csproj" />
     <ProjectReference Include="$(RepoRoot)src\Platform\Microsoft.Testing.Extensions.Retry\Microsoft.Testing.Extensions.Retry.csproj" />
     <ProjectReference Include="$(RepoRoot)src\Platform\Microsoft.Testing.Extensions.TrxReport\Microsoft.Testing.Extensions.TrxReport.csproj" />
     <PackageReference Include="Microsoft.Testing.Extensions.CodeCoverage" GeneratePathProperty="True" />
   </ItemGroup>
 
-  <ItemGroup Condition=" '$(UseVSTestRunner)' == 'false' AND '$(UseMSTestFromSource)' != 'true' ">
+  <ItemGroup Condition=" '$(EnableMSTestRunner)' == 'true' AND '$(UseInternalTestFramework)' != 'true' AND '$(UseMSTestFromSource)' != 'true' ">
     <PackageReference Include="MSTest.Analyzers" />
     <PackageReference Include="MSTest.TestFramework" />
     <PackageReference Include="MSTest.TestAdapter" />
   </ItemGroup>
 
-  <ItemGroup Condition=" '$(UseVSTestRunner)' == 'false' ">
+  <ItemGroup Condition=" '$(EnableMSTestRunner)' == 'true' OR '$(UseInternalTestFramework)' == 'true' ">
     <Using Include="Microsoft.Testing.Platform.Builder" />
     <Using Include="Microsoft.Testing.Platform.Extensions" />
-    <Using Include="Microsoft.Testing.TestInfrastructure" />
-    <Using Include="Microsoft.VisualStudio.TestTools.UnitTesting" />
+    <Using Include="Microsoft.Testing.TestInfrastructure"
+           Condition="'$(UseInternalTestFramework)' != 'true'" />
+    <Using Include="Microsoft.VisualStudio.TestTools.UnitTesting"
+           Condition="'$(UseMSTest)' == 'true'" />
   </ItemGroup>
 
-  <ItemGroup Condition=" '$(UseVSTestRunner)' == 'false' ">
+  <ItemGroup Condition=" '$(EnableMSTestRunner)' == 'true' OR '$(UseInternalTestFramework)' == 'true' ">
     <None Update="*.testconfig.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
@@ -55,4 +56,16 @@
     </None>
   </ItemGroup>
 
+  <ItemGroup Condition=" '$(UseInternalTestFramework)' == 'true' ">
+    <ProjectReference Include="$(RepoRoot)test\Utilities\TestFramework.ForTestingMSTest\TestFramework.ForTestingMSTest.csproj" />
+  </ItemGroup>
+
+  <ItemGroup Condition=" '$(UseInternalTestFramework)' == 'true' ">
+    <ProjectCapability Include="TestContainer" />
+    <ProjectCapability Include="TestingPlatformServer" />
+  </ItemGroup>
+
+  <Import Project="$(RepoRoot)src\Platform\Microsoft.Testing.Platform\buildMultiTargeting\Microsoft.Testing.Platform.props"
+          Condition=" '$(UseInternalTestFramework)' == 'true' " />
+
 </Project>

test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MSTest.Acceptance.IntegrationTests.csproj

@@ -1,11 +1,10 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFrameworks>$(NetCurrent)</TargetFrameworks>
-    <UseVSTestRunner>false</UseVSTestRunner>
-	  <EnableMSTestRunner>true</EnableMSTestRunner>
-    <DefineConstants Condition=" '$(FastAcceptanceTest)' == 'true'">$(DefineConstants);SKIP_INTERMEDIATE_TARGET_FRAMEWORKS</DefineConstants>
+    <EnableMSTestRunner>true</EnableMSTestRunner>
     <OutputType>Exe</OutputType>
+    <DefineConstants Condition=" '$(FastAcceptanceTest)' == 'true'">$(DefineConstants);SKIP_INTERMEDIATE_TARGET_FRAMEWORKS</DefineConstants>
     <UseAppHost>true</UseAppHost>
   </PropertyGroup>
 

test/IntegrationTests/MSTest.IntegrationTests/Extensions/VerifyE2E.cs

@@ -6,6 +6,8 @@
 using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter;
 using Microsoft.VisualStudio.TestPlatform.ObjectModel;
 
+using TestResult = Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult;
+
 namespace MSTest.IntegrationTests;
 
 public static class VerifyE2E

test/IntegrationTests/MSTest.IntegrationTests/MSTest.IntegrationTests.csproj

@@ -1,9 +1,11 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFramework>$(NetFrameworkMinimum)</TargetFramework>
     <IsPackable>false</IsPackable>
     <NoWarn>$(NoWarn),1685</NoWarn>
+    <UseInternalTestFramework>true</UseInternalTestFramework>
+    <OutputType>Exe</OutputType>
   </PropertyGroup>
 
   <ItemGroup>
@@ -17,7 +19,6 @@
   </ItemGroup>
 
   <ItemGroup>
-    <ProjectReference Include="$(RepoRoot)test\Utilities\TestFramework.ForTestingMSTest\TestFramework.ForTestingMSTest.csproj" />
     <ProjectReference Include="$(RepoRoot)src\Adapter\MSTest.TestAdapter\MSTest.TestAdapter.csproj" />
     <ProjectReference Include="$(RepoRoot)src\Analyzers\MSTest.Internal.Analyzers\MSTest.Internal.Analyzers.csproj">
       <OutputItemType>Analyzer</OutputItemType>

test/IntegrationTests/MSTest.IntegrationTests/OutputTests.cs

@@ -6,17 +6,21 @@
 using Microsoft.MSTestV2.CLIAutomation;
 using Microsoft.VisualStudio.TestPlatform.ObjectModel;
 
+using TestResult = Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult;
+
 namespace MSTest.IntegrationTests;
 
 public class OutputTests : CLITestBase
 {
     private const string TestAssetName = "OutputTestProject";
 
-    public async Task OutputIsNotMixedWhenTestsRunInParallel() => await ValidateOutputForClass("UnitTest1");
+#if DEBUG
+    public async Task OutputIsNotMixedWhenTestsRunInParallel() => await ValidateOutputForClassAsync("UnitTest1");
+#endif
 
-    public async Task OutputIsNotMixedWhenAsyncTestsRunInParallel() => await ValidateOutputForClass("UnitTest2");
+    public async Task OutputIsNotMixedWhenAsyncTestsRunInParallel() => await ValidateOutputForClassAsync("UnitTest2");
 
-    private static async Task ValidateOutputForClass(string className)
+    private static async Task ValidateOutputForClassAsync(string className)
     {
         // LogMessageListener uses an implementation of a string writer that captures output per async context.
         // This allows us to capture output from tasks even when they are running in parallel.

test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataExtensibilityTests.cs

@@ -6,6 +6,8 @@
 using Microsoft.MSTestV2.CLIAutomation;
 using Microsoft.VisualStudio.TestPlatform.ObjectModel;
 
+using TestResult = Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult;
+
 namespace MSTest.IntegrationTests;
 
 public class DataExtensibilityTests : CLITestBase

test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DynamicDataTests.cs

@@ -6,6 +6,8 @@
 using Microsoft.MSTestV2.CLIAutomation;
 using Microsoft.VisualStudio.TestPlatform.ObjectModel;
 
+using TestResult = Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult;
+
 namespace MSTest.IntegrationTests;
 
 public class DynamicDataTests : CLITestBase

test/IntegrationTests/MSTest.IntegrationTests/Program.cs

@@ -0,0 +1,21 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using Microsoft.Testing.Extensions;
+
+using TestFramework.ForTestingMSTest;
+
+ITestApplicationBuilder builder = await TestApplication.CreateBuilderAsync(args);
+
+#if ENABLE_CODECOVERAGE
+builder.AddCodeCoverageProvider();
+#endif
+builder.AddTrxReportProvider();
+builder.AddHangDumpProvider();
+builder.AddCrashDumpProvider(ignoreIfNotSupported: true);
+builder.AddRetryProvider();
+
+builder.AddInternalTestFramework();
+
+ITestApplication app = await builder.BuildAsync();
+return await app.RunAsync();

test/IntegrationTests/MSTest.IntegrationTests/Utilities/CLITestBase.discovery.cs

@@ -15,6 +15,8 @@
 
 using TestFramework.ForTestingMSTest;
 
+using TestResult = Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult;
+
 namespace Microsoft.MSTestV2.CLIAutomation;
 
 public partial class CLITestBase : TestContainer
@@ -90,7 +92,7 @@ private sealed class InternalFrameworkHandle : IFrameworkHandle
         private readonly List<string> _messageList = [];
         private readonly ConcurrentDictionary<TestCase, ConcurrentBag<TestResult>> _testResults = new();
 
-        private ConcurrentBag<TestResult> _activeResults = null!;
+        private ConcurrentBag<TestResult> _activeResults = new();
 
         public bool EnableShutdownAfterTestRun { get; set; }
 

test/IntegrationTests/MSTest.IntegrationTests/Utilities/TestCaseFilterFactory.cs

@@ -113,7 +113,8 @@ public TestFilterExpression(string filter, Func<Func<string, object?>, bool> exp
 
         public string TestCaseFilterValue { get; }
 
-        public bool MatchTestCase(TestCase testCase, Func<string, object?> propertyValueProvider) => _expression(propertyValueProvider);
+        public bool MatchTestCase(TestCase testCase, Func<string, object?> propertyValueProvider)
+            => _expression(propertyValueProvider);
     }
 
     private static void MergeExpression(Stack<Expression<Func<Func<string, object?>, bool>>> exp, Operator op)

test/IntegrationTests/MSTest.VstestConsoleWrapper.IntegrationTests/DeploymentTests.cs

@@ -21,7 +21,8 @@ public class DeploymentTests : CLITestBase
     public void ValidateTestSourceDependencyDeployment_net462()
         => ValidateTestSourceDependencyDeployment("net462");
 
-    public void ValidateTestSourceDependencyDeployment_netcoreapp31()
+    [SuppressMessage("CodeQuality", "IDE0051:Remove unused private members", Justification = "Disabled as it's not working on CIA fatal error occurred. The required library hostfxr.dll could not be found.")]
+    private void ValidateTestSourceDependencyDeployment_netcoreapp31()
         => ValidateTestSourceDependencyDeployment("netcoreapp3.1");
 
     private void ValidateTestSourceDependencyDeployment(string targetFramework)
@@ -48,7 +49,8 @@ public void ValidateTestSourceLocationDeployment(string targetFramework)
     public void ValidateDirectoryDeployment_net462()
         => ValidateDirectoryDeployment("net462");
 
-    public void ValidateDirectoryDeployment_netcoreapp31()
+    [SuppressMessage("CodeQuality", "IDE0051:Remove unused private members", Justification = "Disabled as it's not working on CIA fatal error occurred. The required library hostfxr.dll could not be found.")]
+    private void ValidateDirectoryDeployment_netcoreapp31()
         => ValidateDirectoryDeployment("netcoreapp3.1");
 
     public void ValidateDirectoryDeployment(string targetFramework)
@@ -60,7 +62,8 @@ public void ValidateDirectoryDeployment(string targetFramework)
     public void ValidateFileDeployment_net462()
         => ValidateFileDeployment("net462");
 
-    public void ValidateFileDeployment_netcoreapp31()
+    [SuppressMessage("CodeQuality", "IDE0051:Remove unused private members", Justification = "Disabled as it's not working on CIA fatal error occurred. The required library hostfxr.dll could not be found.")]
+    private void ValidateFileDeployment_netcoreapp31()
         => ValidateFileDeployment("netcoreapp3.1");
 
     public void ValidateFileDeployment(string targetFramework)