Add lifecycle callbacks to test host orchestrator

src/Platform/Microsoft.Testing.Extensions.MSBuild/MSBuildExtensions.cs

@@ -26,6 +26,12 @@ public static void AddMSBuild(this ITestApplicationBuilder builder)
                 serviceProvider.GetCommandLineOptions(),
                 serviceProvider.GetTestApplicationCancellationTokenSource()));
 
+        ((TestApplicationBuilder)builder).TestHostOrchestrator.AddTestHostOrchestratorApplicationLifetime(
+            serviceProvider => new MSBuildOrchestratorLifetime(
+                serviceProvider.GetConfiguration(),
+                serviceProvider.GetCommandLineOptions(),
+                serviceProvider.GetTestApplicationCancellationTokenSource()));
+
         CompositeExtensionFactory<MSBuildConsumer> compositeExtensionFactory
             = new(serviceProvider => new MSBuildConsumer(
                 serviceProvider,

src/Platform/Microsoft.Testing.Extensions.MSBuild/MSBuildOrchestratorLifecycleCallbacks.cs

@@ -0,0 +1,70 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using Microsoft.Testing.Extensions.MSBuild.Serializers;
+using Microsoft.Testing.Platform.CommandLine;
+using Microsoft.Testing.Platform.Configurations;
+using Microsoft.Testing.Platform.Extensions.TestHostOrchestrator;
+using Microsoft.Testing.Platform.Helpers;
+using Microsoft.Testing.Platform.IPC;
+using Microsoft.Testing.Platform.IPC.Models;
+using Microsoft.Testing.Platform.IPC.Serializers;
+using Microsoft.Testing.Platform.Services;
+
+namespace Microsoft.Testing.Extensions.MSBuild;
+
+internal sealed class MSBuildOrchestratorLifetime : ITestHostOrchestratorApplicationLifetime
+{
+    private readonly IConfiguration _configuration;
+    private readonly ICommandLineOptions _commandLineOptions;
+    private readonly ITestApplicationCancellationTokenSource _testApplicationCancellationTokenSource;
+
+    public MSBuildOrchestratorLifetime(
+        IConfiguration configuration,
+        ICommandLineOptions commandLineOptions,
+        ITestApplicationCancellationTokenSource testApplicationCancellationTokenSource)
+    {
+        _configuration = configuration;
+        _commandLineOptions = commandLineOptions;
+        _testApplicationCancellationTokenSource = testApplicationCancellationTokenSource;
+    }
+
+    public string Uid => nameof(MSBuildOrchestratorLifetime);
+
+    public string Version => AppVersion.DefaultSemVer;
+
+    public string DisplayName => nameof(MSBuildOrchestratorLifetime);
+
+    public string Description => Resources.ExtensionResources.MSBuildExtensionsDescription;
+
+    public Task<bool> IsEnabledAsync()
+        => Task.FromResult(_commandLineOptions.IsOptionSet(MSBuildConstants.MSBuildNodeOptionKey));
+
+    public async Task BeforeRunAsync(CancellationToken cancellationToken)
+    {
+        if (!_commandLineOptions.TryGetOptionArgumentList(MSBuildConstants.MSBuildNodeOptionKey, out string[]? msbuildInfo))
+        {
+            throw new InvalidOperationException($"MSBuild pipe name not found in the command line, missing {MSBuildConstants.MSBuildNodeOptionKey}");
+        }
+
+        if (msbuildInfo is null || msbuildInfo.Length != 1 || string.IsNullOrEmpty(msbuildInfo[0]))
+        {
+            throw new InvalidOperationException($"MSBuild pipe name not found in the command line, missing argument for {MSBuildConstants.MSBuildNodeOptionKey}");
+        }
+
+        using var pipeClient = new NamedPipeClient(msbuildInfo[0]);
+        pipeClient.RegisterSerializer(new ModuleInfoRequestSerializer(), typeof(ModuleInfoRequest));
+        pipeClient.RegisterSerializer(new VoidResponseSerializer(), typeof(VoidResponse));
+        using var cancellationTokenSource = new CancellationTokenSource(TimeoutHelper.DefaultHangTimeSpanTimeout);
+        using var linkedCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(cancellationTokenSource.Token, _testApplicationCancellationTokenSource.CancellationToken);
+        await pipeClient.ConnectAsync(linkedCancellationToken.Token);
+        await pipeClient.RequestReplyAsync<ModuleInfoRequest, VoidResponse>(
+            new ModuleInfoRequest(
+            RuntimeInformation.FrameworkDescription,
+            RuntimeInformation.ProcessArchitecture.ToString().ToLowerInvariant(),
+            _configuration.GetTestResultDirectory()),
+            _testApplicationCancellationTokenSource.CancellationToken);
+    }
+
+    public Task AfterRunAsync(int exitCode, CancellationToken cancellation) => Task.CompletedTask;
+}

src/Platform/Microsoft.Testing.Platform.MSBuild/Tasks/InvokeTestingPlatformTask.cs

@@ -433,6 +433,10 @@ protected override bool HandleTaskExecutionErrors()
 
     private Task<IResponse> HandleRequestAsync(IRequest request)
     {
+        // For the case, of orchestrator (e.g, Retry), we can get ModuleInfoRequest from the orchestrator itself.
+        // If there is no orchestrator or the orchestrator didn't send ModuleInfoRequest, we will get it from the first test host.
+        // For the case of retry, the request is different between the orchestrator and the test host.
+        // More specifically, the results directory is different (orchestrator points to original, while test host points to the specific retry results directory).
         if (request is ModuleInfoRequest moduleInfo)
         {
             if (_moduleInfo is null)

src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs

@@ -364,6 +364,12 @@ await LogTestHostCreatedAsync(
         {
             policiesService.ProcessRole = TestProcessRole.TestHostOrchestrator;
             await proxyOutputDevice.HandleProcessRoleAsync(TestProcessRole.TestHostOrchestrator);
+
+            // Build and register the test application lifecycle callbacks.
+            ITestHostOrchestratorApplicationLifetime[] orchestratorLifetimes =
+                await ((TestHostOrchestratorManager)TestHostOrchestratorManager).BuildTestHostOrchestratorApplicationLifetimesAsync(serviceProvider);
+            serviceProvider.AddServices(orchestratorLifetimes);
+
             return new TestHostOrchestratorHost(testHostOrchestratorConfiguration, serviceProvider);
         }
 

src/Platform/Microsoft.Testing.Platform/Hosts/TestHostOchestratorHost.cs

@@ -27,7 +27,18 @@ public async Task<int> RunAsync()
         await logger.LogInformationAsync($"Running test orchestrator '{testHostOrchestrator.Uid}'");
         try
         {
+            foreach (ITestHostOrchestratorApplicationLifetime orchestratorLifetime in _serviceProvider.GetServicesInternal<ITestHostOrchestratorApplicationLifetime>())
+            {
+                await orchestratorLifetime.BeforeRunAsync(applicationCancellationToken.CancellationToken);
+            }
+
             exitCode = await testHostOrchestrator.OrchestrateTestHostExecutionAsync();
+
+            foreach (ITestHostOrchestratorApplicationLifetime orchestratorLifetime in _serviceProvider.GetServicesInternal<ITestHostOrchestratorApplicationLifetime>())
+            {
+                await orchestratorLifetime.AfterRunAsync(exitCode, applicationCancellationToken.CancellationToken);
+                await DisposeHelper.DisposeAsync(orchestratorLifetime);
+            }
         }
         catch (OperationCanceledException) when (applicationCancellationToken.CancellationToken.IsCancellationRequested)
         {

src/Platform/Microsoft.Testing.Platform/TestHostOrcherstrator/ITestHostControllersExtension.cs

@@ -0,0 +1,9 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Microsoft.Testing.Platform.Extensions.TestHostOrchestrator;
+
+/// <summary>
+/// Represents an extension for test host orchestrators.
+/// </summary>
+internal interface ITestHostOrchestratorExtension : IExtension;

src/Platform/Microsoft.Testing.Platform/TestHostOrcherstrator/ITestHostOrchestratorApplicationLifetime.cs

@@ -0,0 +1,25 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Microsoft.Testing.Platform.Extensions.TestHostOrchestrator;
+
+// NOTE: The equivalent of this for "test host" is ITestApplicationLifecycleCallbacks, which is an unfortunate naming.
+// If we ever open orchestration before MTP v2 (https://github.com/microsoft/testfx/issues/5733), we should
+// consider if we are okay with this kinda inconsistent naming between test host and test host orchestrator.
+internal interface ITestHostOrchestratorApplicationLifetime : ITestHostOrchestratorExtension
+{
+    /// <summary>
+    /// Executes before the orchestrator runs.
+    /// </summary>
+    /// <param name="cancellationToken">The cancellation token.</param>
+    /// <returns>A task representing the asynchronous operation.</returns>
+    Task BeforeRunAsync(CancellationToken cancellationToken);
+
+    /// <summary>
+    /// Executes after the orchestrator runs.
+    /// </summary>
+    /// <param name="exitCode">The exit code of the orchestrator.</param>
+    /// <param name="cancellation">The cancellation token.</param>
+    /// <returns>A task representing the asynchronous operation.</returns>
+    Task AfterRunAsync(int exitCode, CancellationToken cancellation);
+}

src/Platform/Microsoft.Testing.Platform/TestHostOrcherstrator/ITestHostOrchestratorManager.cs

@@ -9,5 +9,10 @@ internal interface ITestHostOrchestratorManager
 {
     void AddTestHostOrchestrator(Func<IServiceProvider, ITestHostOrchestrator> factory);
 
+    // NOTE: In ITestHostManager, we have AddTestApplicationLifecycleCallbacks, which is an unfortunate naming.
+    // If we ever open orchestration before MTP v2 (https://github.com/microsoft/testfx/issues/5733), we should
+    // consider if we are okay with this kinda inconsistent naming between test host and test host orchestrator.
+    void AddTestHostOrchestratorApplicationLifetime(Func<IServiceProvider, ITestHostOrchestratorApplicationLifetime> testHostOrchestratorApplicationLifetimeFactory);
+
     Task<TestHostOrchestratorConfiguration> BuildAsync(ServiceProvider serviceProvider);
 }

src/Platform/Microsoft.Testing.Platform/TestHostOrcherstrator/TestHostOrchestratorManager.cs

@@ -8,6 +8,7 @@ namespace Microsoft.Testing.Platform.Extensions.TestHostOrchestrator;
 
 internal sealed class TestHostOrchestratorManager : ITestHostOrchestratorManager
 {
+    private readonly List<Func<IServiceProvider, ITestHostOrchestratorApplicationLifetime>> _testHostOrchestratorApplicationLifetimeFactories = [];
     private List<Func<IServiceProvider, ITestHostOrchestrator>>? _factories;
 
     public void AddTestHostOrchestrator(Func<IServiceProvider, ITestHostOrchestrator> factory)
@@ -48,4 +49,37 @@ public async Task<TestHostOrchestratorConfiguration> BuildAsync(ServiceProvider
 
         return new TestHostOrchestratorConfiguration([.. orchestrators]);
     }
+
+    public void AddTestHostOrchestratorApplicationLifetime(Func<IServiceProvider, ITestHostOrchestratorApplicationLifetime> testHostOrchestratorApplicationLifetimeFactory)
+    {
+        Guard.NotNull(testHostOrchestratorApplicationLifetimeFactory);
+        _testHostOrchestratorApplicationLifetimeFactories.Add(testHostOrchestratorApplicationLifetimeFactory);
+    }
+
+    internal async Task<ITestHostOrchestratorApplicationLifetime[]> BuildTestHostOrchestratorApplicationLifetimesAsync(ServiceProvider serviceProvider)
+    {
+        List<ITestHostOrchestratorApplicationLifetime> lifetimes = [];
+        foreach (Func<IServiceProvider, ITestHostOrchestratorApplicationLifetime> testHostOrchestratorApplicationLifetimeFactory in _testHostOrchestratorApplicationLifetimeFactories)
+        {
+            ITestHostOrchestratorApplicationLifetime service = testHostOrchestratorApplicationLifetimeFactory(serviceProvider);
+
+            // Check if we have already extensions of the same type with same id registered
+            if (lifetimes.Any(x => x.Uid == service.Uid))
+            {
+                ITestHostOrchestratorApplicationLifetime currentRegisteredExtension = lifetimes.Single(x => x.Uid == service.Uid);
+                throw new InvalidOperationException(string.Format(CultureInfo.InvariantCulture, PlatformResources.ExtensionWithSameUidAlreadyRegisteredErrorMessage, service.Uid, currentRegisteredExtension.GetType()));
+            }
+
+            // We initialize only if enabled
+            if (await service.IsEnabledAsync())
+            {
+                await service.TryInitializeAsync();
+
+                // Register the extension for usage
+                lifetimes.Add(service);
+            }
+        }
+
+        return [.. lifetimes];
+    }
 }

test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/RetryFailedTestsTests.cs

@@ -178,6 +178,31 @@ await RetryHelper.RetryAsync(
             }, 3, TimeSpan.FromSeconds(5));
     }
 
+    [TestMethod]
+    public async Task RetryFailedTests_PassingFromFirstTime_UsingOldDotnetTest_MoveFiles_Succeeds()
+    {
+        string resultDirectory = Path.Combine(AssetFixture.TargetAssetPath, Guid.NewGuid().ToString("N"));
+
+        DotnetMuxerResult result = await DotnetCli.RunAsync(
+            $"test \"{AssetFixture.TargetAssetPath}\" -- --retry-failed-tests 1 --results-directory \"{resultDirectory}\"",
+            AcceptanceFixture.NuGetGlobalPackagesFolder.Path,
+            workingDirectory: AssetFixture.TargetAssetPath);
+
+        Assert.AreEqual(ExitCodes.Success, result.ExitCode);
+
+        // File names are on the form: RetryFailedTests_tfm_architecture.log
+        string[] logFilesFromInvokeTestingPlatformTask = Directory.GetFiles(resultDirectory, "RetryFailedTests_*_*.log");
+        Assert.AreEqual(TargetFrameworks.All.Length, logFilesFromInvokeTestingPlatformTask.Length);
+        foreach (string logFile in logFilesFromInvokeTestingPlatformTask)
+        {
+            string logFileContents = File.ReadAllText(logFile);
+            Assert.Contains("Test run summary: Passed!", logFileContents);
+            Assert.Contains("total: 3", logFileContents);
+            Assert.Contains("succeeded: 3", logFileContents);
+            Assert.Contains("Tests suite completed successfully in 1 attempts", logFileContents);
+        }
+    }
+
     public sealed class TestAssetFixture() : TestAssetFixtureBase(AcceptanceFixture.NuGetGlobalPackagesFolder)
     {
         public string TargetAssetPath => GetAssetPath(AssetName);
@@ -200,21 +225,30 @@ public sealed class TestAssetFixture() : TestAssetFixtureBase(AcceptanceFixture.
         <OutputType>Exe</OutputType>
         <UseAppHost>true</UseAppHost>
         <LangVersion>preview</LangVersion>
+        <GenerateTestingPlatformEntryPoint>false</GenerateTestingPlatformEntryPoint>
+        <TestingPlatformDotnetTestSupport>true</TestingPlatformDotnetTestSupport>
+        <TestingPlatformCaptureOutput>false</TestingPlatformCaptureOutput>
     </PropertyGroup>
     <ItemGroup>
         <PackageReference Include="Microsoft.Testing.Extensions.CrashDump" Version="$MicrosoftTestingPlatformVersion$" />
         <PackageReference Include="Microsoft.Testing.Extensions.Retry" Version="$MicrosoftTestingPlatformVersion$" />
         <PackageReference Include="Microsoft.Testing.Extensions.TrxReport" Version="$MicrosoftTestingPlatformVersion$" />
+        <PackageReference Include="Microsoft.Testing.Platform.MSBuild" Version="$MicrosoftTestingPlatformVersion$" />
     </ItemGroup>
 </Project>
 
+#file dotnet.config
+[dotnet.test.runner]
+name= "VSTest"
+
 #file Program.cs
 using Microsoft.Testing.Extensions;
 using Microsoft.Testing.Extensions.TrxReport.Abstractions;
 using Microsoft.Testing.Platform.Builder;
 using Microsoft.Testing.Platform.Capabilities.TestFramework;
 using Microsoft.Testing.Platform.Extensions.Messages;
 using Microsoft.Testing.Platform.Extensions.TestFramework;
+using Microsoft.Testing.Platform.MSBuild;
 using Microsoft.Testing.Platform.Services;
 
 public class Program
@@ -228,6 +262,7 @@ public static async Task<int> Main(string[] args)
         builder.AddCrashDumpProvider();
         builder.AddTrxReportProvider();
         builder.AddRetryProvider();
+        builder.AddMSBuild();
         using ITestApplication app = await builder.BuildAsync();
         return await app.RunAsync();
     }
@@ -268,7 +303,7 @@ public async Task ExecuteRequestAsync(ExecuteRequestContext context)
         string resultDir = Environment.GetEnvironmentVariable("RESULTDIR")!; 
         bool crash = Environment.GetEnvironmentVariable("CRASH") == "1";
 
-        if (await TestMethod1(fail, resultDir, crash))
+        if (TestMethod1(fail, resultDir, crash))
         {
             await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(context.Request.Session.SessionUid,
                 new TestNode() { Uid = "1", DisplayName = "TestMethod1", Properties = new(PassedTestNodeStateProperty.CachedInstance) }));
@@ -279,7 +314,7 @@ public async Task ExecuteRequestAsync(ExecuteRequestContext context)
                 new TestNode() { Uid = "1", DisplayName = "TestMethod1", Properties = new(new FailedTestNodeStateProperty()) }));
         }
 
-        if (await TestMethod2(fail, resultDir))
+        if (TestMethod2(fail, resultDir))
         {
             await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(context.Request.Session.SessionUid,
                 new TestNode() { Uid = "2", DisplayName = "TestMethod2", Properties = new(PassedTestNodeStateProperty.CachedInstance) }));
@@ -290,7 +325,7 @@ public async Task ExecuteRequestAsync(ExecuteRequestContext context)
                 new TestNode() { Uid = "2", DisplayName = "TestMethod2", Properties = new(new FailedTestNodeStateProperty()) }));
         }
 
-        if (await TestMethod3(fail, resultDir))
+        if (TestMethod3(fail, resultDir))
         {
             await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(context.Request.Session.SessionUid,
                 new TestNode() { Uid = "3", DisplayName = "TestMethod3", Properties = new(PassedTestNodeStateProperty.CachedInstance) }));
@@ -304,7 +339,7 @@ public async Task ExecuteRequestAsync(ExecuteRequestContext context)
         context.Complete();
     }
 
-    private async Task<bool> TestMethod1(bool fail, string resultDir, bool crash)
+    private bool TestMethod1(bool fail, string resultDir, bool crash)
     {
         if (crash)
         {
@@ -328,7 +363,7 @@ private async Task<bool> TestMethod1(bool fail, string resultDir, bool crash)
         return assert;        
     }
 
-    private async Task<bool> TestMethod2(bool fail, string resultDir)
+    private bool TestMethod2(bool fail, string resultDir)
     {
         bool envVar = Environment.GetEnvironmentVariable("METHOD2") is null;
         System.Console.WriteLine("envVar " + envVar);
@@ -348,7 +383,7 @@ private async Task<bool> TestMethod2(bool fail, string resultDir)
         return assert;
     }
 
-    private async Task<bool> TestMethod3(bool fail, string resultDir)
+    private bool TestMethod3(bool fail, string resultDir)
     {
         bool envVar = Environment.GetEnvironmentVariable("METHOD3") is null;