From c13f31bc397d84bbd464094172760f5b52e86bd2 Mon Sep 17 00:00:00 2001 From: Amaury Leveugle Date: Mon, 27 Jul 2026 13:46:43 +0200 Subject: [PATCH 1/5] Add E2E test for dotnet test (MTP) live test-host output Covers dotnet/sdk#51615: a test app's stdout/stderr must reach the user while the run is in progress, not only when a test fails. The new TestProjectWithLiveOutput asset writes to the console both before the pipe handshake completes and while the test session runs, and reports a single passing test. The test asserts all three markers appear in the dotnet test output of a fully successful run, and that the in-run output precedes the end-of-run summary. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: eedfe288-df4b-48f1-a612-67ce78b017a5 --- .../TestProjectWithLiveOutput/Program.cs | 55 ++++++++++++++++++ .../TestProjectWithLiveOutput.csproj | 18 ++++++ .../TestProjectWithLiveOutput/global.json | 5 ++ .../GivenDotnetTestForwardsTestHostOutput.cs | 57 +++++++++++++++++++ 4 files changed, 135 insertions(+) create mode 100644 test/TestAssets/TestProjects/TestProjectWithLiveOutput/Program.cs create mode 100644 test/TestAssets/TestProjects/TestProjectWithLiveOutput/TestProjectWithLiveOutput.csproj create mode 100644 test/TestAssets/TestProjects/TestProjectWithLiveOutput/global.json create mode 100644 test/dotnet.Tests/CommandTests/Test/GivenDotnetTestForwardsTestHostOutput.cs diff --git a/test/TestAssets/TestProjects/TestProjectWithLiveOutput/Program.cs b/test/TestAssets/TestProjects/TestProjectWithLiveOutput/Program.cs new file mode 100644 index 000000000000..a6bcb4c33bc2 --- /dev/null +++ b/test/TestAssets/TestProjects/TestProjectWithLiveOutput/Program.cs @@ -0,0 +1,55 @@ +using Microsoft.Testing.Platform.Builder; +using Microsoft.Testing.Platform.Capabilities.TestFramework; +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.Extensions.TestFramework; + +// Written before the test application is built, i.e. before the pipe handshake with the SDK +// completes. It exercises the "buffer until the protocol version is negotiated, then flush" path. +Console.WriteLine("LIVE_OUTPUT_BEFORE_HANDSHAKE"); + +var testApplicationBuilder = await TestApplication.CreateBuilderAsync(args); + +testApplicationBuilder.RegisterTestFramework(_ => new TestFrameworkCapabilities(), (_, __) => new DummyTestAdapter()); + +using var testApplication = await testApplicationBuilder.BuildAsync(); +return await testApplication.RunAsync(); + +public class DummyTestAdapter : ITestFramework, IDataProducer +{ + public string Uid => nameof(DummyTestAdapter); + + public string Version => "2.0.0"; + + public string DisplayName => nameof(DummyTestAdapter); + + public string Description => nameof(DummyTestAdapter); + + public Task IsEnabledAsync() => Task.FromResult(true); + + public Type[] DataTypesProduced => new[] { + typeof(TestNodeUpdateMessage) + }; + + public Task CreateTestSessionAsync(CreateTestSessionContext context) + => Task.FromResult(new CreateTestSessionResult() { IsSuccess = true }); + + public Task CloseTestSessionAsync(CloseTestSessionContext context) + => Task.FromResult(new CloseTestSessionResult() { IsSuccess = true }); + + public async Task ExecuteRequestAsync(ExecuteRequestContext context) + { + // Written while the test session is running, so it must be streamed to the SDK terminal + // as it is produced instead of being buffered and only shown when something fails. + Console.WriteLine("LIVE_OUTPUT_STANDARD_OUTPUT"); + Console.Error.WriteLine("LIVE_OUTPUT_STANDARD_ERROR"); + + await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(context.Request.Session.SessionUid, new TestNode() + { + Uid = "Test0", + DisplayName = "Test0", + Properties = new PropertyBag(new PassedTestNodeStateProperty("OK")), + })); + + context.Complete(); + } +} diff --git a/test/TestAssets/TestProjects/TestProjectWithLiveOutput/TestProjectWithLiveOutput.csproj b/test/TestAssets/TestProjects/TestProjectWithLiveOutput/TestProjectWithLiveOutput.csproj new file mode 100644 index 000000000000..1ba237591546 --- /dev/null +++ b/test/TestAssets/TestProjects/TestProjectWithLiveOutput/TestProjectWithLiveOutput.csproj @@ -0,0 +1,18 @@ + + + + + $(CurrentTargetFramework) + Exe + + enable + enable + + false + true + + + + + + diff --git a/test/TestAssets/TestProjects/TestProjectWithLiveOutput/global.json b/test/TestAssets/TestProjects/TestProjectWithLiveOutput/global.json new file mode 100644 index 000000000000..9009caf0ba8f --- /dev/null +++ b/test/TestAssets/TestProjects/TestProjectWithLiveOutput/global.json @@ -0,0 +1,5 @@ +{ + "test": { + "runner": "Microsoft.Testing.Platform" + } +} diff --git a/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestForwardsTestHostOutput.cs b/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestForwardsTestHostOutput.cs new file mode 100644 index 000000000000..d9552102e3af --- /dev/null +++ b/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestForwardsTestHostOutput.cs @@ -0,0 +1,57 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CommandResult = Microsoft.DotNet.Cli.Utils.CommandResult; +using ExitCodes = Microsoft.NET.TestFramework.ExitCode; + +namespace Microsoft.DotNet.Cli.Test.Tests +{ + /// + /// End-to-end coverage for https://github.com/dotnet/sdk/issues/51615: the console output of a + /// test application must reach the user while the run is in progress, not only when a test fails. + /// + [TestClass] + public class GivenDotnetTestForwardsTestHostOutput : SdkTest + { + private const string OutputBeforeHandshake = "LIVE_OUTPUT_BEFORE_HANDSHAKE"; + private const string StandardOutputDuringRun = "LIVE_OUTPUT_STANDARD_OUTPUT"; + private const string StandardErrorDuringRun = "LIVE_OUTPUT_STANDARD_ERROR"; + + [DataRow(TestingConstants.Debug)] + [DataRow(TestingConstants.Release)] + [TestMethod] + public void RunTestProjectWritingToConsole_ShouldForwardOutputEvenWhenAllTestsPass(string configuration) + { + TestAsset testInstance = TestAssetsManager.CopyTestAsset("TestProjectWithLiveOutput", Guid.NewGuid().ToString()) + .WithSource(); + + CommandResult result = new DotnetTestCommand(Log, disableNewOutput: false) + .WithWorkingDirectory(testInstance.Path) + .Execute("-c", configuration); + + // The test app writes to the console before the handshake and while the test session runs. + // Both must show up even though the run succeeds and no output-related option was passed. + result.StdOut + .Should().Contain(OutputBeforeHandshake) + .And.Contain(StandardOutputDuringRun) + .And.Contain(StandardErrorDuringRun); + + if (!SdkTestContext.IsLocalized()) + { + result.StdOut + .Should().Contain("Test run summary: Passed!") + .And.Contain("total: 1") + .And.Contain("succeeded: 1") + .And.Contain("failed: 0") + .And.Contain("skipped: 0"); + + // The output is streamed as it is produced, so it precedes the end-of-run summary + // instead of being replayed as part of a failure report. + result.StdOut!.IndexOf(StandardOutputDuringRun, StringComparison.Ordinal) + .Should().BeLessThan(result.StdOut!.IndexOf("Test run summary:", StringComparison.Ordinal)); + } + + result.ExitCode.Should().Be(ExitCodes.Success); + } + } +} From e88ead7a0fb03a9328914c335509e67f6f092bb9 Mon Sep 17 00:00:00 2001 From: Amaury Leveugle Date: Mon, 27 Jul 2026 14:57:11 +0200 Subject: [PATCH 2/5] Prove liveness with a sentinel handshake and cover the output device path Addresses review feedback on the previous commit: - The ordering assertion on the captured stdout could not fail: the child's output streams are fully drained before the summary is emitted, so even a buffer-until-exit implementation satisfied it. Replaced with a real observation: the test app blocks until a sentinel file appears, and the test creates that file from CommandOutputHandler when it sees the marker on the live standard output of 'dotnet test'. Output that is not forwarded live therefore deadlocks the app until its own timeout expires and fails the run. - The pre-handshake marker did not guard the flush-on-negotiation path, because a later line flushed the whole buffer anyway. Moved it to a dedicated asset that writes only before the handshake and produces nothing afterwards. - Added coverage for output written through the platform's IOutputDevice, which reaches the SDK as protocol 1.3.0 display messages. Session, warning and error messages are covered; plain informational text is deliberately discarded by the host under the pipe protocol. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: eedfe288-df4b-48f1-a612-67ce78b017a5 --- .../TestProjectWithLiveOutput/Program.cs | 57 +++++++++++-- .../Program.cs | 53 ++++++++++++ ...estProjectWithOutputBeforeHandshake.csproj | 18 ++++ .../global.json | 5 ++ .../GivenDotnetTestForwardsTestHostOutput.cs | 83 +++++++++++++++---- 5 files changed, 190 insertions(+), 26 deletions(-) create mode 100644 test/TestAssets/TestProjects/TestProjectWithOutputBeforeHandshake/Program.cs create mode 100644 test/TestAssets/TestProjects/TestProjectWithOutputBeforeHandshake/TestProjectWithOutputBeforeHandshake.csproj create mode 100644 test/TestAssets/TestProjects/TestProjectWithOutputBeforeHandshake/global.json diff --git a/test/TestAssets/TestProjects/TestProjectWithLiveOutput/Program.cs b/test/TestAssets/TestProjects/TestProjectWithLiveOutput/Program.cs index a6bcb4c33bc2..dd015928be53 100644 --- a/test/TestAssets/TestProjects/TestProjectWithLiveOutput/Program.cs +++ b/test/TestAssets/TestProjects/TestProjectWithLiveOutput/Program.cs @@ -1,21 +1,28 @@ using Microsoft.Testing.Platform.Builder; using Microsoft.Testing.Platform.Capabilities.TestFramework; using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.Extensions.OutputDevice; using Microsoft.Testing.Platform.Extensions.TestFramework; - -// Written before the test application is built, i.e. before the pipe handshake with the SDK -// completes. It exercises the "buffer until the protocol version is negotiated, then flush" path. -Console.WriteLine("LIVE_OUTPUT_BEFORE_HANDSHAKE"); +using Microsoft.Testing.Platform.OutputDevice; +using Microsoft.Testing.Platform.Services; var testApplicationBuilder = await TestApplication.CreateBuilderAsync(args); -testApplicationBuilder.RegisterTestFramework(_ => new TestFrameworkCapabilities(), (_, __) => new DummyTestAdapter()); +testApplicationBuilder.RegisterTestFramework(_ => new TestFrameworkCapabilities(), (_, serviceProvider) => new DummyTestAdapter(serviceProvider.GetOutputDevice())); using var testApplication = await testApplicationBuilder.BuildAsync(); return await testApplication.RunAsync(); -public class DummyTestAdapter : ITestFramework, IDataProducer +public class DummyTestAdapter(IOutputDevice outputDevice) : ITestFramework, IDataProducer, IOutputDeviceDataProducer { + // Set by the test to a path that does not exist yet. The test creates the file as soon as it + // observes LIVE_OUTPUT_STANDARD_OUTPUT on the standard output of 'dotnet test' while the + // command is still running, so this app can only see the file appear if its own console output + // really was forwarded live rather than buffered until it exits. + private const string SentinelPathEnvironmentVariable = "LIVE_OUTPUT_SENTINEL_PATH"; + + private static readonly TimeSpan SentinelTimeout = TimeSpan.FromSeconds(60); + public string Uid => nameof(DummyTestAdapter); public string Version => "2.0.0"; @@ -38,18 +45,50 @@ public Task CloseTestSessionAsync(CloseTestSessionContex public async Task ExecuteRequestAsync(ExecuteRequestContext context) { - // Written while the test session is running, so it must be streamed to the SDK terminal - // as it is produced instead of being buffered and only shown when something fails. Console.WriteLine("LIVE_OUTPUT_STANDARD_OUTPUT"); Console.Error.WriteLine("LIVE_OUTPUT_STANDARD_ERROR"); + // Text written through the platform's output device does not reach the console directly: + // it is forwarded to 'dotnet test' over the pipe protocol and rendered by its reporter. + // Only durable session messages, warnings and errors cross the wire - plain informational + // text is deliberately discarded by the host under the pipe protocol. + await outputDevice.DisplayAsync(this, new SessionMessageOutputDeviceData("LIVE_OUTPUT_SESSION_MESSAGE"), context.CancellationToken); + await outputDevice.DisplayAsync(this, new WarningMessageOutputDeviceData("LIVE_OUTPUT_WARNING_MESSAGE"), context.CancellationToken); + await outputDevice.DisplayAsync(this, new ErrorMessageOutputDeviceData("LIVE_OUTPUT_ERROR_MESSAGE"), context.CancellationToken); + + string? failureReason = await WaitForOutputToBeObservedAsync(); + await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(context.Request.Session.SessionUid, new TestNode() { Uid = "Test0", DisplayName = "Test0", - Properties = new PropertyBag(new PassedTestNodeStateProperty("OK")), + Properties = new PropertyBag(failureReason is null + ? new PassedTestNodeStateProperty("OK") + : new FailedTestNodeStateProperty(failureReason)), })); context.Complete(); } + + private static async Task WaitForOutputToBeObservedAsync() + { + string? sentinelPath = Environment.GetEnvironmentVariable(SentinelPathEnvironmentVariable); + if (string.IsNullOrEmpty(sentinelPath)) + { + return $"{SentinelPathEnvironmentVariable} is not set."; + } + + DateTime deadline = DateTime.UtcNow + SentinelTimeout; + while (DateTime.UtcNow < deadline) + { + if (File.Exists(sentinelPath)) + { + return null; + } + + await Task.Delay(50); + } + + return $"The standard output of this test app was not observed within {SentinelTimeout.TotalSeconds} seconds while it was still running, so it was not forwarded live."; + } } diff --git a/test/TestAssets/TestProjects/TestProjectWithOutputBeforeHandshake/Program.cs b/test/TestAssets/TestProjects/TestProjectWithOutputBeforeHandshake/Program.cs new file mode 100644 index 000000000000..ca0c6837a4d2 --- /dev/null +++ b/test/TestAssets/TestProjects/TestProjectWithOutputBeforeHandshake/Program.cs @@ -0,0 +1,53 @@ +using Microsoft.Testing.Platform.Builder; +using Microsoft.Testing.Platform.Capabilities.TestFramework; +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.Extensions.TestFramework; + +// This is the only console output the app produces, and it is written before the pipe handshake +// with 'dotnet test' can have completed. The SDK has to buffer it until it knows the negotiated +// protocol version and then flush it. Producing no further output afterwards is what makes this +// asset a guard: the flush cannot be masked by a later line flushing the whole buffer, and since +// the run succeeds the output is never replayed as part of a failure summary either. +Console.WriteLine("OUTPUT_BEFORE_HANDSHAKE"); + +var testApplicationBuilder = await TestApplication.CreateBuilderAsync(args); + +testApplicationBuilder.RegisterTestFramework(_ => new TestFrameworkCapabilities(), (_, __) => new DummyTestAdapter()); + +using var testApplication = await testApplicationBuilder.BuildAsync(); +return await testApplication.RunAsync(); + +public class DummyTestAdapter : ITestFramework, IDataProducer +{ + public string Uid => nameof(DummyTestAdapter); + + public string Version => "2.0.0"; + + public string DisplayName => nameof(DummyTestAdapter); + + public string Description => nameof(DummyTestAdapter); + + public Task IsEnabledAsync() => Task.FromResult(true); + + public Type[] DataTypesProduced => new[] { + typeof(TestNodeUpdateMessage) + }; + + public Task CreateTestSessionAsync(CreateTestSessionContext context) + => Task.FromResult(new CreateTestSessionResult() { IsSuccess = true }); + + public Task CloseTestSessionAsync(CloseTestSessionContext context) + => Task.FromResult(new CloseTestSessionResult() { IsSuccess = true }); + + public async Task ExecuteRequestAsync(ExecuteRequestContext context) + { + await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(context.Request.Session.SessionUid, new TestNode() + { + Uid = "Test0", + DisplayName = "Test0", + Properties = new PropertyBag(new PassedTestNodeStateProperty("OK")), + })); + + context.Complete(); + } +} diff --git a/test/TestAssets/TestProjects/TestProjectWithOutputBeforeHandshake/TestProjectWithOutputBeforeHandshake.csproj b/test/TestAssets/TestProjects/TestProjectWithOutputBeforeHandshake/TestProjectWithOutputBeforeHandshake.csproj new file mode 100644 index 000000000000..1ba237591546 --- /dev/null +++ b/test/TestAssets/TestProjects/TestProjectWithOutputBeforeHandshake/TestProjectWithOutputBeforeHandshake.csproj @@ -0,0 +1,18 @@ + + + + + $(CurrentTargetFramework) + Exe + + enable + enable + + false + true + + + + + + diff --git a/test/TestAssets/TestProjects/TestProjectWithOutputBeforeHandshake/global.json b/test/TestAssets/TestProjects/TestProjectWithOutputBeforeHandshake/global.json new file mode 100644 index 000000000000..9009caf0ba8f --- /dev/null +++ b/test/TestAssets/TestProjects/TestProjectWithOutputBeforeHandshake/global.json @@ -0,0 +1,5 @@ +{ + "test": { + "runner": "Microsoft.Testing.Platform" + } +} diff --git a/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestForwardsTestHostOutput.cs b/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestForwardsTestHostOutput.cs index d9552102e3af..eb5eba58a06e 100644 --- a/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestForwardsTestHostOutput.cs +++ b/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestForwardsTestHostOutput.cs @@ -7,34 +7,59 @@ namespace Microsoft.DotNet.Cli.Test.Tests { /// - /// End-to-end coverage for https://github.com/dotnet/sdk/issues/51615: the console output of a - /// test application must reach the user while the run is in progress, not only when a test fails. + /// End-to-end coverage for https://github.com/dotnet/sdk/issues/51615: what a test application + /// writes to the console must reach the user while the run is in progress, and not only be + /// replayed when a test fails. /// [TestClass] public class GivenDotnetTestForwardsTestHostOutput : SdkTest { - private const string OutputBeforeHandshake = "LIVE_OUTPUT_BEFORE_HANDSHAKE"; - private const string StandardOutputDuringRun = "LIVE_OUTPUT_STANDARD_OUTPUT"; - private const string StandardErrorDuringRun = "LIVE_OUTPUT_STANDARD_ERROR"; + private const string SentinelPathEnvironmentVariable = "LIVE_OUTPUT_SENTINEL_PATH"; + private const string StandardOutputMarker = "LIVE_OUTPUT_STANDARD_OUTPUT"; + private const string StandardErrorMarker = "LIVE_OUTPUT_STANDARD_ERROR"; + private const string OutputDeviceSessionMessageMarker = "LIVE_OUTPUT_SESSION_MESSAGE"; + private const string OutputDeviceWarningMarker = "LIVE_OUTPUT_WARNING_MESSAGE"; + private const string OutputDeviceErrorMarker = "LIVE_OUTPUT_ERROR_MESSAGE"; + private const string OutputBeforeHandshakeMarker = "OUTPUT_BEFORE_HANDSHAKE"; [DataRow(TestingConstants.Debug)] [DataRow(TestingConstants.Release)] [TestMethod] - public void RunTestProjectWritingToConsole_ShouldForwardOutputEvenWhenAllTestsPass(string configuration) + public void RunTestProjectWritingToConsole_ShouldForwardOutputWhileTheRunIsInProgress(string configuration) { TestAsset testInstance = TestAssetsManager.CopyTestAsset("TestProjectWithLiveOutput", Guid.NewGuid().ToString()) .WithSource(); - CommandResult result = new DotnetTestCommand(Log, disableNewOutput: false) + // The test app blocks until this file exists, and the file is only created once the + // marker it wrote has been observed on the live standard output of 'dotnet test'. + // An implementation that buffers the test app's output until it exits therefore + // deadlocks the app until its own timeout expires, which fails the run. + string sentinelPath = Path.Combine(testInstance.Path, "live-output-observed.sentinel"); + + var command = new DotnetTestCommand(Log, disableNewOutput: false) .WithWorkingDirectory(testInstance.Path) - .Execute("-c", configuration); + .WithEnvironmentVariable(SentinelPathEnvironmentVariable, sentinelPath); + command.CommandOutputHandler = line => + { + if (line.Contains(StandardOutputMarker, StringComparison.Ordinal) && !File.Exists(sentinelPath)) + { + File.WriteAllText(sentinelPath, string.Empty); + } + }; - // The test app writes to the console before the handshake and while the test session runs. - // Both must show up even though the run succeeds and no output-related option was passed. + CommandResult result = command.Execute("-c", configuration); + + // The run succeeds, so nothing replays the test app's output as part of a failure + // report: the markers can only be present because they were forwarded as produced. + // The output device markers additionally cover the text the test app routes through + // the platform's IOutputDevice, which reaches the SDK as protocol 1.3.0 display + // messages and is rendered at its informational, warning and error levels. result.StdOut - .Should().Contain(OutputBeforeHandshake) - .And.Contain(StandardOutputDuringRun) - .And.Contain(StandardErrorDuringRun); + .Should().Contain(StandardOutputMarker) + .And.Contain(StandardErrorMarker) + .And.Contain(OutputDeviceSessionMessageMarker) + .And.Contain(OutputDeviceWarningMarker) + .And.Contain(OutputDeviceErrorMarker); if (!SdkTestContext.IsLocalized()) { @@ -44,11 +69,35 @@ public void RunTestProjectWritingToConsole_ShouldForwardOutputEvenWhenAllTestsPa .And.Contain("succeeded: 1") .And.Contain("failed: 0") .And.Contain("skipped: 0"); + } + + result.ExitCode.Should().Be(ExitCodes.Success); + } + + [DataRow(TestingConstants.Debug)] + [DataRow(TestingConstants.Release)] + [TestMethod] + public void RunTestProjectWritingToConsoleBeforeHandshake_ShouldForwardOutputOnceProtocolIsNegotiated(string configuration) + { + TestAsset testInstance = TestAssetsManager.CopyTestAsset("TestProjectWithOutputBeforeHandshake", Guid.NewGuid().ToString()) + .WithSource(); - // The output is streamed as it is produced, so it precedes the end-of-run summary - // instead of being replayed as part of a failure report. - result.StdOut!.IndexOf(StandardOutputDuringRun, StringComparison.Ordinal) - .Should().BeLessThan(result.StdOut!.IndexOf("Test run summary:", StringComparison.Ordinal)); + CommandResult result = new DotnetTestCommand(Log, disableNewOutput: false) + .WithWorkingDirectory(testInstance.Path) + .Execute("-c", configuration); + + // The test app only writes before the handshake completes, so this output has to be + // buffered until the protocol version is known and then flushed. + result.StdOut.Should().Contain(OutputBeforeHandshakeMarker); + + if (!SdkTestContext.IsLocalized()) + { + result.StdOut + .Should().Contain("Test run summary: Passed!") + .And.Contain("total: 1") + .And.Contain("succeeded: 1") + .And.Contain("failed: 0") + .And.Contain("skipped: 0"); } result.ExitCode.Should().Be(ExitCodes.Success); From ecd5ec0acba9af6d0ec14d6fedbe015addd881df Mon Sep 17 00:00:00 2001 From: Amaury Leveugle Date: Mon, 27 Jul 2026 15:14:27 +0200 Subject: [PATCH 3/5] Keep the liveness assertion load-bearing under retry and I/O failures Two problems found reviewing the sentinel handshake: - Execute retries the command on transient failures without re-copying the test asset, so a sentinel created by an earlier attempt would still be there on the next one. The app would then return immediately and the test would go green having proven nothing. Clear the sentinel per attempt from ProcessStartedHandler, which runs right after the process starts. - The output handler runs on the only thread draining the command's standard output. An exception from the file write would stop that drain, so the child would block once the pipe buffer filled and the run would hang instead of failing. Capture the exception and assert on it after the run. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: eedfe288-df4b-48f1-a612-67ce78b017a5 --- .../GivenDotnetTestForwardsTestHostOutput.cs | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestForwardsTestHostOutput.cs b/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestForwardsTestHostOutput.cs index eb5eba58a06e..69ba19bd7873 100644 --- a/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestForwardsTestHostOutput.cs +++ b/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestForwardsTestHostOutput.cs @@ -36,19 +36,37 @@ public void RunTestProjectWritingToConsole_ShouldForwardOutputWhileTheRunIsInPro // deadlocks the app until its own timeout expires, which fails the run. string sentinelPath = Path.Combine(testInstance.Path, "live-output-observed.sentinel"); + Exception? sentinelWriteFailure = null; var command = new DotnetTestCommand(Log, disableNewOutput: false) .WithWorkingDirectory(testInstance.Path) .WithEnvironmentVariable(SentinelPathEnvironmentVariable, sentinelPath); + + // Execute retries the command on transient failures without re-copying the asset, so + // clear the sentinel on every attempt: a file left behind by an earlier attempt would + // let the app proceed immediately and quietly void what this test is proving. + command.ProcessStartedHandler = _ => File.Delete(sentinelPath); command.CommandOutputHandler = line => { if (line.Contains(StandardOutputMarker, StringComparison.Ordinal) && !File.Exists(sentinelPath)) { - File.WriteAllText(sentinelPath, string.Empty); + try + { + File.WriteAllText(sentinelPath, string.Empty); + } + catch (Exception ex) + { + // This runs on the only thread draining the command's standard output. + // Letting the exception escape would stop that drain and hang the run, so + // record it and let the assertion below report it. + sentinelWriteFailure ??= ex; + } } }; CommandResult result = command.Execute("-c", configuration); + sentinelWriteFailure.Should().BeNull("the sentinel file should be writable"); + // The run succeeds, so nothing replays the test app's output as part of a failure // report: the markers can only be present because they were forwarded as produced. // The output device markers additionally cover the text the test app routes through From 20013c7f0b0e7baa9a483c78d21e468ad231a7ce Mon Sep 17 00:00:00 2001 From: Amaury Leveugle Date: Mon, 27 Jul 2026 15:23:10 +0200 Subject: [PATCH 4/5] Guard the per-attempt sentinel delete against I/O failures The delete runs after the command's process has started but before it is registered with the process reaper, so an exception escaping it would leave that process orphaned. Record it like the write failure and assert on it. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: eedfe288-df4b-48f1-a612-67ce78b017a5 --- .../GivenDotnetTestForwardsTestHostOutput.cs | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestForwardsTestHostOutput.cs b/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestForwardsTestHostOutput.cs index 69ba19bd7873..1664c9c849bc 100644 --- a/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestForwardsTestHostOutput.cs +++ b/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestForwardsTestHostOutput.cs @@ -36,7 +36,7 @@ public void RunTestProjectWritingToConsole_ShouldForwardOutputWhileTheRunIsInPro // deadlocks the app until its own timeout expires, which fails the run. string sentinelPath = Path.Combine(testInstance.Path, "live-output-observed.sentinel"); - Exception? sentinelWriteFailure = null; + Exception? sentinelFailure = null; var command = new DotnetTestCommand(Log, disableNewOutput: false) .WithWorkingDirectory(testInstance.Path) .WithEnvironmentVariable(SentinelPathEnvironmentVariable, sentinelPath); @@ -44,7 +44,19 @@ public void RunTestProjectWritingToConsole_ShouldForwardOutputWhileTheRunIsInPro // Execute retries the command on transient failures without re-copying the asset, so // clear the sentinel on every attempt: a file left behind by an earlier attempt would // let the app proceed immediately and quietly void what this test is proving. - command.ProcessStartedHandler = _ => File.Delete(sentinelPath); + // This runs after the command's process has started but before it is registered with + // the process reaper, so an exception escaping here would leave that process orphaned. + command.ProcessStartedHandler = _ => + { + try + { + File.Delete(sentinelPath); + } + catch (Exception ex) + { + sentinelFailure ??= ex; + } + }; command.CommandOutputHandler = line => { if (line.Contains(StandardOutputMarker, StringComparison.Ordinal) && !File.Exists(sentinelPath)) @@ -58,14 +70,14 @@ public void RunTestProjectWritingToConsole_ShouldForwardOutputWhileTheRunIsInPro // This runs on the only thread draining the command's standard output. // Letting the exception escape would stop that drain and hang the run, so // record it and let the assertion below report it. - sentinelWriteFailure ??= ex; + sentinelFailure ??= ex; } } }; CommandResult result = command.Execute("-c", configuration); - sentinelWriteFailure.Should().BeNull("the sentinel file should be writable"); + sentinelFailure.Should().BeNull("the sentinel file should be writable and deletable"); // The run succeeds, so nothing replays the test app's output as part of a failure // report: the markers can only be present because they were forwarded as produced. From f89e058e01d7e971314d648b81fd5a4c406d082e Mon Sep 17 00:00:00 2001 From: Amaury Leveugle Date: Wed, 29 Jul 2026 13:02:39 +0200 Subject: [PATCH 5/5] Use a file-scoped namespace in the new E2E test file Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: c8782400-17e4-4447-b71f-06288a76e89e --- .../GivenDotnetTestForwardsTestHostOutput.cs | 211 +++++++++--------- 1 file changed, 105 insertions(+), 106 deletions(-) diff --git a/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestForwardsTestHostOutput.cs b/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestForwardsTestHostOutput.cs index 1664c9c849bc..025047df7066 100644 --- a/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestForwardsTestHostOutput.cs +++ b/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestForwardsTestHostOutput.cs @@ -4,133 +4,132 @@ using CommandResult = Microsoft.DotNet.Cli.Utils.CommandResult; using ExitCodes = Microsoft.NET.TestFramework.ExitCode; -namespace Microsoft.DotNet.Cli.Test.Tests +namespace Microsoft.DotNet.Cli.Test.Tests; + +/// +/// End-to-end coverage for https://github.com/dotnet/sdk/issues/51615: what a test application +/// writes to the console must reach the user while the run is in progress, and not only be +/// replayed when a test fails. +/// +[TestClass] +public class GivenDotnetTestForwardsTestHostOutput : SdkTest { - /// - /// End-to-end coverage for https://github.com/dotnet/sdk/issues/51615: what a test application - /// writes to the console must reach the user while the run is in progress, and not only be - /// replayed when a test fails. - /// - [TestClass] - public class GivenDotnetTestForwardsTestHostOutput : SdkTest + private const string SentinelPathEnvironmentVariable = "LIVE_OUTPUT_SENTINEL_PATH"; + private const string StandardOutputMarker = "LIVE_OUTPUT_STANDARD_OUTPUT"; + private const string StandardErrorMarker = "LIVE_OUTPUT_STANDARD_ERROR"; + private const string OutputDeviceSessionMessageMarker = "LIVE_OUTPUT_SESSION_MESSAGE"; + private const string OutputDeviceWarningMarker = "LIVE_OUTPUT_WARNING_MESSAGE"; + private const string OutputDeviceErrorMarker = "LIVE_OUTPUT_ERROR_MESSAGE"; + private const string OutputBeforeHandshakeMarker = "OUTPUT_BEFORE_HANDSHAKE"; + + [DataRow(TestingConstants.Debug)] + [DataRow(TestingConstants.Release)] + [TestMethod] + public void RunTestProjectWritingToConsole_ShouldForwardOutputWhileTheRunIsInProgress(string configuration) { - private const string SentinelPathEnvironmentVariable = "LIVE_OUTPUT_SENTINEL_PATH"; - private const string StandardOutputMarker = "LIVE_OUTPUT_STANDARD_OUTPUT"; - private const string StandardErrorMarker = "LIVE_OUTPUT_STANDARD_ERROR"; - private const string OutputDeviceSessionMessageMarker = "LIVE_OUTPUT_SESSION_MESSAGE"; - private const string OutputDeviceWarningMarker = "LIVE_OUTPUT_WARNING_MESSAGE"; - private const string OutputDeviceErrorMarker = "LIVE_OUTPUT_ERROR_MESSAGE"; - private const string OutputBeforeHandshakeMarker = "OUTPUT_BEFORE_HANDSHAKE"; - - [DataRow(TestingConstants.Debug)] - [DataRow(TestingConstants.Release)] - [TestMethod] - public void RunTestProjectWritingToConsole_ShouldForwardOutputWhileTheRunIsInProgress(string configuration) + TestAsset testInstance = TestAssetsManager.CopyTestAsset("TestProjectWithLiveOutput", Guid.NewGuid().ToString()) + .WithSource(); + + // The test app blocks until this file exists, and the file is only created once the + // marker it wrote has been observed on the live standard output of 'dotnet test'. + // An implementation that buffers the test app's output until it exits therefore + // deadlocks the app until its own timeout expires, which fails the run. + string sentinelPath = Path.Combine(testInstance.Path, "live-output-observed.sentinel"); + + Exception? sentinelFailure = null; + var command = new DotnetTestCommand(Log, disableNewOutput: false) + .WithWorkingDirectory(testInstance.Path) + .WithEnvironmentVariable(SentinelPathEnvironmentVariable, sentinelPath); + + // Execute retries the command on transient failures without re-copying the asset, so + // clear the sentinel on every attempt: a file left behind by an earlier attempt would + // let the app proceed immediately and quietly void what this test is proving. + // This runs after the command's process has started but before it is registered with + // the process reaper, so an exception escaping here would leave that process orphaned. + command.ProcessStartedHandler = _ => { - TestAsset testInstance = TestAssetsManager.CopyTestAsset("TestProjectWithLiveOutput", Guid.NewGuid().ToString()) - .WithSource(); - - // The test app blocks until this file exists, and the file is only created once the - // marker it wrote has been observed on the live standard output of 'dotnet test'. - // An implementation that buffers the test app's output until it exits therefore - // deadlocks the app until its own timeout expires, which fails the run. - string sentinelPath = Path.Combine(testInstance.Path, "live-output-observed.sentinel"); - - Exception? sentinelFailure = null; - var command = new DotnetTestCommand(Log, disableNewOutput: false) - .WithWorkingDirectory(testInstance.Path) - .WithEnvironmentVariable(SentinelPathEnvironmentVariable, sentinelPath); - - // Execute retries the command on transient failures without re-copying the asset, so - // clear the sentinel on every attempt: a file left behind by an earlier attempt would - // let the app proceed immediately and quietly void what this test is proving. - // This runs after the command's process has started but before it is registered with - // the process reaper, so an exception escaping here would leave that process orphaned. - command.ProcessStartedHandler = _ => + try + { + File.Delete(sentinelPath); + } + catch (Exception ex) + { + sentinelFailure ??= ex; + } + }; + command.CommandOutputHandler = line => + { + if (line.Contains(StandardOutputMarker, StringComparison.Ordinal) && !File.Exists(sentinelPath)) { try { - File.Delete(sentinelPath); + File.WriteAllText(sentinelPath, string.Empty); } catch (Exception ex) { + // This runs on the only thread draining the command's standard output. + // Letting the exception escape would stop that drain and hang the run, so + // record it and let the assertion below report it. sentinelFailure ??= ex; } - }; - command.CommandOutputHandler = line => - { - if (line.Contains(StandardOutputMarker, StringComparison.Ordinal) && !File.Exists(sentinelPath)) - { - try - { - File.WriteAllText(sentinelPath, string.Empty); - } - catch (Exception ex) - { - // This runs on the only thread draining the command's standard output. - // Letting the exception escape would stop that drain and hang the run, so - // record it and let the assertion below report it. - sentinelFailure ??= ex; - } - } - }; + } + }; - CommandResult result = command.Execute("-c", configuration); + CommandResult result = command.Execute("-c", configuration); - sentinelFailure.Should().BeNull("the sentinel file should be writable and deletable"); + sentinelFailure.Should().BeNull("the sentinel file should be writable and deletable"); - // The run succeeds, so nothing replays the test app's output as part of a failure - // report: the markers can only be present because they were forwarded as produced. - // The output device markers additionally cover the text the test app routes through - // the platform's IOutputDevice, which reaches the SDK as protocol 1.3.0 display - // messages and is rendered at its informational, warning and error levels. - result.StdOut - .Should().Contain(StandardOutputMarker) - .And.Contain(StandardErrorMarker) - .And.Contain(OutputDeviceSessionMessageMarker) - .And.Contain(OutputDeviceWarningMarker) - .And.Contain(OutputDeviceErrorMarker); + // The run succeeds, so nothing replays the test app's output as part of a failure + // report: the markers can only be present because they were forwarded as produced. + // The output device markers additionally cover the text the test app routes through + // the platform's IOutputDevice, which reaches the SDK as protocol 1.3.0 display + // messages and is rendered at its informational, warning and error levels. + result.StdOut + .Should().Contain(StandardOutputMarker) + .And.Contain(StandardErrorMarker) + .And.Contain(OutputDeviceSessionMessageMarker) + .And.Contain(OutputDeviceWarningMarker) + .And.Contain(OutputDeviceErrorMarker); - if (!SdkTestContext.IsLocalized()) - { - result.StdOut - .Should().Contain("Test run summary: Passed!") - .And.Contain("total: 1") - .And.Contain("succeeded: 1") - .And.Contain("failed: 0") - .And.Contain("skipped: 0"); - } - - result.ExitCode.Should().Be(ExitCodes.Success); + if (!SdkTestContext.IsLocalized()) + { + result.StdOut + .Should().Contain("Test run summary: Passed!") + .And.Contain("total: 1") + .And.Contain("succeeded: 1") + .And.Contain("failed: 0") + .And.Contain("skipped: 0"); } - [DataRow(TestingConstants.Debug)] - [DataRow(TestingConstants.Release)] - [TestMethod] - public void RunTestProjectWritingToConsoleBeforeHandshake_ShouldForwardOutputOnceProtocolIsNegotiated(string configuration) - { - TestAsset testInstance = TestAssetsManager.CopyTestAsset("TestProjectWithOutputBeforeHandshake", Guid.NewGuid().ToString()) - .WithSource(); + result.ExitCode.Should().Be(ExitCodes.Success); + } - CommandResult result = new DotnetTestCommand(Log, disableNewOutput: false) - .WithWorkingDirectory(testInstance.Path) - .Execute("-c", configuration); + [DataRow(TestingConstants.Debug)] + [DataRow(TestingConstants.Release)] + [TestMethod] + public void RunTestProjectWritingToConsoleBeforeHandshake_ShouldForwardOutputOnceProtocolIsNegotiated(string configuration) + { + TestAsset testInstance = TestAssetsManager.CopyTestAsset("TestProjectWithOutputBeforeHandshake", Guid.NewGuid().ToString()) + .WithSource(); - // The test app only writes before the handshake completes, so this output has to be - // buffered until the protocol version is known and then flushed. - result.StdOut.Should().Contain(OutputBeforeHandshakeMarker); + CommandResult result = new DotnetTestCommand(Log, disableNewOutput: false) + .WithWorkingDirectory(testInstance.Path) + .Execute("-c", configuration); - if (!SdkTestContext.IsLocalized()) - { - result.StdOut - .Should().Contain("Test run summary: Passed!") - .And.Contain("total: 1") - .And.Contain("succeeded: 1") - .And.Contain("failed: 0") - .And.Contain("skipped: 0"); - } + // The test app only writes before the handshake completes, so this output has to be + // buffered until the protocol version is known and then flushed. + result.StdOut.Should().Contain(OutputBeforeHandshakeMarker); - result.ExitCode.Should().Be(ExitCodes.Success); + if (!SdkTestContext.IsLocalized()) + { + result.StdOut + .Should().Contain("Test run summary: Passed!") + .And.Contain("total: 1") + .And.Contain("succeeded: 1") + .And.Contain("failed: 0") + .And.Contain("skipped: 0"); } + + result.ExitCode.Should().Be(ExitCodes.Success); } }