From 83d6f8701d5fcce092c02d796adf01554bb13a32 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Tue, 1 Sep 2026 13:22:28 +0100 Subject: [PATCH 01/26] feat: add programmatic reporting settings Resolve reporting switches lazily so values configured in before-test-discovery hooks apply after reporter registration. Environment variables retain higher precedence. --- docs/docs/guides/html-report.md | 8 ++- docs/docs/guides/report-aggregation.md | 6 +- docs/docs/reference/command-line-flags.md | 3 +- docs/docs/reference/environment-variables.md | 23 ++++++- .../reference/programmatic-configuration.md | 12 ++++ src/TUnit.Core/Settings/ReportingSettings.cs | 28 ++++++++ src/TUnit.Core/Settings/TUnitSettings.cs | 5 ++ .../Reporters/Html/HtmlReporter.cs | 42 ++++++++++-- .../HtmlReporterConfigurationTests.cs | 65 +++++++++++++++++++ ...Has_No_API_Changes.DotNet10_0.verified.txt | 7 ++ ..._Has_No_API_Changes.DotNet8_0.verified.txt | 7 ++ ..._Has_No_API_Changes.DotNet9_0.verified.txt | 7 ++ ...ary_Has_No_API_Changes.Net4_7.verified.txt | 7 ++ tests/TUnit.UnitTests/TUnitSettingsTests.cs | 24 +++++++ 14 files changed, 230 insertions(+), 14 deletions(-) create mode 100644 src/TUnit.Core/Settings/ReportingSettings.cs create mode 100644 tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs diff --git a/docs/docs/guides/html-report.md b/docs/docs/guides/html-report.md index 325e24cd760..16c02002948 100644 --- a/docs/docs/guides/html-report.md +++ b/docs/docs/guides/html-report.md @@ -21,7 +21,7 @@ The OS and runtime version are included automatically so that matrix builds (mul Open it in any modern browser. The report is fully self-contained (single HTML file) and works offline. -A machine-readable JSON sidecar (`{AssemblyName}-{os}-{tfm}.tunit-report.json`) is written alongside the HTML report. It powers [report aggregation](/docs/guides/report-aggregation) — merging reports from multiple test projects into one — and can be disabled with `TUNIT_DISABLE_JSON_REPORT=true`. +A machine-readable JSON sidecar (`{AssemblyName}-{os}-{tfm}.tunit-report.json`) is written alongside the HTML report. It powers [report aggregation](/docs/guides/report-aggregation) — merging reports from multiple test projects into one — and can be disabled with `TUNIT_DISABLE_JSON_REPORT=true` or `context.Settings.Reporting.JsonReportEnabled = false`. Running many test projects and want **one combined report instead of one per project**? See [Aggregated Reports](/docs/guides/report-aggregation). @@ -53,6 +53,8 @@ export TUNIT_DISABLE_HTML_REPORTER=true Accepts: `true`, `1`, `yes` (case-insensitive). +For version-controlled project configuration, set `context.Settings.Reporting.HtmlReportEnabled = false` in a `[Before(HookType.TestDiscovery)]` hook instead. + ### Deprecated: `--report-html` Flag The `--report-html` flag is deprecated since the report is now generated by default. Using it will show a deprecation warning but will not cause an error. @@ -135,6 +137,8 @@ This is useful if you: The report file and the `GITHUB_STEP_SUMMARY` are still generated. +This can also be configured in code with `context.Settings.Reporting.ArtifactUploadEnabled = false`. + ### Viewing the Report After the workflow run completes: @@ -234,7 +238,7 @@ The collector uses **smart sampling**: spans from known test traces are fully re ### Report Not Generated -- Check that `TUNIT_DISABLE_HTML_REPORTER` is not set in your environment +- Check that `TUNIT_DISABLE_HTML_REPORTER` is not set and `context.Settings.Reporting.HtmlReportEnabled` is not `false` - Verify that the `TestResults/` directory is writable - Check the console output for any warning messages about report generation failures diff --git a/docs/docs/guides/report-aggregation.md b/docs/docs/guides/report-aggregation.md index 6cc1e5881b6..f399d78b215 100644 --- a/docs/docs/guides/report-aggregation.md +++ b/docs/docs/guides/report-aggregation.md @@ -12,7 +12,7 @@ Report aggregation merges all of that into **one combined HTML report and one Gi ## How It Works -1. Alongside every HTML report, TUnit writes a machine-readable sidecar: `{AssemblyName}-{os}-{tfm}.tunit-report.json`. This is on by default (disable with `TUNIT_DISABLE_JSON_REPORT=true`). +1. Alongside every HTML report, TUnit writes a machine-readable sidecar: `{AssemblyName}-{os}-{tfm}.tunit-report.json`. This is on by default (disable with `TUNIT_DISABLE_JSON_REPORT=true` or `context.Settings.Reporting.JsonReportEnabled = false`). 2. With aggregation enabled, each test process also copies its sidecar into a directory shared by all sibling processes. 3. As each process finishes, it takes a cross-process lock, reads *all* sidecars present so far, and regenerates the merged HTML report and the summary block. The last process to finish naturally leaves the complete aggregate — no process ever needs to know whether it is the last one. @@ -166,11 +166,11 @@ tunit-report merge --directory [options] | --- | --- | | `TUNIT_AGGREGATE_REPORTS` | Unset (default) — cooperative merge wherever a shared directory is resolvable (GitHub Actions, or explicit `TUNIT_AGGREGATE_DIR`); silently off otherwise. `defer` — persist sidecars + merged HTML only; no summary blocks (multi-step scenarios). `off` (also `false`/`0`/`no`/`disabled`/`none`) — no aggregation. | | `TUNIT_AGGREGATE_DIR` | Shared directory for sidecars and the merged report. Required outside GitHub Actions; optional override on GitHub Actions. | -| `TUNIT_DISABLE_JSON_REPORT` | Disables the JSON sidecar written next to the HTML report. Note: sidecars are what aggregation and `tunit-report` consume. | +| `TUNIT_DISABLE_JSON_REPORT` | Disables the JSON sidecar written next to the HTML report. Programmatic equivalent: `context.Settings.Reporting.JsonReportEnabled = false`. Note: sidecars are what aggregation and `tunit-report` consume. | ## Notes & Limitations -- Aggregation is driven by the HTML reporter's data pipeline — if you set `TUNIT_DISABLE_HTML_REPORTER`, no sidecars are produced and there is nothing to merge. +- Aggregation is driven by the HTML reporter's data pipeline — if you set `TUNIT_DISABLE_HTML_REPORTER` or `context.Settings.Reporting.HtmlReportEnabled = false`, no sidecars are produced and there is nothing to merge. - With cooperative mode (the default) across *multiple steps in the same job*, each step appends its own progressively-larger block (earlier steps' blocks can't be rewritten). Use `defer` + the tool for that layout, or `off` to restore per-suite blocks. - Suites are identified per assembly + OS + TFM, so multi-targeted projects appear as separate rows (e.g. `MyTests (.NET 8.0.x)` / `MyTests (.NET 9.0.x)`). - The GitHub step summary is capped at 1 MB by GitHub; the aggregated block replaces N per-suite blocks, so it usually *reduces* summary size. diff --git a/docs/docs/reference/command-line-flags.md b/docs/docs/reference/command-line-flags.md index 2e86de5f3e6..ef2d60f2ca9 100644 --- a/docs/docs/reference/command-line-flags.md +++ b/docs/docs/reference/command-line-flags.md @@ -137,7 +137,8 @@ Please note that for the coverage and trx report, you need to install [additiona --report-html (Deprecated) The HTML report is now generated by default. - Disable it with the TUNIT_DISABLE_HTML_REPORTER environment variable. + Disable it with TUNIT_DISABLE_HTML_REPORTER or set + context.Settings.Reporting.HtmlReportEnabled = false. --report-html-filename Path for the HTML test report file diff --git a/docs/docs/reference/environment-variables.md b/docs/docs/reference/environment-variables.md index 0f32f77ac35..f78d7e2b142 100644 --- a/docs/docs/reference/environment-variables.md +++ b/docs/docs/reference/environment-variables.md @@ -60,9 +60,11 @@ Accepts truthy values: `true`, `1`, `yes` (case-insensitive). **Use case:** When you don't need the HTML report or want to reduce disk I/O. The report is written to `TestResults/{AssemblyName}-report.html` by default. +**Programmatic equivalent:** `context.Settings.Reporting.HtmlReportEnabled = false` + ### TUNIT_DISABLE_ARTIFACT_UPLOAD -Skips the autmoatic upload of the html report but still generates the files. +Skips automatic upload of the HTML report but still generates the files. ```bash export TUNIT_DISABLE_ARTIFACT_UPLOAD=true @@ -77,6 +79,20 @@ GitHub changed the server-side behaviour of `upload-artifacts` since v4 and hasn Forgejo and gitea don't implement this new artifact endpoint but the runners set `GITHUB_ACTIONS=true`. In this case it attempts to upload the report a few times until it eventually backs off after 30s. +**Programmatic equivalent:** `context.Settings.Reporting.ArtifactUploadEnabled = false` + +### TUNIT_DISABLE_JSON_REPORT + +Disables the machine-readable JSON sidecar written alongside the HTML report. + +```bash +export TUNIT_DISABLE_JSON_REPORT=true +``` + +Accepts truthy values: `true`, `1`, `yes` (case-insensitive). + +**Programmatic equivalent:** `context.Settings.Reporting.JsonReportEnabled = false` + ### TUNIT_DISABLE_JUNIT_REPORTER Disables the JUnit XML reporter. @@ -284,8 +300,9 @@ When the same setting is configured in multiple places, TUnit follows this prior | `TUNIT_DISABLE_GITHUB_REPORTER` | - | Disables GitHub reporter | | `TUNIT_DISABLE_JUNIT_REPORTER` | - | Disables JUnit reporter | | `TUNIT_ENABLE_JUNIT_REPORTER` | - | Enables JUnit reporter | -| `TUNIT_DISABLE_HTML_REPORTER` | - | Disables HTML report generation | -| `TUNIT_DISABLE_ARTIFACT_UPLOAD` | - | Keeps the HTML report file but skips the GitHub Actions artifact upload | +| `TUNIT_DISABLE_HTML_REPORTER` | - | Disables HTML report generation (`context.Settings.Reporting.HtmlReportEnabled = false`) | +| `TUNIT_DISABLE_JSON_REPORT` | - | Disables the machine-readable JSON sidecar (`context.Settings.Reporting.JsonReportEnabled = false`) | +| `TUNIT_DISABLE_ARTIFACT_UPLOAD` | - | Keeps the HTML report file but skips the GitHub Actions artifact upload (`context.Settings.Reporting.ArtifactUploadEnabled = false`) | | `JUNIT_XML_OUTPUT_PATH` | - | JUnit output path | | `TUNIT_MAX_PARALLEL_TESTS` | `--maximum-parallel-tests` | Max parallel tests | | `TUNIT_EXECUTION_MODE` | `--reflection` | Selects source-generation (`sourcegeneration`/`aot`) or `reflection` execution mode | diff --git a/docs/docs/reference/programmatic-configuration.md b/docs/docs/reference/programmatic-configuration.md index 9c08e4973e6..ad9ee85c755 100644 --- a/docs/docs/reference/programmatic-configuration.md +++ b/docs/docs/reference/programmatic-configuration.md @@ -15,6 +15,7 @@ Settings are organized into logical groups: - `Parallelism` — concurrent test execution limits - `Execution` — runtime behavior such as fail-fast - `Display` — output and display options +- `Reporting` — HTML report generation and publishing - `Mocks` — defaults for TUnit.Mocks when the package is referenced ## Usage @@ -33,6 +34,7 @@ public class TestSetup context.Settings.Timeouts.DefaultTestTimeout = TimeSpan.FromMinutes(5); context.Settings.Timeouts.DefaultHookTimeout = TimeSpan.FromMinutes(2); context.Settings.Execution.FailFast = true; + context.Settings.Reporting.HtmlReportEnabled = false; context.Settings.Mocks.DefaultMode = MockBehavior.Strict; return Task.CompletedTask; @@ -73,6 +75,16 @@ Settings are accessed exclusively through `context.Settings` in the discovery ho |---|---|---|---| | `FailFast` | `bool` | `false` | Cancels the remaining test run after the first test failure. | +### `context.Settings.Reporting` + +| Property | Type | Default | Description | +|---|---|---|---| +| `HtmlReportEnabled` | `bool` | `true` | Generates the HTML test report. | +| `JsonReportEnabled` | `bool` | `true` | Generates the machine-readable JSON sidecar used by report aggregation. | +| `ArtifactUploadEnabled` | `bool` | `true` | Uploads the HTML report as an artifact when supported by the CI environment. | + +The corresponding `TUNIT_DISABLE_HTML_REPORTER`, `TUNIT_DISABLE_JSON_REPORT`, and `TUNIT_DISABLE_ARTIFACT_UPLOAD` environment variables take precedence over these values. + ### `context.Settings.Mocks` Available when `TUnit.Mocks` is referenced. diff --git a/src/TUnit.Core/Settings/ReportingSettings.cs b/src/TUnit.Core/Settings/ReportingSettings.cs new file mode 100644 index 00000000000..a198a358b27 --- /dev/null +++ b/src/TUnit.Core/Settings/ReportingSettings.cs @@ -0,0 +1,28 @@ +namespace TUnit.Core.Settings; + +/// +/// Controls built-in report generation and publishing. +/// +public sealed class ReportingSettings +{ + internal ReportingSettings() { } + + /// + /// Whether to generate the HTML test report. Default: true. + /// Precedence: TUNIT_DISABLE_HTML_REPORTER → TUnitSettings → built-in default. + /// + public bool HtmlReportEnabled { get; set; } = true; + + /// + /// Whether to generate the machine-readable JSON report sidecar. Default: true. + /// Precedence: TUNIT_DISABLE_JSON_REPORT → TUnitSettings → built-in default. + /// + public bool JsonReportEnabled { get; set; } = true; + + /// + /// Whether to upload the HTML report as an artifact when supported by the CI environment. + /// Default: true. + /// Precedence: TUNIT_DISABLE_ARTIFACT_UPLOAD → TUnitSettings → built-in default. + /// + public bool ArtifactUploadEnabled { get; set; } = true; +} diff --git a/src/TUnit.Core/Settings/TUnitSettings.cs b/src/TUnit.Core/Settings/TUnitSettings.cs index cc15574a89b..7ca50b384a0 100644 --- a/src/TUnit.Core/Settings/TUnitSettings.cs +++ b/src/TUnit.Core/Settings/TUnitSettings.cs @@ -38,4 +38,9 @@ internal TUnitSettings() { } /// Controls test run behavior. /// public ExecutionSettings Execution { get; } = new(); + + /// + /// Controls report generation and publishing. + /// + public ReportingSettings Reporting { get; } = new(); } diff --git a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs index 4f3f4a51846..1056ee885e6 100644 --- a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs +++ b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs @@ -12,6 +12,7 @@ using Microsoft.Testing.Platform.Services; using Microsoft.Testing.Platform.TestHost; using TUnit.Core; +using TUnit.Core.Settings; using TUnit.Engine.Configuration; using TUnit.Engine.Constants; using TUnit.Engine.Exceptions; @@ -43,7 +44,7 @@ internal sealed class HtmlReporter(IExtension extension) : IDataConsumer, IDataP public async Task IsEnabledAsync() { - if (IsTruthyEnv(Environment.GetEnvironmentVariable(EnvironmentConstants.DisableHtmlReporter))) + if (!IsHtmlReportEnabled()) { return false; } @@ -61,6 +62,13 @@ public async Task IsEnabledAsync() public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationToken cancellationToken) { + // IsEnabledAsync runs before user discovery hooks. Read this setting again here so + // context.Settings changes made by those hooks still suppress collection and output. + if (!IsHtmlReportEnabled()) + { + return Task.CompletedTask; + } + var testNodeUpdateMessage = (TestNodeUpdateMessage)value; // Keep only the update we'll report per test: a final-state update always wins over a // non-final one, otherwise the latest wins. The engine emits a single final update per @@ -90,8 +98,11 @@ private static bool HasFinalState(TestNodeUpdateMessage update) public Task BeforeRunAsync(CancellationToken cancellationToken) { #if NET - _activityCollector = new ActivityCollector(); - _activityCollector.Start(); + if (IsHtmlReportEnabled()) + { + _activityCollector = new ActivityCollector(); + _activityCollector.Start(); + } #endif return Task.CompletedTask; } @@ -110,6 +121,15 @@ public async Task OnTestSessionFinishingAsync(ITestSessionContext testSessionCon _activityCollector?.Stop(); #endif + if (!IsHtmlReportEnabled()) + { +#if NET + TraceRegistry.Clear(); +#endif + _updates.Clear(); + return; + } + if (_updates.Count == 0) { #if NET @@ -170,7 +190,7 @@ private async Task TryWriteSidecarAndAggregateAsync(ReportData reportData, strin // Serialized once; the same bytes back both the local sidecar and the shared copy. var sidecarBytes = ReportDataJson.SerializeToBytes(reportData); - if (!IsTruthyEnv(Environment.GetEnvironmentVariable(EnvironmentConstants.DisableJsonReport))) + if (IsJsonReportEnabled()) { try { @@ -238,6 +258,18 @@ private static bool IsTruthyEnv(string? value) value.Equals("1", StringComparison.Ordinal) || value.Equals("yes", StringComparison.OrdinalIgnoreCase)); + internal static bool IsHtmlReportEnabled() + => !IsTruthyEnv(Environment.GetEnvironmentVariable(EnvironmentConstants.DisableHtmlReporter)) + && TUnitSettings.Default.Reporting.HtmlReportEnabled; + + internal static bool IsJsonReportEnabled() + => !IsTruthyEnv(Environment.GetEnvironmentVariable(EnvironmentConstants.DisableJsonReport)) + && TUnitSettings.Default.Reporting.JsonReportEnabled; + + internal static bool IsArtifactUploadEnabled() + => !IsTruthyEnv(Environment.GetEnvironmentVariable(EnvironmentConstants.DisableArtifactUpload)) + && TUnitSettings.Default.Reporting.ArtifactUploadEnabled; + // Default HTML report is "{name}-{os}-{tfm}-report.html"; the sidecar drops the // "-report" stem so the default pair reads "{name}-{os}-{tfm}.tunit-report.json". internal static string GetSidecarPath(string htmlOutputPath) @@ -857,7 +889,7 @@ private static bool IsFileLocked(IOException exception) return null; } - if (IsTruthyEnv(Environment.GetEnvironmentVariable(EnvironmentConstants.DisableArtifactUpload))) + if (!IsArtifactUploadEnabled()) { return null; } diff --git a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs new file mode 100644 index 00000000000..339841d99dc --- /dev/null +++ b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs @@ -0,0 +1,65 @@ +using TUnit.Core.Settings; +using TUnit.Engine.Reporters.Html; + +namespace TUnit.Engine.Tests; + +[NotInParallel] +public class HtmlReporterConfigurationTests +{ + private bool _htmlReportEnabled; + private bool _jsonReportEnabled; + private bool _artifactUploadEnabled; + private string? _disableHtmlReporter; + private string? _disableJsonReport; + private string? _disableArtifactUpload; + + [Before(HookType.Test)] + public void SnapshotConfiguration() + { + _htmlReportEnabled = TUnitSettings.Default.Reporting.HtmlReportEnabled; + _jsonReportEnabled = TUnitSettings.Default.Reporting.JsonReportEnabled; + _artifactUploadEnabled = TUnitSettings.Default.Reporting.ArtifactUploadEnabled; + _disableHtmlReporter = Environment.GetEnvironmentVariable("TUNIT_DISABLE_HTML_REPORTER"); + _disableJsonReport = Environment.GetEnvironmentVariable("TUNIT_DISABLE_JSON_REPORT"); + _disableArtifactUpload = Environment.GetEnvironmentVariable("TUNIT_DISABLE_ARTIFACT_UPLOAD"); + + Environment.SetEnvironmentVariable("TUNIT_DISABLE_HTML_REPORTER", null); + Environment.SetEnvironmentVariable("TUNIT_DISABLE_JSON_REPORT", null); + Environment.SetEnvironmentVariable("TUNIT_DISABLE_ARTIFACT_UPLOAD", null); + } + + [After(HookType.Test)] + public void RestoreConfiguration() + { + TUnitSettings.Default.Reporting.HtmlReportEnabled = _htmlReportEnabled; + TUnitSettings.Default.Reporting.JsonReportEnabled = _jsonReportEnabled; + TUnitSettings.Default.Reporting.ArtifactUploadEnabled = _artifactUploadEnabled; + Environment.SetEnvironmentVariable("TUNIT_DISABLE_HTML_REPORTER", _disableHtmlReporter); + Environment.SetEnvironmentVariable("TUNIT_DISABLE_JSON_REPORT", _disableJsonReport); + Environment.SetEnvironmentVariable("TUNIT_DISABLE_ARTIFACT_UPLOAD", _disableArtifactUpload); + } + + [Test] + public async Task Programmatic_Settings_Can_Disable_Html_Reporting_Features() + { + TUnitSettings.Default.Reporting.HtmlReportEnabled = false; + TUnitSettings.Default.Reporting.JsonReportEnabled = false; + TUnitSettings.Default.Reporting.ArtifactUploadEnabled = false; + + await Assert.That(HtmlReporter.IsHtmlReportEnabled()).IsFalse(); + await Assert.That(HtmlReporter.IsJsonReportEnabled()).IsFalse(); + await Assert.That(HtmlReporter.IsArtifactUploadEnabled()).IsFalse(); + } + + [Test] + public async Task Disable_Environment_Variables_Take_Precedence() + { + Environment.SetEnvironmentVariable("TUNIT_DISABLE_HTML_REPORTER", "true"); + Environment.SetEnvironmentVariable("TUNIT_DISABLE_JSON_REPORT", "1"); + Environment.SetEnvironmentVariable("TUNIT_DISABLE_ARTIFACT_UPLOAD", "yes"); + + await Assert.That(HtmlReporter.IsHtmlReportEnabled()).IsFalse(); + await Assert.That(HtmlReporter.IsJsonReportEnabled()).IsFalse(); + await Assert.That(HtmlReporter.IsArtifactUploadEnabled()).IsFalse(); + } +} diff --git a/tests/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet10_0.verified.txt b/tests/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet10_0.verified.txt index 863682c5d02..2c000ddceac 100644 --- a/tests/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet10_0.verified.txt +++ b/tests/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet10_0.verified.txt @@ -3046,11 +3046,18 @@ namespace .Settings { public int? MaximumParallelTests { get; set; } } + public sealed class ReportingSettings + { + public bool ArtifactUploadEnabled { get; set; } + public bool HtmlReportEnabled { get; set; } + public bool JsonReportEnabled { get; set; } + } public sealed class TUnitSettings { public . Display { get; } public . Execution { get; } public . Parallelism { get; } + public . Reporting { get; } public . Timeouts { get; } } public sealed class TimeoutSettings diff --git a/tests/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet8_0.verified.txt b/tests/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet8_0.verified.txt index df987e2c496..7829d687fc6 100644 --- a/tests/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet8_0.verified.txt +++ b/tests/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet8_0.verified.txt @@ -3046,11 +3046,18 @@ namespace .Settings { public int? MaximumParallelTests { get; set; } } + public sealed class ReportingSettings + { + public bool ArtifactUploadEnabled { get; set; } + public bool HtmlReportEnabled { get; set; } + public bool JsonReportEnabled { get; set; } + } public sealed class TUnitSettings { public . Display { get; } public . Execution { get; } public . Parallelism { get; } + public . Reporting { get; } public . Timeouts { get; } } public sealed class TimeoutSettings diff --git a/tests/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet9_0.verified.txt b/tests/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet9_0.verified.txt index 8657b96496c..11d504d7622 100644 --- a/tests/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet9_0.verified.txt +++ b/tests/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet9_0.verified.txt @@ -3046,11 +3046,18 @@ namespace .Settings { public int? MaximumParallelTests { get; set; } } + public sealed class ReportingSettings + { + public bool ArtifactUploadEnabled { get; set; } + public bool HtmlReportEnabled { get; set; } + public bool JsonReportEnabled { get; set; } + } public sealed class TUnitSettings { public . Display { get; } public . Execution { get; } public . Parallelism { get; } + public . Reporting { get; } public . Timeouts { get; } } public sealed class TimeoutSettings diff --git a/tests/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.Net4_7.verified.txt b/tests/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.Net4_7.verified.txt index bc6b0a77b30..fb14bdc8ccc 100644 --- a/tests/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.Net4_7.verified.txt +++ b/tests/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.Net4_7.verified.txt @@ -2968,11 +2968,18 @@ namespace .Settings { public int? MaximumParallelTests { get; set; } } + public sealed class ReportingSettings + { + public bool ArtifactUploadEnabled { get; set; } + public bool HtmlReportEnabled { get; set; } + public bool JsonReportEnabled { get; set; } + } public sealed class TUnitSettings { public . Display { get; } public . Execution { get; } public . Parallelism { get; } + public . Reporting { get; } public . Timeouts { get; } } public sealed class TimeoutSettings diff --git a/tests/TUnit.UnitTests/TUnitSettingsTests.cs b/tests/TUnit.UnitTests/TUnitSettingsTests.cs index 287fdbd63b5..054ba8ff403 100644 --- a/tests/TUnit.UnitTests/TUnitSettingsTests.cs +++ b/tests/TUnit.UnitTests/TUnitSettingsTests.cs @@ -18,6 +18,9 @@ public class TUnitSettingsTests private int? _savedMaximumParallelTests; private bool _savedDetailedStackTrace; private bool _savedFailFast; + private bool _savedHtmlReportEnabled; + private bool _savedJsonReportEnabled; + private bool _savedArtifactUploadEnabled; [Before(HookType.Test)] public void SnapshotSettings() @@ -29,6 +32,9 @@ public void SnapshotSettings() _savedMaximumParallelTests = TUnitSettings.Default.Parallelism.MaximumParallelTests; _savedDetailedStackTrace = TUnitSettings.Default.Display.DetailedStackTrace; _savedFailFast = TUnitSettings.Default.Execution.FailFast; + _savedHtmlReportEnabled = TUnitSettings.Default.Reporting.HtmlReportEnabled; + _savedJsonReportEnabled = TUnitSettings.Default.Reporting.JsonReportEnabled; + _savedArtifactUploadEnabled = TUnitSettings.Default.Reporting.ArtifactUploadEnabled; } [After(HookType.Test)] @@ -41,6 +47,9 @@ public void RestoreSettings() TUnitSettings.Default.Parallelism.MaximumParallelTests = _savedMaximumParallelTests; TUnitSettings.Default.Display.DetailedStackTrace = _savedDetailedStackTrace; TUnitSettings.Default.Execution.FailFast = _savedFailFast; + TUnitSettings.Default.Reporting.HtmlReportEnabled = _savedHtmlReportEnabled; + TUnitSettings.Default.Reporting.JsonReportEnabled = _savedJsonReportEnabled; + TUnitSettings.Default.Reporting.ArtifactUploadEnabled = _savedArtifactUploadEnabled; } [Test] @@ -53,6 +62,9 @@ public async Task Defaults_Are_Correct() await Assert.That(TUnitSettings.Default.Parallelism.MaximumParallelTests).IsNull(); await Assert.That(TUnitSettings.Default.Display.DetailedStackTrace).IsFalse(); await Assert.That(TUnitSettings.Default.Execution.FailFast).IsFalse(); + await Assert.That(TUnitSettings.Default.Reporting.HtmlReportEnabled).IsTrue(); + await Assert.That(TUnitSettings.Default.Reporting.JsonReportEnabled).IsTrue(); + await Assert.That(TUnitSettings.Default.Reporting.ArtifactUploadEnabled).IsTrue(); } [Test] @@ -62,6 +74,18 @@ public async Task Settings_Can_Be_Modified() await Assert.That(TUnitSettings.Default.Timeouts.DefaultTestTimeout).IsEqualTo(TimeSpan.FromMinutes(10)); } + [Test] + public async Task Reporting_Settings_Can_Be_Modified() + { + TUnitSettings.Default.Reporting.HtmlReportEnabled = false; + TUnitSettings.Default.Reporting.JsonReportEnabled = false; + TUnitSettings.Default.Reporting.ArtifactUploadEnabled = false; + + await Assert.That(TUnitSettings.Default.Reporting.HtmlReportEnabled).IsFalse(); + await Assert.That(TUnitSettings.Default.Reporting.JsonReportEnabled).IsFalse(); + await Assert.That(TUnitSettings.Default.Reporting.ArtifactUploadEnabled).IsFalse(); + } + // Covers TestCoordinator's `test.Timeout ?? TUnitSettings...ExplicitDefaultTestTimeout` fallback: // when the user never assigns DefaultTestTimeout, tests without [Timeout] skip the // TimeoutHelper wrapper entirely (the right-hand side of the coalesce is null). From 4412d1d2f925cb5dd802443e58573b99718b1923 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Tue, 1 Sep 2026 21:55:14 +0100 Subject: [PATCH 02/26] fix: honor reporting settings end to end --- .../Reporters/Html/HtmlReporter.cs | 77 +++++++++++++++---- .../HtmlReporterConfigurationTests.cs | 75 ++++++++++++++++++ .../ReportingSettingsTests.cs | 44 +++++++++++ .../ReportingSettingsTests.cs | 25 ++++++ 4 files changed, 204 insertions(+), 17 deletions(-) create mode 100644 tests/TUnit.Engine.Tests/ReportingSettingsTests.cs create mode 100644 tests/TUnit.TestProject/ReportingSettingsTests.cs diff --git a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs index 1056ee885e6..27119c93a57 100644 --- a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs +++ b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs @@ -28,6 +28,8 @@ namespace TUnit.Engine.Reporters.Html; internal sealed class HtmlReporter(IExtension extension) : IDataConsumer, IDataProducer, ITestHostApplicationLifetime, ITestSessionLifetimeHandler, IFilterReceiver, IDisposable { + private const int HtmlReportEnabledUnresolved = -1; + // System.Text.Json's Utf8JsonWriter limits a single string token to int.MaxValue / 6 characters. // Truncate large outputs early so report generation never fails for test suites with excessive logging. internal const int MaxOutputLength = 1 * 1024 * 1024; // 1 MB @@ -37,6 +39,7 @@ internal sealed class HtmlReporter(IExtension extension) : IDataConsumer, IDataP private string _resultsDirectory = "TestResults"; private readonly ConcurrentDictionary _updates = []; private GitHubReporter? _githubReporter; + private int _htmlReportEnabledAfterDiscovery = HtmlReportEnabledUnresolved; #if NET private ActivityCollector? _activityCollector; @@ -62,9 +65,7 @@ public async Task IsEnabledAsync() public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationToken cancellationToken) { - // IsEnabledAsync runs before user discovery hooks. Read this setting again here so - // context.Settings changes made by those hooks still suppress collection and output. - if (!IsHtmlReportEnabled()) + if (!IsHtmlReportEnabledForRun()) { return Task.CompletedTask; } @@ -118,10 +119,10 @@ public async Task OnTestSessionFinishingAsync(ITestSessionContext testSessionCon try { #if NET - _activityCollector?.Stop(); + StopActivityCollection(); #endif - if (!IsHtmlReportEnabled()) + if (!IsHtmlReportEnabledForRun()) { #if NET TraceRegistry.Clear(); @@ -185,21 +186,23 @@ public async Task OnTestSessionFinishingAsync(ITestSessionContext testSessionCon } } - private async Task TryWriteSidecarAndAggregateAsync(ReportData reportData, string htmlOutputPath, CancellationToken cancellationToken) + internal async Task TryWriteSidecarAndAggregateAsync(ReportData reportData, string htmlOutputPath, CancellationToken cancellationToken) { + if (!IsJsonReportEnabled()) + { + return; + } + // Serialized once; the same bytes back both the local sidecar and the shared copy. var sidecarBytes = ReportDataJson.SerializeToBytes(reportData); - if (IsJsonReportEnabled()) + try { - try - { - AtomicFile.WriteAllBytes(GetSidecarPath(htmlOutputPath), sidecarBytes); - } - catch (Exception ex) - { - Console.WriteLine($"Warning: Failed to write JSON report sidecar: {ex.Message}"); - } + AtomicFile.WriteAllBytes(GetSidecarPath(htmlOutputPath), sidecarBytes); + } + catch (Exception ex) + { + Console.WriteLine($"Warning: Failed to write JSON report sidecar: {ex.Message}"); } var aggregator = ReportAggregator.TryCreateFromEnvironment(Environment.GetEnvironmentVariable); @@ -270,6 +273,36 @@ internal static bool IsArtifactUploadEnabled() => !IsTruthyEnv(Environment.GetEnvironmentVariable(EnvironmentConstants.DisableArtifactUpload)) && TUnitSettings.Default.Reporting.ArtifactUploadEnabled; + private bool IsHtmlReportEnabledForRun() + { + var resolved = Volatile.Read(ref _htmlReportEnabledAfterDiscovery); + if (resolved != HtmlReportEnabledUnresolved) + { + return resolved == 1; + } + + var enabled = IsHtmlReportEnabled(); + resolved = Interlocked.CompareExchange( + ref _htmlReportEnabledAfterDiscovery, + enabled ? 1 : 0, + HtmlReportEnabledUnresolved); + + if (resolved != HtmlReportEnabledUnresolved) + { + return resolved == 1; + } + +#if NET + if (!enabled) + { + StopActivityCollection(); + TraceRegistry.Clear(); + } +#endif + + return enabled; + } + // Default HTML report is "{name}-{os}-{tfm}-report.html"; the sidecar drops the // "-report" stem so the default pair reads "{name}-{os}-{tfm}.tunit-report.json". internal static string GetSidecarPath(string htmlOutputPath) @@ -285,7 +318,7 @@ internal static string GetSidecarPath(string htmlOutputPath) internal async Task PublishArtifactAsync(string outputPath, SessionUid sessionUid, CancellationToken cancellationToken) { - if (_messageBus is null) + if (_messageBus is null || !IsArtifactUploadEnabled()) { return; } @@ -302,10 +335,20 @@ internal async Task PublishArtifactAsync(string outputPath, SessionUid sessionUi public void Dispose() { #if NET - _activityCollector?.Dispose(); + StopActivityCollection(); #endif } +#if NET + internal bool IsActivityCollectionActive => _activityCollector is not null; + + private void StopActivityCollection() + { + _activityCollector?.Dispose(); + _activityCollector = null; + } +#endif + public string? Filter { get; set; } internal void SetOutputPath(string path) diff --git a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs index 339841d99dc..7716d7aaf9e 100644 --- a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs +++ b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs @@ -1,3 +1,5 @@ +using Microsoft.Testing.Platform.TestHost; +using Shouldly; using TUnit.Core.Settings; using TUnit.Engine.Reporters.Html; @@ -12,6 +14,8 @@ public class HtmlReporterConfigurationTests private string? _disableHtmlReporter; private string? _disableJsonReport; private string? _disableArtifactUpload; + private string? _aggregateReports; + private string? _aggregateDirectory; [Before(HookType.Test)] public void SnapshotConfiguration() @@ -22,10 +26,14 @@ public void SnapshotConfiguration() _disableHtmlReporter = Environment.GetEnvironmentVariable("TUNIT_DISABLE_HTML_REPORTER"); _disableJsonReport = Environment.GetEnvironmentVariable("TUNIT_DISABLE_JSON_REPORT"); _disableArtifactUpload = Environment.GetEnvironmentVariable("TUNIT_DISABLE_ARTIFACT_UPLOAD"); + _aggregateReports = Environment.GetEnvironmentVariable("TUNIT_AGGREGATE_REPORTS"); + _aggregateDirectory = Environment.GetEnvironmentVariable("TUNIT_AGGREGATE_DIR"); Environment.SetEnvironmentVariable("TUNIT_DISABLE_HTML_REPORTER", null); Environment.SetEnvironmentVariable("TUNIT_DISABLE_JSON_REPORT", null); Environment.SetEnvironmentVariable("TUNIT_DISABLE_ARTIFACT_UPLOAD", null); + Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_REPORTS", null); + Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_DIR", null); } [After(HookType.Test)] @@ -37,6 +45,8 @@ public void RestoreConfiguration() Environment.SetEnvironmentVariable("TUNIT_DISABLE_HTML_REPORTER", _disableHtmlReporter); Environment.SetEnvironmentVariable("TUNIT_DISABLE_JSON_REPORT", _disableJsonReport); Environment.SetEnvironmentVariable("TUNIT_DISABLE_ARTIFACT_UPLOAD", _disableArtifactUpload); + Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_REPORTS", _aggregateReports); + Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_DIR", _aggregateDirectory); } [Test] @@ -62,4 +72,69 @@ public async Task Disable_Environment_Variables_Take_Precedence() await Assert.That(HtmlReporter.IsJsonReportEnabled()).IsFalse(); await Assert.That(HtmlReporter.IsArtifactUploadEnabled()).IsFalse(); } + + [Test] + public async Task Disabled_Html_Report_Stops_Activity_Collection_After_Discovery() + { + using var reporter = new HtmlReporter(new MockExtension()); + await reporter.BeforeRunAsync(CancellationToken.None); + reporter.IsActivityCollectionActive.ShouldBeTrue(); + + TUnitSettings.Default.Reporting.HtmlReportEnabled = false; + await reporter.ConsumeAsync(reporter, null!, CancellationToken.None); + + reporter.IsActivityCollectionActive.ShouldBeFalse(); + } + + [Test] + public async Task Disabled_Artifact_Upload_Does_Not_Publish_Session_File_Artifact() + { + TUnitSettings.Default.Reporting.ArtifactUploadEnabled = false; + var reporter = new HtmlReporter(new MockExtension()); + var messageBus = new CapturingMessageBus(); + reporter.SetMessageBus(messageBus); + + await reporter.PublishArtifactAsync("report.html", new SessionUid("session"), CancellationToken.None); + + messageBus.Published.ShouldBeEmpty(); + } + + [Test] + public async Task Disabled_Json_Report_Does_Not_Write_Local_Or_Aggregation_Sidecars() + { + TUnitSettings.Default.Reporting.JsonReportEnabled = false; + var tempDirectory = Path.Combine(Path.GetTempPath(), $"tunit-report-test-{Guid.NewGuid():N}"); + var aggregationDirectory = Path.Combine(tempDirectory, "aggregate"); + var htmlPath = Path.Combine(tempDirectory, "report.html"); + Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_REPORTS", "true"); + Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_DIR", aggregationDirectory); + + try + { + var reporter = new HtmlReporter(new MockExtension()); + await reporter.TryWriteSidecarAndAggregateAsync(CreateReportData(), htmlPath, CancellationToken.None); + + File.Exists(HtmlReporter.GetSidecarPath(htmlPath)).ShouldBeFalse(); + Directory.Exists(aggregationDirectory).ShouldBeFalse(); + } + finally + { + if (Directory.Exists(tempDirectory)) + { + Directory.Delete(tempDirectory, recursive: true); + } + } + } + + private static ReportData CreateReportData() => new() + { + AssemblyName = "Tests", + MachineName = "machine", + Timestamp = DateTimeOffset.UtcNow.ToString("O"), + TUnitVersion = "1.0.0", + OperatingSystem = "test", + RuntimeVersion = "test", + Summary = new ReportSummary(), + Groups = [], + }; } diff --git a/tests/TUnit.Engine.Tests/ReportingSettingsTests.cs b/tests/TUnit.Engine.Tests/ReportingSettingsTests.cs new file mode 100644 index 00000000000..76841a2383b --- /dev/null +++ b/tests/TUnit.Engine.Tests/ReportingSettingsTests.cs @@ -0,0 +1,44 @@ +using Shouldly; +using TUnit.Engine.Tests.Enums; + +namespace TUnit.Engine.Tests; + +public class ReportingSettingsTests(TestMode testMode) : InvokableTestBase(testMode) +{ + [Test] + public async Task Discovery_Hook_Can_Disable_Reporting() + { + var tempDirectory = Path.Combine(Path.GetTempPath(), $"tunit-report-settings-{Guid.NewGuid():N}"); + var reportPath = Path.Combine(tempDirectory, "report.html"); + var aggregationDirectory = Path.Combine(tempDirectory, "aggregate"); + + try + { + var options = new RunOptions() + .WithArgument("--report-html-filename") + .WithArgument(reportPath) + .WithEnvironmentVariable("TUNIT_DISABLE_HTML_REPORTER", "false") + .WithEnvironmentVariable("TUNIT_DISABLE_JSON_REPORT", "false") + .WithEnvironmentVariable("TUNIT_DISABLE_ARTIFACT_UPLOAD", "false") + .WithEnvironmentVariable("TUNIT_AGGREGATE_REPORTS", "true") + .WithEnvironmentVariable("TUNIT_AGGREGATE_DIR", aggregationDirectory) + .WithEnvironmentVariable("TUNIT_TEST_DISABLE_REPORTING_FROM_DISCOVERY_HOOK", "true"); + + await RunTestsWithFilter( + "/*/*/ReportingSettingsTests/*", + [ + result => result.ResultSummary.Counters.Passed.ShouldBe(1), + _ => File.Exists(reportPath).ShouldBeFalse(), + _ => Directory.Exists(aggregationDirectory).ShouldBeFalse(), + ], + options); + } + finally + { + if (Directory.Exists(tempDirectory)) + { + Directory.Delete(tempDirectory, recursive: true); + } + } + } +} diff --git a/tests/TUnit.TestProject/ReportingSettingsTests.cs b/tests/TUnit.TestProject/ReportingSettingsTests.cs new file mode 100644 index 00000000000..f01f5f703df --- /dev/null +++ b/tests/TUnit.TestProject/ReportingSettingsTests.cs @@ -0,0 +1,25 @@ +namespace TUnit.TestProject; + +public static class ReportingSettingsHooks +{ + internal const string DisableReportingEnvironmentVariable = "TUNIT_TEST_DISABLE_REPORTING_FROM_DISCOVERY_HOOK"; + + [Before(TestDiscovery)] + public static void ConfigureReporting(BeforeTestDiscoveryContext context) + { + if (Environment.GetEnvironmentVariable(DisableReportingEnvironmentVariable) is not "true") + { + return; + } + + context.Settings.Reporting.HtmlReportEnabled = false; + context.Settings.Reporting.JsonReportEnabled = false; + context.Settings.Reporting.ArtifactUploadEnabled = false; + } +} + +public class ReportingSettingsTests +{ + [Test] + public void Test() { } +} From 79f1ccfa03d1cb3b444d09d6f0ab275f3e70c9db Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Tue, 1 Sep 2026 22:02:50 +0100 Subject: [PATCH 03/26] fix: preserve report trace data --- src/TUnit.Engine/Reporters/Html/HtmlReporter.cs | 17 +++++++++++++---- .../HtmlReporterConfigurationTests.cs | 4 ++-- tests/TUnit.Engine.Tests/HtmlReporterTests.cs | 16 ++++++++++++++++ 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs index 27119c93a57..22a9efabce1 100644 --- a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs +++ b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs @@ -184,6 +184,12 @@ public async Task OnTestSessionFinishingAsync(ITestSessionContext testSessionCon { Console.WriteLine($"Warning: HTML report generation failed: {ex.Message}"); } + finally + { +#if NET + DisposeActivityCollection(); +#endif + } } internal async Task TryWriteSidecarAndAggregateAsync(ReportData reportData, string htmlOutputPath, CancellationToken cancellationToken) @@ -295,7 +301,7 @@ private bool IsHtmlReportEnabledForRun() #if NET if (!enabled) { - StopActivityCollection(); + DisposeActivityCollection(); TraceRegistry.Clear(); } #endif @@ -335,14 +341,17 @@ internal async Task PublishArtifactAsync(string outputPath, SessionUid sessionUi public void Dispose() { #if NET - StopActivityCollection(); + DisposeActivityCollection(); #endif } #if NET - internal bool IsActivityCollectionActive => _activityCollector is not null; + internal bool HasActivityCollector => _activityCollector is not null; + + internal void StopActivityCollection() + => _activityCollector?.Stop(); - private void StopActivityCollection() + private void DisposeActivityCollection() { _activityCollector?.Dispose(); _activityCollector = null; diff --git a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs index 7716d7aaf9e..a1c4e6cbfc5 100644 --- a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs +++ b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs @@ -78,12 +78,12 @@ public async Task Disabled_Html_Report_Stops_Activity_Collection_After_Discovery { using var reporter = new HtmlReporter(new MockExtension()); await reporter.BeforeRunAsync(CancellationToken.None); - reporter.IsActivityCollectionActive.ShouldBeTrue(); + reporter.HasActivityCollector.ShouldBeTrue(); TUnitSettings.Default.Reporting.HtmlReportEnabled = false; await reporter.ConsumeAsync(reporter, null!, CancellationToken.None); - reporter.IsActivityCollectionActive.ShouldBeFalse(); + reporter.HasActivityCollector.ShouldBeFalse(); } [Test] diff --git a/tests/TUnit.Engine.Tests/HtmlReporterTests.cs b/tests/TUnit.Engine.Tests/HtmlReporterTests.cs index 5d7d646d22e..e4e615ce878 100644 --- a/tests/TUnit.Engine.Tests/HtmlReporterTests.cs +++ b/tests/TUnit.Engine.Tests/HtmlReporterTests.cs @@ -30,6 +30,22 @@ public void HtmlReporter_DataTypesProduced_Contains_SessionFileArtifact() producer.DataTypesProduced.ShouldContain(typeof(SessionFileArtifact)); } + [Test] + public async Task Stopping_Activity_Collection_Preserves_Spans_For_Report_Data() + { + using var reporter = new HtmlReporter(new MockExtension()); + await reporter.BeforeRunAsync(CancellationToken.None); + + var activity = TUnitActivitySource.StartLifecycleActivity(TUnitActivitySource.SpanTestSession); + TUnitActivitySource.StopActivity(activity); + reporter.StopActivityCollection(); + + var reportData = reporter.BuildReportData(); + + reportData.Spans.ShouldNotBeNull(); + reportData.Spans.ShouldContain(span => span.SpanType == TUnitActivitySource.SpanTestSession); + } + [Test] public async Task PublishArtifactAsync_Publishes_SessionFileArtifact_When_SessionContext_Set_And_File_Exists() { From 40ee3628754fe104cbd51765e93c14443bb61a39 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Tue, 1 Sep 2026 22:12:24 +0100 Subject: [PATCH 04/26] fix: scope reporting state to sessions --- .../Reporters/Html/HtmlReporter.cs | 7 +++-- .../HtmlReporterConfigurationTests.cs | 29 ++++++++++++++----- tests/TUnit.Engine.Tests/HtmlReporterTests.cs | 4 +-- .../ReportingSettingsTests.cs | 5 ++-- .../ReportingSettingsTests.cs | 7 +++-- 5 files changed, 37 insertions(+), 15 deletions(-) diff --git a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs index 22a9efabce1..18fa2fe51b1 100644 --- a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs +++ b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs @@ -112,7 +112,10 @@ public Task AfterRunAsync(int exitCode, CancellationToken cancellation) => Task.CompletedTask; // All work happens in OnTestSessionFinishingAsync. public Task OnTestSessionStartingAsync(ITestSessionContext testSessionContext) - => Task.CompletedTask; + { + Volatile.Write(ref _htmlReportEnabledAfterDiscovery, HtmlReportEnabledUnresolved); + return Task.CompletedTask; + } public async Task OnTestSessionFinishingAsync(ITestSessionContext testSessionContext) { @@ -279,7 +282,7 @@ internal static bool IsArtifactUploadEnabled() => !IsTruthyEnv(Environment.GetEnvironmentVariable(EnvironmentConstants.DisableArtifactUpload)) && TUnitSettings.Default.Reporting.ArtifactUploadEnabled; - private bool IsHtmlReportEnabledForRun() + internal bool IsHtmlReportEnabledForRun() { var resolved = Volatile.Read(ref _htmlReportEnabledAfterDiscovery); if (resolved != HtmlReportEnabledUnresolved) diff --git a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs index a1c4e6cbfc5..1e840e9a284 100644 --- a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs +++ b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs @@ -74,33 +74,47 @@ public async Task Disable_Environment_Variables_Take_Precedence() } [Test] - public async Task Disabled_Html_Report_Stops_Activity_Collection_After_Discovery() + public async Task Disabled_Html_Report_Stops_Activity_Collection_After_Discovery(CancellationToken cancellationToken) { using var reporter = new HtmlReporter(new MockExtension()); - await reporter.BeforeRunAsync(CancellationToken.None); + await reporter.BeforeRunAsync(cancellationToken); reporter.HasActivityCollector.ShouldBeTrue(); TUnitSettings.Default.Reporting.HtmlReportEnabled = false; - await reporter.ConsumeAsync(reporter, null!, CancellationToken.None); + await reporter.ConsumeAsync(reporter, null!, cancellationToken); reporter.HasActivityCollector.ShouldBeFalse(); } [Test] - public async Task Disabled_Artifact_Upload_Does_Not_Publish_Session_File_Artifact() + public async Task Html_Report_Setting_Is_Resolved_Per_Session() + { + using var reporter = new HtmlReporter(new MockExtension()); + + await reporter.OnTestSessionStartingAsync(null!); + TUnitSettings.Default.Reporting.HtmlReportEnabled = false; + reporter.IsHtmlReportEnabledForRun().ShouldBeFalse(); + + await reporter.OnTestSessionStartingAsync(null!); + TUnitSettings.Default.Reporting.HtmlReportEnabled = true; + reporter.IsHtmlReportEnabledForRun().ShouldBeTrue(); + } + + [Test] + public async Task Disabled_Artifact_Upload_Does_Not_Publish_Session_File_Artifact(CancellationToken cancellationToken) { TUnitSettings.Default.Reporting.ArtifactUploadEnabled = false; var reporter = new HtmlReporter(new MockExtension()); var messageBus = new CapturingMessageBus(); reporter.SetMessageBus(messageBus); - await reporter.PublishArtifactAsync("report.html", new SessionUid("session"), CancellationToken.None); + await reporter.PublishArtifactAsync("report.html", new SessionUid("session"), cancellationToken); messageBus.Published.ShouldBeEmpty(); } [Test] - public async Task Disabled_Json_Report_Does_Not_Write_Local_Or_Aggregation_Sidecars() + public async Task Disabled_Json_Report_Does_Not_Write_Local_Or_Aggregation_Sidecars(CancellationToken cancellationToken) { TUnitSettings.Default.Reporting.JsonReportEnabled = false; var tempDirectory = Path.Combine(Path.GetTempPath(), $"tunit-report-test-{Guid.NewGuid():N}"); @@ -111,8 +125,9 @@ public async Task Disabled_Json_Report_Does_Not_Write_Local_Or_Aggregation_Sidec try { + Directory.CreateDirectory(tempDirectory); var reporter = new HtmlReporter(new MockExtension()); - await reporter.TryWriteSidecarAndAggregateAsync(CreateReportData(), htmlPath, CancellationToken.None); + await reporter.TryWriteSidecarAndAggregateAsync(CreateReportData(), htmlPath, cancellationToken); File.Exists(HtmlReporter.GetSidecarPath(htmlPath)).ShouldBeFalse(); Directory.Exists(aggregationDirectory).ShouldBeFalse(); diff --git a/tests/TUnit.Engine.Tests/HtmlReporterTests.cs b/tests/TUnit.Engine.Tests/HtmlReporterTests.cs index e4e615ce878..af33e1cc31b 100644 --- a/tests/TUnit.Engine.Tests/HtmlReporterTests.cs +++ b/tests/TUnit.Engine.Tests/HtmlReporterTests.cs @@ -31,10 +31,10 @@ public void HtmlReporter_DataTypesProduced_Contains_SessionFileArtifact() } [Test] - public async Task Stopping_Activity_Collection_Preserves_Spans_For_Report_Data() + public async Task Stopping_Activity_Collection_Preserves_Spans_For_Report_Data(CancellationToken cancellationToken) { using var reporter = new HtmlReporter(new MockExtension()); - await reporter.BeforeRunAsync(CancellationToken.None); + await reporter.BeforeRunAsync(cancellationToken); var activity = TUnitActivitySource.StartLifecycleActivity(TUnitActivitySource.SpanTestSession); TUnitActivitySource.StopActivity(activity); diff --git a/tests/TUnit.Engine.Tests/ReportingSettingsTests.cs b/tests/TUnit.Engine.Tests/ReportingSettingsTests.cs index 76841a2383b..1ca19d4ecfe 100644 --- a/tests/TUnit.Engine.Tests/ReportingSettingsTests.cs +++ b/tests/TUnit.Engine.Tests/ReportingSettingsTests.cs @@ -6,7 +6,7 @@ namespace TUnit.Engine.Tests; public class ReportingSettingsTests(TestMode testMode) : InvokableTestBase(testMode) { [Test] - public async Task Discovery_Hook_Can_Disable_Reporting() + public async Task Discovery_Hook_Can_Disable_Reporting(CancellationToken cancellationToken) { var tempDirectory = Path.Combine(Path.GetTempPath(), $"tunit-report-settings-{Guid.NewGuid():N}"); var reportPath = Path.Combine(tempDirectory, "report.html"); @@ -22,7 +22,8 @@ public async Task Discovery_Hook_Can_Disable_Reporting() .WithEnvironmentVariable("TUNIT_DISABLE_ARTIFACT_UPLOAD", "false") .WithEnvironmentVariable("TUNIT_AGGREGATE_REPORTS", "true") .WithEnvironmentVariable("TUNIT_AGGREGATE_DIR", aggregationDirectory) - .WithEnvironmentVariable("TUNIT_TEST_DISABLE_REPORTING_FROM_DISCOVERY_HOOK", "true"); + .WithEnvironmentVariable("TUNIT_TEST_DISABLE_REPORTING_FROM_DISCOVERY_HOOK", "true") + .WithGracefulCancellationToken(cancellationToken); await RunTestsWithFilter( "/*/*/ReportingSettingsTests/*", diff --git a/tests/TUnit.TestProject/ReportingSettingsTests.cs b/tests/TUnit.TestProject/ReportingSettingsTests.cs index f01f5f703df..5bba1a327da 100644 --- a/tests/TUnit.TestProject/ReportingSettingsTests.cs +++ b/tests/TUnit.TestProject/ReportingSettingsTests.cs @@ -5,8 +5,10 @@ public static class ReportingSettingsHooks internal const string DisableReportingEnvironmentVariable = "TUNIT_TEST_DISABLE_REPORTING_FROM_DISCOVERY_HOOK"; [Before(TestDiscovery)] - public static void ConfigureReporting(BeforeTestDiscoveryContext context) + public static void ConfigureReporting(BeforeTestDiscoveryContext context, CancellationToken cancellationToken) { + cancellationToken.ThrowIfCancellationRequested(); + if (Environment.GetEnvironmentVariable(DisableReportingEnvironmentVariable) is not "true") { return; @@ -21,5 +23,6 @@ public static void ConfigureReporting(BeforeTestDiscoveryContext context) public class ReportingSettingsTests { [Test] - public void Test() { } + public void Test(CancellationToken cancellationToken) + => cancellationToken.ThrowIfCancellationRequested(); } From a3e041018144425702a001f5427997dcd2c928b9 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Tue, 1 Sep 2026 22:26:46 +0100 Subject: [PATCH 05/26] fix: clear stale report state --- .../Reporters/Aggregation/ReportAggregator.cs | 12 +++++- .../Reporters/Html/HtmlReporter.cs | 27 ++++++++++++- .../HtmlReporterConfigurationTests.cs | 40 +++++++++++++++++-- 3 files changed, 73 insertions(+), 6 deletions(-) diff --git a/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs b/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs index acba21e926d..d51a5e0bee0 100644 --- a/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs +++ b/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs @@ -130,12 +130,14 @@ internal string WriteSidecar(byte[] sidecarUtf8Json, string assemblyName, string { System.IO.Directory.CreateDirectory(Directory); - var fileName = $"{PathValidator.SanitizeFileName(assemblyName)}-{ShortHash(suiteSalt)}{ReportDataJson.SidecarExtension}"; - var path = Path.Combine(Directory, fileName); + var path = GetSidecarPath(assemblyName, suiteSalt); AtomicFile.WriteAllBytes(path, sidecarUtf8Json); return path; } + internal void DeleteSidecar(string assemblyName, string suiteSalt) + => File.Delete(GetSidecarPath(assemblyName, suiteSalt)); + /// /// Reads every sidecar currently present in the shared directory. Unreadable or /// foreign files are skipped — a crashed sibling must not break the merge. @@ -231,4 +233,10 @@ private static string ShortHash(string value) var hash = sha.ComputeHash(Encoding.UTF8.GetBytes(value)); return BitConverter.ToString(hash, 0, 4).Replace("-", "").ToLowerInvariant(); } + + private string GetSidecarPath(string assemblyName, string suiteSalt) + { + var fileName = $"{PathValidator.SanitizeFileName(assemblyName)}-{ShortHash(suiteSalt)}{ReportDataJson.SidecarExtension}"; + return Path.Combine(Directory, fileName); + } } diff --git a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs index 18fa2fe51b1..92ea4bf6135 100644 --- a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs +++ b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs @@ -113,6 +113,7 @@ public Task AfterRunAsync(int exitCode, CancellationToken cancellation) public Task OnTestSessionStartingAsync(ITestSessionContext testSessionContext) { + _updates.Clear(); Volatile.Write(ref _htmlReportEnabledAfterDiscovery, HtmlReportEnabledUnresolved); return Task.CompletedTask; } @@ -197,8 +198,11 @@ public async Task OnTestSessionFinishingAsync(ITestSessionContext testSessionCon internal async Task TryWriteSidecarAndAggregateAsync(ReportData reportData, string htmlOutputPath, CancellationToken cancellationToken) { + var aggregator = ReportAggregator.TryCreateFromEnvironment(Environment.GetEnvironmentVariable); + if (!IsJsonReportEnabled()) { + DeleteSidecars(reportData, htmlOutputPath, aggregator); return; } @@ -214,7 +218,6 @@ internal async Task TryWriteSidecarAndAggregateAsync(ReportData reportData, stri Console.WriteLine($"Warning: Failed to write JSON report sidecar: {ex.Message}"); } - var aggregator = ReportAggregator.TryCreateFromEnvironment(Environment.GetEnvironmentVariable); if (aggregator is null) { return; @@ -263,6 +266,28 @@ internal async Task TryWriteSidecarAndAggregateAsync(ReportData reportData, stri } } + private static void DeleteSidecars(ReportData reportData, string htmlOutputPath, ReportAggregator? aggregator) + { + TryDeleteSidecar(() => File.Delete(GetSidecarPath(htmlOutputPath))); + + if (aggregator is not null) + { + TryDeleteSidecar(() => aggregator.DeleteSidecar(reportData.AssemblyName, htmlOutputPath)); + } + } + + private static void TryDeleteSidecar(Action deleteSidecar) + { + try + { + deleteSidecar(); + } + catch (Exception ex) + { + Console.WriteLine($"Warning: Failed to remove disabled JSON report sidecar: {ex.Message}"); + } + } + // The truthy vocabulary shared by the TUNIT_DISABLE_* switches. private static bool IsTruthyEnv(string? value) => value is not null && diff --git a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs index 1e840e9a284..b4e62c1fab4 100644 --- a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs +++ b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs @@ -1,6 +1,8 @@ +using Microsoft.Testing.Platform.Extensions.Messages; using Microsoft.Testing.Platform.TestHost; using Shouldly; using TUnit.Core.Settings; +using TUnit.Engine.Reporters.Aggregation; using TUnit.Engine.Reporters.Html; namespace TUnit.Engine.Tests; @@ -87,19 +89,36 @@ public async Task Disabled_Html_Report_Stops_Activity_Collection_After_Discovery } [Test] - public async Task Html_Report_Setting_Is_Resolved_Per_Session() + public async Task Html_Report_Setting_Is_Resolved_Per_Session(CancellationToken cancellationToken) { using var reporter = new HtmlReporter(new MockExtension()); + cancellationToken.ThrowIfCancellationRequested(); await reporter.OnTestSessionStartingAsync(null!); TUnitSettings.Default.Reporting.HtmlReportEnabled = false; reporter.IsHtmlReportEnabledForRun().ShouldBeFalse(); + cancellationToken.ThrowIfCancellationRequested(); await reporter.OnTestSessionStartingAsync(null!); TUnitSettings.Default.Reporting.HtmlReportEnabled = true; reporter.IsHtmlReportEnabledForRun().ShouldBeTrue(); } + [Test] + public async Task Test_Updates_Are_Cleared_Between_Sessions(CancellationToken cancellationToken) + { + using var reporter = new HtmlReporter(new MockExtension()); + + await reporter.OnTestSessionStartingAsync(null!); + await reporter.ConsumeAsync(reporter, CreatePassedUpdate("first"), cancellationToken); + reporter.BuildReportData().Groups.SelectMany(x => x.Tests).Single().Id.ShouldBe("first"); + + await reporter.OnTestSessionStartingAsync(null!); + await reporter.ConsumeAsync(reporter, CreatePassedUpdate("second"), cancellationToken); + + reporter.BuildReportData().Groups.SelectMany(x => x.Tests).Single().Id.ShouldBe("second"); + } + [Test] public async Task Disabled_Artifact_Upload_Does_Not_Publish_Session_File_Artifact(CancellationToken cancellationToken) { @@ -116,7 +135,6 @@ public async Task Disabled_Artifact_Upload_Does_Not_Publish_Session_File_Artifac [Test] public async Task Disabled_Json_Report_Does_Not_Write_Local_Or_Aggregation_Sidecars(CancellationToken cancellationToken) { - TUnitSettings.Default.Reporting.JsonReportEnabled = false; var tempDirectory = Path.Combine(Path.GetTempPath(), $"tunit-report-test-{Guid.NewGuid():N}"); var aggregationDirectory = Path.Combine(tempDirectory, "aggregate"); var htmlPath = Path.Combine(tempDirectory, "report.html"); @@ -127,10 +145,17 @@ public async Task Disabled_Json_Report_Does_Not_Write_Local_Or_Aggregation_Sidec { Directory.CreateDirectory(tempDirectory); var reporter = new HtmlReporter(new MockExtension()); + + TUnitSettings.Default.Reporting.JsonReportEnabled = true; + await reporter.TryWriteSidecarAndAggregateAsync(CreateReportData(), htmlPath, cancellationToken); + File.Exists(HtmlReporter.GetSidecarPath(htmlPath)).ShouldBeTrue(); + Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExtension}").ShouldNotBeEmpty(); + + TUnitSettings.Default.Reporting.JsonReportEnabled = false; await reporter.TryWriteSidecarAndAggregateAsync(CreateReportData(), htmlPath, cancellationToken); File.Exists(HtmlReporter.GetSidecarPath(htmlPath)).ShouldBeFalse(); - Directory.Exists(aggregationDirectory).ShouldBeFalse(); + Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExtension}").ShouldBeEmpty(); } finally { @@ -152,4 +177,13 @@ public async Task Disabled_Json_Report_Does_Not_Write_Local_Or_Aggregation_Sidec Summary = new ReportSummary(), Groups = [], }; + + private static TestNodeUpdateMessage CreatePassedUpdate(string id) => new( + new SessionUid("session"), + new TestNode + { + Uid = new TestNodeUid(id), + DisplayName = id, + Properties = new PropertyBag(PassedTestNodeStateProperty.CachedInstance), + }); } From 39e3ac7fafe86dec9383addf95c0a120f21bf0f8 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Tue, 1 Sep 2026 22:38:01 +0100 Subject: [PATCH 06/26] fix: refresh aggregate after sidecar cleanup --- src/TUnit.Engine/Reporters/GitHubReporter.cs | 8 ++ .../Reporters/Html/HtmlReporter.cs | 73 +++++++++++++------ .../TUnit.Engine.Tests/GitHubReporterTests.cs | 12 +++ .../HtmlReporterConfigurationTests.cs | 29 +++++--- 4 files changed, 92 insertions(+), 30 deletions(-) diff --git a/src/TUnit.Engine/Reporters/GitHubReporter.cs b/src/TUnit.Engine/Reporters/GitHubReporter.cs index cf3c21d5b0b..1e1e3d752e1 100644 --- a/src/TUnit.Engine/Reporters/GitHubReporter.cs +++ b/src/TUnit.Engine/Reporters/GitHubReporter.cs @@ -494,6 +494,14 @@ internal void WriteAggregatedSummary(IReadOnlyList suites, stri GitHubSummaryRegion.ReplaceOrAppend(_outputSummaryFilePath, content, MaxFileSizeInBytes); } + internal void ClearAggregatedSummary() + { + if (_outputSummaryFilePath is not null) + { + GitHubSummaryRegion.ReplaceOrAppend(_outputSummaryFilePath, string.Empty, MaxFileSizeInBytes); + } + } + private async Task WriteFile(string contents) { var fileInfo = new FileInfo(_outputSummaryFilePath); diff --git a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs index 92ea4bf6135..bd52883ea47 100644 --- a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs +++ b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs @@ -202,7 +202,7 @@ internal async Task TryWriteSidecarAndAggregateAsync(ReportData reportData, stri if (!IsJsonReportEnabled()) { - DeleteSidecars(reportData, htmlOutputPath, aggregator); + await DeleteSidecarsAndRefreshAggregateAsync(reportData, htmlOutputPath, aggregator, cancellationToken); return; } @@ -244,47 +244,78 @@ internal async Task TryWriteSidecarAndAggregateAsync(ReportData reportData, stri return; } - var suites = aggregator.ReadAllSidecars(); - if (suites.Count == 0) - { - return; - } + RefreshAggregatedOutputs(aggregator); + } + catch (Exception ex) + { + Console.WriteLine($"Warning: Report aggregation failed: {ex.Message}"); + } + } - aggregator.WriteMergedHtml(suites); - Console.WriteLine($"Merged HTML test report ({suites.Count} {(suites.Count == 1 ? "suite" : "suites")} so far) written to: {aggregator.MergedReportPath}"); + private async Task DeleteSidecarsAndRefreshAggregateAsync( + ReportData reportData, + string htmlOutputPath, + ReportAggregator? aggregator, + CancellationToken cancellationToken) + { + TryDeleteReportFile(() => File.Delete(GetSidecarPath(htmlOutputPath))); - // The step summary is rewritten in the same lock cycle: one env parse, one - // lock acquisition and one sidecar scan per process for both merged outputs. - if (aggregator.Mode == AggregationMode.Cooperative) + if (aggregator is null) + { + return; + } + + try + { + using var aggregationLock = await aggregator.AcquireLockAsync(cancellationToken); + if (aggregationLock is null) { - _githubReporter?.WriteAggregatedSummary(suites, aggregator.MergedReportPath); + return; } + + TryDeleteReportFile(() => aggregator.DeleteSidecar(reportData.AssemblyName, htmlOutputPath)); + RefreshAggregatedOutputs(aggregator); } - catch (Exception ex) + catch (Exception ex) when (ex is not OperationCanceledException) { - Console.WriteLine($"Warning: Report aggregation failed: {ex.Message}"); + Console.WriteLine($"Warning: Report aggregation cleanup failed: {ex.Message}"); } } - private static void DeleteSidecars(ReportData reportData, string htmlOutputPath, ReportAggregator? aggregator) + private void RefreshAggregatedOutputs(ReportAggregator aggregator) { - TryDeleteSidecar(() => File.Delete(GetSidecarPath(htmlOutputPath))); + var suites = aggregator.ReadAllSidecars(); + if (suites.Count == 0) + { + TryDeleteReportFile(() => File.Delete(aggregator.MergedReportPath)); + if (aggregator.Mode == AggregationMode.Cooperative) + { + _githubReporter?.ClearAggregatedSummary(); + } + + return; + } + + aggregator.WriteMergedHtml(suites); + Console.WriteLine($"Merged HTML test report ({suites.Count} {(suites.Count == 1 ? "suite" : "suites")} so far) written to: {aggregator.MergedReportPath}"); - if (aggregator is not null) + // The step summary is rewritten in the same lock cycle: one env parse, one + // lock acquisition and one sidecar scan per process for both merged outputs. + if (aggregator.Mode == AggregationMode.Cooperative) { - TryDeleteSidecar(() => aggregator.DeleteSidecar(reportData.AssemblyName, htmlOutputPath)); + _githubReporter?.WriteAggregatedSummary(suites, aggregator.MergedReportPath); } } - private static void TryDeleteSidecar(Action deleteSidecar) + private static void TryDeleteReportFile(Action deleteFile) { try { - deleteSidecar(); + deleteFile(); } catch (Exception ex) { - Console.WriteLine($"Warning: Failed to remove disabled JSON report sidecar: {ex.Message}"); + Console.WriteLine($"Warning: Failed to remove disabled report file: {ex.Message}"); } } diff --git a/tests/TUnit.Engine.Tests/GitHubReporterTests.cs b/tests/TUnit.Engine.Tests/GitHubReporterTests.cs index 38295421c26..34e297d6974 100644 --- a/tests/TUnit.Engine.Tests/GitHubReporterTests.cs +++ b/tests/TUnit.Engine.Tests/GitHubReporterTests.cs @@ -3,6 +3,7 @@ using Shouldly; using TUnit.Engine.Exceptions; using TUnit.Engine.Reporters; +using TUnit.Engine.Reporters.Aggregation; namespace TUnit.Engine.Tests; @@ -114,6 +115,17 @@ public async Task IsEnabledAsync_Should_Return_False_When_GITHUB_ACTIONS_Is_Not_ result.ShouldBeFalse(); } + [Test] + public async Task ClearAggregatedSummary_Removes_Stale_Content() + { + var (reporter, outputFile) = await SetupReporter(); + GitHubSummaryRegion.ReplaceOrAppend(outputFile, "stale aggregate"); + + reporter.ClearAggregatedSummary(); + + File.ReadAllText(outputFile).ShouldNotContain("stale aggregate"); + } + [Test] public async Task AfterRunAsync_Groups_Failures_By_Exception_Type() { diff --git a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs index b4e62c1fab4..09bdd0fb532 100644 --- a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs +++ b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs @@ -133,11 +133,13 @@ public async Task Disabled_Artifact_Upload_Does_Not_Publish_Session_File_Artifac } [Test] - public async Task Disabled_Json_Report_Does_Not_Write_Local_Or_Aggregation_Sidecars(CancellationToken cancellationToken) + public async Task Disabled_Json_Report_Removes_Stale_Aggregation_Outputs(CancellationToken cancellationToken) { var tempDirectory = Path.Combine(Path.GetTempPath(), $"tunit-report-test-{Guid.NewGuid():N}"); var aggregationDirectory = Path.Combine(tempDirectory, "aggregate"); - var htmlPath = Path.Combine(tempDirectory, "report.html"); + var removedHtmlPath = Path.Combine(tempDirectory, "removed-report.html"); + var remainingHtmlPath = Path.Combine(tempDirectory, "remaining-report.html"); + var mergedReportPath = Path.Combine(aggregationDirectory, ReportDataJson.MergedReportFileName); Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_REPORTS", "true"); Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_DIR", aggregationDirectory); @@ -147,15 +149,24 @@ public async Task Disabled_Json_Report_Does_Not_Write_Local_Or_Aggregation_Sidec var reporter = new HtmlReporter(new MockExtension()); TUnitSettings.Default.Reporting.JsonReportEnabled = true; - await reporter.TryWriteSidecarAndAggregateAsync(CreateReportData(), htmlPath, cancellationToken); - File.Exists(HtmlReporter.GetSidecarPath(htmlPath)).ShouldBeTrue(); - Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExtension}").ShouldNotBeEmpty(); + await reporter.TryWriteSidecarAndAggregateAsync(CreateReportData("RemovedSuiteMarker"), removedHtmlPath, cancellationToken); + await reporter.TryWriteSidecarAndAggregateAsync(CreateReportData("RemainingSuiteMarker"), remainingHtmlPath, cancellationToken); + Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExtension}").Length.ShouldBe(2); TUnitSettings.Default.Reporting.JsonReportEnabled = false; - await reporter.TryWriteSidecarAndAggregateAsync(CreateReportData(), htmlPath, cancellationToken); + await reporter.TryWriteSidecarAndAggregateAsync(CreateReportData("RemovedSuiteMarker"), removedHtmlPath, cancellationToken); - File.Exists(HtmlReporter.GetSidecarPath(htmlPath)).ShouldBeFalse(); + File.Exists(HtmlReporter.GetSidecarPath(removedHtmlPath)).ShouldBeFalse(); + Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExtension}").Length.ShouldBe(1); + var mergedReport = File.ReadAllText(mergedReportPath); + mergedReport.ShouldNotContain("RemovedSuiteMarker"); + mergedReport.ShouldContain("RemainingSuiteMarker"); + + await reporter.TryWriteSidecarAndAggregateAsync(CreateReportData("RemainingSuiteMarker"), remainingHtmlPath, cancellationToken); + + File.Exists(HtmlReporter.GetSidecarPath(remainingHtmlPath)).ShouldBeFalse(); Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExtension}").ShouldBeEmpty(); + File.Exists(mergedReportPath).ShouldBeFalse(); } finally { @@ -166,9 +177,9 @@ public async Task Disabled_Json_Report_Does_Not_Write_Local_Or_Aggregation_Sidec } } - private static ReportData CreateReportData() => new() + private static ReportData CreateReportData(string assemblyName = "Tests") => new() { - AssemblyName = "Tests", + AssemblyName = assemblyName, MachineName = "machine", Timestamp = DateTimeOffset.UtcNow.ToString("O"), TUnitVersion = "1.0.0", From b44648ad57d961e65831c39b69bda19ad76691db Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Tue, 1 Sep 2026 22:55:03 +0100 Subject: [PATCH 07/26] fix: recreate tracing per test session --- .../Reporters/Aggregation/ReportAggregator.cs | 4 +- .../Reporters/Html/HtmlReporter.cs | 69 ++++++++++++------- .../HtmlReporterConfigurationTests.cs | 30 +++++++- tests/TUnit.Engine.Tests/HtmlReporterTests.cs | 3 +- 4 files changed, 79 insertions(+), 27 deletions(-) diff --git a/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs b/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs index d51a5e0bee0..786c3dee842 100644 --- a/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs +++ b/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs @@ -136,7 +136,9 @@ internal string WriteSidecar(byte[] sidecarUtf8Json, string assemblyName, string } internal void DeleteSidecar(string assemblyName, string suiteSalt) - => File.Delete(GetSidecarPath(assemblyName, suiteSalt)); + { + File.Delete(GetSidecarPath(assemblyName, suiteSalt)); + } /// /// Reads every sidecar currently present in the shared directory. Unreadable or diff --git a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs index bd52883ea47..e97435b0d88 100644 --- a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs +++ b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs @@ -38,6 +38,7 @@ internal sealed class HtmlReporter(IExtension extension) : IDataConsumer, IDataP private IMessageBus? _messageBus; private string _resultsDirectory = "TestResults"; private readonly ConcurrentDictionary _updates = []; + private readonly object _htmlReportStateLock = new(); private GitHubReporter? _githubReporter; private int _htmlReportEnabledAfterDiscovery = HtmlReportEnabledUnresolved; @@ -98,13 +99,6 @@ private static bool HasFinalState(TestNodeUpdateMessage update) public Task BeforeRunAsync(CancellationToken cancellationToken) { -#if NET - if (IsHtmlReportEnabled()) - { - _activityCollector = new ActivityCollector(); - _activityCollector.Start(); - } -#endif return Task.CompletedTask; } @@ -113,8 +107,19 @@ public Task AfterRunAsync(int exitCode, CancellationToken cancellation) public Task OnTestSessionStartingAsync(ITestSessionContext testSessionContext) { - _updates.Clear(); - Volatile.Write(ref _htmlReportEnabledAfterDiscovery, HtmlReportEnabledUnresolved); + lock (_htmlReportStateLock) + { + _updates.Clear(); + Volatile.Write(ref _htmlReportEnabledAfterDiscovery, HtmlReportEnabledUnresolved); +#if NET + DisposeActivityCollection(); + if (IsHtmlReportEnabled()) + { + StartActivityCollection(); + } +#endif + } + return Task.CompletedTask; } @@ -346,26 +351,31 @@ internal bool IsHtmlReportEnabledForRun() return resolved == 1; } - var enabled = IsHtmlReportEnabled(); - resolved = Interlocked.CompareExchange( - ref _htmlReportEnabledAfterDiscovery, - enabled ? 1 : 0, - HtmlReportEnabledUnresolved); - - if (resolved != HtmlReportEnabledUnresolved) + lock (_htmlReportStateLock) { - return resolved == 1; - } + resolved = Volatile.Read(ref _htmlReportEnabledAfterDiscovery); + if (resolved != HtmlReportEnabledUnresolved) + { + return resolved == 1; + } + + var enabled = IsHtmlReportEnabled(); #if NET - if (!enabled) - { - DisposeActivityCollection(); - TraceRegistry.Clear(); - } + if (enabled) + { + StartActivityCollection(); + } + else + { + DisposeActivityCollection(); + TraceRegistry.Clear(); + } #endif - return enabled; + Volatile.Write(ref _htmlReportEnabledAfterDiscovery, enabled ? 1 : 0); + return enabled; + } } // Default HTML report is "{name}-{os}-{tfm}-report.html"; the sidecar drops the @@ -407,6 +417,17 @@ public void Dispose() #if NET internal bool HasActivityCollector => _activityCollector is not null; + private void StartActivityCollection() + { + if (_activityCollector is not null) + { + return; + } + + _activityCollector = new ActivityCollector(); + _activityCollector.Start(); + } + internal void StopActivityCollection() => _activityCollector?.Stop(); diff --git a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs index 09bdd0fb532..6c0f27f180f 100644 --- a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs +++ b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs @@ -2,6 +2,7 @@ using Microsoft.Testing.Platform.TestHost; using Shouldly; using TUnit.Core.Settings; +using TUnit.Engine.Reporters; using TUnit.Engine.Reporters.Aggregation; using TUnit.Engine.Reporters.Html; @@ -79,7 +80,8 @@ public async Task Disable_Environment_Variables_Take_Precedence() public async Task Disabled_Html_Report_Stops_Activity_Collection_After_Discovery(CancellationToken cancellationToken) { using var reporter = new HtmlReporter(new MockExtension()); - await reporter.BeforeRunAsync(cancellationToken); + cancellationToken.ThrowIfCancellationRequested(); + await reporter.OnTestSessionStartingAsync(null!); reporter.HasActivityCollector.ShouldBeTrue(); TUnitSettings.Default.Reporting.HtmlReportEnabled = false; @@ -104,6 +106,32 @@ public async Task Html_Report_Setting_Is_Resolved_Per_Session(CancellationToken reporter.IsHtmlReportEnabledForRun().ShouldBeTrue(); } + [Test] + public async Task Activity_Collection_Is_Recreated_Between_Sessions(CancellationToken cancellationToken) + { + TUnitSettings.Default.Reporting.HtmlReportEnabled = true; + using var reporter = new HtmlReporter(new MockExtension()); + + cancellationToken.ThrowIfCancellationRequested(); + await reporter.OnTestSessionStartingAsync(null!); + reporter.HasActivityCollector.ShouldBeTrue(); + + cancellationToken.ThrowIfCancellationRequested(); + await reporter.OnTestSessionFinishingAsync(null!); + reporter.HasActivityCollector.ShouldBeFalse(); + + cancellationToken.ThrowIfCancellationRequested(); + await reporter.OnTestSessionStartingAsync(null!); + reporter.HasActivityCollector.ShouldBeTrue(); + + var activity = TUnitActivitySource.StartLifecycleActivity(TUnitActivitySource.SpanTestSession); + TUnitActivitySource.StopActivity(activity); + reporter.StopActivityCollection(); + var spans = reporter.BuildReportData().Spans; + spans.ShouldNotBeNull(); + spans.ShouldContain(span => span.SpanType == TUnitActivitySource.SpanTestSession); + } + [Test] public async Task Test_Updates_Are_Cleared_Between_Sessions(CancellationToken cancellationToken) { diff --git a/tests/TUnit.Engine.Tests/HtmlReporterTests.cs b/tests/TUnit.Engine.Tests/HtmlReporterTests.cs index af33e1cc31b..05123ed29cc 100644 --- a/tests/TUnit.Engine.Tests/HtmlReporterTests.cs +++ b/tests/TUnit.Engine.Tests/HtmlReporterTests.cs @@ -34,7 +34,8 @@ public void HtmlReporter_DataTypesProduced_Contains_SessionFileArtifact() public async Task Stopping_Activity_Collection_Preserves_Spans_For_Report_Data(CancellationToken cancellationToken) { using var reporter = new HtmlReporter(new MockExtension()); - await reporter.BeforeRunAsync(cancellationToken); + cancellationToken.ThrowIfCancellationRequested(); + await reporter.OnTestSessionStartingAsync(null!); var activity = TUnitActivitySource.StartLifecycleActivity(TUnitActivitySource.SpanTestSession); TUnitActivitySource.StopActivity(activity); From 731cf026828fa4b945cd594e12db8bb2f2b4e7f6 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Tue, 1 Sep 2026 23:04:04 +0100 Subject: [PATCH 08/26] fix: harden reporter session lifecycle Serialize shared aggregation mutations, clean stale report outputs when HTML is disabled, and start trace collection before discovery can re-enable reporting. --- .../Reporters/Html/HtmlReporter.cs | 45 +++++--- .../HtmlReporterConfigurationTests.cs | 100 ++++++++++++++++++ 2 files changed, 128 insertions(+), 17 deletions(-) diff --git a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs index e97435b0d88..919733ffe39 100644 --- a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs +++ b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs @@ -113,10 +113,10 @@ public Task OnTestSessionStartingAsync(ITestSessionContext testSessionContext) Volatile.Write(ref _htmlReportEnabledAfterDiscovery, HtmlReportEnabledUnresolved); #if NET DisposeActivityCollection(); - if (IsHtmlReportEnabled()) - { - StartActivityCollection(); - } + // Discovery hooks may re-enable reporting for this session. Start before + // discovery so those early spans are retained, then resolve the setting + // on the first post-discovery update. + StartActivityCollection(); #endif } @@ -137,6 +137,13 @@ public async Task OnTestSessionFinishingAsync(ITestSessionContext testSessionCon TraceRegistry.Clear(); #endif _updates.Clear(); + var disabledOutputPath = _outputPath ?? GetDefaultOutputPath(); + var aggregator = ReportAggregator.TryCreateFromEnvironment(Environment.GetEnvironmentVariable); + await DeleteSidecarsAndRefreshAggregateAsync( + GetAssemblyName(), + disabledOutputPath, + aggregator, + testSessionContext?.CancellationToken ?? CancellationToken.None); return; } @@ -207,7 +214,7 @@ internal async Task TryWriteSidecarAndAggregateAsync(ReportData reportData, stri if (!IsJsonReportEnabled()) { - await DeleteSidecarsAndRefreshAggregateAsync(reportData, htmlOutputPath, aggregator, cancellationToken); + await DeleteSidecarsAndRefreshAggregateAsync(reportData.AssemblyName, htmlOutputPath, aggregator, cancellationToken); return; } @@ -230,6 +237,15 @@ internal async Task TryWriteSidecarAndAggregateAsync(ReportData reportData, stri try { + // Serialize shared sidecar mutation and merged-output regeneration as one + // transaction. A disabled sibling can otherwise delete this sidecar after + // it is written but before this process acquires the aggregation lock. + using var aggregationLock = await aggregator.AcquireLockAsync(cancellationToken); + if (aggregationLock is null) + { + return; + } + aggregator.WriteSidecar(sidecarBytes, reportData.AssemblyName, suiteSalt: htmlOutputPath); // Aggregation is committed for this suite: whatever happens below, the classic @@ -241,14 +257,6 @@ internal async Task TryWriteSidecarAndAggregateAsync(ReportData reportData, stri _githubReporter.SuppressPerSuiteSummary = true; } - // Every finishing process regenerates the merged outputs from all sidecars - // present so far; the last one to finish leaves the complete aggregate. - using var aggregationLock = await aggregator.AcquireLockAsync(cancellationToken); - if (aggregationLock is null) - { - return; - } - RefreshAggregatedOutputs(aggregator); } catch (Exception ex) @@ -258,7 +266,7 @@ internal async Task TryWriteSidecarAndAggregateAsync(ReportData reportData, stri } private async Task DeleteSidecarsAndRefreshAggregateAsync( - ReportData reportData, + string assemblyName, string htmlOutputPath, ReportAggregator? aggregator, CancellationToken cancellationToken) @@ -278,7 +286,7 @@ private async Task DeleteSidecarsAndRefreshAggregateAsync( return; } - TryDeleteReportFile(() => aggregator.DeleteSidecar(reportData.AssemblyName, htmlOutputPath)); + TryDeleteReportFile(() => aggregator.DeleteSidecar(assemblyName, htmlOutputPath)); RefreshAggregatedOutputs(aggregator); } catch (Exception ex) when (ex is not OperationCanceledException) @@ -471,7 +479,7 @@ internal void SetResultsDirectory(string path) internal ReportData BuildReportData() { - var assemblyName = Assembly.GetEntryAssembly()?.GetName().Name ?? "TestResults"; + var assemblyName = GetAssemblyName(); var tunitVersion = typeof(HtmlReporter).Assembly.GetName().Version?.ToString() ?? "unknown"; // Get the last update with a final state for each test @@ -911,13 +919,16 @@ private static (string Status, ReportExceptionData? Exception, string? SkipReaso private string GetDefaultOutputPath() { - var assemblyName = Assembly.GetEntryAssembly()?.GetName().Name ?? "TestResults"; + var assemblyName = GetAssemblyName(); var sanitizedName = PathValidator.SanitizeFileName(assemblyName); var os = GetShortOsName(); var tfm = GetShortFrameworkName(); return Path.GetFullPath(Path.Combine(_resultsDirectory, $"{sanitizedName}-{os}-{tfm}-report.html")); } + private static string GetAssemblyName() + => Assembly.GetEntryAssembly()?.GetName().Name ?? "TestResults"; + private static string GetShortOsName() { #if NET diff --git a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs index 6c0f27f180f..c03d9eed426 100644 --- a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs +++ b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs @@ -106,6 +106,27 @@ public async Task Html_Report_Setting_Is_Resolved_Per_Session(CancellationToken reporter.IsHtmlReportEnabledForRun().ShouldBeTrue(); } + [Test] + public async Task Activity_Collection_Starts_Before_Discovery_Reenables_Reporting(CancellationToken cancellationToken) + { + TUnitSettings.Default.Reporting.HtmlReportEnabled = false; + using var reporter = new HtmlReporter(new MockExtension()); + + cancellationToken.ThrowIfCancellationRequested(); + await reporter.OnTestSessionStartingAsync(null!); + reporter.HasActivityCollector.ShouldBeTrue(); + + var activity = TUnitActivitySource.StartLifecycleActivity(TUnitActivitySource.SpanTestSession); + TUnitSettings.Default.Reporting.HtmlReportEnabled = true; + reporter.IsHtmlReportEnabledForRun().ShouldBeTrue(); + TUnitActivitySource.StopActivity(activity); + + reporter.StopActivityCollection(); + var spans = reporter.BuildReportData().Spans; + spans.ShouldNotBeNull(); + spans.ShouldContain(span => span.SpanType == TUnitActivitySource.SpanTestSession); + } + [Test] public async Task Activity_Collection_Is_Recreated_Between_Sessions(CancellationToken cancellationToken) { @@ -205,6 +226,85 @@ public async Task Disabled_Json_Report_Removes_Stale_Aggregation_Outputs(Cancell } } + [Test] + public async Task Disabled_Html_Report_Removes_Stale_Aggregation_Outputs(CancellationToken cancellationToken) + { + var tempDirectory = Path.Combine(Path.GetTempPath(), $"tunit-report-test-{Guid.NewGuid():N}"); + var aggregationDirectory = Path.Combine(tempDirectory, "aggregate"); + var disabledHtmlPath = Path.Combine(tempDirectory, "disabled-report.html"); + var remainingHtmlPath = Path.Combine(tempDirectory, "remaining-report.html"); + var mergedReportPath = Path.Combine(aggregationDirectory, ReportDataJson.MergedReportFileName); + Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_REPORTS", "true"); + Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_DIR", aggregationDirectory); + + try + { + Directory.CreateDirectory(tempDirectory); + using var reporter = new HtmlReporter(new MockExtension()); + reporter.SetOutputPath(disabledHtmlPath); + var disabledAssemblyName = reporter.BuildReportData().AssemblyName; + + TUnitSettings.Default.Reporting.JsonReportEnabled = true; + await reporter.TryWriteSidecarAndAggregateAsync(CreateReportData(disabledAssemblyName), disabledHtmlPath, cancellationToken); + await reporter.TryWriteSidecarAndAggregateAsync(CreateReportData("RemainingSuiteMarker"), remainingHtmlPath, cancellationToken); + + TUnitSettings.Default.Reporting.HtmlReportEnabled = false; + await reporter.OnTestSessionFinishingAsync(null!); + + File.Exists(HtmlReporter.GetSidecarPath(disabledHtmlPath)).ShouldBeFalse(); + Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExtension}").Length.ShouldBe(1); + var mergedReport = File.ReadAllText(mergedReportPath); + mergedReport.ShouldNotContain(disabledAssemblyName); + mergedReport.ShouldContain("RemainingSuiteMarker"); + } + finally + { + if (Directory.Exists(tempDirectory)) + { + Directory.Delete(tempDirectory, recursive: true); + } + } + } + + [Test] + public async Task Shared_Sidecar_Write_Waits_For_Aggregation_Lock(CancellationToken cancellationToken) + { + var tempDirectory = Path.Combine(Path.GetTempPath(), $"tunit-report-test-{Guid.NewGuid():N}"); + var aggregationDirectory = Path.Combine(tempDirectory, "aggregate"); + var htmlPath = Path.Combine(tempDirectory, "suite-report.html"); + Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_REPORTS", "true"); + Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_DIR", aggregationDirectory); + + try + { + Directory.CreateDirectory(tempDirectory); + using var reporter = new HtmlReporter(new MockExtension()); + var aggregator = ReportAggregator.TryCreateFromEnvironment(Environment.GetEnvironmentVariable); + var aggregationLock = await aggregator!.AcquireLockAsync(cancellationToken); + aggregationLock.ShouldNotBeNull(); + + Task writeTask; + using (aggregationLock!) + { + writeTask = reporter.TryWriteSidecarAndAggregateAsync(CreateReportData(), htmlPath, cancellationToken); + + File.Exists(HtmlReporter.GetSidecarPath(htmlPath)).ShouldBeTrue(); + Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExtension}").ShouldBeEmpty(); + } + + await writeTask; + + Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExtension}").Length.ShouldBe(1); + } + finally + { + if (Directory.Exists(tempDirectory)) + { + Directory.Delete(tempDirectory, recursive: true); + } + } + } + private static ReportData CreateReportData(string assemblyName = "Tests") => new() { AssemblyName = assemblyName, From c47ac637caa561266ce98a64e26f0114f4783472 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Tue, 1 Sep 2026 23:11:50 +0100 Subject: [PATCH 09/26] fix: preserve aggregation across sessions Keep sidecars available during lock contention and reset GitHub summary suppression before each session. --- .../Reporters/Html/HtmlReporter.cs | 29 +++++++++----- .../HtmlReporterConfigurationTests.cs | 39 ++++++++++++++++++- 2 files changed, 56 insertions(+), 12 deletions(-) diff --git a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs index 919733ffe39..49fae2a3a08 100644 --- a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs +++ b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs @@ -110,6 +110,10 @@ public Task OnTestSessionStartingAsync(ITestSessionContext testSessionContext) lock (_htmlReportStateLock) { _updates.Clear(); + if (_githubReporter is not null) + { + _githubReporter.SuppressPerSuiteSummary = false; + } Volatile.Write(ref _htmlReportEnabledAfterDiscovery, HtmlReportEnabledUnresolved); #if NET DisposeActivityCollection(); @@ -237,16 +241,10 @@ internal async Task TryWriteSidecarAndAggregateAsync(ReportData reportData, stri try { - // Serialize shared sidecar mutation and merged-output regeneration as one - // transaction. A disabled sibling can otherwise delete this sidecar after - // it is written but before this process acquires the aggregation lock. - using var aggregationLock = await aggregator.AcquireLockAsync(cancellationToken); - if (aggregationLock is null) - { - return; - } - - aggregator.WriteSidecar(sidecarBytes, reportData.AssemblyName, suiteSalt: htmlOutputPath); + // Publish before waiting so lock timeout cannot drop this suite from a later + // sibling's merge. Once the wait ends, restore a sidecar that concurrent + // disabled cleanup may have removed. + var sharedSidecarPath = aggregator.WriteSidecar(sidecarBytes, reportData.AssemblyName, suiteSalt: htmlOutputPath); // Aggregation is committed for this suite: whatever happens below, the classic // per-suite block must not be appended on top of the aggregated one. (If we @@ -257,6 +255,17 @@ internal async Task TryWriteSidecarAndAggregateAsync(ReportData reportData, stri _githubReporter.SuppressPerSuiteSummary = true; } + using var aggregationLock = await aggregator.AcquireLockAsync(cancellationToken); + if (!File.Exists(sharedSidecarPath)) + { + aggregator.WriteSidecar(sidecarBytes, reportData.AssemblyName, suiteSalt: htmlOutputPath); + } + + if (aggregationLock is null) + { + return; + } + RefreshAggregatedOutputs(aggregator); } catch (Exception ex) diff --git a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs index c03d9eed426..c5d0e9c025d 100644 --- a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs +++ b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs @@ -168,6 +168,39 @@ public async Task Test_Updates_Are_Cleared_Between_Sessions(CancellationToken ca reporter.BuildReportData().Groups.SelectMany(x => x.Tests).Single().Id.ShouldBe("second"); } + [Test] + public async Task GitHub_Summary_Suppression_Is_Reset_Between_Sessions(CancellationToken cancellationToken) + { + var tempDirectory = Path.Combine(Path.GetTempPath(), $"tunit-report-test-{Guid.NewGuid():N}"); + Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_REPORTS", "true"); + Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_DIR", Path.Combine(tempDirectory, "aggregate")); + + try + { + Directory.CreateDirectory(tempDirectory); + using var reporter = new HtmlReporter(new MockExtension()); + var githubReporter = new GitHubReporter(new MockExtension()); + reporter.SetGitHubReporter(githubReporter); + + await reporter.TryWriteSidecarAndAggregateAsync( + CreateReportData(), + Path.Combine(tempDirectory, "suite-report.html"), + cancellationToken); + githubReporter.SuppressPerSuiteSummary.ShouldBeTrue(); + + await reporter.OnTestSessionStartingAsync(null!); + + githubReporter.SuppressPerSuiteSummary.ShouldBeFalse(); + } + finally + { + if (Directory.Exists(tempDirectory)) + { + Directory.Delete(tempDirectory, recursive: true); + } + } + } + [Test] public async Task Disabled_Artifact_Upload_Does_Not_Publish_Session_File_Artifact(CancellationToken cancellationToken) { @@ -267,7 +300,7 @@ public async Task Disabled_Html_Report_Removes_Stale_Aggregation_Outputs(Cancell } [Test] - public async Task Shared_Sidecar_Write_Waits_For_Aggregation_Lock(CancellationToken cancellationToken) + public async Task Shared_Sidecar_Is_Published_While_Waiting_And_Restored_After_Cleanup(CancellationToken cancellationToken) { var tempDirectory = Path.Combine(Path.GetTempPath(), $"tunit-report-test-{Guid.NewGuid():N}"); var aggregationDirectory = Path.Combine(tempDirectory, "aggregate"); @@ -289,12 +322,14 @@ public async Task Shared_Sidecar_Write_Waits_For_Aggregation_Lock(CancellationTo writeTask = reporter.TryWriteSidecarAndAggregateAsync(CreateReportData(), htmlPath, cancellationToken); File.Exists(HtmlReporter.GetSidecarPath(htmlPath)).ShouldBeTrue(); - Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExtension}").ShouldBeEmpty(); + var sharedSidecarPath = Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExtension}").Single(); + File.Delete(sharedSidecarPath); } await writeTask; Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExtension}").Length.ShouldBe(1); + File.ReadAllText(Path.Combine(aggregationDirectory, ReportDataJson.MergedReportFileName)).ShouldContain("Tests"); } finally { From b641c94618ded243eb7d21629799500ed3cf74db Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Tue, 1 Sep 2026 23:25:49 +0100 Subject: [PATCH 10/26] fix: wait for complete report aggregation Final writers must acquire the aggregation lock before returning or completed suites can be omitted permanently. Cancellation and non-contention failures retain the classic summary fallback. --- .../Reporters/Aggregation/ReportAggregator.cs | 25 ++++++++----------- .../Reporters/Html/HtmlReporter.cs | 24 +++++------------- .../HtmlReporterConfigurationTests.cs | 3 +-- 3 files changed, 17 insertions(+), 35 deletions(-) diff --git a/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs b/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs index 786c3dee842..0cdc77bec4a 100644 --- a/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs +++ b/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs @@ -40,8 +40,9 @@ internal sealed class ReportAggregator private const string LockFileName = ".tunit-aggregate.lock"; // Lock contention is expected (N processes finishing together, each holding the lock - // for a full merge), so wait far longer than the file-write retry defaults. - private const int LockMaxAttempts = 60; + // for a full merge). Warn after prolonged contention, but keep waiting: abandoning the + // final merge would leave a completed suite absent from aggregate outputs. + private const int LockContentionWarningAttempt = 60; private const int LockRetryDelayMs = 250; internal AggregationMode Mode { get; } @@ -181,37 +182,31 @@ internal List ReadAllSidecars() /// /// Acquires the cross-process aggregation lock. Every writer performs its whole - /// read-merge-write cycle under this lock, so merges never interleave. Returns - /// when the lock cannot be acquired within the timeout; - /// callers should then skip merging (a later sibling will produce a fresher merge). + /// read-merge-write cycle under this lock, so merges never interleave. Waits until + /// acquisition or cancellation because skipping the final merge loses results. /// - internal async Task AcquireLockAsync(CancellationToken cancellationToken) + internal async Task AcquireLockAsync(CancellationToken cancellationToken) { System.IO.Directory.CreateDirectory(Directory); var lockPath = Path.Combine(Directory, LockFileName); - for (var attempt = 1; attempt <= LockMaxAttempts; attempt++) + for (var attempt = 1; ; attempt++) { cancellationToken.ThrowIfCancellationRequested(); try { return new FileStream(lockPath, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None); } - catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + catch (IOException) { - // Contention (or a permissions hiccup) on the final attempt must still fall - // through to the graceful "skip this merge" path, never escape as a throw. - if (attempt == LockMaxAttempts) + if (attempt == LockContentionWarningAttempt) { - break; + Console.WriteLine("Warning: Report aggregation lock is still unavailable; waiting to preserve complete merged results."); } await Task.Delay(LockRetryDelayMs + Random.Shared.Next(0, 100), cancellationToken); } } - - Console.WriteLine("Warning: Could not acquire the report aggregation lock; skipping merge for this process."); - return null; } /// diff --git a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs index 49fae2a3a08..d36b3c6ecc3 100644 --- a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs +++ b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs @@ -246,27 +246,20 @@ internal async Task TryWriteSidecarAndAggregateAsync(ReportData reportData, stri // disabled cleanup may have removed. var sharedSidecarPath = aggregator.WriteSidecar(sidecarBytes, reportData.AssemblyName, suiteSalt: htmlOutputPath); - // Aggregation is committed for this suite: whatever happens below, the classic - // per-suite block must not be appended on top of the aggregated one. (If we - // fail past this point, a sibling that merges after us still renders this - // suite's results from the sidecar just written.) - if (_githubReporter is not null) - { - _githubReporter.SuppressPerSuiteSummary = true; - } - using var aggregationLock = await aggregator.AcquireLockAsync(cancellationToken); if (!File.Exists(sharedSidecarPath)) { aggregator.WriteSidecar(sidecarBytes, reportData.AssemblyName, suiteSalt: htmlOutputPath); } - if (aggregationLock is null) + RefreshAggregatedOutputs(aggregator); + + // Suppress the classic summary only after this suite is present in refreshed + // aggregate outputs. Acquisition cancellation or I/O failure falls back to it. + if (_githubReporter is not null) { - return; + _githubReporter.SuppressPerSuiteSummary = true; } - - RefreshAggregatedOutputs(aggregator); } catch (Exception ex) { @@ -290,11 +283,6 @@ private async Task DeleteSidecarsAndRefreshAggregateAsync( try { using var aggregationLock = await aggregator.AcquireLockAsync(cancellationToken); - if (aggregationLock is null) - { - return; - } - TryDeleteReportFile(() => aggregator.DeleteSidecar(assemblyName, htmlOutputPath)); RefreshAggregatedOutputs(aggregator); } diff --git a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs index c5d0e9c025d..1fb7345d826 100644 --- a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs +++ b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs @@ -314,10 +314,9 @@ public async Task Shared_Sidecar_Is_Published_While_Waiting_And_Restored_After_C using var reporter = new HtmlReporter(new MockExtension()); var aggregator = ReportAggregator.TryCreateFromEnvironment(Environment.GetEnvironmentVariable); var aggregationLock = await aggregator!.AcquireLockAsync(cancellationToken); - aggregationLock.ShouldNotBeNull(); Task writeTask; - using (aggregationLock!) + using (aggregationLock) { writeTask = reporter.TryWriteSidecarAndAggregateAsync(CreateReportData(), htmlPath, cancellationToken); From 7d1937c16c4f8c9389e7862c266c111132e91203 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Tue, 1 Sep 2026 23:33:26 +0100 Subject: [PATCH 11/26] fix: bound report aggregation waits Lock contention falls back after roughly ten seconds so report generation cannot hang test-suite completion. Per-suite outputs remain available and shared sidecars allow later recovery. --- .../Reporters/Aggregation/ReportAggregator.cs | 22 ++++++++++--------- .../Reporters/Html/HtmlReporter.cs | 10 +++++++++ .../HtmlReporterConfigurationTests.cs | 3 ++- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs b/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs index 0cdc77bec4a..29ce67edb2b 100644 --- a/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs +++ b/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs @@ -39,10 +39,9 @@ internal sealed class ReportAggregator private const string SidecarSearchPattern = "*" + ReportDataJson.SidecarExtension; private const string LockFileName = ".tunit-aggregate.lock"; - // Lock contention is expected (N processes finishing together, each holding the lock - // for a full merge). Warn after prolonged contention, but keep waiting: abandoning the - // final merge would leave a completed suite absent from aggregate outputs. - private const int LockContentionWarningAttempt = 60; + // Bound lock contention to roughly ten seconds. Reporting must never hang test-suite + // completion; timed-out writers leave their sidecar for a later aggregate refresh. + private const int LockMaxAttempts = 30; private const int LockRetryDelayMs = 250; internal AggregationMode Mode { get; } @@ -182,15 +181,15 @@ internal List ReadAllSidecars() /// /// Acquires the cross-process aggregation lock. Every writer performs its whole - /// read-merge-write cycle under this lock, so merges never interleave. Waits until - /// acquisition or cancellation because skipping the final merge loses results. + /// read-merge-write cycle under this lock, so merges never interleave. Returns + /// after a bounded wait so reporting cannot hang the run. /// - internal async Task AcquireLockAsync(CancellationToken cancellationToken) + internal async Task AcquireLockAsync(CancellationToken cancellationToken) { System.IO.Directory.CreateDirectory(Directory); var lockPath = Path.Combine(Directory, LockFileName); - for (var attempt = 1; ; attempt++) + for (var attempt = 1; attempt <= LockMaxAttempts; attempt++) { cancellationToken.ThrowIfCancellationRequested(); try @@ -199,14 +198,17 @@ internal async Task AcquireLockAsync(CancellationToken cancellation } catch (IOException) { - if (attempt == LockContentionWarningAttempt) + if (attempt == LockMaxAttempts) { - Console.WriteLine("Warning: Report aggregation lock is still unavailable; waiting to preserve complete merged results."); + break; } await Task.Delay(LockRetryDelayMs + Random.Shared.Next(0, 100), cancellationToken); } } + + Console.WriteLine("Warning: Report aggregation lock timed out; keeping per-suite reports and deferring aggregate refresh."); + return null; } /// diff --git a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs index d36b3c6ecc3..165a721c920 100644 --- a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs +++ b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs @@ -252,6 +252,11 @@ internal async Task TryWriteSidecarAndAggregateAsync(ReportData reportData, stri aggregator.WriteSidecar(sidecarBytes, reportData.AssemblyName, suiteSalt: htmlOutputPath); } + if (aggregationLock is null) + { + return; + } + RefreshAggregatedOutputs(aggregator); // Suppress the classic summary only after this suite is present in refreshed @@ -283,6 +288,11 @@ private async Task DeleteSidecarsAndRefreshAggregateAsync( try { using var aggregationLock = await aggregator.AcquireLockAsync(cancellationToken); + if (aggregationLock is null) + { + return; + } + TryDeleteReportFile(() => aggregator.DeleteSidecar(assemblyName, htmlOutputPath)); RefreshAggregatedOutputs(aggregator); } diff --git a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs index 1fb7345d826..c5d0e9c025d 100644 --- a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs +++ b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs @@ -314,9 +314,10 @@ public async Task Shared_Sidecar_Is_Published_While_Waiting_And_Restored_After_C using var reporter = new HtmlReporter(new MockExtension()); var aggregator = ReportAggregator.TryCreateFromEnvironment(Environment.GetEnvironmentVariable); var aggregationLock = await aggregator!.AcquireLockAsync(cancellationToken); + aggregationLock.ShouldNotBeNull(); Task writeTask; - using (aggregationLock) + using (aggregationLock!) { writeTask = reporter.TryWriteSidecarAndAggregateAsync(CreateReportData(), htmlPath, cancellationToken); From c3b5a314695968ee6863ce0d4fba6d7fecc15c65 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Tue, 1 Sep 2026 23:54:46 +0100 Subject: [PATCH 12/26] fix: clean reporter state between sessions Stale sidecars and GitHub artifact metadata must not survive cancellation or reused-host session boundaries. --- .../Reporters/Html/HtmlReporter.cs | 14 ++++--- .../HtmlReporterConfigurationTests.cs | 40 ++++++++++++++++++- 2 files changed, 47 insertions(+), 7 deletions(-) diff --git a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs index 165a721c920..b5ef52c1218 100644 --- a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs +++ b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs @@ -113,6 +113,8 @@ public Task OnTestSessionStartingAsync(ITestSessionContext testSessionContext) if (_githubReporter is not null) { _githubReporter.SuppressPerSuiteSummary = false; + _githubReporter.ArtifactUrl = null; + _githubReporter.ShowArtifactUploadTip = false; } Volatile.Write(ref _htmlReportEnabledAfterDiscovery, HtmlReportEnabledUnresolved); #if NET @@ -146,8 +148,7 @@ public async Task OnTestSessionFinishingAsync(ITestSessionContext testSessionCon await DeleteSidecarsAndRefreshAggregateAsync( GetAssemblyName(), disabledOutputPath, - aggregator, - testSessionContext?.CancellationToken ?? CancellationToken.None); + aggregator); return; } @@ -218,7 +219,7 @@ internal async Task TryWriteSidecarAndAggregateAsync(ReportData reportData, stri if (!IsJsonReportEnabled()) { - await DeleteSidecarsAndRefreshAggregateAsync(reportData.AssemblyName, htmlOutputPath, aggregator, cancellationToken); + await DeleteSidecarsAndRefreshAggregateAsync(reportData.AssemblyName, htmlOutputPath, aggregator); return; } @@ -275,8 +276,7 @@ internal async Task TryWriteSidecarAndAggregateAsync(ReportData reportData, stri private async Task DeleteSidecarsAndRefreshAggregateAsync( string assemblyName, string htmlOutputPath, - ReportAggregator? aggregator, - CancellationToken cancellationToken) + ReportAggregator? aggregator) { TryDeleteReportFile(() => File.Delete(GetSidecarPath(htmlOutputPath))); @@ -287,7 +287,9 @@ private async Task DeleteSidecarsAndRefreshAggregateAsync( try { - using var aggregationLock = await aggregator.AcquireLockAsync(cancellationToken); + // Cleanup must survive session cancellation or stale enabled-run sidecars can + // re-enter a sibling's aggregate. Lock acquisition remains time-bounded. + using var aggregationLock = await aggregator.AcquireLockAsync(CancellationToken.None); if (aggregationLock is null) { return; diff --git a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs index c5d0e9c025d..650faafa020 100644 --- a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs +++ b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs @@ -169,7 +169,7 @@ public async Task Test_Updates_Are_Cleared_Between_Sessions(CancellationToken ca } [Test] - public async Task GitHub_Summary_Suppression_Is_Reset_Between_Sessions(CancellationToken cancellationToken) + public async Task GitHub_Report_State_Is_Reset_Between_Sessions(CancellationToken cancellationToken) { var tempDirectory = Path.Combine(Path.GetTempPath(), $"tunit-report-test-{Guid.NewGuid():N}"); Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_REPORTS", "true"); @@ -187,10 +187,14 @@ await reporter.TryWriteSidecarAndAggregateAsync( Path.Combine(tempDirectory, "suite-report.html"), cancellationToken); githubReporter.SuppressPerSuiteSummary.ShouldBeTrue(); + githubReporter.ArtifactUrl = "https://example.com/old-artifact"; + githubReporter.ShowArtifactUploadTip = true; await reporter.OnTestSessionStartingAsync(null!); githubReporter.SuppressPerSuiteSummary.ShouldBeFalse(); + githubReporter.ArtifactUrl.ShouldBeNull(); + githubReporter.ShowArtifactUploadTip.ShouldBeFalse(); } finally { @@ -259,6 +263,40 @@ public async Task Disabled_Json_Report_Removes_Stale_Aggregation_Outputs(Cancell } } + [Test] + public async Task Cancelled_Session_Still_Removes_Disabled_Report_Sidecars(CancellationToken cancellationToken) + { + var tempDirectory = Path.Combine(Path.GetTempPath(), $"tunit-report-test-{Guid.NewGuid():N}"); + var aggregationDirectory = Path.Combine(tempDirectory, "aggregate"); + var htmlPath = Path.Combine(tempDirectory, "cancelled-report.html"); + Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_REPORTS", "true"); + Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_DIR", aggregationDirectory); + + try + { + Directory.CreateDirectory(tempDirectory); + using var reporter = new HtmlReporter(new MockExtension()); + + TUnitSettings.Default.Reporting.JsonReportEnabled = true; + await reporter.TryWriteSidecarAndAggregateAsync(CreateReportData(), htmlPath, cancellationToken); + + using var cancelled = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cancelled.Cancel(); + TUnitSettings.Default.Reporting.JsonReportEnabled = false; + await reporter.TryWriteSidecarAndAggregateAsync(CreateReportData(), htmlPath, cancelled.Token); + + File.Exists(HtmlReporter.GetSidecarPath(htmlPath)).ShouldBeFalse(); + Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExtension}").ShouldBeEmpty(); + } + finally + { + if (Directory.Exists(tempDirectory)) + { + Directory.Delete(tempDirectory, recursive: true); + } + } + } + [Test] public async Task Disabled_Html_Report_Removes_Stale_Aggregation_Outputs(CancellationToken cancellationToken) { From 3bc6607eabbfa0ac31da6ea65a9e869ddb73e7e3 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Wed, 2 Sep 2026 00:35:32 +0100 Subject: [PATCH 13/26] fix: persist aggregation timeout state Disabled-suite tombstones keep stale sidecars out of later merges without unbounded waits. Shared sidecars publish only under the lock, and reused reporters clear all session accumulators. --- .../Reporters/Aggregation/ReportAggregator.cs | 22 ++++++++++++ src/TUnit.Engine/Reporters/GitHubReporter.cs | 10 ++++++ .../Reporters/Html/HtmlReporter.cs | 36 +++++++++---------- .../TUnit.Engine.Tests/GitHubReporterTests.cs | 25 +++++++++++++ .../HtmlReporterConfigurationTests.cs | 34 ++++++++++++++++-- 5 files changed, 104 insertions(+), 23 deletions(-) diff --git a/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs b/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs index 29ce67edb2b..dfcf8724e55 100644 --- a/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs +++ b/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs @@ -38,6 +38,7 @@ internal sealed class ReportAggregator { private const string SidecarSearchPattern = "*" + ReportDataJson.SidecarExtension; private const string LockFileName = ".tunit-aggregate.lock"; + private const string DisabledMarkerExtension = ".disabled"; // Bound lock contention to roughly ten seconds. Reporting must never hang test-suite // completion; timed-out writers leave their sidecar for a later aggregate refresh. @@ -140,6 +141,17 @@ internal void DeleteSidecar(string assemblyName, string suiteSalt) File.Delete(GetSidecarPath(assemblyName, suiteSalt)); } + internal void MarkSidecarDisabled(string assemblyName, string suiteSalt) + { + System.IO.Directory.CreateDirectory(Directory); + AtomicFile.WriteAllBytes(GetDisabledMarkerPath(assemblyName, suiteSalt), []); + } + + internal void ClearDisabledMarker(string assemblyName, string suiteSalt) + { + File.Delete(GetDisabledMarkerPath(assemblyName, suiteSalt)); + } + /// /// Reads every sidecar currently present in the shared directory. Unreadable or /// foreign files are skipped — a crashed sibling must not break the merge. @@ -162,6 +174,11 @@ internal List ReadAllSidecars() { try { + if (File.Exists(file + DisabledMarkerExtension)) + { + continue; + } + var bytes = File.ReadAllBytes(file); if (seenDigests.Add(Convert.ToBase64String(sha.ComputeHash(bytes))) && ReportDataJson.TryDeserialize((ReadOnlyMemory)bytes) is { } data) @@ -233,6 +250,11 @@ private static string ShortHash(string value) return BitConverter.ToString(hash, 0, 4).Replace("-", "").ToLowerInvariant(); } + private string GetDisabledMarkerPath(string assemblyName, string suiteSalt) + { + return GetSidecarPath(assemblyName, suiteSalt) + DisabledMarkerExtension; + } + private string GetSidecarPath(string assemblyName, string suiteSalt) { var fileName = $"{PathValidator.SanitizeFileName(assemblyName)}-{ShortHash(suiteSalt)}{ReportDataJson.SidecarExtension}"; diff --git a/src/TUnit.Engine/Reporters/GitHubReporter.cs b/src/TUnit.Engine/Reporters/GitHubReporter.cs index 1e1e3d752e1..9348d130211 100644 --- a/src/TUnit.Engine/Reporters/GitHubReporter.cs +++ b/src/TUnit.Engine/Reporters/GitHubReporter.cs @@ -470,6 +470,16 @@ public async Task AfterRunAsync(int exitCode, CancellationToken cancellation) /// internal bool SuppressPerSuiteSummary { get; set; } + internal void ResetSessionState() + { + _latestUpdates.Clear(); + _terminalStateCounts.Clear(); + ArtifactUrl = null; + ShowArtifactUploadTip = false; + SuppressPerSuiteSummary = false; + _runStopwatch = Stopwatch.StartNew(); + } + /// /// Renders the aggregated block and rewrites the marked summary region. Called by /// HtmlReporter inside the aggregation lock, so the whole merge is one lock cycle. diff --git a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs index b5ef52c1218..52fb3e20b35 100644 --- a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs +++ b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs @@ -110,12 +110,7 @@ public Task OnTestSessionStartingAsync(ITestSessionContext testSessionContext) lock (_htmlReportStateLock) { _updates.Clear(); - if (_githubReporter is not null) - { - _githubReporter.SuppressPerSuiteSummary = false; - _githubReporter.ArtifactUrl = null; - _githubReporter.ShowArtifactUploadTip = false; - } + _githubReporter?.ResetSessionState(); Volatile.Write(ref _htmlReportEnabledAfterDiscovery, HtmlReportEnabledUnresolved); #if NET DisposeActivityCollection(); @@ -242,30 +237,23 @@ internal async Task TryWriteSidecarAndAggregateAsync(ReportData reportData, stri try { - // Publish before waiting so lock timeout cannot drop this suite from a later - // sibling's merge. Once the wait ends, restore a sidecar that concurrent - // disabled cleanup may have removed. - var sharedSidecarPath = aggregator.WriteSidecar(sidecarBytes, reportData.AssemblyName, suiteSalt: htmlOutputPath); - using var aggregationLock = await aggregator.AcquireLockAsync(cancellationToken); - if (!File.Exists(sharedSidecarPath)) - { - aggregator.WriteSidecar(sidecarBytes, reportData.AssemblyName, suiteSalt: htmlOutputPath); - } - if (aggregationLock is null) { return; } - RefreshAggregatedOutputs(aggregator); + aggregator.WriteSidecar(sidecarBytes, reportData.AssemblyName, suiteSalt: htmlOutputPath); + aggregator.ClearDisabledMarker(reportData.AssemblyName, htmlOutputPath); - // Suppress the classic summary only after this suite is present in refreshed - // aggregate outputs. Acquisition cancellation or I/O failure falls back to it. + // Once the shared sidecar is durable and enabled, later sibling refreshes own + // the summary even if this process fails while refreshing it. if (_githubReporter is not null) { _githubReporter.SuppressPerSuiteSummary = true; } + + RefreshAggregatedOutputs(aggregator); } catch (Exception ex) { @@ -287,16 +275,24 @@ private async Task DeleteSidecarsAndRefreshAggregateAsync( try { + // Durable marker prevents every later merge from reading stale suite data even + // when this process cannot acquire the lock to delete it immediately. + aggregator.MarkSidecarDisabled(assemblyName, htmlOutputPath); + // Cleanup must survive session cancellation or stale enabled-run sidecars can // re-enter a sibling's aggregate. Lock acquisition remains time-bounded. using var aggregationLock = await aggregator.AcquireLockAsync(CancellationToken.None); if (aggregationLock is null) { + // An enabled writer may have cleared the first marker while holding the + // lock. Reassert disabled state because this cleanup completed later. + aggregator.MarkSidecarDisabled(assemblyName, htmlOutputPath); return; } - TryDeleteReportFile(() => aggregator.DeleteSidecar(assemblyName, htmlOutputPath)); + aggregator.DeleteSidecar(assemblyName, htmlOutputPath); RefreshAggregatedOutputs(aggregator); + aggregator.ClearDisabledMarker(assemblyName, htmlOutputPath); } catch (Exception ex) when (ex is not OperationCanceledException) { diff --git a/tests/TUnit.Engine.Tests/GitHubReporterTests.cs b/tests/TUnit.Engine.Tests/GitHubReporterTests.cs index 34e297d6974..d8c31e7d26b 100644 --- a/tests/TUnit.Engine.Tests/GitHubReporterTests.cs +++ b/tests/TUnit.Engine.Tests/GitHubReporterTests.cs @@ -126,6 +126,31 @@ public async Task ClearAggregatedSummary_Removes_Stale_Content() File.ReadAllText(outputFile).ShouldNotContain("stale aggregate"); } + [Test] + public async Task ResetSessionState_Clears_Test_And_Presentation_State() + { + var (reporter, outputFile) = await SetupReporter(); + await FeedTestMessages(reporter, + CreatePassedTestMessage("retry", "CurrentTest", "Tests"), + CreatePassedTestMessage("retry", "CurrentTest", "Tests"), + CreatePassedTestMessage("stale", "StaleTest", "Tests")); + reporter.ArtifactUrl = "https://example.com/old-artifact"; + reporter.ShowArtifactUploadTip = true; + reporter.SuppressPerSuiteSummary = true; + + reporter.ResetSessionState(); + await FeedTestMessages(reporter, CreatePassedTestMessage("retry", "CurrentTest", "Tests")); + await reporter.AfterRunAsync(0, CancellationToken.None); + + var output = await File.ReadAllTextAsync(outputFile); + output.ShouldContain("**1 tests**"); + output.ShouldNotContain("StaleTest"); + output.ShouldNotContain("flaky"); + reporter.ArtifactUrl.ShouldBeNull(); + reporter.ShowArtifactUploadTip.ShouldBeFalse(); + reporter.SuppressPerSuiteSummary.ShouldBeFalse(); + } + [Test] public async Task AfterRunAsync_Groups_Failures_By_Exception_Type() { diff --git a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs index 650faafa020..e2aa49d8c6f 100644 --- a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs +++ b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs @@ -297,6 +297,35 @@ public async Task Cancelled_Session_Still_Removes_Disabled_Report_Sidecars(Cance } } + [Test] + public async Task Disabled_Marker_Excludes_Stale_Shared_Sidecar(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var tempDirectory = Path.Combine(Path.GetTempPath(), $"tunit-report-test-{Guid.NewGuid():N}"); + Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_REPORTS", "true"); + Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_DIR", tempDirectory); + + try + { + var aggregator = ReportAggregator.TryCreateFromEnvironment(Environment.GetEnvironmentVariable)!; + var reportData = CreateReportData("DisabledSuiteMarker"); + aggregator.WriteSidecar(ReportDataJson.SerializeToBytes(reportData), reportData.AssemblyName, "suite"); + + aggregator.MarkSidecarDisabled(reportData.AssemblyName, "suite"); + aggregator.ReadAllSidecars().ShouldBeEmpty(); + + aggregator.ClearDisabledMarker(reportData.AssemblyName, "suite"); + aggregator.ReadAllSidecars().Single().AssemblyName.ShouldBe("DisabledSuiteMarker"); + } + finally + { + if (Directory.Exists(tempDirectory)) + { + Directory.Delete(tempDirectory, recursive: true); + } + } + } + [Test] public async Task Disabled_Html_Report_Removes_Stale_Aggregation_Outputs(CancellationToken cancellationToken) { @@ -338,7 +367,7 @@ public async Task Disabled_Html_Report_Removes_Stale_Aggregation_Outputs(Cancell } [Test] - public async Task Shared_Sidecar_Is_Published_While_Waiting_And_Restored_After_Cleanup(CancellationToken cancellationToken) + public async Task Shared_Sidecar_Is_Published_Only_After_Aggregation_Lock(CancellationToken cancellationToken) { var tempDirectory = Path.Combine(Path.GetTempPath(), $"tunit-report-test-{Guid.NewGuid():N}"); var aggregationDirectory = Path.Combine(tempDirectory, "aggregate"); @@ -360,8 +389,7 @@ public async Task Shared_Sidecar_Is_Published_While_Waiting_And_Restored_After_C writeTask = reporter.TryWriteSidecarAndAggregateAsync(CreateReportData(), htmlPath, cancellationToken); File.Exists(HtmlReporter.GetSidecarPath(htmlPath)).ShouldBeTrue(); - var sharedSidecarPath = Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExtension}").Single(); - File.Delete(sharedSidecarPath); + Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExtension}").ShouldBeEmpty(); } await writeTask; From 9975f690481b6b37a7bc7b14d23c97e2ed555b32 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Wed, 2 Sep 2026 00:45:18 +0100 Subject: [PATCH 14/26] fix: preserve timed-out aggregation sidecars Excluded sidecars remain durable during bounded lock waits, then become visible for sibling or deferred merges without racing summary suppression. --- .../Reporters/Aggregation/ReportAggregator.cs | 15 +++--- .../Reporters/Aggregation/ReportDataJson.cs | 1 + .../Reporters/Html/HtmlReporter.cs | 54 +++++++++++++------ src/TUnit.Reporting.Tool/Program.cs | 5 ++ .../HtmlReporterConfigurationTests.cs | 9 ++-- 5 files changed, 57 insertions(+), 27 deletions(-) diff --git a/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs b/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs index dfcf8724e55..383710f8cf6 100644 --- a/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs +++ b/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs @@ -38,7 +38,6 @@ internal sealed class ReportAggregator { private const string SidecarSearchPattern = "*" + ReportDataJson.SidecarExtension; private const string LockFileName = ".tunit-aggregate.lock"; - private const string DisabledMarkerExtension = ".disabled"; // Bound lock contention to roughly ten seconds. Reporting must never hang test-suite // completion; timed-out writers leave their sidecar for a later aggregate refresh. @@ -141,15 +140,15 @@ internal void DeleteSidecar(string assemblyName, string suiteSalt) File.Delete(GetSidecarPath(assemblyName, suiteSalt)); } - internal void MarkSidecarDisabled(string assemblyName, string suiteSalt) + internal void ExcludeSidecar(string assemblyName, string suiteSalt) { System.IO.Directory.CreateDirectory(Directory); - AtomicFile.WriteAllBytes(GetDisabledMarkerPath(assemblyName, suiteSalt), []); + AtomicFile.WriteAllBytes(GetExclusionMarkerPath(assemblyName, suiteSalt), []); } - internal void ClearDisabledMarker(string assemblyName, string suiteSalt) + internal void IncludeSidecar(string assemblyName, string suiteSalt) { - File.Delete(GetDisabledMarkerPath(assemblyName, suiteSalt)); + File.Delete(GetExclusionMarkerPath(assemblyName, suiteSalt)); } /// @@ -174,7 +173,7 @@ internal List ReadAllSidecars() { try { - if (File.Exists(file + DisabledMarkerExtension)) + if (File.Exists(file + ReportDataJson.SidecarExclusionExtension)) { continue; } @@ -250,9 +249,9 @@ private static string ShortHash(string value) return BitConverter.ToString(hash, 0, 4).Replace("-", "").ToLowerInvariant(); } - private string GetDisabledMarkerPath(string assemblyName, string suiteSalt) + private string GetExclusionMarkerPath(string assemblyName, string suiteSalt) { - return GetSidecarPath(assemblyName, suiteSalt) + DisabledMarkerExtension; + return GetSidecarPath(assemblyName, suiteSalt) + ReportDataJson.SidecarExclusionExtension; } private string GetSidecarPath(string assemblyName, string suiteSalt) diff --git a/src/TUnit.Engine/Reporters/Aggregation/ReportDataJson.cs b/src/TUnit.Engine/Reporters/Aggregation/ReportDataJson.cs index ee2ebecf3dd..26ad8d6ce51 100644 --- a/src/TUnit.Engine/Reporters/Aggregation/ReportDataJson.cs +++ b/src/TUnit.Engine/Reporters/Aggregation/ReportDataJson.cs @@ -25,6 +25,7 @@ internal static class ReportDataJson /// File extension shared by every sidecar so aggregators can discover them. internal const string SidecarExtension = ".tunit-report.json"; + internal const string SidecarExclusionExtension = ".excluded"; /// Merged HTML report filename, shared by the engine and the tool's default output. internal const string MergedReportFileName = "merged-report.html"; diff --git a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs index 52fb3e20b35..828724afef9 100644 --- a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs +++ b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs @@ -237,23 +237,47 @@ internal async Task TryWriteSidecarAndAggregateAsync(ReportData reportData, stri try { - using var aggregationLock = await aggregator.AcquireLockAsync(cancellationToken); - if (aggregationLock is null) + // Keep the sidecar invisible while lock ownership is unresolved. If the wait + // times out, the bytes still exist for a later/deferred merge without letting + // a concurrent merge race summary-suppression state. + aggregator.ExcludeSidecar(reportData.AssemblyName, htmlOutputPath); + var sharedSidecarPath = aggregator.WriteSidecar(sidecarBytes, reportData.AssemblyName, suiteSalt: htmlOutputPath); + + IDisposable? aggregationLock; + try { - return; + aggregationLock = await aggregator.AcquireLockAsync(cancellationToken); } - - aggregator.WriteSidecar(sidecarBytes, reportData.AssemblyName, suiteSalt: htmlOutputPath); - aggregator.ClearDisabledMarker(reportData.AssemblyName, htmlOutputPath); - - // Once the shared sidecar is durable and enabled, later sibling refreshes own - // the summary even if this process fails while refreshing it. - if (_githubReporter is not null) + catch (OperationCanceledException) { - _githubReporter.SuppressPerSuiteSummary = true; + // Publication already started. Complete the timeout-style handoff so the + // durable sidecar is not left permanently excluded. + aggregationLock = null; } - RefreshAggregatedOutputs(aggregator); + using (aggregationLock) + { + if (!File.Exists(sharedSidecarPath)) + { + aggregator.WriteSidecar(sidecarBytes, reportData.AssemblyName, suiteSalt: htmlOutputPath); + } + + // Suppress before making the sidecar visible: a sibling can merge it as soon + // as the exclusion marker disappears. + if (_githubReporter is not null) + { + _githubReporter.SuppressPerSuiteSummary = true; + } + + aggregator.IncludeSidecar(reportData.AssemblyName, htmlOutputPath); + + if (aggregationLock is null) + { + return; + } + + RefreshAggregatedOutputs(aggregator); + } } catch (Exception ex) { @@ -277,7 +301,7 @@ private async Task DeleteSidecarsAndRefreshAggregateAsync( { // Durable marker prevents every later merge from reading stale suite data even // when this process cannot acquire the lock to delete it immediately. - aggregator.MarkSidecarDisabled(assemblyName, htmlOutputPath); + aggregator.ExcludeSidecar(assemblyName, htmlOutputPath); // Cleanup must survive session cancellation or stale enabled-run sidecars can // re-enter a sibling's aggregate. Lock acquisition remains time-bounded. @@ -286,13 +310,13 @@ private async Task DeleteSidecarsAndRefreshAggregateAsync( { // An enabled writer may have cleared the first marker while holding the // lock. Reassert disabled state because this cleanup completed later. - aggregator.MarkSidecarDisabled(assemblyName, htmlOutputPath); + aggregator.ExcludeSidecar(assemblyName, htmlOutputPath); return; } aggregator.DeleteSidecar(assemblyName, htmlOutputPath); RefreshAggregatedOutputs(aggregator); - aggregator.ClearDisabledMarker(assemblyName, htmlOutputPath); + aggregator.IncludeSidecar(assemblyName, htmlOutputPath); } catch (Exception ex) when (ex is not OperationCanceledException) { diff --git a/src/TUnit.Reporting.Tool/Program.cs b/src/TUnit.Reporting.Tool/Program.cs index b05183e5dc0..732a6b9f008 100644 --- a/src/TUnit.Reporting.Tool/Program.cs +++ b/src/TUnit.Reporting.Tool/Program.cs @@ -155,6 +155,11 @@ private static (List Suites, int Skipped) LoadSidecars(string direct var enumeration = new EnumerationOptions { RecurseSubdirectories = true, IgnoreInaccessible = true }; foreach (var file in Directory.EnumerateFiles(directory, "*" + ReportDataJson.SidecarExtension, enumeration)) { + if (File.Exists(file + ReportDataJson.SidecarExclusionExtension)) + { + continue; + } + byte[] bytes; try { diff --git a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs index e2aa49d8c6f..e86b4b21cad 100644 --- a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs +++ b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs @@ -311,10 +311,10 @@ public async Task Disabled_Marker_Excludes_Stale_Shared_Sidecar(CancellationToke var reportData = CreateReportData("DisabledSuiteMarker"); aggregator.WriteSidecar(ReportDataJson.SerializeToBytes(reportData), reportData.AssemblyName, "suite"); - aggregator.MarkSidecarDisabled(reportData.AssemblyName, "suite"); + aggregator.ExcludeSidecar(reportData.AssemblyName, "suite"); aggregator.ReadAllSidecars().ShouldBeEmpty(); - aggregator.ClearDisabledMarker(reportData.AssemblyName, "suite"); + aggregator.IncludeSidecar(reportData.AssemblyName, "suite"); aggregator.ReadAllSidecars().Single().AssemblyName.ShouldBe("DisabledSuiteMarker"); } finally @@ -367,7 +367,7 @@ public async Task Disabled_Html_Report_Removes_Stale_Aggregation_Outputs(Cancell } [Test] - public async Task Shared_Sidecar_Is_Published_Only_After_Aggregation_Lock(CancellationToken cancellationToken) + public async Task Shared_Sidecar_Remains_Excluded_Until_Lock_Wait_Completes(CancellationToken cancellationToken) { var tempDirectory = Path.Combine(Path.GetTempPath(), $"tunit-report-test-{Guid.NewGuid():N}"); var aggregationDirectory = Path.Combine(tempDirectory, "aggregate"); @@ -389,7 +389,8 @@ public async Task Shared_Sidecar_Is_Published_Only_After_Aggregation_Lock(Cancel writeTask = reporter.TryWriteSidecarAndAggregateAsync(CreateReportData(), htmlPath, cancellationToken); File.Exists(HtmlReporter.GetSidecarPath(htmlPath)).ShouldBeTrue(); - Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExtension}").ShouldBeEmpty(); + Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExtension}").Length.ShouldBe(1); + aggregator.ReadAllSidecars().ShouldBeEmpty(); } await writeTask; From 7fd08a7c657e02c32cdfb8793ea0e9f643f0eab7 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Thu, 3 Sep 2026 11:15:32 +0100 Subject: [PATCH 15/26] fix(reporting): harden sidecar recovery --- .../Reporters/Aggregation/ReportAggregator.cs | 52 +++++++++- .../Reporters/Aggregation/ReportDataJson.cs | 28 ++++++ .../Reporters/Html/HtmlReporter.cs | 37 ++++--- src/TUnit.Reporting.Tool/Program.cs | 4 +- .../HtmlReporterConfigurationTests.cs | 98 +++++++++++++++++++ 5 files changed, 203 insertions(+), 16 deletions(-) diff --git a/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs b/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs index 383710f8cf6..5b4e4e6765d 100644 --- a/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs +++ b/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs @@ -151,6 +151,27 @@ internal void IncludeSidecar(string assemblyName, string suiteSalt) File.Delete(GetExclusionMarkerPath(assemblyName, suiteSalt)); } + internal IDisposable BeginSidecarPublication(string assemblyName, string suiteSalt) + { + System.IO.Directory.CreateDirectory(Directory); + var markerPath = GetPublishingMarkerPath(assemblyName, suiteSalt); + var stream = new FileStream(markerPath, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None); + return new PublicationMarker(stream, markerPath); + } + + internal bool HasSidecarState(string assemblyName, string suiteSalt) + { + if (!System.IO.Directory.Exists(Directory)) + { + return false; + } + + var sidecarPath = GetSidecarPath(assemblyName, suiteSalt); + return File.Exists(sidecarPath) + || File.Exists(sidecarPath + ReportDataJson.SidecarExclusionExtension) + || File.Exists(sidecarPath + ReportDataJson.SidecarPublishingExtension); + } + /// /// Reads every sidecar currently present in the shared directory. Unreadable or /// foreign files are skipped — a crashed sibling must not break the merge. @@ -173,7 +194,7 @@ internal List ReadAllSidecars() { try { - if (File.Exists(file + ReportDataJson.SidecarExclusionExtension)) + if (ReportDataJson.ShouldSkipSidecar(file)) { continue; } @@ -195,6 +216,30 @@ internal List ReadAllSidecars() return results; } + private sealed class PublicationMarker(FileStream stream, string path) : IDisposable + { + private FileStream? _stream = stream; + + public void Dispose() + { + var ownedStream = Interlocked.Exchange(ref _stream, null); + if (ownedStream is null) + { + return; + } + + ownedStream.Dispose(); + try + { + File.Delete(path); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + // A reader can recover the unlocked marker after this best-effort delete. + } + } + } + /// /// Acquires the cross-process aggregation lock. Every writer performs its whole /// read-merge-write cycle under this lock, so merges never interleave. Returns @@ -254,6 +299,11 @@ private string GetExclusionMarkerPath(string assemblyName, string suiteSalt) return GetSidecarPath(assemblyName, suiteSalt) + ReportDataJson.SidecarExclusionExtension; } + private string GetPublishingMarkerPath(string assemblyName, string suiteSalt) + { + return GetSidecarPath(assemblyName, suiteSalt) + ReportDataJson.SidecarPublishingExtension; + } + private string GetSidecarPath(string assemblyName, string suiteSalt) { var fileName = $"{PathValidator.SanitizeFileName(assemblyName)}-{ShortHash(suiteSalt)}{ReportDataJson.SidecarExtension}"; diff --git a/src/TUnit.Engine/Reporters/Aggregation/ReportDataJson.cs b/src/TUnit.Engine/Reporters/Aggregation/ReportDataJson.cs index 26ad8d6ce51..81524904c2b 100644 --- a/src/TUnit.Engine/Reporters/Aggregation/ReportDataJson.cs +++ b/src/TUnit.Engine/Reporters/Aggregation/ReportDataJson.cs @@ -26,10 +26,38 @@ internal static class ReportDataJson /// File extension shared by every sidecar so aggregators can discover them. internal const string SidecarExtension = ".tunit-report.json"; internal const string SidecarExclusionExtension = ".excluded"; + internal const string SidecarPublishingExtension = ".publishing"; /// Merged HTML report filename, shared by the engine and the tool's default output. internal const string MergedReportFileName = "merged-report.html"; + internal static bool ShouldSkipSidecar(string sidecarPath) + { + if (File.Exists(sidecarPath + SidecarExclusionExtension)) + { + return true; + } + + var publishingMarkerPath = sidecarPath + SidecarPublishingExtension; + if (!File.Exists(publishingMarkerPath)) + { + return false; + } + + try + { + using (new FileStream(publishingMarkerPath, FileMode.Open, FileAccess.ReadWrite, FileShare.None)) + { + } + File.Delete(publishingMarkerPath); + return false; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return true; + } + } + /// /// Serializes straight to UTF-8 bytes — callers write the same payload to more than one /// file, so producing bytes once avoids a UTF-8 → string → UTF-8 round trip per copy. diff --git a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs index 828724afef9..80ff3535c50 100644 --- a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs +++ b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs @@ -152,6 +152,16 @@ await DeleteSidecarsAndRefreshAggregateAsync( #if NET TraceRegistry.Clear(); #endif + if (!IsJsonReportEnabled()) + { + var emptyOutputPath = _outputPath ?? GetDefaultOutputPath(); + var aggregator = ReportAggregator.TryCreateFromEnvironment(Environment.GetEnvironmentVariable); + await DeleteSidecarsAndRefreshAggregateAsync( + GetAssemblyName(), + emptyOutputPath, + aggregator); + } + return; } @@ -237,10 +247,9 @@ internal async Task TryWriteSidecarAndAggregateAsync(ReportData reportData, stri try { - // Keep the sidecar invisible while lock ownership is unresolved. If the wait - // times out, the bytes still exist for a later/deferred merge without letting - // a concurrent merge race summary-suppression state. - aggregator.ExcludeSidecar(reportData.AssemblyName, htmlOutputPath); + // Atomic bytes stay hidden while publication ownership is active. A later + // process can recover the marker immediately if this publisher terminates. + using var publicationMarker = aggregator.BeginSidecarPublication(reportData.AssemblyName, htmlOutputPath); var sharedSidecarPath = aggregator.WriteSidecar(sidecarBytes, reportData.AssemblyName, suiteSalt: htmlOutputPath); IDisposable? aggregationLock; @@ -250,8 +259,6 @@ internal async Task TryWriteSidecarAndAggregateAsync(ReportData reportData, stri } catch (OperationCanceledException) { - // Publication already started. Complete the timeout-style handoff so the - // durable sidecar is not left permanently excluded. aggregationLock = null; } @@ -262,14 +269,7 @@ internal async Task TryWriteSidecarAndAggregateAsync(ReportData reportData, stri aggregator.WriteSidecar(sidecarBytes, reportData.AssemblyName, suiteSalt: htmlOutputPath); } - // Suppress before making the sidecar visible: a sibling can merge it as soon - // as the exclusion marker disappears. - if (_githubReporter is not null) - { - _githubReporter.SuppressPerSuiteSummary = true; - } - - aggregator.IncludeSidecar(reportData.AssemblyName, htmlOutputPath); + publicationMarker.Dispose(); if (aggregationLock is null) { @@ -277,6 +277,10 @@ internal async Task TryWriteSidecarAndAggregateAsync(ReportData reportData, stri } RefreshAggregatedOutputs(aggregator); + if (_githubReporter is not null) + { + _githubReporter.SuppressPerSuiteSummary = true; + } } } catch (Exception ex) @@ -297,6 +301,11 @@ private async Task DeleteSidecarsAndRefreshAggregateAsync( return; } + if (!aggregator.HasSidecarState(assemblyName, htmlOutputPath)) + { + return; + } + try { // Durable marker prevents every later merge from reading stale suite data even diff --git a/src/TUnit.Reporting.Tool/Program.cs b/src/TUnit.Reporting.Tool/Program.cs index 732a6b9f008..9847528f3c8 100644 --- a/src/TUnit.Reporting.Tool/Program.cs +++ b/src/TUnit.Reporting.Tool/Program.cs @@ -155,7 +155,9 @@ private static (List Suites, int Skipped) LoadSidecars(string direct var enumeration = new EnumerationOptions { RecurseSubdirectories = true, IgnoreInaccessible = true }; foreach (var file in Directory.EnumerateFiles(directory, "*" + ReportDataJson.SidecarExtension, enumeration)) { - if (File.Exists(file + ReportDataJson.SidecarExclusionExtension)) + // This command is the designated final merge, so no publisher should still + // be active. Recover any publication marker left by a terminated process. + if (ReportDataJson.ShouldSkipSidecar(file)) { continue; } diff --git a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs index e86b4b21cad..c5bee7bce22 100644 --- a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs +++ b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs @@ -407,6 +407,104 @@ public async Task Shared_Sidecar_Remains_Excluded_Until_Lock_Wait_Completes(Canc } } + [Test] + public async Task Empty_Session_With_Disabled_Json_Removes_Stale_Sidecars(CancellationToken cancellationToken) + { + var tempDirectory = Path.Combine(Path.GetTempPath(), $"tunit-report-test-{Guid.NewGuid():N}"); + var aggregationDirectory = Path.Combine(tempDirectory, "aggregate"); + var htmlPath = Path.Combine(tempDirectory, "suite-report.html"); + Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_REPORTS", "true"); + Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_DIR", aggregationDirectory); + + try + { + Directory.CreateDirectory(tempDirectory); + using var reporter = new HtmlReporter(new MockExtension()); + reporter.SetOutputPath(htmlPath); + var assemblyName = reporter.BuildReportData().AssemblyName; + + await reporter.TryWriteSidecarAndAggregateAsync(CreateReportData(assemblyName), htmlPath, cancellationToken); + TUnitSettings.Default.Reporting.JsonReportEnabled = false; + + await reporter.OnTestSessionFinishingAsync(null!); + + File.Exists(HtmlReporter.GetSidecarPath(htmlPath)).ShouldBeFalse(); + Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExtension}").ShouldBeEmpty(); + } + finally + { + if (Directory.Exists(tempDirectory)) + { + Directory.Delete(tempDirectory, recursive: true); + } + } + } + + [Test] + public async Task Abandoned_Publication_Marker_Is_Recovered(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var tempDirectory = Path.Combine(Path.GetTempPath(), $"tunit-report-test-{Guid.NewGuid():N}"); + Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_REPORTS", "true"); + Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_DIR", tempDirectory); + + try + { + var aggregator = ReportAggregator.TryCreateFromEnvironment(Environment.GetEnvironmentVariable)!; + var reportData = CreateReportData(); + using var publicationMarker = aggregator.BeginSidecarPublication(reportData.AssemblyName, "suite"); + var sidecarPath = aggregator.WriteSidecar(ReportDataJson.SerializeToBytes(reportData), reportData.AssemblyName, "suite"); + + aggregator.ReadAllSidecars().ShouldBeEmpty(); + publicationMarker.Dispose(); + File.WriteAllBytes(sidecarPath + ReportDataJson.SidecarPublishingExtension, []); + + aggregator.ReadAllSidecars().Single().AssemblyName.ShouldBe("Tests"); + File.Exists(sidecarPath + ReportDataJson.SidecarPublishingExtension).ShouldBeFalse(); + } + finally + { + if (Directory.Exists(tempDirectory)) + { + Directory.Delete(tempDirectory, recursive: true); + } + } + } + + [Test] + public async Task Cancelled_Lock_Wait_Keeps_Per_Suite_Summary(CancellationToken cancellationToken) + { + var tempDirectory = Path.Combine(Path.GetTempPath(), $"tunit-report-test-{Guid.NewGuid():N}"); + Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_REPORTS", "true"); + Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_DIR", tempDirectory); + + try + { + using var reporter = new HtmlReporter(new MockExtension()); + var githubReporter = new GitHubReporter(new MockExtension()); + reporter.SetGitHubReporter(githubReporter); + var aggregator = ReportAggregator.TryCreateFromEnvironment(Environment.GetEnvironmentVariable)!; + using var aggregationLock = await aggregator.AcquireLockAsync(cancellationToken); + using var cancelled = new CancellationTokenSource(); + cancelled.Cancel(); + + await reporter.TryWriteSidecarAndAggregateAsync( + CreateReportData(), + Path.Combine(tempDirectory, "suite-report.html"), + cancelled.Token); + + githubReporter.SuppressPerSuiteSummary.ShouldBeFalse(); + aggregator.ReadAllSidecars().Single().AssemblyName.ShouldBe("Tests"); + } + finally + { + if (Directory.Exists(tempDirectory)) + { + Directory.Delete(tempDirectory, recursive: true); + } + } + } + private static ReportData CreateReportData(string assemblyName = "Tests") => new() { AssemblyName = assemblyName, From 27e5bfa8441add8a326171e2636dd8ec80202154 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Thu, 3 Sep 2026 11:42:01 +0100 Subject: [PATCH 16/26] fix(reporting): revive enabled sidecars --- src/TUnit.Engine/Reporters/Html/HtmlReporter.cs | 4 ++++ tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs index 80ff3535c50..b1382b5cdcf 100644 --- a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs +++ b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs @@ -269,6 +269,10 @@ internal async Task TryWriteSidecarAndAggregateAsync(ReportData reportData, stri aggregator.WriteSidecar(sidecarBytes, reportData.AssemblyName, suiteSalt: htmlOutputPath); } + // A disabled predecessor may have left a durable exclusion after its + // cleanup timed out. The new bytes become authoritative when this + // publication is exposed, even if aggregate refresh was cancelled. + aggregator.IncludeSidecar(reportData.AssemblyName, htmlOutputPath); publicationMarker.Dispose(); if (aggregationLock is null) diff --git a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs index c5bee7bce22..03216d55ac5 100644 --- a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs +++ b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs @@ -487,10 +487,12 @@ public async Task Cancelled_Lock_Wait_Keeps_Per_Suite_Summary(CancellationToken using var aggregationLock = await aggregator.AcquireLockAsync(cancellationToken); using var cancelled = new CancellationTokenSource(); cancelled.Cancel(); + var htmlPath = Path.Combine(tempDirectory, "suite-report.html"); + aggregator.ExcludeSidecar("Tests", htmlPath); await reporter.TryWriteSidecarAndAggregateAsync( CreateReportData(), - Path.Combine(tempDirectory, "suite-report.html"), + htmlPath, cancelled.Token); githubReporter.SuppressPerSuiteSummary.ShouldBeFalse(); From 4a0cbe0b34681962c1e45ff9508aaff590b1e928 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Thu, 3 Sep 2026 11:49:06 +0100 Subject: [PATCH 17/26] fix(reporting): serialize sidecar cleanup --- .../Reporters/Aggregation/ReportAggregator.cs | 3 ++ .../Reporters/Html/HtmlReporter.cs | 32 ++++++++--------- .../HtmlReporterConfigurationTests.cs | 36 +++++++++++++++++-- 3 files changed, 53 insertions(+), 18 deletions(-) diff --git a/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs b/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs index 5b4e4e6765d..b4e80683998 100644 --- a/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs +++ b/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs @@ -151,6 +151,9 @@ internal void IncludeSidecar(string assemblyName, string suiteSalt) File.Delete(GetExclusionMarkerPath(assemblyName, suiteSalt)); } + internal bool IsSidecarExcluded(string assemblyName, string suiteSalt) => + File.Exists(GetExclusionMarkerPath(assemblyName, suiteSalt)); + internal IDisposable BeginSidecarPublication(string assemblyName, string suiteSalt) { System.IO.Directory.CreateDirectory(Directory); diff --git a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs index b1382b5cdcf..fa138799765 100644 --- a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs +++ b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs @@ -247,10 +247,9 @@ internal async Task TryWriteSidecarAndAggregateAsync(ReportData reportData, stri try { - // Atomic bytes stay hidden while publication ownership is active. A later - // process can recover the marker immediately if this publisher terminates. + // Shared publication is serialized with cleanup. A later process can + // recover the marker immediately if this publisher terminates. using var publicationMarker = aggregator.BeginSidecarPublication(reportData.AssemblyName, htmlOutputPath); - var sharedSidecarPath = aggregator.WriteSidecar(sidecarBytes, reportData.AssemblyName, suiteSalt: htmlOutputPath); IDisposable? aggregationLock; try @@ -262,24 +261,20 @@ internal async Task TryWriteSidecarAndAggregateAsync(ReportData reportData, stri aggregationLock = null; } - using (aggregationLock) + if (aggregationLock is null) { - if (!File.Exists(sharedSidecarPath)) - { - aggregator.WriteSidecar(sidecarBytes, reportData.AssemblyName, suiteSalt: htmlOutputPath); - } + return; + } + using (aggregationLock) + { // A disabled predecessor may have left a durable exclusion after its - // cleanup timed out. The new bytes become authoritative when this - // publication is exposed, even if aggregate refresh was cancelled. + // cleanup timed out. Publish and expose the replacement atomically + // with respect to cleanup so that predecessor cannot delete it. + aggregator.WriteSidecar(sidecarBytes, reportData.AssemblyName, suiteSalt: htmlOutputPath); aggregator.IncludeSidecar(reportData.AssemblyName, htmlOutputPath); publicationMarker.Dispose(); - if (aggregationLock is null) - { - return; - } - RefreshAggregatedOutputs(aggregator); if (_githubReporter is not null) { @@ -327,7 +322,12 @@ private async Task DeleteSidecarsAndRefreshAggregateAsync( return; } - aggregator.DeleteSidecar(assemblyName, htmlOutputPath); + // An enabled publisher that acquired the lock first clears this marker. + // In that case its fresh sidecar supersedes this older cleanup request. + if (aggregator.IsSidecarExcluded(assemblyName, htmlOutputPath)) + { + aggregator.DeleteSidecar(assemblyName, htmlOutputPath); + } RefreshAggregatedOutputs(aggregator); aggregator.IncludeSidecar(assemblyName, htmlOutputPath); } diff --git a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs index 03216d55ac5..f5644007947 100644 --- a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs +++ b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs @@ -389,7 +389,8 @@ public async Task Shared_Sidecar_Remains_Excluded_Until_Lock_Wait_Completes(Canc writeTask = reporter.TryWriteSidecarAndAggregateAsync(CreateReportData(), htmlPath, cancellationToken); File.Exists(HtmlReporter.GetSidecarPath(htmlPath)).ShouldBeTrue(); - Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExtension}").Length.ShouldBe(1); + Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExtension}").ShouldBeEmpty(); + Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarPublishingExtension}").Length.ShouldBe(1); aggregator.ReadAllSidecars().ShouldBeEmpty(); } @@ -475,8 +476,9 @@ public async Task Abandoned_Publication_Marker_Is_Recovered(CancellationToken ca public async Task Cancelled_Lock_Wait_Keeps_Per_Suite_Summary(CancellationToken cancellationToken) { var tempDirectory = Path.Combine(Path.GetTempPath(), $"tunit-report-test-{Guid.NewGuid():N}"); + var aggregationDirectory = Path.Combine(tempDirectory, "aggregate"); Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_REPORTS", "true"); - Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_DIR", tempDirectory); + Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_DIR", aggregationDirectory); try { @@ -496,7 +498,37 @@ await reporter.TryWriteSidecarAndAggregateAsync( cancelled.Token); githubReporter.SuppressPerSuiteSummary.ShouldBeFalse(); + aggregator.ReadAllSidecars().ShouldBeEmpty(); + Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExclusionExtension}").Length.ShouldBe(1); + } + finally + { + if (Directory.Exists(tempDirectory)) + { + Directory.Delete(tempDirectory, recursive: true); + } + } + } + + [Test] + public async Task Enabled_Publication_Clears_Stale_Exclusion(CancellationToken cancellationToken) + { + var tempDirectory = Path.Combine(Path.GetTempPath(), $"tunit-report-test-{Guid.NewGuid():N}"); + var aggregationDirectory = Path.Combine(tempDirectory, "aggregate"); + var htmlPath = Path.Combine(tempDirectory, "suite-report.html"); + Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_REPORTS", "true"); + Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_DIR", aggregationDirectory); + + try + { + using var reporter = new HtmlReporter(new MockExtension()); + var aggregator = ReportAggregator.TryCreateFromEnvironment(Environment.GetEnvironmentVariable)!; + aggregator.ExcludeSidecar("Tests", htmlPath); + + await reporter.TryWriteSidecarAndAggregateAsync(CreateReportData(), htmlPath, cancellationToken); + aggregator.ReadAllSidecars().Single().AssemblyName.ShouldBe("Tests"); + Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExclusionExtension}").ShouldBeEmpty(); } finally { From dd1c6f6222301e3bfae6e1e602120bc5198947ac Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Thu, 3 Sep 2026 11:55:29 +0100 Subject: [PATCH 18/26] fix(reporting): persist sidecars before merge --- .../Reporters/Aggregation/ReportAggregator.cs | 30 +++++++++++-- .../Reporters/Html/HtmlReporter.cs | 42 +++++++++++-------- .../HtmlReporterConfigurationTests.cs | 9 ++-- 3 files changed, 57 insertions(+), 24 deletions(-) diff --git a/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs b/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs index b4e80683998..297e04383b8 100644 --- a/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs +++ b/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs @@ -151,9 +151,6 @@ internal void IncludeSidecar(string assemblyName, string suiteSalt) File.Delete(GetExclusionMarkerPath(assemblyName, suiteSalt)); } - internal bool IsSidecarExcluded(string assemblyName, string suiteSalt) => - File.Exists(GetExclusionMarkerPath(assemblyName, suiteSalt)); - internal IDisposable BeginSidecarPublication(string assemblyName, string suiteSalt) { System.IO.Directory.CreateDirectory(Directory); @@ -162,6 +159,33 @@ internal IDisposable BeginSidecarPublication(string assemblyName, string suiteSa return new PublicationMarker(stream, markerPath); } + internal async Task AcquireSidecarPublicationAsync( + string assemblyName, + string suiteSalt, + CancellationToken cancellationToken) + { + for (var attempt = 1; attempt <= LockMaxAttempts; attempt++) + { + cancellationToken.ThrowIfCancellationRequested(); + try + { + return BeginSidecarPublication(assemblyName, suiteSalt); + } + catch (IOException) + { + if (attempt == LockMaxAttempts) + { + break; + } + + await Task.Delay(LockRetryDelayMs + Random.Shared.Next(0, 100), cancellationToken); + } + } + + Console.WriteLine("Warning: Report sidecar publication lock timed out; keeping the local per-suite report."); + return null; + } + internal bool HasSidecarState(string assemblyName, string suiteSalt) { if (!System.IO.Directory.Exists(Directory)) diff --git a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs index fa138799765..428881f0626 100644 --- a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs +++ b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs @@ -247,9 +247,20 @@ internal async Task TryWriteSidecarAndAggregateAsync(ReportData reportData, stri try { - // Shared publication is serialized with cleanup. A later process can - // recover the marker immediately if this publisher terminates. - using var publicationMarker = aggregator.BeginSidecarPublication(reportData.AssemblyName, htmlOutputPath); + // Serialize shared publication with cleanup. Persist and expose the latest + // sidecar before waiting for the aggregate lock, so a cancelled or timed-out + // refresh still leaves evidence for a later merge. + using var publicationMarker = await aggregator.AcquireSidecarPublicationAsync( + reportData.AssemblyName, + htmlOutputPath, + CancellationToken.None); + if (publicationMarker is null) + { + return; + } + + aggregator.WriteSidecar(sidecarBytes, reportData.AssemblyName, suiteSalt: htmlOutputPath); + aggregator.IncludeSidecar(reportData.AssemblyName, htmlOutputPath); IDisposable? aggregationLock; try @@ -268,11 +279,6 @@ internal async Task TryWriteSidecarAndAggregateAsync(ReportData reportData, stri using (aggregationLock) { - // A disabled predecessor may have left a durable exclusion after its - // cleanup timed out. Publish and expose the replacement atomically - // with respect to cleanup so that predecessor cannot delete it. - aggregator.WriteSidecar(sidecarBytes, reportData.AssemblyName, suiteSalt: htmlOutputPath); - aggregator.IncludeSidecar(reportData.AssemblyName, htmlOutputPath); publicationMarker.Dispose(); RefreshAggregatedOutputs(aggregator); @@ -307,6 +313,16 @@ private async Task DeleteSidecarsAndRefreshAggregateAsync( try { + using var publicationMarker = await aggregator.AcquireSidecarPublicationAsync( + assemblyName, + htmlOutputPath, + CancellationToken.None); + if (publicationMarker is null) + { + aggregator.ExcludeSidecar(assemblyName, htmlOutputPath); + return; + } + // Durable marker prevents every later merge from reading stale suite data even // when this process cannot acquire the lock to delete it immediately. aggregator.ExcludeSidecar(assemblyName, htmlOutputPath); @@ -316,18 +332,10 @@ private async Task DeleteSidecarsAndRefreshAggregateAsync( using var aggregationLock = await aggregator.AcquireLockAsync(CancellationToken.None); if (aggregationLock is null) { - // An enabled writer may have cleared the first marker while holding the - // lock. Reassert disabled state because this cleanup completed later. - aggregator.ExcludeSidecar(assemblyName, htmlOutputPath); return; } - // An enabled publisher that acquired the lock first clears this marker. - // In that case its fresh sidecar supersedes this older cleanup request. - if (aggregator.IsSidecarExcluded(assemblyName, htmlOutputPath)) - { - aggregator.DeleteSidecar(assemblyName, htmlOutputPath); - } + aggregator.DeleteSidecar(assemblyName, htmlOutputPath); RefreshAggregatedOutputs(aggregator); aggregator.IncludeSidecar(assemblyName, htmlOutputPath); } diff --git a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs index f5644007947..78a701e00bf 100644 --- a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs +++ b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs @@ -367,7 +367,7 @@ public async Task Disabled_Html_Report_Removes_Stale_Aggregation_Outputs(Cancell } [Test] - public async Task Shared_Sidecar_Remains_Excluded_Until_Lock_Wait_Completes(CancellationToken cancellationToken) + public async Task Shared_Sidecar_Stays_Hidden_Until_Lock_Wait_Completes(CancellationToken cancellationToken) { var tempDirectory = Path.Combine(Path.GetTempPath(), $"tunit-report-test-{Guid.NewGuid():N}"); var aggregationDirectory = Path.Combine(tempDirectory, "aggregate"); @@ -389,7 +389,7 @@ public async Task Shared_Sidecar_Remains_Excluded_Until_Lock_Wait_Completes(Canc writeTask = reporter.TryWriteSidecarAndAggregateAsync(CreateReportData(), htmlPath, cancellationToken); File.Exists(HtmlReporter.GetSidecarPath(htmlPath)).ShouldBeTrue(); - Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExtension}").ShouldBeEmpty(); + Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExtension}").Length.ShouldBe(1); Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarPublishingExtension}").Length.ShouldBe(1); aggregator.ReadAllSidecars().ShouldBeEmpty(); } @@ -498,8 +498,9 @@ await reporter.TryWriteSidecarAndAggregateAsync( cancelled.Token); githubReporter.SuppressPerSuiteSummary.ShouldBeFalse(); - aggregator.ReadAllSidecars().ShouldBeEmpty(); - Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExclusionExtension}").Length.ShouldBe(1); + aggregator.ReadAllSidecars().Single().AssemblyName.ShouldBe("Tests"); + Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExclusionExtension}").ShouldBeEmpty(); + File.Exists(Path.Combine(aggregationDirectory, ReportDataJson.MergedReportFileName)).ShouldBeFalse(); } finally { From f359c6378c21d461eb39b591c1e18b991d59bbfb Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Thu, 3 Sep 2026 12:40:57 +0100 Subject: [PATCH 19/26] fix(reporting): keep publication lock stable --- .../Reporters/Aggregation/ReportAggregator.cs | 29 ++----------------- .../Reporters/Aggregation/ReportDataJson.cs | 9 +++--- src/TUnit.Reporting.Tool/Program.cs | 2 +- .../HtmlReporterConfigurationTests.cs | 14 +++++---- 4 files changed, 17 insertions(+), 37 deletions(-) diff --git a/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs b/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs index 297e04383b8..2eb4c57ab47 100644 --- a/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs +++ b/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs @@ -154,9 +154,8 @@ internal void IncludeSidecar(string assemblyName, string suiteSalt) internal IDisposable BeginSidecarPublication(string assemblyName, string suiteSalt) { System.IO.Directory.CreateDirectory(Directory); - var markerPath = GetPublishingMarkerPath(assemblyName, suiteSalt); - var stream = new FileStream(markerPath, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None); - return new PublicationMarker(stream, markerPath); + var lockPath = GetPublishingMarkerPath(assemblyName, suiteSalt); + return new FileStream(lockPath, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None); } internal async Task AcquireSidecarPublicationAsync( @@ -243,30 +242,6 @@ internal List ReadAllSidecars() return results; } - private sealed class PublicationMarker(FileStream stream, string path) : IDisposable - { - private FileStream? _stream = stream; - - public void Dispose() - { - var ownedStream = Interlocked.Exchange(ref _stream, null); - if (ownedStream is null) - { - return; - } - - ownedStream.Dispose(); - try - { - File.Delete(path); - } - catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) - { - // A reader can recover the unlocked marker after this best-effort delete. - } - } - } - /// /// Acquires the cross-process aggregation lock. Every writer performs its whole /// read-merge-write cycle under this lock, so merges never interleave. Returns diff --git a/src/TUnit.Engine/Reporters/Aggregation/ReportDataJson.cs b/src/TUnit.Engine/Reporters/Aggregation/ReportDataJson.cs index 81524904c2b..7039e50887f 100644 --- a/src/TUnit.Engine/Reporters/Aggregation/ReportDataJson.cs +++ b/src/TUnit.Engine/Reporters/Aggregation/ReportDataJson.cs @@ -38,18 +38,19 @@ internal static bool ShouldSkipSidecar(string sidecarPath) return true; } - var publishingMarkerPath = sidecarPath + SidecarPublishingExtension; - if (!File.Exists(publishingMarkerPath)) + var publicationLockPath = sidecarPath + SidecarPublishingExtension; + if (!File.Exists(publicationLockPath)) { return false; } try { - using (new FileStream(publishingMarkerPath, FileMode.Open, FileAccess.ReadWrite, FileShare.None)) + // Lock file is stable: deleting it after releasing the handle lets another + // process lock the old inode while a third process creates and locks a new one. + using (new FileStream(publicationLockPath, FileMode.Open, FileAccess.ReadWrite, FileShare.None)) { } - File.Delete(publishingMarkerPath); return false; } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) diff --git a/src/TUnit.Reporting.Tool/Program.cs b/src/TUnit.Reporting.Tool/Program.cs index 9847528f3c8..3801ef33211 100644 --- a/src/TUnit.Reporting.Tool/Program.cs +++ b/src/TUnit.Reporting.Tool/Program.cs @@ -156,7 +156,7 @@ private static (List Suites, int Skipped) LoadSidecars(string direct foreach (var file in Directory.EnumerateFiles(directory, "*" + ReportDataJson.SidecarExtension, enumeration)) { // This command is the designated final merge, so no publisher should still - // be active. Recover any publication marker left by a terminated process. + // be active. Skip any suite whose stable publication lock is still held. if (ReportDataJson.ShouldSkipSidecar(file)) { continue; diff --git a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs index 78a701e00bf..b7b729c6b49 100644 --- a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs +++ b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs @@ -442,7 +442,7 @@ public async Task Empty_Session_With_Disabled_Json_Removes_Stale_Sidecars(Cancel } [Test] - public async Task Abandoned_Publication_Marker_Is_Recovered(CancellationToken cancellationToken) + public async Task Publication_Lock_File_Is_Stable_And_Reused(CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); var tempDirectory = Path.Combine(Path.GetTempPath(), $"tunit-report-test-{Guid.NewGuid():N}"); @@ -453,15 +453,19 @@ public async Task Abandoned_Publication_Marker_Is_Recovered(CancellationToken ca { var aggregator = ReportAggregator.TryCreateFromEnvironment(Environment.GetEnvironmentVariable)!; var reportData = CreateReportData(); - using var publicationMarker = aggregator.BeginSidecarPublication(reportData.AssemblyName, "suite"); + using var publicationLock = aggregator.BeginSidecarPublication(reportData.AssemblyName, "suite"); var sidecarPath = aggregator.WriteSidecar(ReportDataJson.SerializeToBytes(reportData), reportData.AssemblyName, "suite"); aggregator.ReadAllSidecars().ShouldBeEmpty(); - publicationMarker.Dispose(); - File.WriteAllBytes(sidecarPath + ReportDataJson.SidecarPublishingExtension, []); + publicationLock.Dispose(); aggregator.ReadAllSidecars().Single().AssemblyName.ShouldBe("Tests"); - File.Exists(sidecarPath + ReportDataJson.SidecarPublishingExtension).ShouldBeFalse(); + File.Exists(sidecarPath + ReportDataJson.SidecarPublishingExtension).ShouldBeTrue(); + + using (aggregator.BeginSidecarPublication(reportData.AssemblyName, "suite")) + { + aggregator.ReadAllSidecars().ShouldBeEmpty(); + } } finally { From 80782106ada3918e67eceba479b6de02f22a72c8 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Thu, 3 Sep 2026 13:37:36 +0100 Subject: [PATCH 20/26] fix(reporting): preserve fallback ordering Suppress per-suite summaries after durable defer publication. Defer disabled cleanup when the publication lock cannot be acquired. --- .../Reporters/Html/HtmlReporter.cs | 5 +- .../HtmlReporterConfigurationTests.cs | 73 +++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs index 428881f0626..0128a29ef98 100644 --- a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs +++ b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs @@ -261,6 +261,10 @@ internal async Task TryWriteSidecarAndAggregateAsync(ReportData reportData, stri aggregator.WriteSidecar(sidecarBytes, reportData.AssemblyName, suiteSalt: htmlOutputPath); aggregator.IncludeSidecar(reportData.AssemblyName, htmlOutputPath); + if (aggregator.Mode == AggregationMode.Defer && _githubReporter is not null) + { + _githubReporter.SuppressPerSuiteSummary = true; + } IDisposable? aggregationLock; try @@ -319,7 +323,6 @@ private async Task DeleteSidecarsAndRefreshAggregateAsync( CancellationToken.None); if (publicationMarker is null) { - aggregator.ExcludeSidecar(assemblyName, htmlOutputPath); return; } diff --git a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs index b7b729c6b49..1e054251d93 100644 --- a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs +++ b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs @@ -515,6 +515,79 @@ await reporter.TryWriteSidecarAndAggregateAsync( } } + [Test] + public async Task Cancelled_Defer_Lock_Wait_Suppresses_Per_Suite_Summary(CancellationToken cancellationToken) + { + var tempDirectory = Path.Combine(Path.GetTempPath(), $"tunit-report-test-{Guid.NewGuid():N}"); + var aggregationDirectory = Path.Combine(tempDirectory, "aggregate"); + Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_REPORTS", "defer"); + Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_DIR", aggregationDirectory); + + try + { + using var reporter = new HtmlReporter(new MockExtension()); + var githubReporter = new GitHubReporter(new MockExtension()); + reporter.SetGitHubReporter(githubReporter); + var aggregator = ReportAggregator.TryCreateFromEnvironment(Environment.GetEnvironmentVariable)!; + using var aggregationLock = await aggregator.AcquireLockAsync(cancellationToken); + using var cancelled = new CancellationTokenSource(); + cancelled.Cancel(); + var htmlPath = Path.Combine(tempDirectory, "suite-report.html"); + + await reporter.TryWriteSidecarAndAggregateAsync( + CreateReportData(), + htmlPath, + cancelled.Token); + + githubReporter.SuppressPerSuiteSummary.ShouldBeTrue(); + aggregator.ReadAllSidecars().Single().AssemblyName.ShouldBe("Tests"); + File.Exists(Path.Combine(aggregationDirectory, ReportDataJson.MergedReportFileName)).ShouldBeFalse(); + } + finally + { + if (Directory.Exists(tempDirectory)) + { + Directory.Delete(tempDirectory, recursive: true); + } + } + } + + [Test] + [Timeout(30_000)] + public async Task Disabled_Cleanup_Does_Not_Exclude_Without_Publication_Lock(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var tempDirectory = Path.Combine(Path.GetTempPath(), $"tunit-report-test-{Guid.NewGuid():N}"); + var aggregationDirectory = Path.Combine(tempDirectory, "aggregate"); + var htmlPath = Path.Combine(tempDirectory, "suite-report.html"); + Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_REPORTS", "true"); + Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_DIR", aggregationDirectory); + + try + { + using var reporter = new HtmlReporter(new MockExtension()); + var aggregator = ReportAggregator.TryCreateFromEnvironment(Environment.GetEnvironmentVariable)!; + var reportData = CreateReportData(); + aggregator.WriteSidecar(ReportDataJson.SerializeToBytes(reportData), reportData.AssemblyName, htmlPath); + TUnitSettings.Default.Reporting.JsonReportEnabled = false; + + using (aggregator.BeginSidecarPublication(reportData.AssemblyName, htmlPath)) + { + await reporter.TryWriteSidecarAndAggregateAsync(reportData, htmlPath, cancellationToken); + } + + aggregator.ReadAllSidecars().Single().AssemblyName.ShouldBe("Tests"); + Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExclusionExtension}").ShouldBeEmpty(); + } + finally + { + if (Directory.Exists(tempDirectory)) + { + Directory.Delete(tempDirectory, recursive: true); + } + } + } + [Test] public async Task Enabled_Publication_Clears_Stale_Exclusion(CancellationToken cancellationToken) { From b52e7d602cd6c04acd21520ecfc77858337beb9f Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Thu, 3 Sep 2026 21:43:42 +0100 Subject: [PATCH 21/26] fix(reporting): preserve contended sidecars --- .../Reporters/Aggregation/ReportAggregator.cs | 40 ++++++++----------- .../Reporters/Html/HtmlReporter.cs | 12 ++++-- .../HtmlReporterConfigurationTests.cs | 36 +++++++++++++++++ 3 files changed, 61 insertions(+), 27 deletions(-) diff --git a/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs b/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs index 2eb4c57ab47..ff0527f3dd5 100644 --- a/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs +++ b/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs @@ -163,26 +163,11 @@ internal IDisposable BeginSidecarPublication(string assemblyName, string suiteSa string suiteSalt, CancellationToken cancellationToken) { - for (var attempt = 1; attempt <= LockMaxAttempts; attempt++) - { - cancellationToken.ThrowIfCancellationRequested(); - try - { - return BeginSidecarPublication(assemblyName, suiteSalt); - } - catch (IOException) - { - if (attempt == LockMaxAttempts) - { - break; - } - - await Task.Delay(LockRetryDelayMs + Random.Shared.Next(0, 100), cancellationToken); - } - } - - Console.WriteLine("Warning: Report sidecar publication lock timed out; keeping the local per-suite report."); - return null; + System.IO.Directory.CreateDirectory(Directory); + return await AcquireFileLockAsync( + GetPublishingMarkerPath(assemblyName, suiteSalt), + cancellationToken, + "Warning: Report sidecar publication lock timed out; keeping the local per-suite report."); } internal bool HasSidecarState(string assemblyName, string suiteSalt) @@ -250,8 +235,17 @@ internal List ReadAllSidecars() internal async Task AcquireLockAsync(CancellationToken cancellationToken) { System.IO.Directory.CreateDirectory(Directory); - var lockPath = Path.Combine(Directory, LockFileName); + return await AcquireFileLockAsync( + Path.Combine(Directory, LockFileName), + cancellationToken, + "Warning: Report aggregation lock timed out; keeping per-suite reports and deferring aggregate refresh."); + } + private static async Task AcquireFileLockAsync( + string lockPath, + CancellationToken cancellationToken, + string timeoutWarning) + { for (var attempt = 1; attempt <= LockMaxAttempts; attempt++) { cancellationToken.ThrowIfCancellationRequested(); @@ -259,7 +253,7 @@ internal List ReadAllSidecars() { return new FileStream(lockPath, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None); } - catch (IOException) + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { if (attempt == LockMaxAttempts) { @@ -270,7 +264,7 @@ internal List ReadAllSidecars() } } - Console.WriteLine("Warning: Report aggregation lock timed out; keeping per-suite reports and deferring aggregate refresh."); + Console.WriteLine(timeoutWarning); return null; } diff --git a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs index 0128a29ef98..5862e383e21 100644 --- a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs +++ b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs @@ -247,9 +247,14 @@ internal async Task TryWriteSidecarAndAggregateAsync(ReportData reportData, stri try { - // Serialize shared publication with cleanup. Persist and expose the latest - // sidecar before waiting for the aggregate lock, so a cancelled or timed-out - // refresh still leaves evidence for a later merge. + // Persist atomically before waiting for the publication lock. Its marker keeps + // readers away during a concurrent mutation, while a lock timeout still leaves + // durable data that becomes visible when the current publisher releases it. + aggregator.WriteSidecar(sidecarBytes, reportData.AssemblyName, suiteSalt: htmlOutputPath); + + // Serialize exclusion changes with cleanup. Expose the latest sidecar before + // waiting for the aggregate lock, so a cancelled or timed-out refresh still + // leaves evidence for a later merge. using var publicationMarker = await aggregator.AcquireSidecarPublicationAsync( reportData.AssemblyName, htmlOutputPath, @@ -259,7 +264,6 @@ internal async Task TryWriteSidecarAndAggregateAsync(ReportData reportData, stri return; } - aggregator.WriteSidecar(sidecarBytes, reportData.AssemblyName, suiteSalt: htmlOutputPath); aggregator.IncludeSidecar(reportData.AssemblyName, htmlOutputPath); if (aggregator.Mode == AggregationMode.Defer && _githubReporter is not null) { diff --git a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs index 1e054251d93..73f32892e59 100644 --- a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs +++ b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs @@ -588,6 +588,42 @@ public async Task Disabled_Cleanup_Does_Not_Exclude_Without_Publication_Lock(Can } } + [Test] + [Timeout(30_000)] + public async Task Enabled_Publication_Contention_Preserves_Shared_Sidecar(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var tempDirectory = Path.Combine(Path.GetTempPath(), $"tunit-report-test-{Guid.NewGuid():N}"); + var aggregationDirectory = Path.Combine(tempDirectory, "aggregate"); + var htmlPath = Path.Combine(tempDirectory, "suite-report.html"); + Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_REPORTS", "true"); + Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_DIR", aggregationDirectory); + + try + { + using var reporter = new HtmlReporter(new MockExtension()); + var aggregator = ReportAggregator.TryCreateFromEnvironment(Environment.GetEnvironmentVariable)!; + var reportData = CreateReportData("ContendedSuiteMarker"); + + using (aggregator.BeginSidecarPublication(reportData.AssemblyName, htmlPath)) + { + await reporter.TryWriteSidecarAndAggregateAsync(reportData, htmlPath, cancellationToken); + + Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExtension}").Length.ShouldBe(1); + aggregator.ReadAllSidecars().ShouldBeEmpty(); + } + + aggregator.ReadAllSidecars().Single().AssemblyName.ShouldBe("ContendedSuiteMarker"); + } + finally + { + if (Directory.Exists(tempDirectory)) + { + Directory.Delete(tempDirectory, recursive: true); + } + } + } + [Test] public async Task Enabled_Publication_Clears_Stale_Exclusion(CancellationToken cancellationToken) { From 30accdea108c4eeef95caea242262d1eb89e67e1 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Thu, 3 Sep 2026 23:07:42 +0100 Subject: [PATCH 22/26] fix: preserve newer report publications --- .../Reporters/Aggregation/ReportAggregator.cs | 16 ++-- .../Reporters/Aggregation/ReportDataJson.cs | 22 +++++- .../Reporters/Html/HtmlReporter.cs | 7 +- .../HtmlReporterConfigurationTests.cs | 78 +++++++++++++++++-- 4 files changed, 103 insertions(+), 20 deletions(-) diff --git a/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs b/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs index ff0527f3dd5..99e7bb99687 100644 --- a/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs +++ b/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs @@ -132,18 +132,16 @@ internal string WriteSidecar(byte[] sidecarUtf8Json, string assemblyName, string var path = GetSidecarPath(assemblyName, suiteSalt); AtomicFile.WriteAllBytes(path, sidecarUtf8Json); + AtomicFile.WriteAllBytes(GetGenerationMarkerPath(assemblyName, suiteSalt), Guid.NewGuid().ToByteArray()); return path; } - internal void DeleteSidecar(string assemblyName, string suiteSalt) - { - File.Delete(GetSidecarPath(assemblyName, suiteSalt)); - } - internal void ExcludeSidecar(string assemblyName, string suiteSalt) { System.IO.Directory.CreateDirectory(Directory); - AtomicFile.WriteAllBytes(GetExclusionMarkerPath(assemblyName, suiteSalt), []); + var generationPath = GetGenerationMarkerPath(assemblyName, suiteSalt); + var generation = File.Exists(generationPath) ? File.ReadAllBytes(generationPath) : []; + AtomicFile.WriteAllBytes(GetExclusionMarkerPath(assemblyName, suiteSalt), generation); } internal void IncludeSidecar(string assemblyName, string suiteSalt) @@ -180,6 +178,7 @@ internal bool HasSidecarState(string assemblyName, string suiteSalt) var sidecarPath = GetSidecarPath(assemblyName, suiteSalt); return File.Exists(sidecarPath) || File.Exists(sidecarPath + ReportDataJson.SidecarExclusionExtension) + || File.Exists(sidecarPath + ReportDataJson.SidecarGenerationExtension) || File.Exists(sidecarPath + ReportDataJson.SidecarPublishingExtension); } @@ -300,6 +299,11 @@ private string GetPublishingMarkerPath(string assemblyName, string suiteSalt) return GetSidecarPath(assemblyName, suiteSalt) + ReportDataJson.SidecarPublishingExtension; } + private string GetGenerationMarkerPath(string assemblyName, string suiteSalt) + { + return GetSidecarPath(assemblyName, suiteSalt) + ReportDataJson.SidecarGenerationExtension; + } + private string GetSidecarPath(string assemblyName, string suiteSalt) { var fileName = $"{PathValidator.SanitizeFileName(assemblyName)}-{ShortHash(suiteSalt)}{ReportDataJson.SidecarExtension}"; diff --git a/src/TUnit.Engine/Reporters/Aggregation/ReportDataJson.cs b/src/TUnit.Engine/Reporters/Aggregation/ReportDataJson.cs index 7039e50887f..4dd407266da 100644 --- a/src/TUnit.Engine/Reporters/Aggregation/ReportDataJson.cs +++ b/src/TUnit.Engine/Reporters/Aggregation/ReportDataJson.cs @@ -26,6 +26,7 @@ internal static class ReportDataJson /// File extension shared by every sidecar so aggregators can discover them. internal const string SidecarExtension = ".tunit-report.json"; internal const string SidecarExclusionExtension = ".excluded"; + internal const string SidecarGenerationExtension = ".generation"; internal const string SidecarPublishingExtension = ".publishing"; /// Merged HTML report filename, shared by the engine and the tool's default output. @@ -33,9 +34,26 @@ internal static class ReportDataJson internal static bool ShouldSkipSidecar(string sidecarPath) { - if (File.Exists(sidecarPath + SidecarExclusionExtension)) + var exclusionPath = sidecarPath + SidecarExclusionExtension; + if (File.Exists(exclusionPath)) { - return true; + var generationPath = sidecarPath + SidecarGenerationExtension; + if (!File.Exists(generationPath)) + { + // Exclusions written before generation tracking apply to their legacy + // sidecar. A later publication writes a generation and supersedes them. + return true; + } + + try + { + return File.ReadAllBytes(exclusionPath).AsSpan() + .SequenceEqual(File.ReadAllBytes(generationPath)); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return true; + } } var publicationLockPath = sidecarPath + SidecarPublishingExtension; diff --git a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs index 5862e383e21..b72fe2dc065 100644 --- a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs +++ b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs @@ -330,8 +330,8 @@ private async Task DeleteSidecarsAndRefreshAggregateAsync( return; } - // Durable marker prevents every later merge from reading stale suite data even - // when this process cannot acquire the lock to delete it immediately. + // The marker records the generation being disabled. A concurrent enabled run + // writes a new generation, so this cleanup cannot hide its replacement. aggregator.ExcludeSidecar(assemblyName, htmlOutputPath); // Cleanup must survive session cancellation or stale enabled-run sidecars can @@ -342,9 +342,8 @@ private async Task DeleteSidecarsAndRefreshAggregateAsync( return; } - aggregator.DeleteSidecar(assemblyName, htmlOutputPath); + publicationMarker.Dispose(); RefreshAggregatedOutputs(aggregator); - aggregator.IncludeSidecar(assemblyName, htmlOutputPath); } catch (Exception ex) when (ex is not OperationCanceledException) { diff --git a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs index 73f32892e59..2c669eef7f3 100644 --- a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs +++ b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs @@ -233,6 +233,7 @@ public async Task Disabled_Json_Report_Removes_Stale_Aggregation_Outputs(Cancell { Directory.CreateDirectory(tempDirectory); var reporter = new HtmlReporter(new MockExtension()); + var aggregator = ReportAggregator.TryCreateFromEnvironment(Environment.GetEnvironmentVariable)!; TUnitSettings.Default.Reporting.JsonReportEnabled = true; await reporter.TryWriteSidecarAndAggregateAsync(CreateReportData("RemovedSuiteMarker"), removedHtmlPath, cancellationToken); @@ -243,7 +244,8 @@ public async Task Disabled_Json_Report_Removes_Stale_Aggregation_Outputs(Cancell await reporter.TryWriteSidecarAndAggregateAsync(CreateReportData("RemovedSuiteMarker"), removedHtmlPath, cancellationToken); File.Exists(HtmlReporter.GetSidecarPath(removedHtmlPath)).ShouldBeFalse(); - Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExtension}").Length.ShouldBe(1); + Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExtension}").Length.ShouldBe(2); + aggregator.ReadAllSidecars().Single().AssemblyName.ShouldBe("RemainingSuiteMarker"); var mergedReport = File.ReadAllText(mergedReportPath); mergedReport.ShouldNotContain("RemovedSuiteMarker"); mergedReport.ShouldContain("RemainingSuiteMarker"); @@ -251,7 +253,8 @@ public async Task Disabled_Json_Report_Removes_Stale_Aggregation_Outputs(Cancell await reporter.TryWriteSidecarAndAggregateAsync(CreateReportData("RemainingSuiteMarker"), remainingHtmlPath, cancellationToken); File.Exists(HtmlReporter.GetSidecarPath(remainingHtmlPath)).ShouldBeFalse(); - Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExtension}").ShouldBeEmpty(); + Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExtension}").Length.ShouldBe(2); + aggregator.ReadAllSidecars().ShouldBeEmpty(); File.Exists(mergedReportPath).ShouldBeFalse(); } finally @@ -276,6 +279,7 @@ public async Task Cancelled_Session_Still_Removes_Disabled_Report_Sidecars(Cance { Directory.CreateDirectory(tempDirectory); using var reporter = new HtmlReporter(new MockExtension()); + var aggregator = ReportAggregator.TryCreateFromEnvironment(Environment.GetEnvironmentVariable)!; TUnitSettings.Default.Reporting.JsonReportEnabled = true; await reporter.TryWriteSidecarAndAggregateAsync(CreateReportData(), htmlPath, cancellationToken); @@ -286,7 +290,8 @@ public async Task Cancelled_Session_Still_Removes_Disabled_Report_Sidecars(Cance await reporter.TryWriteSidecarAndAggregateAsync(CreateReportData(), htmlPath, cancelled.Token); File.Exists(HtmlReporter.GetSidecarPath(htmlPath)).ShouldBeFalse(); - Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExtension}").ShouldBeEmpty(); + Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExtension}").Length.ShouldBe(1); + aggregator.ReadAllSidecars().ShouldBeEmpty(); } finally { @@ -314,8 +319,12 @@ public async Task Disabled_Marker_Excludes_Stale_Shared_Sidecar(CancellationToke aggregator.ExcludeSidecar(reportData.AssemblyName, "suite"); aggregator.ReadAllSidecars().ShouldBeEmpty(); + var replacement = CreateReportData(reportData.AssemblyName, "replacement-machine"); + aggregator.WriteSidecar(ReportDataJson.SerializeToBytes(replacement), replacement.AssemblyName, "suite"); + aggregator.ReadAllSidecars().Single().MachineName.ShouldBe("replacement-machine"); + aggregator.IncludeSidecar(reportData.AssemblyName, "suite"); - aggregator.ReadAllSidecars().Single().AssemblyName.ShouldBe("DisabledSuiteMarker"); + aggregator.ReadAllSidecars().Single().MachineName.ShouldBe("replacement-machine"); } finally { @@ -341,6 +350,7 @@ public async Task Disabled_Html_Report_Removes_Stale_Aggregation_Outputs(Cancell { Directory.CreateDirectory(tempDirectory); using var reporter = new HtmlReporter(new MockExtension()); + var aggregator = ReportAggregator.TryCreateFromEnvironment(Environment.GetEnvironmentVariable)!; reporter.SetOutputPath(disabledHtmlPath); var disabledAssemblyName = reporter.BuildReportData().AssemblyName; @@ -352,7 +362,8 @@ public async Task Disabled_Html_Report_Removes_Stale_Aggregation_Outputs(Cancell await reporter.OnTestSessionFinishingAsync(null!); File.Exists(HtmlReporter.GetSidecarPath(disabledHtmlPath)).ShouldBeFalse(); - Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExtension}").Length.ShouldBe(1); + Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExtension}").Length.ShouldBe(2); + aggregator.ReadAllSidecars().Single().AssemblyName.ShouldBe("RemainingSuiteMarker"); var mergedReport = File.ReadAllText(mergedReportPath); mergedReport.ShouldNotContain(disabledAssemblyName); mergedReport.ShouldContain("RemainingSuiteMarker"); @@ -421,6 +432,7 @@ public async Task Empty_Session_With_Disabled_Json_Removes_Stale_Sidecars(Cancel { Directory.CreateDirectory(tempDirectory); using var reporter = new HtmlReporter(new MockExtension()); + var aggregator = ReportAggregator.TryCreateFromEnvironment(Environment.GetEnvironmentVariable)!; reporter.SetOutputPath(htmlPath); var assemblyName = reporter.BuildReportData().AssemblyName; @@ -430,7 +442,8 @@ public async Task Empty_Session_With_Disabled_Json_Removes_Stale_Sidecars(Cancel await reporter.OnTestSessionFinishingAsync(null!); File.Exists(HtmlReporter.GetSidecarPath(htmlPath)).ShouldBeFalse(); - Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExtension}").ShouldBeEmpty(); + Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExtension}").Length.ShouldBe(1); + aggregator.ReadAllSidecars().ShouldBeEmpty(); } finally { @@ -624,6 +637,55 @@ public async Task Enabled_Publication_Contention_Preserves_Shared_Sidecar(Cancel } } + [Test] + [Timeout(30_000)] + public async Task Enabled_Publication_Supersedes_In_Flight_Disabled_Cleanup(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var tempDirectory = Path.Combine(Path.GetTempPath(), $"tunit-report-test-{Guid.NewGuid():N}"); + var aggregationDirectory = Path.Combine(tempDirectory, "aggregate"); + var htmlPath = Path.Combine(tempDirectory, "suite-report.html"); + Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_REPORTS", "true"); + Environment.SetEnvironmentVariable("TUNIT_AGGREGATE_DIR", aggregationDirectory); + + try + { + Directory.CreateDirectory(tempDirectory); + using var reporter = new HtmlReporter(new MockExtension()); + var aggregator = ReportAggregator.TryCreateFromEnvironment(Environment.GetEnvironmentVariable)!; + var oldReport = CreateReportData(machineName: "old-machine"); + await reporter.TryWriteSidecarAndAggregateAsync(oldReport, htmlPath, cancellationToken); + + var aggregationLock = await aggregator.AcquireLockAsync(cancellationToken); + aggregationLock.ShouldNotBeNull(); + Task cleanupTask; + Task publicationTask; + using (aggregationLock!) + { + TUnitSettings.Default.Reporting.JsonReportEnabled = false; + cleanupTask = reporter.TryWriteSidecarAndAggregateAsync(oldReport, htmlPath, cancellationToken); + Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExclusionExtension}").Length.ShouldBe(1); + + TUnitSettings.Default.Reporting.JsonReportEnabled = true; + var newReport = CreateReportData(machineName: "new-machine"); + publicationTask = reporter.TryWriteSidecarAndAggregateAsync(newReport, htmlPath, cancellationToken); + } + + await Task.WhenAll(cleanupTask, publicationTask); + + var publishedReport = aggregator.ReadAllSidecars().Single(); + publishedReport.MachineName.ShouldBe("new-machine"); + Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExclusionExtension}").ShouldBeEmpty(); + } + finally + { + if (Directory.Exists(tempDirectory)) + { + Directory.Delete(tempDirectory, recursive: true); + } + } + } + [Test] public async Task Enabled_Publication_Clears_Stale_Exclusion(CancellationToken cancellationToken) { @@ -653,10 +715,10 @@ public async Task Enabled_Publication_Clears_Stale_Exclusion(CancellationToken c } } - private static ReportData CreateReportData(string assemblyName = "Tests") => new() + private static ReportData CreateReportData(string assemblyName = "Tests", string machineName = "machine") => new() { AssemblyName = assemblyName, - MachineName = "machine", + MachineName = machineName, Timestamp = DateTimeOffset.UtcNow.ToString("O"), TUnitVersion = "1.0.0", OperatingSystem = "test", From b2e60ba919c9f8ea651311fb4bdc7f186f3c8ff8 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Thu, 3 Sep 2026 23:12:03 +0100 Subject: [PATCH 23/26] fix: release cleanup lock before merge --- src/TUnit.Engine/Reporters/Html/HtmlReporter.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs index b72fe2dc065..a3b2b7a7872 100644 --- a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs +++ b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs @@ -334,6 +334,11 @@ private async Task DeleteSidecarsAndRefreshAggregateAsync( // writes a new generation, so this cleanup cannot hide its replacement. aggregator.ExcludeSidecar(assemblyName, htmlOutputPath); + // The exclusion is durable and generation-scoped, so do not hold the per-suite + // lock during the bounded aggregate-lock wait. A staged enabled publication + // must be able to acquire it and clear the exclusion before either wait expires. + publicationMarker.Dispose(); + // Cleanup must survive session cancellation or stale enabled-run sidecars can // re-enter a sibling's aggregate. Lock acquisition remains time-bounded. using var aggregationLock = await aggregator.AcquireLockAsync(CancellationToken.None); @@ -342,7 +347,6 @@ private async Task DeleteSidecarsAndRefreshAggregateAsync( return; } - publicationMarker.Dispose(); RefreshAggregatedOutputs(aggregator); } catch (Exception ex) when (ex is not OperationCanceledException) From 4e52379165188500a6064294b452dd7226bd28a0 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Thu, 3 Sep 2026 23:19:12 +0100 Subject: [PATCH 24/26] fix: protect active report publications --- .../Reporters/Aggregation/ReportAggregator.cs | 37 +++++++++++++++++-- .../Reporters/Html/HtmlReporter.cs | 30 +++++++-------- .../HtmlReporterConfigurationTests.cs | 21 ++++++++--- 3 files changed, 64 insertions(+), 24 deletions(-) diff --git a/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs b/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs index 99e7bb99687..d72ea8a7ef5 100644 --- a/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs +++ b/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs @@ -136,12 +136,31 @@ internal string WriteSidecar(byte[] sidecarUtf8Json, string assemblyName, string return path; } + internal byte[]? ReadSidecarGeneration(string assemblyName, string suiteSalt) + { + var generationPath = GetGenerationMarkerPath(assemblyName, suiteSalt); + return File.Exists(generationPath) ? File.ReadAllBytes(generationPath) : null; + } + internal void ExcludeSidecar(string assemblyName, string suiteSalt) { System.IO.Directory.CreateDirectory(Directory); - var generationPath = GetGenerationMarkerPath(assemblyName, suiteSalt); - var generation = File.Exists(generationPath) ? File.ReadAllBytes(generationPath) : []; - AtomicFile.WriteAllBytes(GetExclusionMarkerPath(assemblyName, suiteSalt), generation); + var currentGeneration = ReadSidecarGeneration(assemblyName, suiteSalt); + AtomicFile.WriteAllBytes(GetExclusionMarkerPath(assemblyName, suiteSalt), currentGeneration ?? []); + } + + internal void ExcludeSidecarIfGenerationMatches(string assemblyName, string suiteSalt, byte[]? expectedGeneration) + { + System.IO.Directory.CreateDirectory(Directory); + var currentGeneration = ReadSidecarGeneration(assemblyName, suiteSalt); + if (expectedGeneration is null + ? currentGeneration is not null + : currentGeneration is null || !expectedGeneration.AsSpan().SequenceEqual(currentGeneration)) + { + return; + } + + AtomicFile.WriteAllBytes(GetExclusionMarkerPath(assemblyName, suiteSalt), currentGeneration ?? []); } internal void IncludeSidecar(string assemblyName, string suiteSalt) @@ -156,6 +175,18 @@ internal IDisposable BeginSidecarPublication(string assemblyName, string suiteSa return new FileStream(lockPath, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None); } + internal IDisposable? TryAcquireSidecarPublication(string assemblyName, string suiteSalt) + { + try + { + return BeginSidecarPublication(assemblyName, suiteSalt); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return null; + } + } + internal async Task AcquireSidecarPublicationAsync( string assemblyName, string suiteSalt, diff --git a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs index a3b2b7a7872..b3e217c8d08 100644 --- a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs +++ b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs @@ -247,23 +247,23 @@ internal async Task TryWriteSidecarAndAggregateAsync(ReportData reportData, stri try { - // Persist atomically before waiting for the publication lock. Its marker keeps - // readers away during a concurrent mutation, while a lock timeout still leaves - // durable data that becomes visible when the current publisher releases it. - aggregator.WriteSidecar(sidecarBytes, reportData.AssemblyName, suiteSalt: htmlOutputPath); - - // Serialize exclusion changes with cleanup. Expose the latest sidecar before - // waiting for the aggregate lock, so a cancelled or timed-out refresh still - // leaves evidence for a later merge. + // Serialize the shared write with cleanup so cleanup cannot mistake an in-flight + // enabled generation for the stale generation it intended to remove. using var publicationMarker = await aggregator.AcquireSidecarPublicationAsync( reportData.AssemblyName, htmlOutputPath, CancellationToken.None); if (publicationMarker is null) { + // Publication-lock contention must not discard completed suite results. + // Persist atomically after the bounded wait; generation-aware exclusions + // make this replacement visible when the current publisher releases it. + aggregator.WriteSidecar(sidecarBytes, reportData.AssemblyName, suiteSalt: htmlOutputPath); + aggregator.IncludeSidecar(reportData.AssemblyName, htmlOutputPath); return; } + aggregator.WriteSidecar(sidecarBytes, reportData.AssemblyName, suiteSalt: htmlOutputPath); aggregator.IncludeSidecar(reportData.AssemblyName, htmlOutputPath); if (aggregator.Mode == AggregationMode.Defer && _githubReporter is not null) { @@ -321,18 +321,18 @@ private async Task DeleteSidecarsAndRefreshAggregateAsync( try { - using var publicationMarker = await aggregator.AcquireSidecarPublicationAsync( - assemblyName, - htmlOutputPath, - CancellationToken.None); + var expectedGeneration = aggregator.ReadSidecarGeneration(assemblyName, htmlOutputPath); + using var publicationMarker = aggregator.TryAcquireSidecarPublication(assemblyName, htmlOutputPath); if (publicationMarker is null) { + // An enabled publication may already have installed its generation while + // holding this lock. Cleanup must not wait, acquire next, and hide it. return; } - // The marker records the generation being disabled. A concurrent enabled run - // writes a new generation, so this cleanup cannot hide its replacement. - aggregator.ExcludeSidecar(assemblyName, htmlOutputPath); + // Exclude only the generation observed before the lock attempt. A publisher that + // won the per-suite lock in the meantime installed a newer generation and survives. + aggregator.ExcludeSidecarIfGenerationMatches(assemblyName, htmlOutputPath, expectedGeneration); // The exclusion is durable and generation-scoped, so do not hold the per-suite // lock during the bounded aggregate-lock wait. A staged enabled publication diff --git a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs index 2c669eef7f3..dea8baeb01a 100644 --- a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs +++ b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs @@ -315,16 +315,23 @@ public async Task Disabled_Marker_Excludes_Stale_Shared_Sidecar(CancellationToke var aggregator = ReportAggregator.TryCreateFromEnvironment(Environment.GetEnvironmentVariable)!; var reportData = CreateReportData("DisabledSuiteMarker"); aggregator.WriteSidecar(ReportDataJson.SerializeToBytes(reportData), reportData.AssemblyName, "suite"); - - aggregator.ExcludeSidecar(reportData.AssemblyName, "suite"); - aggregator.ReadAllSidecars().ShouldBeEmpty(); + var staleGeneration = aggregator.ReadSidecarGeneration(reportData.AssemblyName, "suite"); var replacement = CreateReportData(reportData.AssemblyName, "replacement-machine"); aggregator.WriteSidecar(ReportDataJson.SerializeToBytes(replacement), replacement.AssemblyName, "suite"); + aggregator.ExcludeSidecarIfGenerationMatches(reportData.AssemblyName, "suite", staleGeneration); + Directory.GetFiles(tempDirectory, $"*{ReportDataJson.SidecarExclusionExtension}").ShouldBeEmpty(); aggregator.ReadAllSidecars().Single().MachineName.ShouldBe("replacement-machine"); + aggregator.ExcludeSidecar(reportData.AssemblyName, "suite"); + aggregator.ReadAllSidecars().ShouldBeEmpty(); + + var latest = CreateReportData(reportData.AssemblyName, "latest-machine"); + aggregator.WriteSidecar(ReportDataJson.SerializeToBytes(latest), latest.AssemblyName, "suite"); + aggregator.ReadAllSidecars().Single().MachineName.ShouldBe("latest-machine"); + aggregator.IncludeSidecar(reportData.AssemblyName, "suite"); - aggregator.ReadAllSidecars().Single().MachineName.ShouldBe("replacement-machine"); + aggregator.ReadAllSidecars().Single().MachineName.ShouldBe("latest-machine"); } finally { @@ -567,7 +574,7 @@ await reporter.TryWriteSidecarAndAggregateAsync( [Test] [Timeout(30_000)] - public async Task Disabled_Cleanup_Does_Not_Exclude_Without_Publication_Lock(CancellationToken cancellationToken) + public async Task Disabled_Cleanup_Does_Not_Exclude_Active_Publication(CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); var tempDirectory = Path.Combine(Path.GetTempPath(), $"tunit-report-test-{Guid.NewGuid():N}"); @@ -586,10 +593,12 @@ public async Task Disabled_Cleanup_Does_Not_Exclude_Without_Publication_Lock(Can using (aggregator.BeginSidecarPublication(reportData.AssemblyName, htmlPath)) { + var replacement = CreateReportData(reportData.AssemblyName, "active-publisher"); + aggregator.WriteSidecar(ReportDataJson.SerializeToBytes(replacement), replacement.AssemblyName, htmlPath); await reporter.TryWriteSidecarAndAggregateAsync(reportData, htmlPath, cancellationToken); } - aggregator.ReadAllSidecars().Single().AssemblyName.ShouldBe("Tests"); + aggregator.ReadAllSidecars().Single().MachineName.ShouldBe("active-publisher"); Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExclusionExtension}").ShouldBeEmpty(); } finally From 4eb30d433dd4846e01071f39b0aafa4492ea2300 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Thu, 3 Sep 2026 23:28:55 +0100 Subject: [PATCH 25/26] fix: isolate timed-out report writes --- .../Reporters/Aggregation/ReportAggregator.cs | 46 ++++++++++++++----- .../Reporters/Aggregation/ReportDataJson.cs | 29 +++++++++++- .../Reporters/Html/HtmlReporter.cs | 18 +++++--- src/TUnit.Reporting.Tool/Program.cs | 4 +- .../HtmlReporterConfigurationTests.cs | 18 ++++++-- 5 files changed, 89 insertions(+), 26 deletions(-) diff --git a/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs b/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs index d72ea8a7ef5..2994ffc9552 100644 --- a/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs +++ b/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs @@ -132,27 +132,45 @@ internal string WriteSidecar(byte[] sidecarUtf8Json, string assemblyName, string var path = GetSidecarPath(assemblyName, suiteSalt); AtomicFile.WriteAllBytes(path, sidecarUtf8Json); - AtomicFile.WriteAllBytes(GetGenerationMarkerPath(assemblyName, suiteSalt), Guid.NewGuid().ToByteArray()); + AtomicFile.WriteAllBytes(GetGenerationMarkerPath(path), Guid.NewGuid().ToByteArray()); return path; } - internal byte[]? ReadSidecarGeneration(string assemblyName, string suiteSalt) + internal void WritePendingSidecar(byte[] sidecarUtf8Json, string assemblyName, string suiteSalt) { - var generationPath = GetGenerationMarkerPath(assemblyName, suiteSalt); + System.IO.Directory.CreateDirectory(Directory); + var path = GetPendingSidecarPath(assemblyName, suiteSalt); + AtomicFile.WriteAllBytes(path, sidecarUtf8Json); + AtomicFile.WriteAllBytes(GetGenerationMarkerPath(path), Guid.NewGuid().ToByteArray()); + } + + internal void DeletePendingSidecar(string assemblyName, string suiteSalt) + { + var path = GetPendingSidecarPath(assemblyName, suiteSalt); + File.Delete(path); + File.Delete(GetGenerationMarkerPath(path)); + } + + internal byte[]? ReadEffectiveSidecarGeneration(string assemblyName, string suiteSalt) + { + var pendingPath = GetPendingSidecarPath(assemblyName, suiteSalt); + var generationPath = File.Exists(pendingPath) + ? GetGenerationMarkerPath(pendingPath) + : GetGenerationMarkerPath(GetSidecarPath(assemblyName, suiteSalt)); return File.Exists(generationPath) ? File.ReadAllBytes(generationPath) : null; } internal void ExcludeSidecar(string assemblyName, string suiteSalt) { System.IO.Directory.CreateDirectory(Directory); - var currentGeneration = ReadSidecarGeneration(assemblyName, suiteSalt); + var currentGeneration = ReadEffectiveSidecarGeneration(assemblyName, suiteSalt); AtomicFile.WriteAllBytes(GetExclusionMarkerPath(assemblyName, suiteSalt), currentGeneration ?? []); } internal void ExcludeSidecarIfGenerationMatches(string assemblyName, string suiteSalt, byte[]? expectedGeneration) { System.IO.Directory.CreateDirectory(Directory); - var currentGeneration = ReadSidecarGeneration(assemblyName, suiteSalt); + var currentGeneration = ReadEffectiveSidecarGeneration(assemblyName, suiteSalt); if (expectedGeneration is null ? currentGeneration is not null : currentGeneration is null || !expectedGeneration.AsSpan().SequenceEqual(currentGeneration)) @@ -207,9 +225,12 @@ internal bool HasSidecarState(string assemblyName, string suiteSalt) } var sidecarPath = GetSidecarPath(assemblyName, suiteSalt); + var pendingPath = GetPendingSidecarPath(assemblyName, suiteSalt); return File.Exists(sidecarPath) + || File.Exists(pendingPath) || File.Exists(sidecarPath + ReportDataJson.SidecarExclusionExtension) - || File.Exists(sidecarPath + ReportDataJson.SidecarGenerationExtension) + || File.Exists(GetGenerationMarkerPath(sidecarPath)) + || File.Exists(GetGenerationMarkerPath(pendingPath)) || File.Exists(sidecarPath + ReportDataJson.SidecarPublishingExtension); } @@ -231,7 +252,9 @@ internal List ReadAllSidecars() // as the tunit-report tool does. var seenDigests = new HashSet(StringComparer.Ordinal); using var sha = SHA256.Create(); - foreach (var file in System.IO.Directory.GetFiles(Directory, SidecarSearchPattern)) + var effectiveSidecars = ReportDataJson.SelectEffectiveSidecars( + System.IO.Directory.GetFiles(Directory, SidecarSearchPattern)); + foreach (var file in effectiveSidecars) { try { @@ -330,10 +353,11 @@ private string GetPublishingMarkerPath(string assemblyName, string suiteSalt) return GetSidecarPath(assemblyName, suiteSalt) + ReportDataJson.SidecarPublishingExtension; } - private string GetGenerationMarkerPath(string assemblyName, string suiteSalt) - { - return GetSidecarPath(assemblyName, suiteSalt) + ReportDataJson.SidecarGenerationExtension; - } + private static string GetGenerationMarkerPath(string sidecarPath) + => sidecarPath + ReportDataJson.SidecarGenerationExtension; + + private string GetPendingSidecarPath(string assemblyName, string suiteSalt) + => ReportDataJson.GetPendingSidecarPath(GetSidecarPath(assemblyName, suiteSalt)); private string GetSidecarPath(string assemblyName, string suiteSalt) { diff --git a/src/TUnit.Engine/Reporters/Aggregation/ReportDataJson.cs b/src/TUnit.Engine/Reporters/Aggregation/ReportDataJson.cs index 4dd407266da..ed44b926f17 100644 --- a/src/TUnit.Engine/Reporters/Aggregation/ReportDataJson.cs +++ b/src/TUnit.Engine/Reporters/Aggregation/ReportDataJson.cs @@ -25,6 +25,7 @@ internal static class ReportDataJson /// File extension shared by every sidecar so aggregators can discover them. internal const string SidecarExtension = ".tunit-report.json"; + internal const string SidecarPendingSegment = ".pending"; internal const string SidecarExclusionExtension = ".excluded"; internal const string SidecarGenerationExtension = ".generation"; internal const string SidecarPublishingExtension = ".publishing"; @@ -34,7 +35,8 @@ internal static class ReportDataJson internal static bool ShouldSkipSidecar(string sidecarPath) { - var exclusionPath = sidecarPath + SidecarExclusionExtension; + var canonicalPath = GetCanonicalSidecarPath(sidecarPath); + var exclusionPath = canonicalPath + SidecarExclusionExtension; if (File.Exists(exclusionPath)) { var generationPath = sidecarPath + SidecarGenerationExtension; @@ -56,7 +58,14 @@ internal static bool ShouldSkipSidecar(string sidecarPath) } } - var publicationLockPath = sidecarPath + SidecarPublishingExtension; + // Pending sidecars are complete atomic files written only after a publisher's + // per-suite lock wait expires. The canonical publisher cannot mutate them. + if (IsPendingSidecar(sidecarPath)) + { + return false; + } + + var publicationLockPath = canonicalPath + SidecarPublishingExtension; if (!File.Exists(publicationLockPath)) { return false; @@ -77,6 +86,22 @@ internal static bool ShouldSkipSidecar(string sidecarPath) } } + internal static bool IsPendingSidecar(string sidecarPath) + => sidecarPath.EndsWith(SidecarPendingSegment + SidecarExtension, StringComparison.Ordinal); + + internal static string GetCanonicalSidecarPath(string sidecarPath) + => IsPendingSidecar(sidecarPath) + ? sidecarPath[..^(SidecarPendingSegment.Length + SidecarExtension.Length)] + SidecarExtension + : sidecarPath; + + internal static string GetPendingSidecarPath(string canonicalSidecarPath) + => canonicalSidecarPath[..^SidecarExtension.Length] + SidecarPendingSegment + SidecarExtension; + + internal static IEnumerable SelectEffectiveSidecars(IEnumerable sidecarPaths) + => sidecarPaths + .GroupBy(GetCanonicalSidecarPath, StringComparer.Ordinal) + .Select(group => group.FirstOrDefault(IsPendingSidecar) ?? group.First()); + /// /// Serializes straight to UTF-8 bytes — callers write the same payload to more than one /// file, so producing bytes once avoids a UTF-8 → string → UTF-8 round trip per copy. diff --git a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs index b3e217c8d08..7ff1cf76618 100644 --- a/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs +++ b/src/TUnit.Engine/Reporters/Html/HtmlReporter.cs @@ -256,15 +256,19 @@ internal async Task TryWriteSidecarAndAggregateAsync(ReportData reportData, stri if (publicationMarker is null) { // Publication-lock contention must not discard completed suite results. - // Persist atomically after the bounded wait; generation-aware exclusions - // make this replacement visible when the current publisher releases it. + // A separate pending slot cannot overwrite the active publisher's canonical + // generation and remains available to this or any later aggregate refresh. + aggregator.WritePendingSidecar(sidecarBytes, reportData.AssemblyName, suiteSalt: htmlOutputPath); + } + else + { + // Owning the lock makes it safe for a newer normal publication to replace + // any pending timeout result left by an earlier publisher. + aggregator.DeletePendingSidecar(reportData.AssemblyName, htmlOutputPath); aggregator.WriteSidecar(sidecarBytes, reportData.AssemblyName, suiteSalt: htmlOutputPath); aggregator.IncludeSidecar(reportData.AssemblyName, htmlOutputPath); - return; } - aggregator.WriteSidecar(sidecarBytes, reportData.AssemblyName, suiteSalt: htmlOutputPath); - aggregator.IncludeSidecar(reportData.AssemblyName, htmlOutputPath); if (aggregator.Mode == AggregationMode.Defer && _githubReporter is not null) { _githubReporter.SuppressPerSuiteSummary = true; @@ -287,7 +291,7 @@ internal async Task TryWriteSidecarAndAggregateAsync(ReportData reportData, stri using (aggregationLock) { - publicationMarker.Dispose(); + publicationMarker?.Dispose(); RefreshAggregatedOutputs(aggregator); if (_githubReporter is not null) @@ -321,7 +325,7 @@ private async Task DeleteSidecarsAndRefreshAggregateAsync( try { - var expectedGeneration = aggregator.ReadSidecarGeneration(assemblyName, htmlOutputPath); + var expectedGeneration = aggregator.ReadEffectiveSidecarGeneration(assemblyName, htmlOutputPath); using var publicationMarker = aggregator.TryAcquireSidecarPublication(assemblyName, htmlOutputPath); if (publicationMarker is null) { diff --git a/src/TUnit.Reporting.Tool/Program.cs b/src/TUnit.Reporting.Tool/Program.cs index 3801ef33211..f533a5d0960 100644 --- a/src/TUnit.Reporting.Tool/Program.cs +++ b/src/TUnit.Reporting.Tool/Program.cs @@ -153,7 +153,9 @@ private static (List Suites, int Skipped) LoadSidecars(string direct // abort the walk — SearchOption.AllDirectories would throw mid-enumeration, before // the per-file guard below ever ran. var enumeration = new EnumerationOptions { RecurseSubdirectories = true, IgnoreInaccessible = true }; - foreach (var file in Directory.EnumerateFiles(directory, "*" + ReportDataJson.SidecarExtension, enumeration)) + var effectiveSidecars = ReportDataJson.SelectEffectiveSidecars( + Directory.EnumerateFiles(directory, "*" + ReportDataJson.SidecarExtension, enumeration)); + foreach (var file in effectiveSidecars) { // This command is the designated final merge, so no publisher should still // be active. Skip any suite whose stable publication lock is still held. diff --git a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs index dea8baeb01a..3b4e53c6d28 100644 --- a/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs +++ b/tests/TUnit.Engine.Tests/HtmlReporterConfigurationTests.cs @@ -315,7 +315,7 @@ public async Task Disabled_Marker_Excludes_Stale_Shared_Sidecar(CancellationToke var aggregator = ReportAggregator.TryCreateFromEnvironment(Environment.GetEnvironmentVariable)!; var reportData = CreateReportData("DisabledSuiteMarker"); aggregator.WriteSidecar(ReportDataJson.SerializeToBytes(reportData), reportData.AssemblyName, "suite"); - var staleGeneration = aggregator.ReadSidecarGeneration(reportData.AssemblyName, "suite"); + var staleGeneration = aggregator.ReadEffectiveSidecarGeneration(reportData.AssemblyName, "suite"); var replacement = CreateReportData(reportData.AssemblyName, "replacement-machine"); aggregator.WriteSidecar(ReportDataJson.SerializeToBytes(replacement), replacement.AssemblyName, "suite"); @@ -625,17 +625,25 @@ public async Task Enabled_Publication_Contention_Preserves_Shared_Sidecar(Cancel { using var reporter = new HtmlReporter(new MockExtension()); var aggregator = ReportAggregator.TryCreateFromEnvironment(Environment.GetEnvironmentVariable)!; - var reportData = CreateReportData("ContendedSuiteMarker"); + var reportData = CreateReportData("ContendedSuiteMarker", "timeout-publisher"); using (aggregator.BeginSidecarPublication(reportData.AssemblyName, htmlPath)) { + var activeReport = CreateReportData(reportData.AssemblyName, "active-publisher"); + aggregator.WriteSidecar(ReportDataJson.SerializeToBytes(activeReport), activeReport.AssemblyName, htmlPath); await reporter.TryWriteSidecarAndAggregateAsync(reportData, htmlPath, cancellationToken); - Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExtension}").Length.ShouldBe(1); - aggregator.ReadAllSidecars().ShouldBeEmpty(); + Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExtension}").Length.ShouldBe(2); + aggregator.ReadAllSidecars().Single().MachineName.ShouldBe("timeout-publisher"); + File.Exists(Path.Combine(aggregationDirectory, ReportDataJson.MergedReportFileName)).ShouldBeTrue(); } - aggregator.ReadAllSidecars().Single().AssemblyName.ShouldBe("ContendedSuiteMarker"); + aggregator.ReadAllSidecars().Single().MachineName.ShouldBe("timeout-publisher"); + + var newerReport = CreateReportData(reportData.AssemblyName, "newer-publisher"); + await reporter.TryWriteSidecarAndAggregateAsync(newerReport, htmlPath, cancellationToken); + Directory.GetFiles(aggregationDirectory, $"*{ReportDataJson.SidecarExtension}").Length.ShouldBe(1); + aggregator.ReadAllSidecars().Single().MachineName.ShouldBe("newer-publisher"); } finally { From e969ae2b6d99ef26baec1f1350ddb4c6632ad01f Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Thu, 3 Sep 2026 23:41:41 +0100 Subject: [PATCH 26/26] fix: embed report publication generation --- .../Reporters/Aggregation/ReportAggregator.cs | 33 ++++----- .../Reporters/Aggregation/ReportDataJson.cs | 67 +++++++++++++------ .../Reporters/Html/HtmlReportDataModel.cs | 4 ++ src/TUnit.Reporting.Tool/Program.cs | 7 +- .../TUnit.UnitTests/ReportAggregationTests.cs | 1 + 5 files changed, 72 insertions(+), 40 deletions(-) diff --git a/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs b/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs index 2994ffc9552..72aac1d2ade 100644 --- a/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs +++ b/src/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs @@ -132,7 +132,6 @@ internal string WriteSidecar(byte[] sidecarUtf8Json, string assemblyName, string var path = GetSidecarPath(assemblyName, suiteSalt); AtomicFile.WriteAllBytes(path, sidecarUtf8Json); - AtomicFile.WriteAllBytes(GetGenerationMarkerPath(path), Guid.NewGuid().ToByteArray()); return path; } @@ -141,44 +140,44 @@ internal void WritePendingSidecar(byte[] sidecarUtf8Json, string assemblyName, s System.IO.Directory.CreateDirectory(Directory); var path = GetPendingSidecarPath(assemblyName, suiteSalt); AtomicFile.WriteAllBytes(path, sidecarUtf8Json); - AtomicFile.WriteAllBytes(GetGenerationMarkerPath(path), Guid.NewGuid().ToByteArray()); } internal void DeletePendingSidecar(string assemblyName, string suiteSalt) { var path = GetPendingSidecarPath(assemblyName, suiteSalt); File.Delete(path); - File.Delete(GetGenerationMarkerPath(path)); } - internal byte[]? ReadEffectiveSidecarGeneration(string assemblyName, string suiteSalt) + internal string? ReadEffectiveSidecarGeneration(string assemblyName, string suiteSalt) { var pendingPath = GetPendingSidecarPath(assemblyName, suiteSalt); - var generationPath = File.Exists(pendingPath) - ? GetGenerationMarkerPath(pendingPath) - : GetGenerationMarkerPath(GetSidecarPath(assemblyName, suiteSalt)); - return File.Exists(generationPath) ? File.ReadAllBytes(generationPath) : null; + var sidecarPath = File.Exists(pendingPath) + ? pendingPath + : GetSidecarPath(assemblyName, suiteSalt); + return File.Exists(sidecarPath) + ? ReportDataJson.GetPublicationGeneration(File.ReadAllBytes(sidecarPath)) + : null; } internal void ExcludeSidecar(string assemblyName, string suiteSalt) { System.IO.Directory.CreateDirectory(Directory); var currentGeneration = ReadEffectiveSidecarGeneration(assemblyName, suiteSalt); - AtomicFile.WriteAllBytes(GetExclusionMarkerPath(assemblyName, suiteSalt), currentGeneration ?? []); + AtomicFile.WriteAllText(GetExclusionMarkerPath(assemblyName, suiteSalt), currentGeneration ?? ""); } - internal void ExcludeSidecarIfGenerationMatches(string assemblyName, string suiteSalt, byte[]? expectedGeneration) + internal void ExcludeSidecarIfGenerationMatches(string assemblyName, string suiteSalt, string? expectedGeneration) { System.IO.Directory.CreateDirectory(Directory); var currentGeneration = ReadEffectiveSidecarGeneration(assemblyName, suiteSalt); if (expectedGeneration is null ? currentGeneration is not null - : currentGeneration is null || !expectedGeneration.AsSpan().SequenceEqual(currentGeneration)) + : !expectedGeneration.Equals(currentGeneration, StringComparison.Ordinal)) { return; } - AtomicFile.WriteAllBytes(GetExclusionMarkerPath(assemblyName, suiteSalt), currentGeneration ?? []); + AtomicFile.WriteAllText(GetExclusionMarkerPath(assemblyName, suiteSalt), currentGeneration ?? ""); } internal void IncludeSidecar(string assemblyName, string suiteSalt) @@ -229,8 +228,6 @@ internal bool HasSidecarState(string assemblyName, string suiteSalt) return File.Exists(sidecarPath) || File.Exists(pendingPath) || File.Exists(sidecarPath + ReportDataJson.SidecarExclusionExtension) - || File.Exists(GetGenerationMarkerPath(sidecarPath)) - || File.Exists(GetGenerationMarkerPath(pendingPath)) || File.Exists(sidecarPath + ReportDataJson.SidecarPublishingExtension); } @@ -258,13 +255,14 @@ internal List ReadAllSidecars() { try { - if (ReportDataJson.ShouldSkipSidecar(file)) + if (ReportDataJson.IsSidecarPublicationInProgress(file)) { continue; } var bytes = File.ReadAllBytes(file); - if (seenDigests.Add(Convert.ToBase64String(sha.ComputeHash(bytes))) + if (!ReportDataJson.IsSidecarExcluded(file, bytes) + && seenDigests.Add(Convert.ToBase64String(sha.ComputeHash(bytes))) && ReportDataJson.TryDeserialize((ReadOnlyMemory)bytes) is { } data) { results.Add(data); @@ -353,9 +351,6 @@ private string GetPublishingMarkerPath(string assemblyName, string suiteSalt) return GetSidecarPath(assemblyName, suiteSalt) + ReportDataJson.SidecarPublishingExtension; } - private static string GetGenerationMarkerPath(string sidecarPath) - => sidecarPath + ReportDataJson.SidecarGenerationExtension; - private string GetPendingSidecarPath(string assemblyName, string suiteSalt) => ReportDataJson.GetPendingSidecarPath(GetSidecarPath(assemblyName, suiteSalt)); diff --git a/src/TUnit.Engine/Reporters/Aggregation/ReportDataJson.cs b/src/TUnit.Engine/Reporters/Aggregation/ReportDataJson.cs index ed44b926f17..2088bb690b2 100644 --- a/src/TUnit.Engine/Reporters/Aggregation/ReportDataJson.cs +++ b/src/TUnit.Engine/Reporters/Aggregation/ReportDataJson.cs @@ -27,37 +27,39 @@ internal static class ReportDataJson internal const string SidecarExtension = ".tunit-report.json"; internal const string SidecarPendingSegment = ".pending"; internal const string SidecarExclusionExtension = ".excluded"; - internal const string SidecarGenerationExtension = ".generation"; internal const string SidecarPublishingExtension = ".publishing"; /// Merged HTML report filename, shared by the engine and the tool's default output. internal const string MergedReportFileName = "merged-report.html"; - internal static bool ShouldSkipSidecar(string sidecarPath) + internal static bool IsSidecarExcluded(string sidecarPath, ReadOnlySpan sidecarUtf8Json) { var canonicalPath = GetCanonicalSidecarPath(sidecarPath); var exclusionPath = canonicalPath + SidecarExclusionExtension; - if (File.Exists(exclusionPath)) + if (!File.Exists(exclusionPath)) { - var generationPath = sidecarPath + SidecarGenerationExtension; - if (!File.Exists(generationPath)) - { - // Exclusions written before generation tracking apply to their legacy - // sidecar. A later publication writes a generation and supersedes them. - return true; - } + return false; + } - try - { - return File.ReadAllBytes(exclusionPath).AsSpan() - .SequenceEqual(File.ReadAllBytes(generationPath)); - } - catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) - { - return true; - } + var generation = GetPublicationGeneration(sidecarUtf8Json); + if (generation is null) + { + // Exclusions created for sidecars from before generation tracking remain valid. + return true; } + try + { + return File.ReadAllText(exclusionPath).Equals(generation, StringComparison.Ordinal); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return true; + } + } + + internal static bool IsSidecarPublicationInProgress(string sidecarPath) + { // Pending sidecars are complete atomic files written only after a publisher's // per-suite lock wait expires. The canonical publisher cannot mutate them. if (IsPendingSidecar(sidecarPath)) @@ -65,7 +67,7 @@ internal static bool ShouldSkipSidecar(string sidecarPath) return false; } - var publicationLockPath = canonicalPath + SidecarPublishingExtension; + var publicationLockPath = sidecarPath + SidecarPublishingExtension; if (!File.Exists(publicationLockPath)) { return false; @@ -86,6 +88,29 @@ internal static bool ShouldSkipSidecar(string sidecarPath) } } + internal static string? GetPublicationGeneration(ReadOnlySpan sidecarUtf8Json) + { + try + { + var reader = new Utf8JsonReader(sidecarUtf8Json); + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.PropertyName + && reader.ValueTextEquals("publicationGeneration") + && reader.Read() + && reader.TokenType == JsonTokenType.String) + { + return reader.GetString(); + } + } + } + catch (JsonException) + { + } + + return null; + } + internal static bool IsPendingSidecar(string sidecarPath) => sidecarPath.EndsWith(SidecarPendingSegment + SidecarExtension, StringComparison.Ordinal); @@ -123,6 +148,7 @@ private static void Write(Utf8JsonWriter w, ReportData data) { w.WriteStartObject(); w.WriteNumber("schemaVersion", SchemaVersion); + w.WriteString("publicationGeneration", data.PublicationGeneration ?? Guid.NewGuid().ToString("N")); w.WriteString("assemblyName", data.AssemblyName); w.WriteString("machineName", data.MachineName); w.WriteString("timestamp", data.Timestamp); @@ -395,6 +421,7 @@ private static bool IsMalformedSidecar(Exception ex) return new ReportData { AssemblyName = GetString(root, "assemblyName") ?? "Unknown", + PublicationGeneration = GetString(root, "publicationGeneration"), MachineName = GetString(root, "machineName") ?? "", Timestamp = GetString(root, "timestamp") ?? "", TUnitVersion = GetString(root, "tunitVersion") ?? "", diff --git a/src/TUnit.Engine/Reporters/Html/HtmlReportDataModel.cs b/src/TUnit.Engine/Reporters/Html/HtmlReportDataModel.cs index eac0d3b23a7..95ec9473143 100644 --- a/src/TUnit.Engine/Reporters/Html/HtmlReportDataModel.cs +++ b/src/TUnit.Engine/Reporters/Html/HtmlReportDataModel.cs @@ -12,6 +12,10 @@ internal sealed class ReportData [JsonPropertyName("assemblyName")] public required string AssemblyName { get; init; } + /// Atomic sidecar generation used only by aggregation cleanup. + [JsonIgnore] + public string? PublicationGeneration { get; init; } + [JsonPropertyName("machineName")] public required string MachineName { get; init; } diff --git a/src/TUnit.Reporting.Tool/Program.cs b/src/TUnit.Reporting.Tool/Program.cs index f533a5d0960..82caaeb626e 100644 --- a/src/TUnit.Reporting.Tool/Program.cs +++ b/src/TUnit.Reporting.Tool/Program.cs @@ -159,7 +159,7 @@ private static (List Suites, int Skipped) LoadSidecars(string direct { // This command is the designated final merge, so no publisher should still // be active. Skip any suite whose stable publication lock is still held. - if (ReportDataJson.ShouldSkipSidecar(file)) + if (ReportDataJson.IsSidecarPublicationInProgress(file)) { continue; } @@ -177,6 +177,11 @@ private static (List Suites, int Skipped) LoadSidecars(string direct continue; } + if (ReportDataJson.IsSidecarExcluded(file, bytes)) + { + continue; + } + if (!seenDigests.Add(Convert.ToBase64String(sha.ComputeHash(bytes)))) { continue; diff --git a/tests/TUnit.UnitTests/ReportAggregationTests.cs b/tests/TUnit.UnitTests/ReportAggregationTests.cs index c739508d64c..dd5a38bf963 100644 --- a/tests/TUnit.UnitTests/ReportAggregationTests.cs +++ b/tests/TUnit.UnitTests/ReportAggregationTests.cs @@ -17,6 +17,7 @@ public async Task Serialize_Then_Deserialize_RoundTrips_AllFields() await Assert.That(restored).IsNotNull(); await Assert.That(restored!.AssemblyName).IsEqualTo("My.Tests"); + await Assert.That(restored.PublicationGeneration).IsNotNull(); await Assert.That(restored.MachineName).IsEqualTo(original.MachineName); await Assert.That(restored.Timestamp).IsEqualTo(original.Timestamp); await Assert.That(restored.TUnitVersion).IsEqualTo(original.TUnitVersion);