diff --git a/TUnit.Core/Attributes/TestMetadata/ClassTimelineAttribute.Constants.cs b/TUnit.Core/Attributes/TestMetadata/ClassTimelineAttribute.Constants.cs new file mode 100644 index 00000000000..ae9f8e9d71d --- /dev/null +++ b/TUnit.Core/Attributes/TestMetadata/ClassTimelineAttribute.Constants.cs @@ -0,0 +1,14 @@ +namespace TUnit.Core; + +// Constants-only partial, source-linked into TUnit.Reporting.Tool (the linked +// HtmlReportGenerator reads the property key back out of report sidecars). Keeping the +// key here — away from the attribute's behaviour and its base types — lets the tool +// compile just this file, so the engine and the tool can never disagree on the value. +public sealed partial class ClassTimelineAttribute +{ + /// + /// Custom-property key used to round-trip the chosen into + /// TestDetails.CustomProperties so the HTML reporter can read it back per class. + /// + internal const string ClassTimelinePropertyKey = "tunit.report.timeline"; +} diff --git a/TUnit.Core/Attributes/TestMetadata/ClassTimelineAttribute.cs b/TUnit.Core/Attributes/TestMetadata/ClassTimelineAttribute.cs index 8d521d7709a..e5ab268beef 100644 --- a/TUnit.Core/Attributes/TestMetadata/ClassTimelineAttribute.cs +++ b/TUnit.Core/Attributes/TestMetadata/ClassTimelineAttribute.cs @@ -29,7 +29,7 @@ namespace TUnit.Core; /// /// [AttributeUsage(AttributeTargets.Class | AttributeTargets.Assembly)] -public sealed class ClassTimelineAttribute(TimelineMode mode) : TUnitAttribute, ITestDiscoveryEventReceiver, IScopedAttribute +public sealed partial class ClassTimelineAttribute(TimelineMode mode) : TUnitAttribute, ITestDiscoveryEventReceiver, IScopedAttribute { /// The timeline rendering mode to apply. public TimelineMode Mode { get; } = mode; @@ -47,9 +47,4 @@ public ValueTask OnTestDiscovered(DiscoveredTestContext context) return default; } - /// - /// Custom-property key used to round-trip the chosen into - /// TestDetails.CustomProperties so the HTML reporter can read it back per class. - /// - internal const string ClassTimelinePropertyKey = "tunit.report.timeline"; } diff --git a/TUnit.Core/SpanData.cs b/TUnit.Core/SpanData.cs index 35f4ddc46e3..3d384de0a04 100644 --- a/TUnit.Core/SpanData.cs +++ b/TUnit.Core/SpanData.cs @@ -2,7 +2,9 @@ namespace TUnit.Core; -internal sealed class SpanData +// A record so report merging can rewrite single properties via `with` without a +// hand-maintained copy that silently drops newly added members. +internal sealed record SpanData { [JsonPropertyName("traceId")] public required string TraceId { get; init; } diff --git a/TUnit.Core/TUnitActivitySource.ReportConstants.cs b/TUnit.Core/TUnitActivitySource.ReportConstants.cs new file mode 100644 index 00000000000..5b0f668957f --- /dev/null +++ b/TUnit.Core/TUnitActivitySource.ReportConstants.cs @@ -0,0 +1,23 @@ +#if NET + +namespace TUnit.Core; + +// Span/tag constants consumed by the HTML report pipeline (HtmlReportGenerator). They live +// in this constants-only partial so TUnit.Reporting.Tool can source-link exactly these +// values without dragging in the ActivitySource statics — which also makes value drift +// between the engine and the tool impossible. Keep this file free of anything but consts. +public static partial class TUnitActivitySource +{ + // Span names used across the engine and HTML report. + internal const string SpanTestSession = "test session"; + internal const string SpanTestAssembly = "test assembly"; + internal const string SpanTestSuite = "test suite"; + internal const string SpanTestCase = "test case"; + internal const string SpanTestBody = "test body"; + + internal const string TagTestClass = "tunit.test.class"; + internal const string TagTestSuiteName = "test.suite.name"; + internal const string TagTraceScope = "tunit.trace.scope"; +} + +#endif diff --git a/TUnit.Core/TUnitActivitySource.cs b/TUnit.Core/TUnitActivitySource.cs index 3883249b38e..0d8045a72d2 100644 --- a/TUnit.Core/TUnitActivitySource.cs +++ b/TUnit.Core/TUnitActivitySource.cs @@ -4,7 +4,7 @@ namespace TUnit.Core; -public static class TUnitActivitySource +public static partial class TUnitActivitySource { private static readonly string Version = typeof(TUnitActivitySource).Assembly.GetName().Version?.ToString() ?? "0.0.0"; @@ -28,12 +28,8 @@ public static class TUnitActivitySource internal static readonly ActivitySource Source = new(SourceName, Version); internal static readonly ActivitySource LifecycleSource = new(LifecycleSourceName, Version); - // Span names used across the engine and HTML report. - internal const string SpanTestSession = "test session"; - internal const string SpanTestAssembly = "test assembly"; - internal const string SpanTestSuite = "test suite"; - internal const string SpanTestCase = "test case"; - internal const string SpanTestBody = "test body"; + // Span names live in TUnitActivitySource.ReportConstants.cs — a constants-only partial + // that TUnit.Reporting.Tool source-links (see that file's header before moving members). // Tag and baggage keys used across init/dispose spans, HTML report, and cross-boundary correlation. @@ -45,19 +41,16 @@ public static class TUnitActivitySource public const string TagTestId = "tunit.test.id"; internal const string TagSessionId = "tunit.session.id"; internal const string TagTestFilter = "tunit.filter"; - internal const string TagTestClass = "tunit.test.class"; internal const string TagTestMethod = "tunit.test.method"; internal const string TagTestNodeUid = "tunit.test.node_uid"; internal const string TagTestCategories = "tunit.test.categories"; internal const string TagTestCount = "tunit.test.count"; internal const string TagClassNamespace = "tunit.class.namespace"; internal const string TagTestCaseName = "test.case.name"; - internal const string TagTestSuiteName = "test.suite.name"; internal const string TagAssemblyName = "tunit.assembly.name"; internal const string TagTestCaseResultStatus = "test.case.result.status"; internal const string TagTestRetryAttempt = "tunit.test.retry_attempt"; internal const string TagTestSkipReason = "tunit.test.skip_reason"; - internal const string TagTraceScope = "tunit.trace.scope"; /// /// Returns a human-readable type name suitable for span labels. diff --git a/TUnit.Engine/Configuration/EnvironmentConstants.cs b/TUnit.Engine/Configuration/EnvironmentConstants.cs index f9717826aa0..16ef17949d3 100644 --- a/TUnit.Engine/Configuration/EnvironmentConstants.cs +++ b/TUnit.Engine/Configuration/EnvironmentConstants.cs @@ -12,6 +12,17 @@ internal static class EnvironmentConstants // TUnit-specific: how long (in days) the auto-uploaded HTML report artifact is kept public const string ArtifactRetentionDays = "TUNIT_ARTIFACT_RETENTION_DAYS"; + // TUnit-specific: cross-process report aggregation (issue #4522). + // ON by default wherever a shared directory is resolvable (GitHub Actions, or explicit + // TUNIT_AGGREGATE_DIR). Values: off/false/0/no/disabled/none = disable; + // defer = persist sidecars + merged HTML only (final summary via `tunit-report merge`); + // anything else (or unset) = cooperative merge. + public const string AggregateReports = "TUNIT_AGGREGATE_REPORTS"; + // Shared directory for sidecars/merged outputs; auto-derived on GitHub Actions when unset. + public const string AggregateDirectory = "TUNIT_AGGREGATE_DIR"; + // Opts out of the machine-readable JSON sidecar written next to the HTML report. + public const string DisableJsonReport = "TUNIT_DISABLE_JSON_REPORT"; + // TUnit-specific: Execution public const string ExecutionMode = "TUNIT_EXECUTION_MODE"; public const string MaxParallelTests = "TUNIT_MAX_PARALLEL_TESTS"; @@ -44,6 +55,11 @@ internal static class EnvironmentConstants // Used to clamp TUNIT_ARTIFACT_RETENTION_DAYS so the API does not reject the request. public const string GitHubRetentionDays = "GITHUB_RETENTION_DAYS"; + // GitHub Actions context (for report aggregation scoping) + public const string RunnerTemp = "RUNNER_TEMP"; + public const string GitHubRunAttempt = "GITHUB_RUN_ATTEMPT"; + public const string GitHubJob = "GITHUB_JOB"; + // GitHub Actions context (for CI metadata in reports) public const string GitHubSha = "GITHUB_SHA"; public const string GitHubRef = "GITHUB_REF"; diff --git a/TUnit.Engine/Reporters/Aggregation/AggregatedSummaryWriter.cs b/TUnit.Engine/Reporters/Aggregation/AggregatedSummaryWriter.cs new file mode 100644 index 00000000000..d7029ee1444 --- /dev/null +++ b/TUnit.Engine/Reporters/Aggregation/AggregatedSummaryWriter.cs @@ -0,0 +1,293 @@ +using System.Net; +using System.Text; +using TUnit.Engine.Reporters.Html; + +namespace TUnit.Engine.Reporters.Aggregation; + +/// +/// Renders one Markdown summary block covering every suite persisted so far — used by the +/// cooperative in-engine merge (each finishing process rewrites the block; the last one +/// leaves the complete aggregate) and by the tunit-report tool's +/// --github-summary. Works purely on sidecar so it needs +/// no live test-node state. +/// +internal static class AggregatedSummaryWriter +{ + // Keeps the step summary within GitHub's 1 MB limit; GitHubReporter's per-suite + // rendering references this same cap so both views truncate identically. + internal const int MaxTestsPerGroup = 50; + + internal static string Render( + IReadOnlyList suites, + bool collapsible = true, + string? serverUrl = null, + string? mergedReportHint = null) + { + var labels = ReportDataMerger.BuildSuiteLabels(suites); + + var totals = new ReportSummary(); + // Bounds parsed once per suite; both the whole-run duration and each table row + // derive from them, so no timestamp is parsed twice. + var bounds = new (long StartMs, long EndMs)?[suites.Count]; + var earliest = long.MaxValue; + var latest = long.MinValue; + for (var i = 0; i < suites.Count; i++) + { + totals.Add(suites[i].Summary); + bounds[i] = ReportDataMerger.ComputeWallClockBounds(suites[i]); + if (bounds[i] is { } b) + { + if (b.StartMs < earliest) earliest = b.StartMs; + if (b.EndMs > latest) latest = b.EndMs; + } + } + + var hasFailures = totals.TotalUnsuccessful > 0; + var statusEmoji = hasFailures ? "❌" : "✅"; + var passRate = totals.Total > 0 ? (double)totals.Passed / totals.Total * 100 : 0; + var wallMs = earliest == long.MaxValue + ? suites.Max(static s => s.TotalDurationMs) + : latest - earliest; + var suiteWord = suites.Count == 1 ? "suite" : "suites"; + + var sb = new StringBuilder(); + sb.AppendLine($"### {statusEmoji} TUnit Test Results — {suites.Count} {suiteWord}"); + sb.AppendLine(); + sb.AppendLine($"**{totals.Total} tests** across **{suites.Count} {suiteWord}** in **{FormatDuration(wallMs)}** — **{passRate:F1}%** passed"); + sb.AppendLine(); + + if (totals.Passed != totals.Total) + { + var segments = new List { $"✅ {totals.Passed} passed" }; + if (totals.Failed > 0) segments.Add($"❌ {totals.Failed} failed"); + if (totals.Skipped > 0) segments.Add($"⏭️ {totals.Skipped} skipped"); + if (totals.TimedOut > 0) segments.Add($"⏱️ {totals.TimedOut} timed out"); + if (totals.Cancelled > 0) segments.Add($"🚫 {totals.Cancelled} cancelled"); + sb.AppendLine(string.Join(" · ", segments)); + sb.AppendLine(); + } + + AppendSuiteTable(sb, suites, labels, bounds); + + if (totals.Flaky > 0) + { + AppendFlakySection(sb, suites, labels); + } + + if (hasFailures) + { + AppendFailuresByCause(sb, suites, labels, collapsible, serverUrl); + } + + if (!string.IsNullOrEmpty(mergedReportHint)) + { + sb.AppendLine(); + sb.AppendLine($"> {mergedReportHint}"); + } + + sb.AppendLine(); + sb.AppendLine("---"); + return sb.ToString(); + } + + private static void AppendSuiteTable( + StringBuilder sb, IReadOnlyList suites, string[] labels, (long StartMs, long EndMs)?[] bounds) + { + var anyReportLink = suites.Any(static s => !string.IsNullOrEmpty(s.ArtifactUrl)); + + sb.AppendLine(anyReportLink + ? "| Suite | Tests | ✅ | ❌ | ⏭️ | Duration | Report |" + : "| Suite | Tests | ✅ | ❌ | ⏭️ | Duration |"); + sb.AppendLine(anyReportLink + ? "| --- | ---: | ---: | ---: | ---: | ---: | --- |" + : "| --- | ---: | ---: | ---: | ---: | ---: |"); + + for (var i = 0; i < suites.Count; i++) + { + var s = suites[i].Summary; + var emoji = s.TotalUnsuccessful > 0 ? "❌" : "✅"; + // Wall clock derived from the suite's own test timestamps; TotalDurationMs is + // only the fallback — it can come from a session span that measures differently. + var duration = bounds[i] is { } b ? b.EndMs - b.StartMs : suites[i].TotalDurationMs; + var row = $"| {emoji} `{labels[i]}` | {s.Total} | {s.Passed} | {s.TotalUnsuccessful} | {s.Skipped} | {FormatDuration(duration)} |"; + if (anyReportLink) + { + row += string.IsNullOrEmpty(suites[i].ArtifactUrl) ? " |" : $" [View]({suites[i].ArtifactUrl}) |"; + } + sb.AppendLine(row); + } + sb.AppendLine(); + } + + private static void AppendFlakySection(StringBuilder sb, IReadOnlyList suites, string[] labels) + { + var flakyTests = new List<(string Suite, string Name, int Attempts, double DurationMs)>(); + for (var i = 0; i < suites.Count; i++) + { + foreach (var group in suites[i].Groups) + { + foreach (var test in group.Tests) + { + if (test.RetryAttempt > 0 && test.Status == "passed") + { + flakyTests.Add((labels[i], $"{test.ClassName}.{test.DisplayName}", test.RetryAttempt + 1, test.DurationMs)); + } + } + } + } + + if (flakyTests.Count == 0) + { + return; + } + + sb.AppendLine($"> **⚠️ {flakyTests.Count} flaky {(flakyTests.Count == 1 ? "test" : "tests")}** passed after retry:"); + foreach (var (suite, name, attempts, durationMs) in flakyTests) + { + sb.AppendLine($"> - `{name}` ({suite}) — {attempts} attempts ({FormatDuration(durationMs)})"); + } + sb.AppendLine(); + } + + private static void AppendFailuresByCause( + StringBuilder sb, IReadOnlyList suites, string[] labels, bool collapsible, string? serverUrl) + { + var failures = new List<(string Suite, ReportTestResult Test, ReportData Owner)>(); + for (var i = 0; i < suites.Count; i++) + { + foreach (var group in suites[i].Groups) + { + foreach (var test in group.Tests) + { + if (test.Status is "failed" or "error" or "timedOut") + { + failures.Add((labels[i], test, suites[i])); + } + } + } + } + + if (failures.Count == 0) + { + return; + } + + var grouped = failures + .GroupBy(static f => ExceptionLabel(f.Test)) + .OrderByDescending(static g => g.Count()) + .ToArray(); + + var diagParts = grouped.Take(3).Select(static g => + { + var topSuite = g.GroupBy(static x => x.Suite).OrderByDescending(static c => c.Count()).First(); + return $"{g.Count()} × `{g.Key}` in `{topSuite.Key}`"; + }); + sb.AppendLine($"> **Quick diagnosis:** {string.Join(", ", diagParts)}"); + sb.AppendLine(); + + sb.AppendLine("#### Failures by Cause"); + sb.AppendLine(); + + foreach (var group in grouped) + { + var entries = group.ToList(); + var label = $"{group.Key} ({entries.Count} {(entries.Count == 1 ? "test" : "tests")})"; + + if (collapsible) + { + sb.AppendLine("
"); + sb.AppendLine($"{label}"); + } + else + { + sb.AppendLine($"**{label}**"); + } + + sb.AppendLine(); + sb.AppendLine("| Test | Suite | Duration |"); + sb.AppendLine("| --- | --- | --- |"); + + var displayCount = Math.Min(entries.Count, MaxTestsPerGroup); + for (var i = 0; i < displayCount; i++) + { + var (suite, test, owner) = entries[i]; + var sourcePart = BuildSourceLink(test, owner, serverUrl) is { } link ? $" {link}" : ""; + sb.AppendLine($"| `{test.ClassName}.{test.DisplayName}`{sourcePart} | `{suite}` | {FormatDuration(test.DurationMs)} |"); + } + + if (entries.Count > MaxTestsPerGroup) + { + sb.AppendLine($"| *...and {entries.Count - MaxTestsPerGroup} more* | | |"); + } + + var commonError = entries + .Select(static e => e.Test.Exception?.Message) + .Where(static m => !string.IsNullOrWhiteSpace(m)) + .GroupBy(static m => m) + .OrderByDescending(static g => g.Count()) + .FirstOrDefault() + ?.Key; + + if (commonError is not null) + { + sb.AppendLine(); + sb.AppendLine("**Common error:**"); + sb.AppendLine($"
{WebUtility.HtmlEncode(Truncate(commonError, 1000))}
"); + } + + if (collapsible) + { + sb.AppendLine(); + sb.AppendLine("
"); + } + + sb.AppendLine(); + } + } + + private static string ExceptionLabel(ReportTestResult test) + { + if (test.Status == "timedOut") + { + return "Timeout"; + } + + var type = test.Exception?.Type; + if (string.IsNullOrEmpty(type)) + { + return "Unknown"; + } + + // Sidecars store the exception's FullName; the summary groups by short name, + // matching the per-suite GitHub summary. + var lastDot = type!.LastIndexOf('.'); + return lastDot >= 0 ? type.Substring(lastDot + 1) : type; + } + + private static string? BuildSourceLink(ReportTestResult test, ReportData owner, string? serverUrl) + { + if (string.IsNullOrEmpty(serverUrl) + || string.IsNullOrEmpty(owner.RepositorySlug) + || string.IsNullOrEmpty(owner.CommitSha) + || string.IsNullOrEmpty(test.SourceRelativePath) + || test.LineNumber is not { } line) + { + return null; + } + + var fileName = Path.GetFileName(test.SourceRelativePath!); + return $"[{fileName}:{line}]({serverUrl!.TrimEnd('/')}/{owner.RepositorySlug}/blob/{owner.CommitSha}/{test.SourceRelativePath}#L{line})"; + } + + private static string Truncate(string value, int maxLength) + => value.Length <= maxLength ? value : value.Substring(0, maxLength) + "…"; + + internal static string FormatDuration(double milliseconds) => milliseconds switch + { + < 1 => "< 1ms", + < 1000 => $"{milliseconds:F0}ms", + < 60_000 => $"{milliseconds / 1000:F1}s", + < 3_600_000 => $"{(int)(milliseconds / 60_000)}m {(int)(milliseconds % 60_000 / 1000)}s", + _ => $"{(int)(milliseconds / 3_600_000)}h {(int)(milliseconds % 3_600_000 / 60_000)}m", + }; +} diff --git a/TUnit.Engine/Reporters/Aggregation/AtomicFile.cs b/TUnit.Engine/Reporters/Aggregation/AtomicFile.cs new file mode 100644 index 00000000000..fdeefb5d8ab --- /dev/null +++ b/TUnit.Engine/Reporters/Aggregation/AtomicFile.cs @@ -0,0 +1,74 @@ +using System.IO; +using System.Text; + +namespace TUnit.Engine.Reporters.Aggregation; + +/// +/// Stage-to-temp-then-swap file writes for the aggregation pipeline: concurrent readers +/// never observe a torn file, and a process killed mid-write cannot leave a truncated +/// destination (which for the GitHub step summary would mean losing other tools' content, +/// not just ours). Single implementation shared by the sidecar, merged-report and +/// summary-region writers — and source-linked into TUnit.Reporting.Tool. +/// +internal static class AtomicFile +{ + internal static void WriteAllBytes(string path, byte[] bytes) + { + var tempPath = TempPathFor(path); + File.WriteAllBytes(tempPath, bytes); + if (!TrySwap(tempPath, path)) + { + File.WriteAllBytes(path, bytes); + } + } + + internal static void WriteAllText(string path, string content) + { + var tempPath = TempPathFor(path); + File.WriteAllText(tempPath, content, Encoding.UTF8); + if (!TrySwap(tempPath, path)) + { + File.WriteAllText(path, content, Encoding.UTF8); + } + } + + private static string TempPathFor(string path) + => path + "." + Guid.NewGuid().ToString("N").Substring(0, 8) + ".tmp"; + + // False = the swap isn't possible on this filesystem (e.g. some network mounts); + // callers fall back to an in-place write, accepting the small tear window. + private static bool TrySwap(string tempPath, string path) + { + try + { +#if NET + File.Move(tempPath, path, overwrite: true); +#else + // No overwriting Move downlevel; Replace is atomic but requires the + // destination to exist. Never delete-then-move — a crash between the two + // would lose the existing file's content entirely. + if (File.Exists(path)) + { + File.Replace(tempPath, path, destinationBackupFileName: null, ignoreMetadataErrors: true); + } + else + { + File.Move(tempPath, path); + } +#endif + return true; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or PlatformNotSupportedException) + { + try + { + File.Delete(tempPath); + } + catch (IOException) + { + // Best effort; a stray .tmp is harmless. + } + return false; + } + } +} diff --git a/TUnit.Engine/Reporters/Aggregation/GitHubSummaryRegion.cs b/TUnit.Engine/Reporters/Aggregation/GitHubSummaryRegion.cs new file mode 100644 index 00000000000..444ad03f868 --- /dev/null +++ b/TUnit.Engine/Reporters/Aggregation/GitHubSummaryRegion.cs @@ -0,0 +1,104 @@ +using System.IO; +using System.Text; + +namespace TUnit.Engine.Reporters.Aggregation; + +/// +/// Maintains a single marked region inside a GitHub step summary file. The +/// GITHUB_STEP_SUMMARY file is per-step and freely rewritable until the step ends +/// (the runner reads it once, at step completion), so every finishing sibling process can +/// replace the region with a fresher aggregate — the last one to finish leaves the +/// complete summary, and no process needs to know whether it is last. +/// +/// Safety: content written by anything else (user echo >> lines, other tools) +/// must never be lost. Only the region strictly between TUnit's own invisible HTML-comment +/// markers is ever replaced, the block is matched conservatively (see ), +/// and the rewrite goes through a temp file + atomic replace so a killed process cannot +/// leave a truncated summary. +/// +internal static class GitHubSummaryRegion +{ + internal const string StartMarker = ""; + internal const string EndMarker = ""; + + /// + /// GitHub truncates step summaries above 1 MiB. Lives here (rather than only in + /// EngineDefaults, which isn't source-linked into the tool) so the engine and + /// tunit-report enforce the same cap. + /// + internal const long MaxFileSizeInBytes = 1024 * 1024; + + /// + /// Replaces the marked region in with + /// , appending a new marked region when none exists. + /// Returns false when the file is missing or would exceed + /// . Callers hold the aggregation lock, which + /// serialises all TUnit writers. + /// + internal static bool ReplaceOrAppend(string summaryFilePath, string content, long maxFileSizeInBytes = MaxFileSizeInBytes) + { + if (!File.Exists(summaryFilePath)) + { + return false; + } + + var existing = File.ReadAllText(summaryFilePath, Encoding.UTF8); + var updated = Splice(existing, content); + + if (Encoding.UTF8.GetByteCount(updated) > maxFileSizeInBytes) + { + // Don't leave a previously-written (now stale) block standing — it would + // silently misreport which suites ran. Swap the region for a short notice, + // which also frees whatever space the old block occupied. + updated = Splice(existing, OverflowNotice); + if (Encoding.UTF8.GetByteCount(updated) > maxFileSizeInBytes) + { + Console.WriteLine("Skipping GitHub step summary update: the file is already at GitHub's 1MB size limit."); + return false; + } + + Console.WriteLine("The aggregated summary would exceed GitHub's 1MB step summary limit; wrote an overflow notice instead."); + AtomicFile.WriteAllText(summaryFilePath, updated); + return false; + } + + AtomicFile.WriteAllText(summaryFilePath, updated); + return true; + } + + internal const string OverflowNotice = + "> ⚠️ The TUnit aggregated summary was omitted because it would exceed GitHub's 1 MiB step-summary limit. See the merged HTML report for full results."; + + /// + /// Splices into as a marked block. + /// The block to replace is matched conservatively: find the first end marker, then the + /// last start marker before it — the span between them is guaranteed to be TUnit's own + /// block, because foreign content is only ever appended after our end marker, never + /// inside the pair. Any unpaired (torn) marker is left in place rather than risking a + /// splice that swallows someone else's content; the fresh block is appended instead. + /// + internal static string Splice(string existing, string content) + { + var block = $"{StartMarker}\n{content}\n{EndMarker}\n"; + + var end = existing.IndexOf(EndMarker, StringComparison.Ordinal); + var start = end >= 0 ? existing.LastIndexOf(StartMarker, end, StringComparison.Ordinal) : -1; + + if (end < 0 || start < 0) + { + // No complete block yet (first writer, or a torn/foreign fragment): append. + return existing.Length == 0 || existing.EndsWith("\n", StringComparison.Ordinal) + ? existing + block + : existing + "\n" + block; + } + + var afterEnd = end + EndMarker.Length; + // Swallow the newline our own block writes after the end marker. + if (afterEnd < existing.Length && existing[afterEnd] == '\n') + { + afterEnd++; + } + + return existing.Substring(0, start) + block + existing.Substring(afterEnd); + } +} diff --git a/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs b/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs new file mode 100644 index 00000000000..acba21e926d --- /dev/null +++ b/TUnit.Engine/Reporters/Aggregation/ReportAggregator.cs @@ -0,0 +1,234 @@ +using System.IO; +using System.Security.Cryptography; +using System.Text; +using TUnit.Engine.Configuration; +using TUnit.Engine.Helpers; +using TUnit.Engine.Reporters.Html; + +namespace TUnit.Engine.Reporters.Aggregation; + +internal enum AggregationMode +{ + /// No aggregation — reporters behave per-process, as before. + Disabled, + + /// + /// Sidecars are persisted to the shared directory and every finishing process + /// rewrites the merged HTML report and the marked GitHub step summary region. + /// For sibling processes within a single step (e.g. dotnet test on a solution). + /// + Cooperative, + + /// + /// Sidecars and the merged HTML are persisted, but no step summary is written at all. + /// For pipelines that run test projects as separate steps: a final step runs + /// tunit-report merge --github-summary to emit the single block. + /// + Defer, +} + +/// +/// Cross-process report aggregation (issue #4522). Each TUnit process persists its +/// as a JSON sidecar into a directory shared by all sibling +/// processes of the run, then — under a cross-process file lock — re-renders the merged +/// outputs from every sidecar present so far. The last process to finish naturally leaves +/// the complete aggregate; no process ever needs to know whether it is the last one. +/// +internal sealed class ReportAggregator +{ + private const string SidecarSearchPattern = "*" + ReportDataJson.SidecarExtension; + private const string LockFileName = ".tunit-aggregate.lock"; + + // Lock contention is expected (N processes finishing together, each holding the lock + // for a full merge), so wait far longer than the file-write retry defaults. + private const int LockMaxAttempts = 60; + private const int LockRetryDelayMs = 250; + + internal AggregationMode Mode { get; } + internal string Directory { get; } + internal string MergedReportPath => Path.Combine(Directory, ReportDataJson.MergedReportFileName); + + private ReportAggregator(AggregationMode mode, string directory) + { + Mode = mode; + Directory = directory; + } + + /// + /// Reads TUNIT_AGGREGATE_REPORTS / TUNIT_AGGREGATE_DIR and resolves the shared + /// directory. Aggregation is ON by default wherever a shared directory is resolvable + /// (GitHub Actions, or an explicit TUNIT_AGGREGATE_DIR); plain local runs silently + /// no-op. Returns when aggregation is off or no shared + /// directory can be derived. + /// + internal static ReportAggregator? TryCreateFromEnvironment(Func getEnv) + { + var raw = getEnv(EnvironmentConstants.AggregateReports)?.Trim().ToLowerInvariant(); + + var mode = raw switch + { + "0" or "false" or "no" or "off" or "disabled" or "none" => AggregationMode.Disabled, + "defer" => AggregationMode.Defer, + // Unset, or any affirmative value (1/true/yes/cooperative): cooperative merge. + _ => AggregationMode.Cooperative, + }; + + if (mode == AggregationMode.Disabled) + { + return null; + } + + var directory = ResolveDirectory(getEnv); + if (directory is null) + { + // Only warn when aggregation was explicitly requested — with the on-by-default + // behaviour, every plain local run lands here and must stay silent. + if (!string.IsNullOrEmpty(raw)) + { + Console.WriteLine( + $"Warning: {EnvironmentConstants.AggregateReports} is set but no shared directory could be resolved. " + + $"Set {EnvironmentConstants.AggregateDirectory} to a directory shared by all test processes. Report aggregation is disabled for this run."); + } + return null; + } + + return new ReportAggregator(mode, directory); + } + + private static string? ResolveDirectory(Func getEnv) + { + var explicitDir = getEnv(EnvironmentConstants.AggregateDirectory); + if (!string.IsNullOrWhiteSpace(explicitDir)) + { + return Path.GetFullPath(explicitDir!); + } + + // On GitHub Actions a job-scoped shared directory can be derived automatically: + // RUNNER_TEMP is shared by every process in the job and cleaned between jobs. + // Run id + attempt + job keep re-runs and sibling jobs on the same runner apart. + if (getEnv(EnvironmentConstants.GitHubActions) is "true" + && getEnv(EnvironmentConstants.RunnerTemp) is { Length: > 0 } runnerTemp) + { + var runId = getEnv(EnvironmentConstants.GitHubRunId) ?? "0"; + var attempt = getEnv(EnvironmentConstants.GitHubRunAttempt) ?? "1"; + var job = getEnv(EnvironmentConstants.GitHubJob) ?? "job"; + return Path.Combine(runnerTemp, "tunit-aggregate", + PathValidator.SanitizeFileName($"run-{runId}-{attempt}-{job}")); + } + + return null; + } + + /// + /// Persists this process's already-serialized report data into the shared directory. + /// The file name is stable per suite (assembly + report path hash), so a re-run within + /// the same scope overwrites rather than duplicates. Takes bytes rather than + /// so callers writing the sidecar to more than one location + /// serialize only once. + /// + internal string WriteSidecar(byte[] sidecarUtf8Json, string assemblyName, string suiteSalt) + { + System.IO.Directory.CreateDirectory(Directory); + + var fileName = $"{PathValidator.SanitizeFileName(assemblyName)}-{ShortHash(suiteSalt)}{ReportDataJson.SidecarExtension}"; + var path = Path.Combine(Directory, fileName); + AtomicFile.WriteAllBytes(path, sidecarUtf8Json); + return path; + } + + /// + /// Reads every sidecar currently present in the shared directory. Unreadable or + /// foreign files are skipped — a crashed sibling must not break the merge. + /// + internal List ReadAllSidecars() + { + var results = new List(); + if (!System.IO.Directory.Exists(Directory)) + { + return results; + } + + // When TUNIT_AGGREGATE_DIR is pointed at a directory that also receives the local + // sidecar (e.g. a project's TestResults), the same suite exists twice under two + // names — byte-identical output of one writer, so dedupe on a content hash, same + // as the tunit-report tool does. + var seenDigests = new HashSet(StringComparer.Ordinal); + using var sha = SHA256.Create(); + foreach (var file in System.IO.Directory.GetFiles(Directory, SidecarSearchPattern)) + { + try + { + var bytes = File.ReadAllBytes(file); + if (seenDigests.Add(Convert.ToBase64String(sha.ComputeHash(bytes))) + && ReportDataJson.TryDeserialize((ReadOnlyMemory)bytes) is { } data) + { + results.Add(data); + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + // Locked mid-write by a sibling (it re-merges after us anyway) or not + // readable by this process — one bad file must not abort the whole merge. + } + } + + return results; + } + + /// + /// Acquires the cross-process aggregation lock. Every writer performs its whole + /// read-merge-write cycle under this lock, so merges never interleave. Returns + /// when the lock cannot be acquired within the timeout; + /// callers should then skip merging (a later sibling will produce a fresher merge). + /// + internal async Task AcquireLockAsync(CancellationToken cancellationToken) + { + System.IO.Directory.CreateDirectory(Directory); + var lockPath = Path.Combine(Directory, LockFileName); + + for (var attempt = 1; attempt <= LockMaxAttempts; attempt++) + { + cancellationToken.ThrowIfCancellationRequested(); + try + { + return new FileStream(lockPath, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + // Contention (or a permissions hiccup) on the final attempt must still fall + // through to the graceful "skip this merge" path, never escape as a throw. + if (attempt == LockMaxAttempts) + { + break; + } + + await Task.Delay(LockRetryDelayMs + Random.Shared.Next(0, 100), cancellationToken); + } + } + + Console.WriteLine("Warning: Could not acquire the report aggregation lock; skipping merge for this process."); + return null; + } + + /// + /// Regenerates the merged HTML report from all sidecars present. Caller holds the lock. + /// + internal void WriteMergedHtml(IReadOnlyList suites) + { + if (suites.Count == 0) + { + return; + } + + var merged = ReportDataMerger.Merge(suites); + var html = HtmlReportGenerator.GenerateHtml(merged); + AtomicFile.WriteAllText(MergedReportPath, html); + } + + private static string ShortHash(string value) + { + using var sha = SHA256.Create(); + var hash = sha.ComputeHash(Encoding.UTF8.GetBytes(value)); + return BitConverter.ToString(hash, 0, 4).Replace("-", "").ToLowerInvariant(); + } +} diff --git a/TUnit.Engine/Reporters/Aggregation/ReportDataJson.cs b/TUnit.Engine/Reporters/Aggregation/ReportDataJson.cs new file mode 100644 index 00000000000..ee2ebecf3dd --- /dev/null +++ b/TUnit.Engine/Reporters/Aggregation/ReportDataJson.cs @@ -0,0 +1,492 @@ +using System.IO; +using System.Text; +using System.Text.Json; +using TUnit.Core; +using TUnit.Engine.Reporters.Html; + +namespace TUnit.Engine.Reporters.Aggregation; + +/// +/// Canonical JSON persistence for — the machine-readable +/// sidecar written next to the HTML report and consumed by cross-process aggregation +/// (the in-engine cooperative merge and the tunit-report dotnet tool). +/// +/// This is a different schema from the renderer JSON embedded in the HTML report: +/// that one is lossy (statuses collapsed, times made relative) and shaped for the +/// template's client script. This one round-trips faithfully, +/// using the property names documented by the [JsonPropertyName] attributes on +/// the model. Both sides are hand-written (no reflection serializer) so they stay +/// Native-AOT compatible. Unknown properties are ignored on read for forward compat. +/// +internal static class ReportDataJson +{ + /// Bump when the schema changes shape incompatibly; readers reject newer majors. + internal const int SchemaVersion = 1; + + /// File extension shared by every sidecar so aggregators can discover them. + internal const string SidecarExtension = ".tunit-report.json"; + + /// Merged HTML report filename, shared by the engine and the tool's default output. + internal const string MergedReportFileName = "merged-report.html"; + + /// + /// Serializes straight to UTF-8 bytes — callers write the same payload to more than one + /// file, so producing bytes once avoids a UTF-8 → string → UTF-8 round trip per copy. + /// + internal static byte[] SerializeToBytes(ReportData data) + { + using var ms = new MemoryStream(); + using (var w = new Utf8JsonWriter(ms, new JsonWriterOptions { Indented = false })) + { + Write(w, data); + } + return ms.ToArray(); + } + + internal static string Serialize(ReportData data) + => Encoding.UTF8.GetString(SerializeToBytes(data)); + + private static void Write(Utf8JsonWriter w, ReportData data) + { + w.WriteStartObject(); + w.WriteNumber("schemaVersion", SchemaVersion); + w.WriteString("assemblyName", data.AssemblyName); + w.WriteString("machineName", data.MachineName); + w.WriteString("timestamp", data.Timestamp); + w.WriteString("tunitVersion", data.TUnitVersion); + w.WriteString("operatingSystem", data.OperatingSystem); + w.WriteString("runtimeVersion", data.RuntimeVersion); + if (data.Filter is not null) w.WriteString("filter", data.Filter); + w.WriteNumber("totalDurationMs", data.TotalDurationMs); + if (data.ArtifactUrl is not null) w.WriteString("artifactUrl", data.ArtifactUrl); + if (data.CommitSha is not null) w.WriteString("commitSha", data.CommitSha); + if (data.Branch is not null) w.WriteString("branch", data.Branch); + if (data.PullRequestNumber is not null) w.WriteString("pullRequestNumber", data.PullRequestNumber); + if (data.RepositorySlug is not null) w.WriteString("repositorySlug", data.RepositorySlug); + if (data.SourceLinks is { } links) + { + // Persisted so a merged report keeps clickable source links: the HTML template + // renders them from these URL templates, not from commit/repository fields. + w.WritePropertyName("sourceLinks"); + w.WriteStartObject(); + w.WriteString("lineUrl", links.LineUrl); + w.WriteString("rangeUrl", links.RangeUrl); + if (links.RawUrl is not null) w.WriteString("rawUrl", links.RawUrl); + w.WriteEndObject(); + } + + w.WritePropertyName("summary"); + WriteSummary(w, data.Summary); + + w.WritePropertyName("groups"); + w.WriteStartArray(); + foreach (var g in data.Groups) + { + w.WriteStartObject(); + w.WriteString("className", g.ClassName); + w.WriteString("namespace", g.Namespace); + w.WritePropertyName("summary"); + WriteSummary(w, g.Summary); + w.WritePropertyName("tests"); + w.WriteStartArray(); + foreach (var t in g.Tests) WriteTest(w, t); + w.WriteEndArray(); + w.WriteEndObject(); + } + w.WriteEndArray(); + + if (data.Spans is { Length: > 0 } spans) + { + w.WritePropertyName("spans"); + w.WriteStartArray(); + foreach (var s in spans) WriteSpan(w, s); + w.WriteEndArray(); + } + + w.WriteEndObject(); + } + + private static void WriteSummary(Utf8JsonWriter w, ReportSummary s) + { + w.WriteStartObject(); + w.WriteNumber("total", s.Total); + w.WriteNumber("passed", s.Passed); + w.WriteNumber("failed", s.Failed); + w.WriteNumber("skipped", s.Skipped); + w.WriteNumber("cancelled", s.Cancelled); + w.WriteNumber("timedOut", s.TimedOut); + w.WriteNumber("flaky", s.Flaky); + w.WriteEndObject(); + } + + private static void WriteTest(Utf8JsonWriter w, ReportTestResult t) + { + w.WriteStartObject(); + w.WriteString("id", t.Id); + w.WriteString("displayName", t.DisplayName); + w.WriteString("methodName", t.MethodName); + w.WriteString("className", t.ClassName); + w.WriteString("status", t.Status); + w.WriteNumber("durationMs", t.DurationMs); + if (t.StartTime is not null) w.WriteString("startTime", t.StartTime); + if (t.EndTime is not null) w.WriteString("endTime", t.EndTime); + if (t.Exception is not null) + { + w.WritePropertyName("exception"); + WriteException(w, t.Exception); + } + if (t.Output is not null) w.WriteString("output", t.Output); + if (t.ErrorOutput is not null) w.WriteString("errorOutput", t.ErrorOutput); + if (t.Categories is { Length: > 0 } cats) + { + w.WritePropertyName("categories"); + w.WriteStartArray(); + foreach (var c in cats) w.WriteStringValue(c); + w.WriteEndArray(); + } + if (t.CustomProperties is { Length: > 0 } props) + { + w.WritePropertyName("customProperties"); + w.WriteStartArray(); + foreach (var p in props) WriteKeyValue(w, p); + w.WriteEndArray(); + } + if (t.FilePath is not null) w.WriteString("filePath", t.FilePath); + if (t.LineNumber is { } line) w.WriteNumber("lineNumber", line); + if (t.EndLineNumber is { } endLine) w.WriteNumber("endLineNumber", endLine); + if (t.SourceRelativePath is not null) w.WriteString("sourceRelativePath", t.SourceRelativePath); + if (t.SkipReason is not null) w.WriteString("skipReason", t.SkipReason); + if (t.RetryAttempt != 0) w.WriteNumber("retryAttempt", t.RetryAttempt); + if (t.Attempts is { Length: > 0 } attempts) + { + w.WritePropertyName("attempts"); + w.WriteStartArray(); + foreach (var a in attempts) + { + w.WriteStartObject(); + w.WriteString("status", a.Status); + w.WriteNumber("durationMs", a.DurationMs); + if (a.ExceptionType is not null) w.WriteString("exceptionType", a.ExceptionType); + if (a.ExceptionMessage is not null) w.WriteString("exceptionMessage", a.ExceptionMessage); + if (a.StackTrace is not null) w.WriteString("stackTrace", a.StackTrace); + w.WriteEndObject(); + } + w.WriteEndArray(); + } + if (t.TraceId is not null) w.WriteString("traceId", t.TraceId); + if (t.SpanId is not null) w.WriteString("spanId", t.SpanId); + if (t.AdditionalTraceIds is { Length: > 0 } extra) + { + w.WritePropertyName("additionalTraceIds"); + w.WriteStartArray(); + foreach (var id in extra) w.WriteStringValue(id); + w.WriteEndArray(); + } + w.WriteEndObject(); + } + + private static void WriteException(Utf8JsonWriter w, ReportExceptionData ex) + { + w.WriteStartObject(); + w.WriteString("type", ex.Type); + w.WriteString("message", ex.Message); + if (ex.StackTrace is not null) w.WriteString("stackTrace", ex.StackTrace); + if (ex.InnerException is not null) + { + w.WritePropertyName("innerException"); + WriteException(w, ex.InnerException); + } + w.WriteEndObject(); + } + + private static void WriteSpan(Utf8JsonWriter w, SpanData s) + { + w.WriteStartObject(); + w.WriteString("traceId", s.TraceId); + w.WriteString("spanId", s.SpanId); + if (s.ParentSpanId is not null) w.WriteString("parentSpanId", s.ParentSpanId); + w.WriteString("name", s.Name); + if (s.SpanType is not null) w.WriteString("spanType", s.SpanType); + w.WriteString("source", s.Source); + w.WriteString("kind", s.Kind); + w.WriteNumber("startTimeMs", s.StartTimeMs); + w.WriteNumber("durationMs", s.DurationMs); + w.WriteString("status", s.Status); + if (s.StatusMessage is not null) w.WriteString("statusMessage", s.StatusMessage); + if (s.Tags is { Length: > 0 } tags) + { + w.WritePropertyName("tags"); + w.WriteStartArray(); + foreach (var t in tags) WriteKeyValue(w, t); + w.WriteEndArray(); + } + if (s.Events is { Length: > 0 } events) + { + w.WritePropertyName("events"); + w.WriteStartArray(); + foreach (var e in events) + { + w.WriteStartObject(); + w.WriteString("name", e.Name); + w.WriteNumber("timestampMs", e.TimestampMs); + if (e.Tags is { Length: > 0 } eTags) + { + w.WritePropertyName("tags"); + w.WriteStartArray(); + foreach (var t in eTags) WriteKeyValue(w, t); + w.WriteEndArray(); + } + w.WriteEndObject(); + } + w.WriteEndArray(); + } + if (s.Links is { Length: > 0 } links) + { + w.WritePropertyName("links"); + w.WriteStartArray(); + foreach (var l in links) + { + w.WriteStartObject(); + w.WriteString("traceId", l.TraceId); + w.WriteString("spanId", l.SpanId); + w.WriteEndObject(); + } + w.WriteEndArray(); + } + w.WriteEndObject(); + } + + private static void WriteKeyValue(Utf8JsonWriter w, ReportKeyValue kv) + { + w.WriteStartObject(); + w.WriteString("key", kv.Key); + w.WriteString("value", kv.Value); + w.WriteEndObject(); + } + + /// + /// Reads a sidecar back into . Returns + /// (never throws) for malformed JSON, a missing/newer schemaVersion, or a file + /// that isn't a TUnit report sidecar — a torn or foreign file in the aggregation + /// directory must not take down every sibling process's merge. + /// + internal static ReportData? TryDeserialize(string json) + { + try + { + using var doc = JsonDocument.Parse(json); + return Read(doc); + } + catch (Exception ex) when (IsMalformedSidecar(ex)) + { + return null; + } + } + + /// + internal static ReportData? TryDeserialize(ReadOnlyMemory utf8Json) + { + try + { + using var doc = JsonDocument.Parse(utf8Json); + return Read(doc); + } + catch (Exception ex) when (IsMalformedSidecar(ex)) + { + return null; + } + } + + // JsonException covers syntax errors, but shape errors surface differently: + // TryGetProperty on a non-object nested value (e.g. "groups":[1]) throws + // InvalidOperationException, and out-of-range numbers (e.g. 1e9999) throw + // FormatException. All three mean "not a usable sidecar" — never let one corrupt + // or foreign file abort a merge. + private static bool IsMalformedSidecar(Exception ex) + => ex is JsonException or InvalidOperationException or FormatException; + + private static ReportData? Read(JsonDocument doc) + { + var root = doc.RootElement; + if (root.ValueKind != JsonValueKind.Object) return null; + // TryGetInt32 (not GetInt32): a corrupt/foreign file with e.g. 999999999999 or 1.5 + // must be rejected as incompatible, not escape as a FormatException. + if (!root.TryGetProperty("schemaVersion", out var version) + || version.ValueKind != JsonValueKind.Number + || !version.TryGetInt32(out var schemaVersion) + || schemaVersion > SchemaVersion) + { + return null; + } + + return new ReportData + { + AssemblyName = GetString(root, "assemblyName") ?? "Unknown", + MachineName = GetString(root, "machineName") ?? "", + Timestamp = GetString(root, "timestamp") ?? "", + TUnitVersion = GetString(root, "tunitVersion") ?? "", + OperatingSystem = GetString(root, "operatingSystem") ?? "", + RuntimeVersion = GetString(root, "runtimeVersion") ?? "", + Filter = GetString(root, "filter"), + TotalDurationMs = GetDouble(root, "totalDurationMs"), + ArtifactUrl = GetString(root, "artifactUrl"), + CommitSha = GetString(root, "commitSha"), + Branch = GetString(root, "branch"), + PullRequestNumber = GetString(root, "pullRequestNumber"), + RepositorySlug = GetString(root, "repositorySlug"), + SourceLinks = root.TryGetProperty("sourceLinks", out var sourceLinks) + && sourceLinks.ValueKind == JsonValueKind.Object + && GetString(sourceLinks, "lineUrl") is { } lineUrl + && GetString(sourceLinks, "rangeUrl") is { } rangeUrl + ? new SourceLinkTemplates(lineUrl, rangeUrl, GetString(sourceLinks, "rawUrl")) + : null, + Summary = root.TryGetProperty("summary", out var summary) ? ReadSummary(summary) : new ReportSummary(), + Groups = ReadArray(root, "groups", ReadGroup) ?? [], + Spans = ReadArray(root, "spans", ReadSpan), + }; + } + + private static ReportSummary ReadSummary(JsonElement e) => new() + { + Total = GetInt(e, "total"), + Passed = GetInt(e, "passed"), + Failed = GetInt(e, "failed"), + Skipped = GetInt(e, "skipped"), + Cancelled = GetInt(e, "cancelled"), + TimedOut = GetInt(e, "timedOut"), + Flaky = GetInt(e, "flaky"), + }; + + // Every array in the schema materializes the same way; the mapper is the only + // per-type piece. AOT-safe — plain generic instantiation, no reflection. + private static T[]? ReadArray(JsonElement parent, string name, Func map) + { + if (!parent.TryGetProperty(name, out var array) || array.ValueKind != JsonValueKind.Array) + { + return null; + } + + var result = new T[array.GetArrayLength()]; + var i = 0; + foreach (var item in array.EnumerateArray()) + { + result[i++] = map(item); + } + return result; + } + + private static ReportTestGroup ReadGroup(JsonElement g) => new() + { + ClassName = GetString(g, "className") ?? "UnknownClass", + Namespace = GetString(g, "namespace") ?? "", + Summary = g.TryGetProperty("summary", out var summary) ? ReadSummary(summary) : new ReportSummary(), + Tests = ReadArray(g, "tests", ReadTest) ?? [], + }; + + private static ReportTestResult ReadTest(JsonElement t) => new() + { + Id = GetString(t, "id") ?? "", + DisplayName = GetString(t, "displayName") ?? "", + MethodName = GetString(t, "methodName") ?? "", + ClassName = GetString(t, "className") ?? "UnknownClass", + Status = GetString(t, "status") ?? "unknown", + DurationMs = GetDouble(t, "durationMs"), + StartTime = GetString(t, "startTime"), + EndTime = GetString(t, "endTime"), + Exception = t.TryGetProperty("exception", out var ex) && ex.ValueKind == JsonValueKind.Object + ? ReadException(ex) + : null, + Output = GetString(t, "output"), + ErrorOutput = GetString(t, "errorOutput"), + Categories = ReadArray(t, "categories", ReadString), + CustomProperties = ReadArray(t, "customProperties", ReadKeyValue), + FilePath = GetString(t, "filePath"), + LineNumber = GetNullableInt(t, "lineNumber"), + EndLineNumber = GetNullableInt(t, "endLineNumber"), + SourceRelativePath = GetString(t, "sourceRelativePath"), + SkipReason = GetString(t, "skipReason"), + RetryAttempt = GetInt(t, "retryAttempt"), + Attempts = ReadArray(t, "attempts", ReadAttempt), + TraceId = GetString(t, "traceId"), + SpanId = GetString(t, "spanId"), + AdditionalTraceIds = ReadArray(t, "additionalTraceIds", ReadString), + }; + + private static ReportAttempt ReadAttempt(JsonElement a) => new() + { + Status = GetString(a, "status") ?? "unknown", + DurationMs = GetDouble(a, "durationMs"), + ExceptionType = GetString(a, "exceptionType"), + ExceptionMessage = GetString(a, "exceptionMessage"), + StackTrace = GetString(a, "stackTrace"), + }; + + private static ReportExceptionData ReadException(JsonElement e) => new() + { + Type = GetString(e, "type") ?? "Unknown", + Message = GetString(e, "message") ?? "", + StackTrace = GetString(e, "stackTrace"), + InnerException = e.TryGetProperty("innerException", out var inner) && inner.ValueKind == JsonValueKind.Object + ? ReadException(inner) + : null, + }; + + private static SpanData ReadSpan(JsonElement s) => new() + { + TraceId = GetString(s, "traceId") ?? "", + SpanId = GetString(s, "spanId") ?? "", + ParentSpanId = GetString(s, "parentSpanId"), + Name = GetString(s, "name") ?? "", + SpanType = GetString(s, "spanType"), + Source = GetString(s, "source") ?? "", + Kind = GetString(s, "kind") ?? "", + StartTimeMs = GetDouble(s, "startTimeMs"), + DurationMs = GetDouble(s, "durationMs"), + Status = GetString(s, "status") ?? "Unset", + StatusMessage = GetString(s, "statusMessage"), + Tags = ReadArray(s, "tags", ReadKeyValue), + Events = ReadArray(s, "events", ReadEvent), + Links = ReadArray(s, "links", ReadLink), + }; + + private static SpanEvent ReadEvent(JsonElement e) => new() + { + Name = GetString(e, "name") ?? "", + TimestampMs = GetDouble(e, "timestampMs"), + Tags = ReadArray(e, "tags", ReadKeyValue), + }; + + private static SpanLink ReadLink(JsonElement l) => new() + { + TraceId = GetString(l, "traceId") ?? "", + SpanId = GetString(l, "spanId") ?? "", + }; + + private static ReportKeyValue ReadKeyValue(JsonElement kv) => new() + { + Key = GetString(kv, "key") ?? "", + Value = GetString(kv, "value") ?? "", + }; + + private static string ReadString(JsonElement v) + => v.ValueKind == JsonValueKind.String ? v.GetString() ?? "" : ""; + + private static string? GetString(JsonElement e, string name) + => e.TryGetProperty(name, out var v) && v.ValueKind == JsonValueKind.String ? v.GetString() : null; + + private static double GetDouble(JsonElement e, string name) + => e.TryGetProperty(name, out var v) + && v.ValueKind == JsonValueKind.Number + && v.TryGetDouble(out var d) + // Out-of-range literals like 1e9999 parse to ±Infinity rather than failing; + // a non-finite duration would poison downstream duration math and formatting. + && !double.IsNaN(d) && !double.IsInfinity(d) + ? d + : 0; + + private static int GetInt(JsonElement e, string name) + => e.TryGetProperty(name, out var v) && v.ValueKind == JsonValueKind.Number && v.TryGetInt32(out var i) ? i : 0; + + private static int? GetNullableInt(JsonElement e, string name) + => e.TryGetProperty(name, out var v) && v.ValueKind == JsonValueKind.Number && v.TryGetInt32(out var i) ? i : null; +} diff --git a/TUnit.Engine/Reporters/Aggregation/ReportDataMerger.cs b/TUnit.Engine/Reporters/Aggregation/ReportDataMerger.cs new file mode 100644 index 00000000000..28609f8c350 --- /dev/null +++ b/TUnit.Engine/Reporters/Aggregation/ReportDataMerger.cs @@ -0,0 +1,351 @@ +using System.Globalization; +using TUnit.Core; +using TUnit.Engine.Reporters.Html; + +namespace TUnit.Engine.Reporters.Aggregation; + +/// +/// Merges per-process sidecars (one per test suite / TFM / OS) +/// into a single that HtmlReportGenerator can render as +/// one combined report. Timing works across processes because test start times are +/// absolute wall-clock timestamps; the generator recomputes run bounds from all tests. +/// +internal static class ReportDataMerger +{ + internal static ReportData Merge(IReadOnlyList suites) + { + if (suites.Count == 0) + { + throw new ArgumentException("At least one suite is required", nameof(suites)); + } + + if (suites.Count == 1) + { + return suites[0]; + } + + // Deterministic output regardless of which process finished last. + var ordered = suites + .OrderBy(static s => s.AssemblyName, StringComparer.Ordinal) + .ThenBy(static s => s.RuntimeVersion, StringComparer.Ordinal) + .ThenBy(static s => s.OperatingSystem, StringComparer.Ordinal) + .ToArray(); + + var labels = BuildSuiteLabels(ordered); + + var summary = new ReportSummary(); + var groups = new List(); + var spans = new List(); + // Same class name can exist in several suites (multi-TFM runs of one assembly, + // or genuinely duplicated names across projects) — suffix with the suite label + // so the merged report keeps them apart instead of visually interleaving them. + var classNameCounts = new Dictionary(StringComparer.Ordinal); + foreach (var suite in ordered) + { + foreach (var group in suite.Groups) + { + classNameCounts[group.ClassName] = classNameCounts.GetValueOrDefault(group.ClassName) + 1; + } + } + + for (var suiteIndex = 0; suiteIndex < ordered.Length; suiteIndex++) + { + var suite = ordered[suiteIndex]; + + summary.Add(suite.Summary); + + // Original → disambiguated class names for this suite, so the suite's spans + // can be re-tagged to match: the report's per-class timelines are joined on + // the span's class tag, and a renamed group would otherwise lose its timeline. + Dictionary? renamedClasses = null; + + foreach (var group in suite.Groups) + { + var className = group.ClassName; + if (classNameCounts[className] > 1) + { + var renamed = $"{className} [{labels[suiteIndex]}]"; + (renamedClasses ??= new Dictionary(StringComparer.Ordinal))[className] = renamed; + className = renamed; + } + + groups.Add(new ReportTestGroup + { + ClassName = className, + Namespace = group.Namespace, + Summary = group.Summary, + Tests = PrefixTestIds(group.Tests, suiteIndex, className), + }); + } + + if (suite.Spans is { Length: > 0 } suiteSpans) + { +#if NET + // Timelines only render on modern TFMs (span collection is #if NET), so + // the retag is pointless elsewhere — spans pass through untouched there. + if (renamedClasses is not null) + { + foreach (var span in suiteSpans) + { + spans.Add(RetagClassSpan(span, renamedClasses)); + } + } + else +#endif + { + spans.AddRange(suiteSpans); + } + } + } + + return new ReportData + { + AssemblyName = $"{ordered.Length} Test Suites", + MachineName = JoinDistinct(ordered, static s => s.MachineName), + Timestamp = EarliestTimestamp(ordered), + TUnitVersion = JoinDistinct(ordered, static s => s.TUnitVersion), + OperatingSystem = JoinDistinct(ordered, static s => s.OperatingSystem), + RuntimeVersion = JoinDistinct(ordered, static s => s.RuntimeVersion), + Filter = JoinDistinctOrNull(ordered, static s => s.Filter), + TotalDurationMs = ComputeWallClockDurationMs(ordered), + Summary = summary, + Groups = groups.ToArray(), + Spans = spans.Count > 0 ? spans.ToArray() : null, + // Source-control metadata (and the link templates built from it) applies to the + // whole merged report, so it is only kept when every suite that has it agrees — + // merging sidecars from different commits/runs must not label every test with + // the first suite's commit or link to the wrong revision. + CommitSha = SingleDistinctOrNull(ordered, static s => s.CommitSha), + Branch = SingleDistinctOrNull(ordered, static s => s.Branch), + PullRequestNumber = SingleDistinctOrNull(ordered, static s => s.PullRequestNumber), + RepositorySlug = SingleDistinctOrNull(ordered, static s => s.RepositorySlug), + SourceLinks = SingleDistinctSourceLinksOrNull(ordered), + }; + } + +#if NET + // The report's class-timeline join resolves a suite span's class as + // `FindTagValue(TagTestClass) ?? Name`. Suite spans carry no TagTestClass; the + // collector rewrites their Name from the TagTestSuiteName tag (the simple class + // name), so the effective join key is Name — that's what must follow a rename. + private static SpanData RetagClassSpan(SpanData span, Dictionary renamedClasses) + { + if (!string.Equals(span.SpanType, TUnitActivitySource.SpanTestSuite, StringComparison.Ordinal) + || !renamedClasses.TryGetValue(span.Name, out var renamed)) + { + return span; + } + + var tags = span.Tags; + if (tags is { Length: > 0 }) + { + for (var i = 0; i < tags.Length; i++) + { + if (string.Equals(tags[i].Key, TUnitActivitySource.TagTestSuiteName, StringComparison.Ordinal) + && string.Equals(tags[i].Value, span.Name, StringComparison.Ordinal)) + { + tags = (ReportKeyValue[])tags.Clone(); + tags[i] = new ReportKeyValue { Key = tags[i].Key, Value = renamed }; + break; + } + } + } + + return span with { Name = renamed, Tags = tags }; + } +#endif + + /// + /// Display labels for suites, unique across the set: assembly name alone when unique, + /// progressively disambiguated with runtime, OS and machine when suites collide + /// (e.g. the same assembly run for net8.0 and net9.0). + /// + internal static string[] BuildSuiteLabels(IReadOnlyList suites) + { + var candidates = new Func[] + { + static s => s.AssemblyName, + static s => $"{s.AssemblyName} ({s.RuntimeVersion})", + static s => $"{s.AssemblyName} ({s.RuntimeVersion}, {s.OperatingSystem})", + static s => $"{s.AssemblyName} ({s.RuntimeVersion}, {s.OperatingSystem}, {s.MachineName})", + }; + + foreach (var candidate in candidates) + { + var labels = new string[suites.Count]; + var seen = new HashSet(StringComparer.Ordinal); + var unique = true; + for (var i = 0; i < suites.Count; i++) + { + labels[i] = candidate(suites[i]); + unique &= seen.Add(labels[i]); + } + if (unique) + { + return labels; + } + } + + // Truly identical metadata — fall back to an index suffix. + var indexed = new string[suites.Count]; + for (var i = 0; i < suites.Count; i++) + { + indexed[i] = $"{suites[i].AssemblyName} #{i + 1}"; + } + return indexed; + } + + // Test UIDs are only unique within one process; the merged report uses them as + // dictionary keys (lane assignment, span correlation), so prefix per suite. + // ClassName is re-stamped so per-test class always matches its (possibly + // disambiguated) group header. + private static ReportTestResult[] PrefixTestIds(ReportTestResult[] tests, int suiteIndex, string className) + { + var result = new ReportTestResult[tests.Length]; + for (var i = 0; i < tests.Length; i++) + { + result[i] = tests[i] with + { + Id = $"s{suiteIndex}::{tests[i].Id}", + ClassName = className, + }; + } + return result; + } + + /// + /// A suite's wall-clock bounds in unix ms, from its tests' absolute start timestamps; + /// when no test carries one. Computed once per suite so callers + /// rendering both per-suite and whole-run durations parse each timestamp only once. + /// + internal static (long StartMs, long EndMs)? ComputeWallClockBounds(ReportData suite) + { + var earliest = long.MaxValue; + var latest = long.MinValue; + foreach (var group in suite.Groups) + { + foreach (var test in group.Tests) + { + if (HtmlReportGenerator.TryParseUnixMs(test.StartTime) is not { } startMs) + { + continue; + } + + var endMs = startMs + (long)Math.Round(test.DurationMs); + if (startMs < earliest) earliest = startMs; + if (endMs > latest) latest = endMs; + } + } + return earliest == long.MaxValue ? null : (earliest, latest); + } + + /// + /// Wall-clock duration across all suites: latest test end minus earliest test start. + /// Suites' own values overlap when they ran + /// in parallel, so summing them would overstate; when no test carries timestamps, + /// fall back to the longest single suite. + /// + internal static double ComputeWallClockDurationMs(IReadOnlyList suites) + { + var earliest = long.MaxValue; + var latest = long.MinValue; + double maxSuiteDuration = 0; + foreach (var suite in suites) + { + if (suite.TotalDurationMs > maxSuiteDuration) + { + maxSuiteDuration = suite.TotalDurationMs; + } + + if (ComputeWallClockBounds(suite) is not { } bounds) + { + continue; + } + + if (bounds.StartMs < earliest) earliest = bounds.StartMs; + if (bounds.EndMs > latest) latest = bounds.EndMs; + } + + return earliest == long.MaxValue ? maxSuiteDuration : latest - earliest; + } + + // ReportData.Timestamp is a display string; parse with its exact write format + // (HtmlReporter uses "dd MMM yyyy, HH:mm:ss 'UTC'") to pick the earliest suite. + private static string EarliestTimestamp(IReadOnlyList suites) + { + var best = suites[0].Timestamp; + var bestParsed = DateTimeOffset.MaxValue; + foreach (var suite in suites) + { + if (DateTimeOffset.TryParseExact(suite.Timestamp, "dd MMM yyyy, HH:mm:ss 'UTC'", + CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, + out var parsed) + && parsed < bestParsed) + { + bestParsed = parsed; + best = suite.Timestamp; + } + } + return best; + } + + private static List DistinctNonEmpty(IReadOnlyList suites, Func selector) + { + var distinct = new List(); + foreach (var suite in suites) + { + var value = selector(suite); + if (!string.IsNullOrEmpty(value) && !distinct.Contains(value!)) + { + distinct.Add(value!); + } + } + return distinct; + } + + private static string JoinDistinct(IReadOnlyList suites, Func selector) + { + var distinct = DistinctNonEmpty(suites, selector); + return distinct.Count switch + { + 0 => "", + <= 3 => string.Join(", ", distinct), + _ => $"{distinct[0]}, {distinct[1]} +{distinct.Count - 2} more", + }; + } + + private static string? JoinDistinctOrNull(IReadOnlyList suites, Func selector) + { + var distinct = DistinctNonEmpty(suites, selector); + return distinct.Count == 0 ? null : string.Join("; ", distinct); + } + + /// The single value shared by every suite that has one; null when they disagree. + private static string? SingleDistinctOrNull(IReadOnlyList suites, Func selector) + { + var distinct = DistinctNonEmpty(suites, selector); + return distinct.Count == 1 ? distinct[0] : null; + } + + private static SourceLinkTemplates? SingleDistinctSourceLinksOrNull(IReadOnlyList suites) + { + SourceLinkTemplates? found = null; + foreach (var suite in suites) + { + if (suite.SourceLinks is not { } links) + { + continue; + } + + if (found is null) + { + found = links; + } + else if (found != links) // record value equality + { + return null; + } + } + return found; + } +} diff --git a/TUnit.Engine/Reporters/GitHubReporter.cs b/TUnit.Engine/Reporters/GitHubReporter.cs index 390b1adf8dd..cf3c21d5b0b 100644 --- a/TUnit.Engine/Reporters/GitHubReporter.cs +++ b/TUnit.Engine/Reporters/GitHubReporter.cs @@ -13,6 +13,7 @@ using TUnit.Engine.Extensions; using TUnit.Engine.Framework; using TUnit.Engine.Helpers; +using TUnit.Engine.Reporters.Aggregation; namespace TUnit.Engine.Reporters; @@ -103,11 +104,21 @@ public Task BeforeRunAsync(CancellationToken cancellationToken) return Task.CompletedTask; } - public Task AfterRunAsync(int exitCode, CancellationToken cancellation) + public async Task AfterRunAsync(int exitCode, CancellationToken cancellation) { if (_latestUpdates.IsEmpty) { - return Task.CompletedTask; + return; + } + + // Cross-process aggregation replaces per-suite blocks with one merged block; the + // whole merge (sidecar write, lock, merged HTML, summary region) runs inside + // HtmlReporter's session-finish, which sets this flag and calls back into + // WriteAggregatedSummary below. When the flag is unset (aggregation off, or the + // HTML reporter — which produces the sidecars — is disabled), append classically. + if (SuppressPerSuiteSummary) + { + return; } var targetFramework = Assembly.GetExecutingAssembly() @@ -336,11 +347,13 @@ public Task AfterRunAsync(int exitCode, CancellationToken cancellation) { stringBuilder.AppendLine(); stringBuilder.AppendLine("---"); - return WriteFile(stringBuilder.ToString()); + await WriteFile(stringBuilder.ToString()); + return; } - // Cap per group to keep the GitHub step summary within the 1 MB file-size limit - const int maxTestsPerGroup = 50; + // Cap per group to keep the GitHub step summary within the 1 MB file-size limit; + // shared with the aggregated summary so both views truncate identically. + const int maxTestsPerGroup = AggregatedSummaryWriter.MaxTestsPerGroup; if (failureMessages.Count > 0) { stringBuilder.AppendLine(); @@ -447,7 +460,38 @@ public Task AfterRunAsync(int exitCode, CancellationToken cancellation) stringBuilder.AppendLine(); stringBuilder.AppendLine("---"); - return WriteFile(stringBuilder.ToString()); + await WriteFile(stringBuilder.ToString()); + } + + /// + /// Set by HtmlReporter once this suite's sidecar is persisted to the shared + /// aggregation directory (its session-finish runs before this reporter's + /// AfterRunAsync), so the merged block owns the summary instead of per-suite appends. + /// + internal bool SuppressPerSuiteSummary { get; set; } + + /// + /// Renders the aggregated block and rewrites the marked summary region. Called by + /// HtmlReporter inside the aggregation lock, so the whole merge is one lock cycle. + /// No-ops when this reporter is disabled (not GitHub Actions / no summary file). + /// + internal void WriteAggregatedSummary(IReadOnlyList suites, string mergedReportPath) + { + if (_outputSummaryFilePath is null) + { + return; + } + + var serverUrl = Environment.GetEnvironmentVariable(EnvironmentConstants.GitHubServerUrl) + ?? EnvironmentConstants.GitHubDefaultServerUrl; + + var content = AggregatedSummaryWriter.Render( + suites, + collapsible: _reporterStyle == GitHubReporterStyle.Collapsible, + serverUrl: serverUrl, + mergedReportHint: $"📄 Combined HTML report: `{mergedReportPath}` — upload it as an artifact to keep it after the job."); + + GitHubSummaryRegion.ReplaceOrAppend(_outputSummaryFilePath, content, MaxFileSizeInBytes); } private async Task WriteFile(string contents) @@ -618,15 +662,9 @@ internal void SetReporterStyle(GitHubReporterStyle style) _reporterStyle = style; } - private static string FormatDuration(TimeSpan? duration) => duration switch - { - null => "-", - { TotalMilliseconds: < 1 } => "< 1ms", - { TotalSeconds: < 1 } d => $"{d.TotalMilliseconds:F0}ms", - { TotalMinutes: < 1 } d => $"{d.TotalSeconds:F1}s", - { TotalHours: < 1 } d => $"{d.Minutes}m {d.Seconds}s", - var d => $"{(int)d.Value.TotalHours}h {d.Value.Minutes}m" - }; + // Delegates so the per-suite and aggregated summaries can never render durations differently. + private static string FormatDuration(TimeSpan? duration) + => duration is null ? "-" : AggregatedSummaryWriter.FormatDuration(duration.Value.TotalMilliseconds); private static string GetExceptionTypeName(IProperty? stateProperty) => stateProperty switch { diff --git a/TUnit.Engine/Reporters/Html/HtmlReportDataModel.cs b/TUnit.Engine/Reporters/Html/HtmlReportDataModel.cs index 251dd4d9b2e..eac0d3b23a7 100644 --- a/TUnit.Engine/Reporters/Html/HtmlReportDataModel.cs +++ b/TUnit.Engine/Reporters/Html/HtmlReportDataModel.cs @@ -56,6 +56,16 @@ internal sealed class ReportData [JsonIgnore] public SourceLinkTemplates? SourceLinks { get; init; } + + /// + /// Link to this suite's uploaded HTML report artifact, when the in-process GitHub + /// upload succeeded. Not part of the HTML renderer JSON — persisted only in the + /// aggregation sidecar (see ReportDataJson) so the merged step summary can + /// link each suite's individual report. Mutable because the upload happens after + /// the report data is built. + /// + [JsonIgnore] + public string? ArtifactUrl { get; set; } } internal sealed class ReportSummary @@ -86,6 +96,22 @@ internal sealed class ReportSummary [JsonIgnore] public int TotalFailed => Failed + TimedOut; + + /// Everything that should mark a suite/run as not-green, cancellations included. + [JsonIgnore] + public int TotalUnsuccessful => Failed + TimedOut + Cancelled; + + /// Accumulates another summary into this one (used when merging suites). + public void Add(ReportSummary other) + { + Total += other.Total; + Passed += other.Passed; + Failed += other.Failed; + Skipped += other.Skipped; + Cancelled += other.Cancelled; + TimedOut += other.TimedOut; + Flaky += other.Flaky; + } } internal sealed class ReportTestGroup @@ -103,7 +129,9 @@ internal sealed class ReportTestGroup public required ReportTestResult[] Tests { get; init; } } -internal sealed class ReportTestResult +// A record so consumers (e.g. ReportDataMerger) can rewrite single properties via `with` +// without a hand-maintained copy that silently drops newly added members. +internal sealed record ReportTestResult { [JsonPropertyName("id")] public required string Id { get; init; } diff --git a/TUnit.Engine/Reporters/Html/HtmlReportGenerator.cs b/TUnit.Engine/Reporters/Html/HtmlReportGenerator.cs index a594f0fc469..3dcdb99c665 100644 --- a/TUnit.Engine/Reporters/Html/HtmlReportGenerator.cs +++ b/TUnit.Engine/Reporters/Html/HtmlReportGenerator.cs @@ -829,7 +829,8 @@ private static bool IsDuplicateKey(ReportKeyValue[] items, int index, string key return sb.Length == 0 ? null : sb.ToString(); } - private static long? TryParseUnixMs(string? iso) + // Internal (not private): ReportDataMerger reuses this for its wall-clock bounds scan. + internal static long? TryParseUnixMs(string? iso) { if (string.IsNullOrEmpty(iso)) return null; return DateTimeOffset.TryParse( diff --git a/TUnit.Engine/Reporters/Html/HtmlReporter.cs b/TUnit.Engine/Reporters/Html/HtmlReporter.cs index a2356859dad..a777ba3027f 100644 --- a/TUnit.Engine/Reporters/Html/HtmlReporter.cs +++ b/TUnit.Engine/Reporters/Html/HtmlReporter.cs @@ -19,6 +19,7 @@ using TUnit.Engine.Framework; using TUnit.Engine.Helpers; using TUnit.Engine.Reporters; +using TUnit.Engine.Reporters.Aggregation; #pragma warning disable TPEXP @@ -42,11 +43,7 @@ internal sealed class HtmlReporter(IExtension extension) : IDataConsumer, IDataP public async Task IsEnabledAsync() { - var disableValue = Environment.GetEnvironmentVariable(EnvironmentConstants.DisableHtmlReporter); - if (disableValue is not null && - (disableValue.Equals("true", StringComparison.OrdinalIgnoreCase) || - disableValue.Equals("1", StringComparison.Ordinal) || - disableValue.Equals("yes", StringComparison.OrdinalIgnoreCase))) + if (IsTruthyEnv(Environment.GetEnvironmentVariable(EnvironmentConstants.DisableHtmlReporter))) { return false; } @@ -156,7 +153,11 @@ public async Task OnTestSessionFinishingAsync(ITestSessionContext testSessionCon } // GitHub Actions integration (artifact upload + step summary) - await TryGitHubIntegrationAsync(outputPath, testSessionContext.CancellationToken); + reportData.ArtifactUrl = await TryGitHubIntegrationAsync(outputPath, testSessionContext.CancellationToken); + + // Machine-readable sidecar + cross-process aggregation. Written after the + // GitHub integration so the sidecar carries this suite's artifact URL. + await TryWriteSidecarAndAggregateAsync(reportData, outputPath, testSessionContext.CancellationToken); } catch (Exception ex) { @@ -164,6 +165,92 @@ public async Task OnTestSessionFinishingAsync(ITestSessionContext testSessionCon } } + private async Task TryWriteSidecarAndAggregateAsync(ReportData reportData, string htmlOutputPath, CancellationToken cancellationToken) + { + // Serialized once; the same bytes back both the local sidecar and the shared copy. + var sidecarBytes = ReportDataJson.SerializeToBytes(reportData); + + if (!IsTruthyEnv(Environment.GetEnvironmentVariable(EnvironmentConstants.DisableJsonReport))) + { + try + { + AtomicFile.WriteAllBytes(GetSidecarPath(htmlOutputPath), sidecarBytes); + } + catch (Exception ex) + { + Console.WriteLine($"Warning: Failed to write JSON report sidecar: {ex.Message}"); + } + } + + var aggregator = ReportAggregator.TryCreateFromEnvironment(Environment.GetEnvironmentVariable); + if (aggregator is null) + { + return; + } + + try + { + aggregator.WriteSidecar(sidecarBytes, reportData.AssemblyName, suiteSalt: htmlOutputPath); + + // Aggregation is committed for this suite: whatever happens below, the classic + // per-suite block must not be appended on top of the aggregated one. (If we + // fail past this point, a sibling that merges after us still renders this + // suite's results from the sidecar just written.) + if (_githubReporter is not null) + { + _githubReporter.SuppressPerSuiteSummary = true; + } + + // Every finishing process regenerates the merged outputs from all sidecars + // present so far; the last one to finish leaves the complete aggregate. + using var aggregationLock = await aggregator.AcquireLockAsync(cancellationToken); + if (aggregationLock is null) + { + return; + } + + var suites = aggregator.ReadAllSidecars(); + if (suites.Count == 0) + { + return; + } + + aggregator.WriteMergedHtml(suites); + Console.WriteLine($"Merged HTML test report ({suites.Count} {(suites.Count == 1 ? "suite" : "suites")} so far) written to: {aggregator.MergedReportPath}"); + + // The step summary is rewritten in the same lock cycle: one env parse, one + // lock acquisition and one sidecar scan per process for both merged outputs. + if (aggregator.Mode == AggregationMode.Cooperative) + { + _githubReporter?.WriteAggregatedSummary(suites, aggregator.MergedReportPath); + } + } + catch (Exception ex) + { + Console.WriteLine($"Warning: Report aggregation failed: {ex.Message}"); + } + } + + // The truthy vocabulary shared by the TUNIT_DISABLE_* switches. + private static bool IsTruthyEnv(string? value) + => value is not null && + (value.Equals("true", StringComparison.OrdinalIgnoreCase) || + value.Equals("1", StringComparison.Ordinal) || + value.Equals("yes", StringComparison.OrdinalIgnoreCase)); + + // Default HTML report is "{name}-{os}-{tfm}-report.html"; the sidecar drops the + // "-report" stem so the default pair reads "{name}-{os}-{tfm}.tunit-report.json". + internal static string GetSidecarPath(string htmlOutputPath) + { + const string reportSuffix = "-report.html"; + if (htmlOutputPath.EndsWith(reportSuffix, StringComparison.OrdinalIgnoreCase)) + { + return htmlOutputPath.Substring(0, htmlOutputPath.Length - reportSuffix.Length) + ReportDataJson.SidecarExtension; + } + + return Path.ChangeExtension(htmlOutputPath, null) + ReportDataJson.SidecarExtension; + } + internal async Task PublishArtifactAsync(string outputPath, SessionUid sessionUid, CancellationToken cancellationToken) { if (_messageBus is null) @@ -762,11 +849,12 @@ private static bool IsFileLocked(IOException exception) exception.Message.Contains("access denied", StringComparison.OrdinalIgnoreCase); } - private async Task TryGitHubIntegrationAsync(string filePath, CancellationToken cancellationToken) + /// The uploaded artifact's URL, when the in-process upload succeeded. + private async Task TryGitHubIntegrationAsync(string filePath, CancellationToken cancellationToken) { if (Environment.GetEnvironmentVariable(EnvironmentConstants.GitHubActions) is not "true") { - return; + return null; } var repo = Environment.GetEnvironmentVariable(EnvironmentConstants.GitHubRepository); @@ -800,18 +888,26 @@ private async Task TryGitHubIntegrationAsync(string filePath, CancellationToken } } + string? artifactUrl = null; + if (artifactId is not null && !string.IsNullOrEmpty(repo) && !string.IsNullOrEmpty(runId)) + { + var serverUrl = (Environment.GetEnvironmentVariable(EnvironmentConstants.GitHubServerUrl) ?? EnvironmentConstants.GitHubDefaultServerUrl).TrimEnd('/'); + artifactUrl = $"{serverUrl}/{repo}/actions/runs/{runId}/artifacts/{artifactId}"; + } + if (_githubReporter is not null) { if (!hasRuntimeToken) { _githubReporter.ShowArtifactUploadTip = true; } - else if (artifactId is not null && !string.IsNullOrEmpty(repo) && !string.IsNullOrEmpty(runId)) + else if (artifactUrl is not null) { - var serverUrl = (Environment.GetEnvironmentVariable(EnvironmentConstants.GitHubServerUrl) ?? EnvironmentConstants.GitHubDefaultServerUrl).TrimEnd('/'); - _githubReporter.ArtifactUrl = $"{serverUrl}/{repo}/actions/runs/{runId}/artifacts/{artifactId}"; + _githubReporter.ArtifactUrl = artifactUrl; } } + + return artifactUrl; } private static int? ParseRetentionDays() diff --git a/TUnit.Pipeline/Modules/GetPackageProjectsModule.cs b/TUnit.Pipeline/Modules/GetPackageProjectsModule.cs index cde96d45eb3..588210d1686 100644 --- a/TUnit.Pipeline/Modules/GetPackageProjectsModule.cs +++ b/TUnit.Pipeline/Modules/GetPackageProjectsModule.cs @@ -30,7 +30,8 @@ public class GetPackageProjectsModule : Module> Sourcy.DotNet.Projects.TUnit_Mocks, Sourcy.DotNet.Projects.TUnit_Mocks_Assertions, Sourcy.DotNet.Projects.TUnit_Mocks_Http, - Sourcy.DotNet.Projects.TUnit_Mocks_Logging + Sourcy.DotNet.Projects.TUnit_Mocks_Logging, + Sourcy.DotNet.Projects.TUnit_Reporting_Tool ]; } } diff --git a/TUnit.Pipeline/Modules/PackTUnitFilesModule.cs b/TUnit.Pipeline/Modules/PackTUnitFilesModule.cs index e9b446a0468..294fd6c48ea 100644 --- a/TUnit.Pipeline/Modules/PackTUnitFilesModule.cs +++ b/TUnit.Pipeline/Modules/PackTUnitFilesModule.cs @@ -59,7 +59,11 @@ await context.DotNet() Properties = properties, IncludeSource = project == Sourcy.DotNet.Projects.TUnit_Templates ? false : true, Configuration = "Release", - NoBuild = true, + // The reporting tool is a standalone dotnet tool no test module + // references, so nothing upstream has restored or built it — let + // pack build it (the strong-name race in the header comment doesn't + // apply: it isn't strong-named and nothing else consumes its bits). + NoBuild = project != Sourcy.DotNet.Projects.TUnit_Reporting_Tool, }, new CommandExecutionOptions { LogSettings = new CommandLoggingOptions diff --git a/TUnit.Pipeline/Modules/RunEngineTestsModule.cs b/TUnit.Pipeline/Modules/RunEngineTestsModule.cs index b3ac5eb7765..f209891e856 100644 --- a/TUnit.Pipeline/Modules/RunEngineTestsModule.cs +++ b/TUnit.Pipeline/Modules/RunEngineTestsModule.cs @@ -46,6 +46,11 @@ public class RunEngineTestsModule : Module EnvironmentVariables = new Dictionary { ["TUNIT_DISABLE_GITHUB_REPORTER"] = "true", + // Engine tests spawn hundreds of short-lived child TUnit processes (many + // designed to fail). With aggregation on by default in CI, every child + // would re-merge the job's shared report on exit — pure overhead here, + // and the failures would pollute the job's aggregated summary. + ["TUNIT_AGGREGATE_REPORTS"] = "off", }, LogSettings = new CommandLoggingOptions { diff --git a/TUnit.Reporting.Tool/Program.cs b/TUnit.Reporting.Tool/Program.cs new file mode 100644 index 00000000000..b05183e5dc0 --- /dev/null +++ b/TUnit.Reporting.Tool/Program.cs @@ -0,0 +1,221 @@ +using System.Text; +using TUnit.Engine.Configuration; +using TUnit.Engine.Reporters.Aggregation; +using TUnit.Engine.Reporters.Html; + +namespace TUnit.Reporting.Tool; + +internal static class Program +{ + private const string Usage = + """ + tunit-report — merge TUnit test reports from multiple test projects into one. + + Usage: + tunit-report merge --directory [options] + + Options: + -d, --directory Directory scanned recursively for *.tunit-report.json + sidecars (written by TUnit next to each HTML report). + -o, --output Path for the merged HTML report. + Default: /merged-report.html + --github-summary Also write the merged summary block to the file that + $GITHUB_STEP_SUMMARY points at (or stdout when unset). + --style