Align CTRF retry reporting and runId with ctrf-io/ctrf#58

src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfMergeMode.cs

@@ -0,0 +1,25 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Microsoft.Testing.Extensions.CtrfReport;
+
+/// <summary>
+/// Controls how <see cref="CtrfReportMerger"/> combines the <c>tests[]</c> arrays of its inputs.
+/// </summary>
+internal enum CtrfMergeMode
+{
+    /// <summary>
+    /// Concatenates the inputs, which is correct when they describe disjoint sets of tests (the shard or
+    /// per-module case). This is the default: MTP test UIDs are only unique WITHIN an assembly, so collapsing
+    /// by identity across modules would fuse same-named tests from different assemblies.
+    /// </summary>
+    Concatenate,
+
+    /// <summary>
+    /// Folds rows describing the same logical test into one, which is correct when the inputs are successive
+    /// attempts of the same test module (<c>--retry-failed-tests</c>): the last attempt wins and earlier ones
+    /// become its <c>retryAttempts[]</c>. Inputs MUST be supplied in attempt order, and MUST come from the same
+    /// module for identities to be comparable.
+    /// </summary>
+    CollapseRetryAttempts,
+}

src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.JsonSerializer.cs

@@ -4,6 +4,7 @@
 using System.Text.Json;
 
 using Microsoft.Testing.Platform;
+using Microsoft.Testing.Platform.Helpers;
 
 namespace Microsoft.Testing.Extensions.CtrfReport;
 
@@ -68,6 +69,11 @@ private byte[] BuildCtrfJson(CapturedTestResult[] results, DateTimeOffset finish
             // Bump this constant whenever we update against a newer schema revision.
             writer.WriteString("specVersion", CtrfSpecVersion);
             writer.WriteString("reportId", Guid.NewGuid().ToString("D"));
+            // CTRF 5.4 (`runId`): identifies the logical run this document belongs to. A logical run can span
+            // several documents — most notably the successive processes of `--retry-failed-tests`, where each
+            // attempt writes its own document. ctrf-io/ctrf#58 confirmed that those per-execution documents (and
+            // any document merged from them) SHOULD share a `runId` while each keeps its own `reportId`.
+            writer.WriteString("runId", ResolveRunId());
             writer.WriteString("timestamp", finishTime.ToString("O", CultureInfo.InvariantCulture));
             writer.WriteString(
                 "generatedBy",
@@ -152,4 +158,27 @@ private byte[] BuildCtrfJson(CapturedTestResult[] results, DateTimeOffset finish
 
         return ms.ToArray();
     }
+
+    /// <summary>
+    /// Resolves the CTRF <c>runId</c>: the id of the logical run this document belongs to.
+    /// </summary>
+    /// <remarks>
+    /// The retry orchestrator sets <c>TESTINGPLATFORM_LOGICAL_RUN_ID</c> before launching its attempts, so every
+    /// attempt process stamps the same value; a CI job can set it too, to correlate documents this process cannot
+    /// know about (the modules of a multi-project run, or shards on different machines). Failing that, the
+    /// <c>dotnet test</c> execution id identifies this test application's own process tree — note it is per root
+    /// test application, NOT per <c>dotnet test</c> invocation, so sibling modules legitimately get distinct ids
+    /// (see <c>docs/mstest-runner-protocol/004-protocol-dotnet-test-pipe.md</c>). A fresh id is the last resort:
+    /// an uncorrelated run is a logical run of its own, and CTRF requires the field to be a non-empty string.
+    /// </remarks>
+    private string ResolveRunId()
+    {
+        string? runId = _environment.GetEnvironmentVariable(EnvironmentVariableConstants.TESTINGPLATFORM_LOGICAL_RUN_ID);
+        if (RoslynString.IsNullOrEmpty(runId))
+        {
+            runId = _environment.GetEnvironmentVariable(EnvironmentVariableConstants.TESTINGPLATFORM_DOTNETTEST_EXECUTIONID);
+        }
+
+        return RoslynString.IsNullOrEmpty(runId) ? Guid.NewGuid().ToString("D") : runId!;
+    }
 }

src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.JsonTestWriter.cs

@@ -77,6 +77,9 @@ private static void WriteTest(Utf8JsonWriter writer, CollapsedTestResult c)
 
         if (c.PriorAttempts.Count > 0)
         {
+            // CTRF 9.20/9.21 as clarified by ctrf-io/ctrf#58: `retries` is the number of
+            // re-executions, which equals `retryAttempts.length` because the array holds
+            // attempts 1..N-1 only — so the final attempt's number is `retries + 1`.
             writer.WriteNumber("retries", c.PriorAttempts.Count);
             writer.WritePropertyName("retryAttempts");
             writer.WriteStartArray();

src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.TestCollapsing.cs

@@ -56,6 +56,11 @@ private static List<CollapsedTestResult> CollapseAttempts(CapturedTestResult[] r
         // For each UID, group all captures in arrival order: the latest entry becomes the
         // final test record, earlier entries become `retryAttempts[]`. Preserves the
         // insertion order of first-seen UIDs in the output (stable across runs).
+        //
+        // ctrf-io/ctrf#58 confirmed this is the intended CTRF model: `retryAttempts[]`
+        // is the attempt history PRECEDING the final attempt (attempts 1..N-1, initial
+        // execution included), and the final attempt is excluded because its outcome and
+        // diagnostics are carried by the test object itself.
         var byUid = new Dictionary<string, int>(StringComparer.Ordinal);
         var collapsed = new List<CollapsedTestResult>(results.Length);
         foreach (CapturedTestResult r in results)