diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfMergeMode.cs b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfMergeMode.cs new file mode 100644 index 0000000000..4f33704d09 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfMergeMode.cs @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Testing.Extensions.CtrfReport; + +/// +/// Controls how combines the tests[] arrays of its inputs. +/// +internal enum CtrfMergeMode +{ + /// + /// Concatenates the inputs, which is correct when they describe disjoint sets of tests (the shard or + /// per-module case). This is the default: MTP test UIDs are only unique WITHIN an assembly, so collapsing + /// by identity across modules would fuse same-named tests from different assemblies. + /// + Concatenate, + + /// + /// Folds rows describing the same logical test into one, which is correct when the inputs are successive + /// attempts of the same test module (--retry-failed-tests): the last attempt wins and earlier ones + /// become its retryAttempts[]. Inputs MUST be supplied in attempt order, and MUST come from the same + /// module for identities to be comparable. + /// + CollapseRetryAttempts, +} diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.JsonSerializer.cs b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.JsonSerializer.cs index 63d59b36cf..a19159605a 100644 --- a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.JsonSerializer.cs +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.JsonSerializer.cs @@ -4,6 +4,7 @@ using System.Text.Json; using Microsoft.Testing.Platform; +using Microsoft.Testing.Platform.Helpers; namespace Microsoft.Testing.Extensions.CtrfReport; @@ -68,6 +69,11 @@ private byte[] BuildCtrfJson(CapturedTestResult[] results, DateTimeOffset finish // Bump this constant whenever we update against a newer schema revision. writer.WriteString("specVersion", CtrfSpecVersion); writer.WriteString("reportId", Guid.NewGuid().ToString("D")); + // CTRF 5.4 (`runId`): identifies the logical run this document belongs to. A logical run can span + // several documents — most notably the successive processes of `--retry-failed-tests`, where each + // attempt writes its own document. ctrf-io/ctrf#58 confirmed that those per-execution documents (and + // any document merged from them) SHOULD share a `runId` while each keeps its own `reportId`. + writer.WriteString("runId", ResolveRunId()); writer.WriteString("timestamp", finishTime.ToString("O", CultureInfo.InvariantCulture)); writer.WriteString( "generatedBy", @@ -152,4 +158,27 @@ private byte[] BuildCtrfJson(CapturedTestResult[] results, DateTimeOffset finish return ms.ToArray(); } + + /// + /// Resolves the CTRF runId: the id of the logical run this document belongs to. + /// + /// + /// The retry orchestrator sets TESTINGPLATFORM_LOGICAL_RUN_ID before launching its attempts, so every + /// attempt process stamps the same value; a CI job can set it too, to correlate documents this process cannot + /// know about (the modules of a multi-project run, or shards on different machines). Failing that, the + /// dotnet test execution id identifies this test application's own process tree — note it is per root + /// test application, NOT per dotnet test invocation, so sibling modules legitimately get distinct ids + /// (see docs/mstest-runner-protocol/004-protocol-dotnet-test-pipe.md). A fresh id is the last resort: + /// an uncorrelated run is a logical run of its own, and CTRF requires the field to be a non-empty string. + /// + private string ResolveRunId() + { + string? runId = _environment.GetEnvironmentVariable(EnvironmentVariableConstants.TESTINGPLATFORM_LOGICAL_RUN_ID); + if (RoslynString.IsNullOrEmpty(runId)) + { + runId = _environment.GetEnvironmentVariable(EnvironmentVariableConstants.TESTINGPLATFORM_DOTNETTEST_EXECUTIONID); + } + + return RoslynString.IsNullOrEmpty(runId) ? Guid.NewGuid().ToString("D") : runId!; + } } diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.JsonTestWriter.cs b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.JsonTestWriter.cs index 4e38a72332..fdd706f33c 100644 --- a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.JsonTestWriter.cs +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.JsonTestWriter.cs @@ -77,6 +77,9 @@ private static void WriteTest(Utf8JsonWriter writer, CollapsedTestResult c) if (c.PriorAttempts.Count > 0) { + // CTRF 9.20/9.21 as clarified by ctrf-io/ctrf#58: `retries` is the number of + // re-executions, which equals `retryAttempts.length` because the array holds + // attempts 1..N-1 only — so the final attempt's number is `retries + 1`. writer.WriteNumber("retries", c.PriorAttempts.Count); writer.WritePropertyName("retryAttempts"); writer.WriteStartArray(); diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.TestCollapsing.cs b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.TestCollapsing.cs index 7de772747a..ad54060c00 100644 --- a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.TestCollapsing.cs +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.TestCollapsing.cs @@ -56,6 +56,11 @@ private static List CollapseAttempts(CapturedTestResult[] r // For each UID, group all captures in arrival order: the latest entry becomes the // final test record, earlier entries become `retryAttempts[]`. Preserves the // insertion order of first-seen UIDs in the output (stable across runs). + // + // ctrf-io/ctrf#58 confirmed this is the intended CTRF model: `retryAttempts[]` + // is the attempt history PRECEDING the final attempt (attempts 1..N-1, initial + // execution included), and the final attempt is excluded because its outcome and + // diagnostics are carried by the test object itself. var byUid = new Dictionary(StringComparer.Ordinal); var collapsed = new List(results.Length); foreach (CapturedTestResult r in results) diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportMerger.cs b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportMerger.cs index e38a870766..65f2b32d2f 100644 --- a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportMerger.cs +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportMerger.cs @@ -15,12 +15,27 @@ namespace Microsoft.Testing.Extensions.CtrfReport; /// This is a pure, invocation-agnostic JSON-level merge (no I/O, no clock) that mirrors the /// TRX and JUnit mergers, demonstrating that the same post-processing shape fits a JSON format: /// -/// results.tests[] arrays are concatenated as-is. +/// results.tests[] arrays are concatenated, except that elements which are not Test objects are dropped so one input's malformed row cannot invalidate the merged document; additionally folds successive attempts of the same test into one row. /// results.summary counters are re-derived by counting the merged tests[] (so summary.tests always matches the array length); start/stop use the earliest/latest across inputs, duration is the resulting span. -/// reportFormat and specVersion are taken from the first report; reportId is derived deterministically from the inputs, so identical inputs reproduce the same id (RFC 018 idempotency). +/// reportFormat and specVersion are taken from the first report; reportId is derived deterministically from the inputs AND the merge mode, so identical inputs reproduce the same id (RFC 018 idempotency) while the two modes — which produce materially different documents — get distinct ids. +/// runId is carried over when every input agrees on one, because the merged document describes the same logical run as its inputs while remaining a distinct artifact with its own reportId (see ctrf-io/ctrf#58). /// tool keeps a concrete identity only when every input reported the exact same tool object; otherwise (inputs disagree or any input omits it) a neutral merger identity is used, so one framework is not attributed to another's tests. /// environment keeps the first report's shared fields, but module-specific values under extra (testApplication, exitCode) are dropped rather than presented as describing all merged modules. /// +/// +/// Validity contract. The merger guarantees the shape of what it SYNTHESIZES — the summary, the identity +/// fields, and the retry attempt objects it builds and renumbers — and PRESERVES verbatim what it merely +/// passes through. The only thing it drops is an element that cannot be a Test at all (a non-object), because +/// that alone would break the array's item type for every consumer of the merged file. +/// +/// +/// It deliberately does NOT validate or repair the Test rows themselves, even though an input may carry a +/// row with a missing or wrong-typed required field. Dropping such a row would lose a real result and +/// rewriting it would fabricate an outcome the producer never reported, which are both worse than relaying a +/// defect that belongs to the input document. A corollary is that merging a single document whose identities +/// are already unique leaves its tests[] untouched: the merger combines reports, it is not a CTRF +/// validator or linter. +/// /// internal static class CtrfReportMerger { @@ -32,7 +47,28 @@ internal static class CtrfReportMerger // not by any input report. private const string GeneratedByName = "Microsoft.Testing.Extensions.CtrfReport"; + // Fields a CTRF retry attempt object (section 11) shares with a test object and that carry over verbatim when + // a non-final attempt is folded into 'retryAttempts[]'. 'attempt' and 'status' are handled separately because + // they are required, and 'attemptId' is listed here because an input that assigned one keeps it. + private static readonly string[] RetryAttemptFields = + [ + "attemptId", + "duration", + "message", + "trace", + "line", + "snippet", + "stdout", + "stderr", + "start", + "stop", + "attachments", + ]; + internal static string Merge(IReadOnlyList inputReports) + => Merge(inputReports, CtrfMergeMode.Concatenate); + + internal static string Merge(IReadOnlyList inputReports, CtrfMergeMode mode) { if (inputReports is null) { @@ -64,6 +100,10 @@ internal static string Merge(IReadOnlyList inputReports) JsonNode? firstTool = null; int reportCount = 0; + // The merged document belongs to the same logical run as its inputs only when they all belong to the + // same one, so a run id is carried over only when every input reported the very same value. + var distinctRunIds = new HashSet(StringComparer.Ordinal); + // Collect each input's environment so shared fields can be retained and module- or agent-specific // ones (values that differ across inputs) dropped, rather than attributing the first report's // environment to every merged test. @@ -84,7 +124,7 @@ internal static string Merge(IReadOnlyList inputReports) continue; } - string? format = root["reportFormat"] is JsonValue formatValue && formatValue.TryGetValue(out string? formatText) ? formatText : null; + string? format = ReadString(root, "reportFormat"); if (!string.Equals(format, "CTRF", StringComparison.OrdinalIgnoreCase)) { continue; @@ -94,6 +134,8 @@ internal static string Merge(IReadOnlyList inputReports) reportCount++; acceptedReports.Add(reportJson); + distinctRunIds.Add(ReadString(root, "runId") is { Length: > 0 } runIdText ? runIdText : string.Empty); + if (root["results"]?["environment"] is JsonObject environment) { environments.Add(environment); @@ -104,22 +146,29 @@ internal static string Merge(IReadOnlyList inputReports) { foreach (JsonNode? test in testArray) { - mergedTests.Add(test?.DeepClone()); + // Only a Test object belongs in tests[]: the CTRF schema types the array's items as objects + // with required members, so carrying a malformed element (a bare string, or a JSON null left + // by a lazy producer) through would turn a defect localized to one input into an invalid + // MERGED document — the artifact consumers actually read. This mirrors the check above, where + // an input that fails the CTRF shape test is rejected outright rather than passed along. + if (test is not JsonObject testObject) + { + continue; + } + + mergedTests.Add(testObject.DeepClone()); // Fall back to per-test timing so a summary-less input (which the merger explicitly // supports) still contributes to the merged min/max instead of being dropped or // forcing the merged timestamp back to the Unix epoch. - if (test is not null) + if (TryReadLong(testObject, "start", out long testStart)) + { + earliestStart = Min(earliestStart, testStart); + } + + if (TryReadLong(testObject, "stop", out long testStop)) { - if (TryReadLong(test, "start", out long testStart)) - { - earliestStart = Min(earliestStart, testStart); - } - - if (TryReadLong(test, "stop", out long testStop)) - { - latestStop = Max(latestStop, testStop); - } + latestStop = Max(latestStop, testStop); } } } @@ -157,18 +206,25 @@ internal static string Merge(IReadOnlyList inputReports) long startMs = earliestStart ?? 0; long stopMs = latestStop ?? startMs; + // In retry mode the inputs are successive attempts of the same suite, so the same logical test can + // appear in several of them. Collapse those repeats into one row before counting, otherwise the same + // test would be reported (and counted) several times. + JsonArray tests = mode == CtrfMergeMode.CollapseRetryAttempts + ? CollapseRetryAttempts(mergedTests) + : mergedTests; + // Counters are derived from the merged tests[] rather than trusting each input's summary, so // summary.tests always equals the array length even when an input omitted or under-reported - // its summary. + // its summary. Every element is a Test object: non-objects were rejected during ingestion. long passed = 0, failed = 0, skipped = 0, pending = 0, other = 0, flaky = 0; - foreach (JsonNode? test in mergedTests) + foreach (JsonNode? test in tests) { - if (test is null) + if (test is not JsonObject testObject) { continue; } - switch ((string?)test["status"]) + switch (ReadString(testObject, "status")) { case "passed": passed++; break; case "failed": failed++; break; @@ -177,7 +233,7 @@ internal static string Merge(IReadOnlyList inputReports) default: other++; break; } - if (test["flaky"] is JsonValue flakyValue && flakyValue.TryGetValue(out bool isFlaky) && isFlaky) + if (testObject["flaky"] is JsonValue flakyValue && flakyValue.TryGetValue(out bool isFlaky) && isFlaky) { flaky++; } @@ -185,7 +241,7 @@ internal static string Merge(IReadOnlyList inputReports) var summaryObject = new JsonObject { - ["tests"] = mergedTests.Count, + ["tests"] = tests.Count, ["passed"] = passed, ["failed"] = failed, ["skipped"] = skipped, @@ -221,27 +277,45 @@ internal static string Merge(IReadOnlyList inputReports) resultsObject["environment"] = commonEnvironment; } - resultsObject["tests"] = mergedTests; + resultsObject["tests"] = tests; var merged = new JsonObject { ["reportFormat"] = first["reportFormat"]?.DeepClone() ?? "CTRF", ["specVersion"] = first["specVersion"]?.DeepClone() ?? "0.0.0", - ["reportId"] = CreateDeterministicReportId(acceptedReports), - ["timestamp"] = DateTimeOffset.FromUnixTimeMilliseconds(stopMs).ToString("O", CultureInfo.InvariantCulture), - // The merged document is produced by this merger, not by any input, so stamp its own identity - // rather than carrying the first input's 'generatedBy' (which could report a different producer - // or version when merging reports from different tool versions). - ["generatedBy"] = GeneratedByName, - ["results"] = resultsObject, + ["reportId"] = CreateDeterministicReportId(acceptedReports, mode), }; + // A merged document is a new artifact (hence its own reportId) but it still describes the same logical + // run as the documents it was built from, so it carries their runId — provided they all agree on one. + // Inputs that disagree, or an input with no runId, contribute the empty sentinel and suppress the field + // rather than picking an arbitrary run to represent the whole merge. + if (distinctRunIds.Count == 1 && distinctRunIds.First() is { Length: > 0 } sharedRunId) + { + merged["runId"] = sharedRunId; + } + + merged["timestamp"] = DateTimeOffset.FromUnixTimeMilliseconds(stopMs).ToString("O", CultureInfo.InvariantCulture); + + // The merged document is produced by this merger, not by any input, so stamp its own identity + // rather than carrying the first input's 'generatedBy' (which could report a different producer + // or version when merging reports from different tool versions). + merged["generatedBy"] = GeneratedByName; + merged["results"] = resultsObject; + return merged.ToJsonString(new JsonSerializerOptions { WriteIndented = true }); } + internal static Task MergeToFileAsync( + IReadOnlyList inputPaths, + string outputPath, + CancellationToken cancellationToken) + => MergeToFileAsync(inputPaths, outputPath, CtrfMergeMode.Concatenate, cancellationToken); + internal static async Task MergeToFileAsync( IReadOnlyList inputPaths, string outputPath, + CtrfMergeMode mode, CancellationToken cancellationToken) { if (inputPaths is null) @@ -276,7 +350,7 @@ internal static async Task MergeToFileAsync( #endif } - string merged = Merge(reports); + string merged = Merge(reports, mode); string? outputDirectory = Path.GetDirectoryName(outputPath); if (!RoslynString.IsNullOrEmpty(outputDirectory)) @@ -297,10 +371,289 @@ await MergeOutputFileHelper.WriteViaTemporarySiblingAsync(outputPath, async temp }).ConfigureAwait(false); } - private static bool TryReadLong(JsonNode summary, string propertyName, out long value) + /// + /// Folds successive attempts of the same logical test — the tests[] rows contributed, in attempt order, by + /// the per-attempt documents of one orchestrated retry run — into a single row per test. + /// + /// + /// The retry model confirmed in ctrf-io/ctrf#58: the last attempt's outcome IS the test object, and + /// retryAttempts[] holds attempts 1..N-1 (the initial execution plus any earlier retries), + /// numbered from 1, so retries == retryAttempts.length and the final attempt is retries + 1. + /// The test object therefore keeps the FINAL attempt's duration (and start/stop), rather + /// than a sum across attempts. Attempts an input already recorded in its own retryAttempts[] (in-process + /// retries within a single attempt process) are flattened into the same history so no execution is lost. + /// + private static JsonArray CollapseRetryAttempts(JsonArray tests) + { + var slots = new List<(JsonObject Final, List Priors)>(); + var byIdentity = new Dictionary(StringComparer.Ordinal); + + foreach (JsonNode? test in tests) + { + // Non-objects were already rejected during ingestion; this only re-establishes the type. + if (test is not JsonObject testObject) + { + continue; + } + + // A row we cannot identify gets its own slot: fusing unrelated rows would lose results, whereas an + // uncollapsed duplicate is merely redundant. + if (GetTestIdentity(testObject) is not string identity) + { + slots.Add((testObject, [])); + continue; + } + + if (byIdentity.TryGetValue(identity, out int index)) + { + (JsonObject previousFinal, List priors) = slots[index]; + priors.Add(previousFinal); + slots[index] = (testObject, priors); + } + else + { + byIdentity.Add(identity, slots.Count); + slots.Add((testObject, [])); + } + } + + var collapsed = new JsonArray(); + foreach ((JsonObject final, List priors) in slots) + { + collapsed.Add(BuildCollapsedTest(final, priors)); + } + + return collapsed; + } + + /// + /// Computes the key identifying the logical test a row describes, preferring the CTRF testId, then the + /// legacy id, then the producer-supplied extra.uid, and finally the suite path plus name and the + /// other stable discriminators the Test object offers. Returns when the row carries + /// none of them. + /// + /// + /// + /// The identifier order follows CTRF 9.1: id is a stable test-case identifier that consumers treat as + /// legacy, using testId in preference only when both are present. Ignoring an id-only report + /// would drop it to the heuristic fallback and risk fusing distinct same-named tests. + /// + /// + /// The fallback length-prefixes every component rather than just separating them. A CTRF name or suite + /// segment is an arbitrary non-empty string, so it may itself contain the separator: with plain separation, + /// suite: ["A"], name: "B\u001fC" and suite: ["A", "B"], name: "C" would produce the same key + /// and collapse two unrelated tests into one, silently dropping a result. Length prefixes make the encoding + /// unambiguous whatever the components contain. + /// + /// + /// The fallback also folds in filePath (9.19) and parameters (9.30), because suite plus name + /// alone is not unique: parameterized rows share both while differing only in their parameters, and + /// same-named tests in different files differ only by path. If a producer were to serialize parameters + /// inconsistently between attempts, the effect is that a retry is not folded — a duplicated row rather than a + /// lost result, which is the safe direction to fail in. + /// + /// + private static string? GetTestIdentity(JsonObject test) + { + if (ReadString(test, "testId") is { Length: > 0 } testId) + { + return $"testId\u001f{testId}"; + } + + if (ReadString(test, "id") is { Length: > 0 } id) + { + return $"id\u001f{id}"; + } + + // `extra` is free-form, so a foreign producer may well have written a string or an array there; indexing + // anything but an object by property name throws. + if (test["extra"] is JsonObject extra && ReadString(extra, "uid") is { Length: > 0 } uid) + { + return $"uid\u001f{uid}"; + } + + if (ReadString(test, "name") is not { Length: > 0 } name) + { + return null; + } + + var identity = new StringBuilder("name"); + if (test["suite"] is JsonArray suite) + { + foreach (JsonNode? segment in suite) + { + // A suite segment is normally a string, but the document is untrusted. Fall back to the + // segment's JSON text so a non-string segment still contributes a distinct, deterministic part + // of the key instead of throwing on the string conversion. + AppendIdentityComponent( + identity, + segment is JsonValue segmentValue && segmentValue.TryGetValue(out string? segmentText) + ? segmentText + : segment?.ToJsonString()); + } + } + + AppendIdentityComponent(identity, name); + AppendIdentityComponent(identity, ReadString(test, "filePath")); + AppendIdentityComponent(identity, test["parameters"]?.ToJsonString()); + return identity.ToString(); + } + + private static void AppendIdentityComponent(StringBuilder identity, string? component) + => identity.Append('\u001f').Append(component?.Length ?? -1).Append(':').Append(component); + + private static JsonNode BuildCollapsedTest(JsonObject final, List priors) + { + var collapsed = (JsonObject)final.DeepClone(); + + // Nothing was merged into this row, so it is passed through rather than synthesized: its own + // `retryAttempts[]` (from in-process retries the producer already recorded) stays exactly as written. + // Below, once there ARE priors, the history becomes the merger's own array — it has to be rebuilt and + // renumbered into a contiguous 1..N-1 — which is why those entries are reshaped and these are not. + if (priors.Count == 0) + { + return collapsed; + } + + var history = new JsonArray(); + foreach (JsonObject prior in priors) + { + AppendAttempts(history, prior); + } + + // In-process retries observed by the final attempt precede its own outcome in the history. + if (collapsed["retryAttempts"] is JsonArray finalAttempts) + { + AppendNestedAttempts(history, finalAttempts); + } + + bool anyFailed = false; + for (int i = 0; i < history.Count; i++) + { + if (history[i] is not JsonObject attempt) + { + continue; + } + + attempt["attempt"] = i + 1; + anyFailed |= ReadString(attempt, "status") == "failed"; + } + + collapsed["retryAttempts"] = history; + collapsed["retries"] = history.Count; + + // CTRF 9.22: flaky only when the FINAL status is passed after at least one failed attempt. Recomputed + // (rather than inherited) because an input's own flag only describes the attempt that produced it. + if (ReadString(collapsed, "status") == "passed" && anyFailed) + { + collapsed["flaky"] = true; + } + else + { + collapsed.Remove("flaky"); + } + + return collapsed; + } + + /// + /// Appends the executions a non-final attempt row represents — the attempts it already nested, then its own + /// outcome — to the retry history being built. + /// + private static void AppendAttempts(JsonArray history, JsonObject test) + { + if (test["retryAttempts"] is JsonArray nested) + { + AppendNestedAttempts(history, nested); + } + + history.Add(ToRetryAttempt(test)); + } + + /// + /// Copies the retry attempts an input already recorded into the history being built, projecting each through + /// the same section 11 shaping as a promoted test row so a foreign producer's nested attempt cannot smuggle a + /// wrong-typed status or a disallowed field into the merged history. Entries that are not objects are skipped: + /// they could not describe an execution. + /// + private static void AppendNestedAttempts(JsonArray history, JsonArray nested) + { + foreach (JsonNode? attempt in nested) + { + if (attempt is JsonObject attemptObject) + { + history.Add(ToRetryAttempt(attemptObject)); + } + } + } + + /// + /// Projects a test row onto the retry attempt object of CTRF section 11, which allows a narrower set of + /// fields than a test: everything else (name, suite, tags, labels, ...) either belongs to the collapsed test + /// object or, like rawStatus, moves under extra, the only permitted extension point. + /// + private static JsonNode ToRetryAttempt(JsonObject test) + { + // 'attempt' is assigned by the caller, once the position of this execution in the history is known. + var attempt = new JsonObject + { + ["attempt"] = 1, + ["status"] = ReadStatus(test), + }; + + foreach (string field in RetryAttemptFields) + { + if (test[field] is JsonNode value) + { + attempt[field] = value.DeepClone(); + } + } + + JsonObject? extra = test["extra"] is JsonObject testExtra ? (JsonObject)testExtra.DeepClone() : null; + if (test["rawStatus"] is JsonNode rawStatus) + { + extra ??= []; + extra["rawStatus"] = rawStatus.DeepClone(); + } + + if (extra is { Count: > 0 }) + { + attempt["extra"] = extra; + } + + return attempt; + } + + /// + /// Reads a string property, treating a value of any other JSON type as absent. The explicit + /// (string?) conversion on a THROWS for a non-string value rather than + /// yielding , and every document reaching the merger is untrusted. + /// + private static string? ReadString(JsonObject owner, string propertyName) + => owner[propertyName] is JsonValue value && value.TryGetValue(out string? text) ? text : null; + + /// + /// Reads a CTRF status, normalizing anything outside the vocabulary section 11.3 allows — a wrong-typed + /// value, or a status this producer invented — to other. Copying such a value verbatim into a retry + /// attempt would make the merged document schema-invalid, and other is both a legal status and the + /// bucket the summary already counts it in. + /// + private static string ReadStatus(JsonObject owner) + { + string? status = ReadString(owner, "status"); + return status is "passed" or "failed" or "skipped" or "pending" or "other" ? status : "other"; + } + + /// + /// Reads an integral property from a JSON node, tolerating anything the node may actually be. The node comes + /// straight out of an untrusted document — results.summary is not guaranteed to be an object — so a + /// non-object must read as "absent" rather than throw: indexing a by property name is + /// an . + /// + private static bool TryReadLong(JsonNode node, string propertyName, out long value) { value = 0; - if (summary[propertyName] is not JsonValue jsonValue) + if (node is not JsonObject jsonObject || jsonObject[propertyName] is not JsonValue jsonValue) { return false; } @@ -377,19 +730,25 @@ private static bool TryReadLong(JsonNode summary, string propertyName, out long } /// - /// Derives a stable reportId from the accepted CTRF input reports so identical inputs reproduce - /// the same id on every retry (RFC 018 idempotency) without a random source or reusing an input report's - /// id. Only the payloads that passed CTRF validation are hashed, so a rejected non-CTRF input cannot - /// alter the merged report's identity. A non-cryptographic 128-bit FNV-1a fill is sufficient here — the - /// id only needs to be deterministic and collision-resistant enough to identify a merged report, not - /// secret. + /// Derives a stable reportId from the accepted CTRF input reports and the merge mode, so identical + /// inputs merged the same way reproduce the same id on every retry (RFC 018 idempotency) without a random + /// source or reusing an input report's id, while the same inputs merged a DIFFERENT way — which yields a + /// materially different document — get a distinct id, as CTRF 5.3 requires. Only the payloads that passed + /// CTRF validation are hashed, so a rejected non-CTRF input cannot alter the merged report's identity. A + /// non-cryptographic 128-bit FNV-1a fill is sufficient here — the id only needs to be deterministic and + /// collision-resistant enough to identify a merged report, not secret. /// - private static string CreateDeterministicReportId(IReadOnlyList acceptedReports) + private static string CreateDeterministicReportId(IReadOnlyList acceptedReports, CtrfMergeMode mode) { const ulong fnvPrime = 1099511628211UL; ulong hashLow = 14695981039346656037UL; ulong hashHigh = 0x9E3779B97F4A7C15UL; + // Fold in the merge mode: the same inputs concatenated and collapsed are two materially different + // documents, and CTRF 5.3 wants a distinct reportId for each rather than one id naming both. + hashLow = (hashLow ^ (ulong)mode) * fnvPrime; + hashHigh = (hashHigh ^ ((ulong)mode + 1UL)) * fnvPrime; + foreach (string report in acceptedReports) { foreach (char c in report) diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/InternalAPI/InternalAPI.Unshipped.txt b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/InternalAPI/InternalAPI.Unshipped.txt index 925dc5999d..65be134020 100644 --- a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/InternalAPI/InternalAPI.Unshipped.txt +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/InternalAPI/InternalAPI.Unshipped.txt @@ -1,7 +1,12 @@ #nullable enable virtual Microsoft.Testing.Extensions.ReportGeneratorBase.ArtifactKind.get -> string? +Microsoft.Testing.Extensions.CtrfReport.CtrfMergeMode +Microsoft.Testing.Extensions.CtrfReport.CtrfMergeMode.CollapseRetryAttempts = 1 -> Microsoft.Testing.Extensions.CtrfReport.CtrfMergeMode +Microsoft.Testing.Extensions.CtrfReport.CtrfMergeMode.Concatenate = 0 -> Microsoft.Testing.Extensions.CtrfReport.CtrfMergeMode Microsoft.Testing.Extensions.CtrfReport.CtrfReportMerger static Microsoft.Testing.Extensions.CtrfReport.CtrfReportMerger.Merge(System.Collections.Generic.IReadOnlyList! inputReports) -> string! +static Microsoft.Testing.Extensions.CtrfReport.CtrfReportMerger.Merge(System.Collections.Generic.IReadOnlyList! inputReports, Microsoft.Testing.Extensions.CtrfReport.CtrfMergeMode mode) -> string! +static Microsoft.Testing.Extensions.CtrfReport.CtrfReportMerger.MergeToFileAsync(System.Collections.Generic.IReadOnlyList! inputPaths, string! outputPath, Microsoft.Testing.Extensions.CtrfReport.CtrfMergeMode mode, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task! static Microsoft.Testing.Extensions.CtrfReport.CtrfReportMerger.MergeToFileAsync(System.Collections.Generic.IReadOnlyList! inputPaths, string! outputPath, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task! static Microsoft.Testing.Platform.Resources.PlatformResources.NamedPipeDirectoryNotWritableErrorMessage.get -> string! static Microsoft.Testing.Platform.Resources.PlatformResources.NamedPipePathTooLongErrorMessage.get -> string! diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Microsoft.Testing.Extensions.CtrfReport.csproj b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Microsoft.Testing.Extensions.CtrfReport.csproj index a9f7c6dccd..f65438334b 100644 --- a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Microsoft.Testing.Extensions.CtrfReport.csproj +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/Microsoft.Testing.Extensions.CtrfReport.csproj @@ -52,6 +52,7 @@ This package extends Microsoft Testing Platform to produce test reports in the C + diff --git a/src/Platform/Microsoft.Testing.Extensions.Retry/RetryOrchestrator.cs b/src/Platform/Microsoft.Testing.Extensions.Retry/RetryOrchestrator.cs index 247c50e215..427a589a1b 100644 --- a/src/Platform/Microsoft.Testing.Extensions.Retry/RetryOrchestrator.cs +++ b/src/Platform/Microsoft.Testing.Extensions.Retry/RetryOrchestrator.cs @@ -2,6 +2,7 @@ // Licensed under dual-license. See LICENSE.PLATFORMTOOLS.txt file in the project root for full license information. using Microsoft.Testing.Extensions.Policy.Resources; +using Microsoft.Testing.Platform; using Microsoft.Testing.Platform.CommandLine; using Microsoft.Testing.Platform.Configurations; using Microsoft.Testing.Platform.Extensions.OutputDevice; @@ -84,6 +85,21 @@ public async Task OrchestrateTestHostExecutionAsync(CancellationToken cance environment.SetEnvironmentVariable(EnvironmentVariableConstants.TESTINGPLATFORM_TRX_TESTRUN_ID, Guid.NewGuid().ToString("N")); + // Every attempt is a separate process writing its own report, but together they are one logical run. + // Formats that can express that (CTRF 'runId') need all attempts to agree on a single id, so establish + // one here and let the launched test hosts inherit it. Preference order: + // - an id already set explicitly (a CI job correlating several modules or machines) is kept; + // - otherwise the dotnet test execution id, which already identifies THIS test application's process + // tree, so the attempts stay part of that run instead of forming a separate one; + // - otherwise a fresh id, because a standalone retried run is its own logical run. + if (RoslynString.IsNullOrEmpty(environment.GetEnvironmentVariable(EnvironmentVariableConstants.TESTINGPLATFORM_LOGICAL_RUN_ID))) + { + string? executionId = environment.GetEnvironmentVariable(EnvironmentVariableConstants.TESTINGPLATFORM_DOTNETTEST_EXECUTIONID); + environment.SetEnvironmentVariable( + EnvironmentVariableConstants.TESTINGPLATFORM_LOGICAL_RUN_ID, + RoslynString.IsNullOrEmpty(executionId) ? Guid.NewGuid().ToString("D") : executionId!); + } + ILogger logger = _serviceProvider.GetLoggerFactory().CreateLogger(); IConfiguration configuration = _serviceProvider.GetConfiguration(); diff --git a/src/Platform/Microsoft.Testing.Platform/Helpers/EnvironmentVariableConstants.cs b/src/Platform/Microsoft.Testing.Platform/Helpers/EnvironmentVariableConstants.cs index 83eed5246e..f33e36a843 100644 --- a/src/Platform/Microsoft.Testing.Platform/Helpers/EnvironmentVariableConstants.cs +++ b/src/Platform/Microsoft.Testing.Platform/Helpers/EnvironmentVariableConstants.cs @@ -58,6 +58,16 @@ internal static class EnvironmentVariableConstants // Unhandled Exception public const string TESTINGPLATFORM_EXIT_PROCESS_ON_UNHANDLED_EXCEPTION = nameof(TESTINGPLATFORM_EXIT_PROCESS_ON_UNHANDLED_EXCEPTION); + // Correlates the processes that make up one logical test run, so report formats that can describe a run + // spanning several documents (CTRF 'runId') can tie those documents back together. The retry orchestrator + // sets it before launching its attempts, and any process tree started from there inherits it. + // + // NOTE: a multi-project 'dotnet test' does NOT set this. Each module is a separate root test application + // with its own execution id (see docs/mstest-runner-protocol/004-protocol-dotnet-test-pipe.md), so its + // modules are distinct execution trees. Correlating them — or correlating shards running on different + // machines — requires setting this variable explicitly before launching them. + public const string TESTINGPLATFORM_LOGICAL_RUN_ID = nameof(TESTINGPLATFORM_LOGICAL_RUN_ID); + // Trx public const string TESTINGPLATFORM_TRX_TESTRUN_ID = nameof(TESTINGPLATFORM_TRX_TESTRUN_ID); } diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/CtrfReportTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/CtrfReportTests.cs index 48891a2b24..1d862aaee4 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/CtrfReportTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/CtrfReportTests.cs @@ -135,6 +135,10 @@ private static void AssertCtrfReportShape(string filePath) // - FlakyTest (retried) → first attempt failed, retried successfully: // final status: "passed", retries: 1, // retryAttempts[].status: "failed", flaky: true + // + // The retry shape follows the model confirmed in ctrf-io/ctrf#58: `retryAttempts[]` + // holds attempts 1..N-1 (the final attempt's outcome IS the test object), so + // `retries` equals `retryAttempts.length` and the final attempt is `retries + 1`. string actual = File.ReadAllText(filePath); string normalized = NormalizeCtrfReport(actual); @@ -143,6 +147,7 @@ private static void AssertCtrfReportShape(string filePath) "reportFormat": "CTRF", "specVersion": "0.0.0", "reportId": "", + "runId": "", "timestamp": "", "generatedBy": "Microsoft.Testing.Extensions.CtrfReport@", "results": { @@ -228,6 +233,7 @@ private static string NormalizeCtrfReport(string actual) // anchored, but runtime-variable values are folded into stable tokens. string normalized = actual; normalized = Regex.Replace(normalized, @"""reportId"": ""[^""]+""", @"""reportId"": """""); + normalized = Regex.Replace(normalized, @"""runId"": ""[^""]+""", @"""runId"": """""); normalized = Regex.Replace(normalized, @"""timestamp"": ""[^""]+""", @"""timestamp"": """""); normalized = Regex.Replace(normalized, @"""generatedBy"": ""Microsoft\.Testing\.Extensions\.CtrfReport@[^""]+""", @"""generatedBy"": ""Microsoft.Testing.Extensions.CtrfReport@"""); normalized = Regex.Replace(normalized, @"""start"": \d+", @"""start"": "); diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/RetryFailedTestsTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/RetryFailedTestsTests.cs index e1b87d9ab7..6cb66202e4 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/RetryFailedTestsTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/RetryFailedTestsTests.cs @@ -551,6 +551,92 @@ public async Task RetryFailedTests_WithMinimumExpectedTests_StripsThresholdOnRet testHostResult.AssertOutputDoesNotContain("Minimum expected tests policy violation"); } + internal static IEnumerable<(string Tfm, string? SeededVariable)> GetRunIdMatrix() + { + foreach (string tfm in TargetFrameworks.Net) + { + // The orchestrator resolves the logical run id as: an explicitly set id wins, else the dotnet test + // execution id (which already identifies this test application's process tree), else a fresh one. + // Exercise all three branches. + yield return (tfm, null); + yield return (tfm, EnvironmentVariableConstants.TESTINGPLATFORM_LOGICAL_RUN_ID); + yield return (tfm, EnvironmentVariableConstants.TESTINGPLATFORM_DOTNETTEST_EXECUTIONID); + } + } + + [TestMethod] + [DynamicData(nameof(GetRunIdMatrix))] + public async Task RetryFailedTests_CtrfReports_ShareRunIdButNotReportId(string tfm, string? seededVariable) + { + // Each attempt is a separate process that writes its own CTRF document, but together they are one + // logical run. Per ctrf-io/ctrf#58 those documents SHOULD share a `runId` while each stays a distinct + // artifact with its own `reportId`. This is the only test that exercises the cross-process contract: + // the engine unit tests mock IEnvironment, so they cannot observe the orchestrator's seeding. + var testHost = TestInfrastructure.TestHost.LocateFrom(AssetFixture.TargetAssetPath, AssetName, tfm); + string resultDirectory = Path.Combine(testHost.DirectoryName, Guid.NewGuid().ToString("N")); + + // METHOD1=1 makes TestMethod1 fail on the first attempt and pass on the second, so exactly two + // attempts run and each writes a CTRF report. + Dictionary environmentVariables = new() + { + { EnvironmentVariableConstants.TESTINGPLATFORM_TELEMETRY_OPTOUT, "1" }, + { "METHOD1", "1" }, + { "RESULTDIR", resultDirectory }, + }; + + // When a correlation id is supplied from outside, the attempts must adopt THAT id rather than minting + // their own — that is what lets a CI job tie several modules or machines into one logical run. + string? expectedRunId = null; + if (seededVariable is not null) + { + expectedRunId = $"seeded-{Guid.NewGuid():N}"; + environmentVariables[seededVariable] = expectedRunId; + } + + TestHostResult testHostResult = await testHost.ExecuteAsync( + $"--retry-failed-tests 3 --report-ctrf --results-directory {resultDirectory}", + environmentVariables, + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.Success); + testHostResult.AssertOutputContains("Retry summary: Passed! after 2/4 attempts"); + + // Attempts 1..N-1 stay under Retries//; only the final attempt's report is moved to the top level. + string[] ctrfFiles = + [ + .. Directory.GetFiles(resultDirectory, "*.ctrf.json", SearchOption.AllDirectories).OrderBy(f => f, StringComparer.Ordinal), + ]; + Assert.HasCount(2, ctrfFiles, $"Expected one CTRF report per attempt.{Environment.NewLine}{string.Join(Environment.NewLine, ctrfFiles)}"); + + string[] runIds = [.. ctrfFiles.Select(f => ReadRequiredStringProperty(f, "runId"))]; + string[] reportIds = [.. ctrfFiles.Select(f => ReadRequiredStringProperty(f, "reportId"))]; + + Assert.AreEqual(runIds[0], runIds[1], "Both attempts belong to the same logical run, so they must share a runId."); + Assert.AreNotEqual(reportIds[0], reportIds[1], "Each attempt is a distinct artifact, so it must have its own reportId."); + Assert.AreNotEqual(runIds[0], reportIds[0], "runId and reportId identify different things and must not be the same value."); + + if (expectedRunId is not null) + { + Assert.AreEqual(expectedRunId, runIds[0], $"'{seededVariable}' must be honored instead of minting a new run id."); + } + else + { + Assert.IsTrue(Guid.TryParse(runIds[0], out _), $"An uncorrelated run must mint a GUID run id, got '{runIds[0]}'."); + } + } + + private static string ReadRequiredStringProperty(string filePath, string propertyName) + { + using var document = System.Text.Json.JsonDocument.Parse(File.ReadAllText(filePath)); + Assert.IsTrue( + document.RootElement.TryGetProperty(propertyName, out System.Text.Json.JsonElement value), + $"'{propertyName}' is missing from '{filePath}'."); + + string? text = value.GetString(); + Assert.IsFalse(string.IsNullOrEmpty(text), $"'{propertyName}' must be a non-empty string in '{filePath}'."); + return text!; + } + public sealed class TestAssetFixture() : TestAssetFixtureBase() { public string TargetAssetPath => GetAssetPath(AssetName); @@ -558,7 +644,8 @@ public sealed class TestAssetFixture() : TestAssetFixtureBase() public override (string ID, string Name, string Code) GetAssetsToGenerate() => (AssetName, AssetName, TestCode .PatchTargetFrameworks(TargetFrameworks.All) - .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion)); + .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion) + .PatchCodeWithReplace("$MicrosoftTestingExtensionsCtrfReportVersion$", MicrosoftTestingExtensionsCtrfReportVersion)); private const string TestCode = """ #file RetryFailedTests.csproj @@ -574,6 +661,7 @@ public override (string ID, string Name, string Code) GetAssetsToGenerate() => ( + @@ -611,6 +699,9 @@ public static async Task Main(string[] args) (_,__) => new DummyTestFramework()); builder.AddCrashDumpProvider(); builder.AddTrxReportProvider(); +#pragma warning disable TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. + builder.AddCtrfReportProvider(); +#pragma warning restore TPEXP builder.AddRetryProvider(); builder.AddMSBuild(); builder.AddTreeNodeFilterService(treeNodeFilterExtension); diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportEngineTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportEngineTests.cs index d2b780131e..37060a19b7 100644 --- a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportEngineTests.cs +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportEngineTests.cs @@ -830,6 +830,61 @@ public async Task GenerateReportAsync_SingleLineOutput_EmitsOneArrayEntry() Assert.AreEqual("only-line", stdout[0].GetString()); } + [TestMethod] + public async Task GenerateReportAsync_RunId_UsesTheSharedLogicalRunId() + { + // ctrf-io/ctrf#58: every process of one logical run — notably the successive attempts of + // --retry-failed-tests — must stamp the same runId so consumers can tie those documents back + // together. The retry orchestrator publishes that id through this environment variable. + using var memoryStream = new MemoryFileStream(); + CtrfReportEngine engine = CreateEngine(memoryStream); + _ = _environmentMock.Setup(x => x.GetEnvironmentVariable("TESTINGPLATFORM_LOGICAL_RUN_ID")).Returns("run-42"); + _ = _environmentMock.Setup(x => x.GetEnvironmentVariable("TESTINGPLATFORM_DOTNETTEST_EXECUTIONID")).Returns("execution-7"); + + await engine.GenerateReportAsync([Captured("p1", "Passing test", "passed")]); + + using var document = JsonDocument.Parse(memoryStream.GetUtf8Content()); + Assert.AreEqual("run-42", document.RootElement.GetProperty("runId").GetString()); + + // The run and the document are different things: a correlated run must not leak into the artifact id. + Assert.AreNotEqual("run-42", document.RootElement.GetProperty("reportId").GetString()); + } + + [TestMethod] + public async Task GenerateReportAsync_RunId_FallsBackToTheDotnetTestExecutionId() + { + // The execution id identifies this test application's own process tree. It is per root test application, + // not per 'dotnet test' invocation, so it correlates a module with its child processes — not with the + // sibling modules of a multi-project run, which legitimately report different logical runs. + using var memoryStream = new MemoryFileStream(); + CtrfReportEngine engine = CreateEngine(memoryStream); + _ = _environmentMock.Setup(x => x.GetEnvironmentVariable("TESTINGPLATFORM_LOGICAL_RUN_ID")).Returns((string?)null); + _ = _environmentMock.Setup(x => x.GetEnvironmentVariable("TESTINGPLATFORM_DOTNETTEST_EXECUTIONID")).Returns("execution-7"); + + await engine.GenerateReportAsync([Captured("p1", "Passing test", "passed")]); + + using var document = JsonDocument.Parse(memoryStream.GetUtf8Content()); + Assert.AreEqual("execution-7", document.RootElement.GetProperty("runId").GetString()); + Assert.AreNotEqual("execution-7", document.RootElement.GetProperty("reportId").GetString()); + } + + [TestMethod] + public async Task GenerateReportAsync_RunId_IsGenerated_WhenNothingCorrelatedTheProcess() + { + // A standalone run is its own logical run, and CTRF requires runId to be a non-empty string when present. + using var memoryStream = new MemoryFileStream(); + CtrfReportEngine engine = CreateEngine(memoryStream); + _ = _environmentMock.Setup(x => x.GetEnvironmentVariable("TESTINGPLATFORM_LOGICAL_RUN_ID")).Returns((string?)null); + _ = _environmentMock.Setup(x => x.GetEnvironmentVariable("TESTINGPLATFORM_DOTNETTEST_EXECUTIONID")).Returns((string?)null); + + await engine.GenerateReportAsync([Captured("p1", "Passing test", "passed")]); + + using var document = JsonDocument.Parse(memoryStream.GetUtf8Content()); + string runId = document.RootElement.GetProperty("runId").GetString()!; + Assert.IsTrue(Guid.TryParse(runId, out _), $"Expected a generated id, got '{runId}'."); + Assert.AreNotEqual(document.RootElement.GetProperty("reportId").GetString(), runId); + } + private CtrfReportEngine CreateEngine(MemoryFileStream stream) { _ = _fileSystem.Setup(x => x.ExistFile(It.IsAny())).Returns(false); diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportMergerTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportMergerTests.cs index 7efdd64aba..29438f4564 100644 --- a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportMergerTests.cs +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/CtrfReportMergerTests.cs @@ -490,6 +490,7 @@ private static string BuildReport( string toolName = "MSTest", string? toolVersion = null, string osPlatform = "test", + string? runId = null, IEnumerable? testEntries = null) { var testArray = new JsonArray(); @@ -542,9 +543,528 @@ private static string BuildReport( }, }; + if (runId is not null) + { + report["runId"] = runId; + } + return report.ToJsonString(); } + [TestMethod] + public void Merge_CarriesRunId_WhenEveryInputReportsTheSameOne() + { + // Per-attempt (and per-shard) documents of one logical run share a runId; the merged document + // describes that same logical run, so it keeps the id while getting its own reportId. + string a = BuildReport(runId: "run-42", testEntries: [Test("a", "passed")]); + string b = BuildReport(runId: "run-42", testEntries: [Test("b", "passed")]); + + JsonNode merged = JsonNode.Parse(CtrfReportMerger.Merge([a, b]))!; + + Assert.AreEqual("run-42", (string?)merged["runId"]); + Assert.AreNotEqual("run-42", (string?)merged["reportId"]); + Assert.AreNotEqual((string?)JsonNode.Parse(a)!["reportId"], (string?)merged["reportId"]); + } + + [TestMethod] + public void Merge_OmitsRunId_WhenInputsBelongToDifferentRuns() + { + string a = BuildReport(runId: "run-1"); + string b = BuildReport(runId: "run-2"); + + JsonNode merged = JsonNode.Parse(CtrfReportMerger.Merge([a, b]))!; + + Assert.IsNull(merged["runId"]); + + // Suppressing the run correlation must not suppress the merged document's own identity. + Assert.IsNotNull((string?)merged["reportId"]); + } + + [TestMethod] + public void Merge_OmitsRunId_WhenAnInputHasNone() + { + // An input with no runId may or may not belong to the same run, so claiming the known one would + // assert a correlation the inputs do not support. + string a = BuildReport(runId: "run-1"); + string b = BuildReport(); + + JsonNode merged = JsonNode.Parse(CtrfReportMerger.Merge([a, b]))!; + + Assert.IsNull(merged["runId"]); + Assert.IsNotNull((string?)merged["reportId"]); + } + + [TestMethod] + public void Merge_DefaultMode_DoesNotCollapseRepeatedTests() + { + // Cross-module merges must keep every row: MTP UIDs are only unique within an assembly, so two + // same-named tests can legitimately come from different modules. + string attempt1 = BuildReport(testEntries: [Attempt("t", "failed", uid: "u1")]); + string attempt2 = BuildReport(testEntries: [Attempt("t", "passed", uid: "u1")]); + + JsonNode results = JsonNode.Parse(CtrfReportMerger.Merge([attempt1, attempt2]))!["results"]!; + + Assert.HasCount(2, (JsonArray)results["tests"]!); + Assert.AreEqual(2, (long)results["summary"]!["tests"]!); + } + + [TestMethod] + public void Merge_CollapseRetryAttempts_FoldsEarlierAttemptsIntoRetryHistory() + { + // Three per-attempt documents of one orchestrated retry run: attempts 1 and 2 failed, attempt 3 passed. + string attempt1 = BuildReport(testEntries: [Attempt("flaky test", "failed", uid: "u1", duration: 120, message: "boom 1")]); + string attempt2 = BuildReport(testEntries: [Attempt("flaky test", "failed", uid: "u1", duration: 130, message: "boom 2")]); + string attempt3 = BuildReport(testEntries: [Attempt("flaky test", "passed", uid: "u1", duration: 140)]); + + JsonNode results = JsonNode.Parse(CtrfReportMerger.Merge([attempt1, attempt2, attempt3], CtrfMergeMode.CollapseRetryAttempts))!["results"]!; + + var tests = (JsonArray)results["tests"]!; + Assert.HasCount(1, tests); + + JsonNode test = tests[0]!; + Assert.AreEqual("passed", (string?)test["status"]); + Assert.IsTrue((bool)test["flaky"]!); + + // ctrf-io/ctrf#58: retryAttempts holds attempts 1..N-1, so retries == retryAttempts.length and the + // final attempt is retries + 1. + var retryAttempts = (JsonArray)test["retryAttempts"]!; + Assert.HasCount(2, retryAttempts); + Assert.AreEqual(2, (long)test["retries"]!); + Assert.AreEqual(1, (long)retryAttempts[0]!["attempt"]!); + Assert.AreEqual(2, (long)retryAttempts[1]!["attempt"]!); + Assert.AreEqual("boom 1", (string?)retryAttempts[0]!["message"]); + Assert.AreEqual("boom 2", (string?)retryAttempts[1]!["message"]); + + // The test object carries the FINAL attempt's duration, not the sum across attempts. + Assert.AreEqual(140, (long)test["duration"]!); + Assert.AreEqual(120, (long)retryAttempts[0]!["duration"]!); + } + + [TestMethod] + public void Merge_CollapseRetryAttempts_CountsLogicalTestsOnce() + { + // The scenario from ctrf-io/ctrf#58: a 4-test suite where each attempt re-runs only what failed. + // The merged document describes the logical run, so it reports 4 tests, not the 8 executions. + string attempt1 = BuildReport(testEntries: + [ + Attempt("ok1", "passed", uid: "u1"), + Attempt("ok2", "passed", uid: "u2"), + Attempt("recovers", "failed", uid: "u3"), + Attempt("always fails", "failed", uid: "u4"), + ]); + string attempt2 = BuildReport(testEntries: + [ + Attempt("recovers", "passed", uid: "u3"), + Attempt("always fails", "failed", uid: "u4"), + ]); + string attempt3 = BuildReport(testEntries: [Attempt("always fails", "failed", uid: "u4")]); + string attempt4 = BuildReport(testEntries: [Attempt("always fails", "failed", uid: "u4")]); + + JsonNode results = JsonNode.Parse( + CtrfReportMerger.Merge([attempt1, attempt2, attempt3, attempt4], CtrfMergeMode.CollapseRetryAttempts))!["results"]!; + + JsonNode summary = results["summary"]!; + Assert.AreEqual(4, (long)summary["tests"]!); + Assert.AreEqual(3, (long)summary["passed"]!); + Assert.AreEqual(1, (long)summary["failed"]!); + Assert.AreEqual(1, (long)summary["flaky"]!); + Assert.HasCount(4, (JsonArray)results["tests"]!); + + JsonNode alwaysFails = ((JsonArray)results["tests"]!).Single(t => (string?)t!["name"] == "always fails")!; + Assert.AreEqual("failed", (string?)alwaysFails["status"]); + Assert.AreEqual(3, (long)alwaysFails["retries"]!); + Assert.IsNull(alwaysFails["flaky"]); + + JsonNode neverRetried = ((JsonArray)results["tests"]!).Single(t => (string?)t!["name"] == "ok1")!; + Assert.IsNull(neverRetried["retries"]); + Assert.IsNull(neverRetried["retryAttempts"]); + } + + [TestMethod] + public void Merge_CollapseRetryAttempts_FlattensInProcessRetriesOfEachAttempt() + { + // An attempt process can itself have retried the test in-process; those executions are already in its + // retryAttempts[] and must keep their place in the merged history instead of being dropped. + JsonObject firstAttempt = Attempt("t", "failed", uid: "u1", message: "second execution"); + firstAttempt["retryAttempts"] = new JsonArray(new JsonObject + { + ["attempt"] = 1, + ["status"] = "failed", + ["message"] = "first execution", + }); + + string attempt1 = BuildReport(testEntries: [firstAttempt]); + string attempt2 = BuildReport(testEntries: [Attempt("t", "passed", uid: "u1")]); + + JsonNode test = ((JsonArray)JsonNode.Parse( + CtrfReportMerger.Merge([attempt1, attempt2], CtrfMergeMode.CollapseRetryAttempts))!["results"]!["tests"]!)[0]!; + + var retryAttempts = (JsonArray)test["retryAttempts"]!; + Assert.HasCount(2, retryAttempts); + Assert.AreEqual(3, (long)test["retries"]! + 1, "The final attempt is retries + 1."); + Assert.AreEqual("first execution", (string?)retryAttempts[0]!["message"]); + Assert.AreEqual("second execution", (string?)retryAttempts[1]!["message"]); + Assert.AreEqual(1, (long)retryAttempts[0]!["attempt"]!); + Assert.AreEqual(2, (long)retryAttempts[1]!["attempt"]!); + } + + [TestMethod] + public void Merge_CollapseRetryAttempts_ProjectsAttemptsOntoRetryAttemptShape() + { + // CTRF section 11 forbids unknown fields on a retry attempt, so test-only fields must not leak into it; + // rawStatus has no attempt-level slot and moves under 'extra', the only permitted extension point. + JsonObject failing = Attempt("t", "failed", uid: "u1", message: "boom"); + failing["rawStatus"] = "timedOut"; + failing["suite"] = new JsonArray("NS", "C"); + failing["tags"] = new JsonArray("slow"); + failing["trace"] = "at X()"; + + string attempt1 = BuildReport(testEntries: [failing]); + string attempt2 = BuildReport(testEntries: [Attempt("t", "passed", uid: "u1")]); + + JsonNode test = ((JsonArray)JsonNode.Parse( + CtrfReportMerger.Merge([attempt1, attempt2], CtrfMergeMode.CollapseRetryAttempts))!["results"]!["tests"]!)[0]!; + + JsonNode retryAttempt = ((JsonArray)test["retryAttempts"]!)[0]!; + Assert.AreEqual("failed", (string?)retryAttempt["status"]); + Assert.AreEqual("boom", (string?)retryAttempt["message"]); + Assert.AreEqual("at X()", (string?)retryAttempt["trace"]); + Assert.AreEqual("timedOut", (string?)retryAttempt["extra"]!["rawStatus"]); + Assert.AreEqual("u1", (string?)retryAttempt["extra"]!["uid"]); + Assert.IsNull(retryAttempt["name"]); + Assert.IsNull(retryAttempt["suite"]); + Assert.IsNull(retryAttempt["tags"]); + Assert.IsNull(retryAttempt["rawStatus"]); + } + + [TestMethod] + public void Merge_CollapseRetryAttempts_DropsStaleFlakyFlag_WhenFinalAttemptFails() + { + // An input's own flaky flag only describes the attempt that produced it; a later failure means the + // logical test is not flaky (CTRF 9.22 requires the FINAL status to be passed). + JsonObject recovered = Attempt("t", "passed", uid: "u1"); + recovered["flaky"] = true; + + string attempt1 = BuildReport(testEntries: [recovered]); + string attempt2 = BuildReport(testEntries: [Attempt("t", "failed", uid: "u1")]); + + JsonNode results = JsonNode.Parse( + CtrfReportMerger.Merge([attempt1, attempt2], CtrfMergeMode.CollapseRetryAttempts))!["results"]!; + + JsonNode test = ((JsonArray)results["tests"]!)[0]!; + Assert.AreEqual("failed", (string?)test["status"]); + Assert.IsNull(test["flaky"]); + Assert.AreEqual(0, (long)results["summary"]!["flaky"]!); + } + + [TestMethod] + public void Merge_CollapseRetryAttempts_UsesNameAndSuite_WhenNoIdentifierIsAvailable() + { + // Without testId or extra.uid, the suite path plus name is the only identity available, and two tests + // that only share a name must not be fused. + static JsonObject Named(string name, string status, string suite) + { + JsonObject test = Test(name, status); + test["suite"] = new JsonArray(suite); + return test; + } + + string attempt1 = BuildReport(testEntries: [Named("t", "failed", "A"), Named("t", "failed", "B")]); + string attempt2 = BuildReport(testEntries: [Named("t", "passed", "A"), Named("t", "failed", "B")]); + + var tests = (JsonArray)JsonNode.Parse( + CtrfReportMerger.Merge([attempt1, attempt2], CtrfMergeMode.CollapseRetryAttempts))!["results"]!["tests"]!; + + Assert.HasCount(2, tests); + Assert.AreEqual("passed", (string?)tests[0]!["status"]); + Assert.AreEqual("failed", (string?)tests[1]!["status"]); + Assert.AreEqual(1, (long)tests[0]!["retries"]!); + Assert.AreEqual(1, (long)tests[1]!["retries"]!); + } + + [TestMethod] + [DataRow(false)] + [DataRow(true)] + public void Merge_RejectsMalformedTestRows_SoTheMergedDocumentStaysValid(bool collapseRetryAttempts) + { + // The mode is passed as a bool because CtrfMergeMode is internal and a test method must be public. + CtrfMergeMode mode = collapseRetryAttempts ? CtrfMergeMode.CollapseRetryAttempts : CtrfMergeMode.Concatenate; + // tests[] comes from an untrusted file. The CTRF schema types its items as Test objects, so carrying a + // bare string or a JSON null through would turn a defect localized to one input into an invalid MERGED + // document. Such rows are dropped — the same policy the merger already applies to a whole non-CTRF input + // — and both modes must agree on that, since the merged tests[] is the same array either way. + // `summary` is deliberately a string here too: `results.summary` is equally untrusted, and reading a + // property off a non-object node throws just as `tests[]` did. The well-formed-looking rows carry + // wrong-typed VALUES — a numeric `status`, a numeric `suite` segment, a numeric `status` inside a nested + // `retryAttempts[]` entry — because the explicit (string?) conversion on a JsonNode throws for a + // non-string value instead of yielding null. + string malformed = """ + {"reportFormat":"CTRF","specVersion":"0.0.0","results":{"summary":"broken","tests":[ + "oops", + null, + 42, + {"name":"numeric status","status":7}, + {"name":"numeric suite","status":"failed","suite":["A",7,{"x":1}]}, + {"name":"nested","status":"passed","extra":{"uid":"n1"},"retryAttempts":[{"attempt":1,"status":3}]} + ]}} + """; + string wellFormed = BuildReport(testEntries: [Attempt("t", "passed", uid: "u1")]); + + JsonNode results = JsonNode.Parse(CtrfReportMerger.Merge([malformed, wellFormed], mode))!["results"]!; + + var tests = (JsonArray)results["tests"]!; + Assert.HasCount(4, tests, "The three non-object elements are dropped; wrong-typed values are tolerated."); + Assert.AreEqual("numeric status", (string?)tests[0]!["name"]); + Assert.AreEqual("t", (string?)tests[3]!["name"]); + + // CTRF 8.1: summary.tests equals the tests[] length, and the status buckets must add back up to it — + // a dropped row must not leave a phantom entry in either the array or the counters. An unreadable + // status is classified as 'other' rather than crashing the merge. + JsonNode summary = results["summary"]!; + Assert.AreEqual(4, (long)summary["tests"]!); + Assert.AreEqual(2, (long)summary["passed"]!); + Assert.AreEqual(1, (long)summary["failed"]!); + Assert.AreEqual(1, (long)summary["other"]!, "A non-string status is unclassifiable."); + long bucketSum = (long)summary["passed"]! + (long)summary["failed"]! + (long)summary["skipped"]! + + (long)summary["pending"]! + (long)summary["other"]!; + Assert.AreEqual((long)summary["tests"]!, bucketSum, "Every counted test must land in exactly one status bucket."); + } + + [TestMethod] + public void Merge_CollapseRetryAttempts_ToleratesNonObjectExtra() + { + // `extra` is free-form, so a foreign producer may put a string or an array there. Identity resolution + // must fall back to the suite/name key instead of throwing while indexing it. + static JsonObject WithExtra(string status, JsonNode extra) + { + JsonObject test = Test("t", status); + test["extra"] = extra; + return test; + } + + string attempt1 = BuildReport(testEntries: [WithExtra("failed", "ci-run-14")]); + string attempt2 = BuildReport(testEntries: [WithExtra("passed", new JsonArray("a"))]); + + var tests = (JsonArray)JsonNode.Parse( + CtrfReportMerger.Merge([attempt1, attempt2], CtrfMergeMode.CollapseRetryAttempts))!["results"]!["tests"]!; + + // Both rows share the name identity, so they collapse even though neither carries a usable extra.uid. + Assert.HasCount(1, tests); + Assert.AreEqual("passed", (string?)tests[0]!["status"]); + Assert.AreEqual(1, (long)tests[0]!["retries"]!); + Assert.IsTrue((bool)tests[0]!["flaky"]!); + } + + [TestMethod] + public void Merge_ReportId_IsDeterministicPerModeAndDiffersBetweenModes() + { + // The two modes turn the same inputs into materially different documents (three rows vs one collapsed + // row here), so CTRF 5.3 requires each to get its own reportId — one id must not name both artifacts. + // Determinism per mode (RFC 018 idempotency) must survive that. + string attempt1 = BuildReport(testEntries: [Attempt("t", "failed", uid: "u1")]); + string attempt2 = BuildReport(testEntries: [Attempt("t", "passed", uid: "u1")]); + string[] inputs = [attempt1, attempt2]; + + string concatenated = (string)JsonNode.Parse(CtrfReportMerger.Merge(inputs, CtrfMergeMode.Concatenate))!["reportId"]!; + string collapsed = (string)JsonNode.Parse(CtrfReportMerger.Merge(inputs, CtrfMergeMode.CollapseRetryAttempts))!["reportId"]!; + + Assert.AreNotEqual(concatenated, collapsed, "Two materially different merged documents must not share a reportId."); + + // Re-merging the same inputs the same way reproduces the id. + Assert.AreEqual(concatenated, (string)JsonNode.Parse(CtrfReportMerger.Merge(inputs, CtrfMergeMode.Concatenate))!["reportId"]!); + Assert.AreEqual(collapsed, (string)JsonNode.Parse(CtrfReportMerger.Merge(inputs, CtrfMergeMode.CollapseRetryAttempts))!["reportId"]!); + + // The default overload keeps concatenating, so it must agree with the explicit Concatenate mode. + Assert.AreEqual(concatenated, (string)JsonNode.Parse(CtrfReportMerger.Merge(inputs))!["reportId"]!); + + // CTRF 5.3: reportId MUST be a valid UUID when present. + Assert.IsTrue(Guid.TryParse(collapsed, out _), $"reportId must be a UUID, got '{collapsed}'."); + } + + [TestMethod] + public void Merge_CollapseRetryAttempts_DoesNotFuseTestsWhoseNameContainsTheIdentitySeparator() + { + // A CTRF `name` is an arbitrary non-empty string, so it may contain whatever character the identity key + // uses as a separator. With plain separation, suite ["A"] + name "B\u001fC" and suite ["A","B"] + name + // "C" flatten to the same key, which would fuse two unrelated tests and silently drop a result. + static JsonObject Named(string status, string name, params string[] suite) + { + JsonObject test = Test(name, status); + test["suite"] = new JsonArray([.. suite.Select(s => (JsonNode)JsonValue.Create(s)!)]); + return test; + } + + string report = BuildReport(testEntries: + [ + Named("failed", "B\u001fC", "A"), + Named("passed", "C", "A", "B"), + ]); + + var tests = (JsonArray)JsonNode.Parse( + CtrfReportMerger.Merge([report], CtrfMergeMode.CollapseRetryAttempts))!["results"]!["tests"]!; + + Assert.HasCount(2, tests, "Two distinct tests must not collapse into one."); + Assert.AreEqual("failed", (string?)tests[0]!["status"]); + Assert.AreEqual("passed", (string?)tests[1]!["status"]); + Assert.IsNull(tests[0]!["retries"], "Neither row is a retry of the other."); + Assert.IsNull(tests[1]!["retries"]); + } + + [TestMethod] + public void Merge_CollapseRetryAttempts_UsesLegacyIdWhenTestIdIsAbsent() + { + // CTRF 9.1: `id` is a stable test-case identifier that consumers treat as legacy, preferring `testId` + // only when both are present. An id-only report must therefore collapse on it rather than dropping to + // the suite/name heuristic, which would fuse these two distinct same-named tests. + static JsonObject WithId(string id, string status) + { + JsonObject test = Test("same name", status); + test["id"] = id; + return test; + } + + string attempt1 = BuildReport(testEntries: [WithId("id-1", "failed"), WithId("id-2", "failed")]); + string attempt2 = BuildReport(testEntries: [WithId("id-1", "passed"), WithId("id-2", "failed")]); + + var tests = (JsonArray)JsonNode.Parse( + CtrfReportMerger.Merge([attempt1, attempt2], CtrfMergeMode.CollapseRetryAttempts))!["results"]!["tests"]!; + + Assert.HasCount(2, tests, "Two distinct ids must stay two tests."); + Assert.AreEqual("passed", (string?)tests[0]!["status"]); + Assert.IsTrue((bool)tests[0]!["flaky"]!); + Assert.AreEqual("failed", (string?)tests[1]!["status"]); + Assert.AreEqual(1, (long)tests[1]!["retries"]!); + } + + [TestMethod] + public void Merge_CollapseRetryAttempts_DoesNotFuseRowsThatDifferOnlyByParametersOrFilePath() + { + // Suite plus name is not unique on its own: parameterized rows share both while differing in their + // parameters (CTRF 9.30), and same-named tests in different files differ only by path (9.19). Fusing + // them would silently drop a result. + static JsonObject Row(string status, JsonNode? parameters, string? filePath) + { + JsonObject test = Test("same name", status); + if (parameters is not null) + { + test["parameters"] = parameters; + } + + if (filePath is not null) + { + test["filePath"] = filePath; + } + + return test; + } + + string report = BuildReport(testEntries: + [ + Row("failed", new JsonObject { ["value"] = 1 }, null), + Row("passed", new JsonObject { ["value"] = 2 }, null), + Row("skipped", null, "a.cs"), + Row("passed", null, "b.cs"), + ]); + + var tests = (JsonArray)JsonNode.Parse( + CtrfReportMerger.Merge([report], CtrfMergeMode.CollapseRetryAttempts))!["results"]!["tests"]!; + + Assert.HasCount(4, tests, "Rows differing only by parameters or filePath are distinct tests."); + Assert.AreSequenceEqual( + (string?[])["failed", "passed", "skipped", "passed"], + tests.Select(t => (string?)t!["status"]).ToArray()); + } + + [TestMethod] + public void Merge_CollapseRetryAttempts_KeepsRetryHistorySchemaValid() + { + // CTRF section 11 constrains a retry attempt: `status` must be one of five values, and no unknown field + // may appear outside `extra`. Both a promoted test row and an attempt an input already nested must be + // shaped to that, otherwise one foreign document makes the merged history schema-invalid. + JsonObject firstAttempt = Test("t", "failed"); + firstAttempt["extra"] = new JsonObject { ["uid"] = "u1" }; + firstAttempt["status"] = 7; + firstAttempt["retryAttempts"] = new JsonArray(new JsonObject + { + ["attempt"] = 1, + ["status"] = 3, + ["message"] = "nested", + ["name"] = "a test-only field section 11 forbids", + }); + + JsonObject finalAttempt = Test("t", "passed"); + finalAttempt["extra"] = new JsonObject { ["uid"] = "u1" }; + + string report = BuildReport(testEntries: [firstAttempt, finalAttempt]); + + JsonNode test = ((JsonArray)JsonNode.Parse( + CtrfReportMerger.Merge([report], CtrfMergeMode.CollapseRetryAttempts))!["results"]!["tests"]!)[0]!; + + var retryAttempts = (JsonArray)test["retryAttempts"]!; + Assert.HasCount(2, retryAttempts); + + // The nested attempt keeps its diagnostics but loses the wrong-typed status and the test-only field. + Assert.AreEqual("other", (string?)retryAttempts[0]!["status"], "A non-string status is normalized."); + Assert.AreEqual("nested", (string?)retryAttempts[0]!["message"]); + Assert.IsNull(retryAttempts[0]!["name"], "Section 11 forbids unknown fields outside 'extra'."); + + // The promoted row is normalized the same way. + Assert.AreEqual("other", (string?)retryAttempts[1]!["status"]); + + // Attempt numbers stay a contiguous 1..N-1 sequence after the projection. + Assert.AreEqual(1, (long)retryAttempts[0]!["attempt"]!); + Assert.AreEqual(2, (long)retryAttempts[1]!["attempt"]!); + } + + [TestMethod] + public void Merge_CollapseRetryAttempts_LeavesAnUnmergedRowExactlyAsWritten() + { + // Validity contract: the merger shapes what it synthesizes and relays what it passes through. A test + // that occurs in only one input (for example one that recovered through in-process retries and was + // therefore never re-run by the orchestrator) has nothing merged into it, so its row -- including the + // retryAttempts[] its producer already recorded -- must come out byte-identical. Repairing it here + // would make merging a single document mutate it. + JsonObject row = Test("t", "passed"); + row["extra"] = new JsonObject { ["uid"] = "u1" }; + row["retryAttempts"] = new JsonArray(new JsonObject + { + ["attempt"] = 7, + ["status"] = "failed", + ["message"] = "recorded by the producer", + }); + + string report = BuildReport(testEntries: [row]); + string original = ((JsonArray)JsonNode.Parse(report)!["results"]!["tests"]!)[0]!.ToJsonString(); + + var tests = (JsonArray)JsonNode.Parse( + CtrfReportMerger.Merge([report], CtrfMergeMode.CollapseRetryAttempts))!["results"]!["tests"]!; + + Assert.HasCount(1, tests); + Assert.AreEqual(original, tests[0]!.ToJsonString(), "An unmerged row must be relayed verbatim."); + + // Specifically: the producer's own attempt numbering is not rewritten, and retries/flaky are not invented. + Assert.AreEqual(7, (long)((JsonArray)tests[0]!["retryAttempts"]!)[0]!["attempt"]!); + Assert.IsNull(tests[0]!["retries"]); + } + + private static JsonObject Attempt(string name, string status, string uid, long duration = 1, string? message = null) + { + var test = new JsonObject + { + ["name"] = name, + ["status"] = status, + ["duration"] = duration, + ["extra"] = new JsonObject { ["uid"] = uid }, + }; + + if (message is not null) + { + test["message"] = message; + } + + return test; + } + [TestMethod] public void Merge_WhenAnInputHasNoEnvironment_DropsEnvironment() { diff --git a/test/Utilities/Microsoft.Testing.TestInfrastructure/WellKnownEnvironmentVariables.cs b/test/Utilities/Microsoft.Testing.TestInfrastructure/WellKnownEnvironmentVariables.cs index cf618db08d..5e09a81d42 100644 --- a/test/Utilities/Microsoft.Testing.TestInfrastructure/WellKnownEnvironmentVariables.cs +++ b/test/Utilities/Microsoft.Testing.TestInfrastructure/WellKnownEnvironmentVariables.cs @@ -98,6 +98,11 @@ public static class WellKnownEnvironmentVariables "TESTINGPLATFORM_DOTNETTEST_EXECUTIONID", "DOTNET_CLI_TEST_COMMAND_WORKING_DIRECTORY", + // Logical run correlation. A CI job may set this to tie several modules or machines into one + // logical run, so it must not bleed into child test hosts and make a test observe an id it did + // not choose. Tests that exercise run correlation inject it explicitly. + "TESTINGPLATFORM_LOGICAL_RUN_ID", + // Isolate from the skip banner in case of parent, children tests "TESTINGPLATFORM_CONSOLEOUTPUTDEVICE_SKIP_BANNER",