From 216d9a2a0e7a8526c3a21907fe3bfe599161515d Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Mon, 5 Jun 2023 11:55:43 -0500 Subject: [PATCH 1/4] Show all source files in Solution Explorer --- src/Xunit.StaFact/Xunit.StaFact.csproj | 9 +++++++++ test/Xunit.StaFact.Tests/Xunit.StaFact.Tests.csproj | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/src/Xunit.StaFact/Xunit.StaFact.csproj b/src/Xunit.StaFact/Xunit.StaFact.csproj index 36b98af2..6cbd90e2 100644 --- a/src/Xunit.StaFact/Xunit.StaFact.csproj +++ b/src/Xunit.StaFact/Xunit.StaFact.csproj @@ -14,6 +14,9 @@ Simply use [WpfFact], [WinFormsFact], [StaFact] or the cross-platform [UIFact] o Theory variants of these attributes allow for parameterized testing. Check out the xunit.combinatorial nuget package for pairwise or combinatorial testing with theories. STA xunit testing unit WPF WinForms testing + + + @@ -22,6 +25,12 @@ Theory variants of these attributes allow for parameterized testing. Check out t + + + + + + diff --git a/test/Xunit.StaFact.Tests/Xunit.StaFact.Tests.csproj b/test/Xunit.StaFact.Tests/Xunit.StaFact.Tests.csproj index f0a458ec..1a5bcb67 100644 --- a/test/Xunit.StaFact.Tests/Xunit.StaFact.Tests.csproj +++ b/test/Xunit.StaFact.Tests/Xunit.StaFact.Tests.csproj @@ -2,9 +2,18 @@ net472;net6.0;net6.0-windows + + + + + + + + + From 7eb41571df749889e75dfd9d990d0e3c10481d06 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Mon, 5 Jun 2023 15:14:43 -0500 Subject: [PATCH 2/4] Implement automatic retries using UISettingsAttribute --- .../Sdk.Mac/CocoaFactDiscoverer.cs | 3 +- .../Sdk.Mac/CocoaTheoryDiscoverer.cs | 5 +- .../WinFormsFactDiscoverer.cs | 3 +- .../WinFormsTheoryDiscoverer.cs | 5 +- .../Sdk.WindowsDesktop/WpfFactDiscoverer.cs | 3 +- .../Sdk.WindowsDesktop/WpfTheoryDiscoverer.cs | 5 +- src/Xunit.StaFact/Sdk/StaFactDiscoverer.cs | 3 +- src/Xunit.StaFact/Sdk/StaTheoryDiscoverer.cs | 5 +- src/Xunit.StaFact/Sdk/UIFactDiscoverer.cs | 99 ++++++++++++++++++- src/Xunit.StaFact/Sdk/UITestCase.cs | 31 +++++- src/Xunit.StaFact/Sdk/UITestCaseRunner.cs | 22 ++++- src/Xunit.StaFact/Sdk/UITestRunner.cs | 37 ++++++- src/Xunit.StaFact/Sdk/UITheoryDiscoverer.cs | 5 +- src/Xunit.StaFact/Sdk/UITheoryTestCase.cs | 9 +- .../Sdk/UITheoryTestCaseRunner.cs | 13 ++- src/Xunit.StaFact/UISettingsAttribute.cs | 20 ++++ src/Xunit.StaFact/UISettingsKey.cs | 9 ++ .../net472/PublicAPI.Shipped.txt | 1 - .../net472/PublicAPI.Unshipped.txt | 6 ++ .../net6.0-macos10.15/PublicAPI.Unshipped.txt | 7 +- .../net6.0-windows/PublicAPI.Shipped.txt | 1 - .../net6.0-windows/PublicAPI.Unshipped.txt | 6 ++ .../net6.0/PublicAPI.Shipped.txt | 1 - .../net6.0/PublicAPI.Unshipped.txt | 6 ++ .../netstandard2.0/PublicAPI.Shipped.txt | 1 - .../netstandard2.0/PublicAPI.Unshipped.txt | 6 ++ 26 files changed, 281 insertions(+), 31 deletions(-) create mode 100644 src/Xunit.StaFact/UISettingsAttribute.cs create mode 100644 src/Xunit.StaFact/UISettingsKey.cs diff --git a/src/Xunit.StaFact/Sdk.Mac/CocoaFactDiscoverer.cs b/src/Xunit.StaFact/Sdk.Mac/CocoaFactDiscoverer.cs index 290cc289..c8fa9c7f 100644 --- a/src/Xunit.StaFact/Sdk.Mac/CocoaFactDiscoverer.cs +++ b/src/Xunit.StaFact/Sdk.Mac/CocoaFactDiscoverer.cs @@ -35,6 +35,7 @@ protected override IXunitTestCase CreateTestCase(ITestFrameworkDiscoveryOptions return new ExecutionErrorTestCase(this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, "Async void methods are not supported."); } - return new UITestCase(UITestCase.SyncContextType.Cocoa, this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod); + UISettingsKey settings = UIFactDiscoverer.GetSettings(testMethod, factAttribute); + return new UITestCase(UITestCase.SyncContextType.Cocoa, this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, testMethodArguments: null, settings); } } diff --git a/src/Xunit.StaFact/Sdk.Mac/CocoaTheoryDiscoverer.cs b/src/Xunit.StaFact/Sdk.Mac/CocoaTheoryDiscoverer.cs index 7ce9e4cf..a30c15af 100644 --- a/src/Xunit.StaFact/Sdk.Mac/CocoaTheoryDiscoverer.cs +++ b/src/Xunit.StaFact/Sdk.Mac/CocoaTheoryDiscoverer.cs @@ -19,11 +19,12 @@ public CocoaTheoryDiscoverer(IMessageSink diagnosticMessageSink) protected override IEnumerable CreateTestCasesForDataRow(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute, object[] dataRow) { - yield return new UITestCase(UITestCase.SyncContextType.Cocoa, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, dataRow); + UISettingsKey settings = UIFactDiscoverer.GetSettings(testMethod, theoryAttribute); + yield return new UITestCase(UITestCase.SyncContextType.Cocoa, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, dataRow, settings); } protected override IEnumerable CreateTestCasesForTheory(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute) { - yield return new UITheoryTestCase(UITestCase.SyncContextType.Cocoa, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod); + yield return new UITheoryTestCase(UITestCase.SyncContextType.Cocoa, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, theoryAttribute); } } diff --git a/src/Xunit.StaFact/Sdk.WindowsDesktop/WinFormsFactDiscoverer.cs b/src/Xunit.StaFact/Sdk.WindowsDesktop/WinFormsFactDiscoverer.cs index f364f628..494a6099 100644 --- a/src/Xunit.StaFact/Sdk.WindowsDesktop/WinFormsFactDiscoverer.cs +++ b/src/Xunit.StaFact/Sdk.WindowsDesktop/WinFormsFactDiscoverer.cs @@ -36,8 +36,9 @@ protected override IXunitTestCase CreateTestCase(ITestFrameworkDiscoveryOptions return new ExecutionErrorTestCase(this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, "Async void methods are not supported."); } + UISettingsKey settings = UIFactDiscoverer.GetSettings(testMethod, factAttribute); return RuntimeInformation.IsOSPlatform(OSPlatform.Windows) - ? (IXunitTestCase)new UITestCase(UITestCase.SyncContextType.WinForms, this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod) + ? (IXunitTestCase)new UITestCase(UITestCase.SyncContextType.WinForms, this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, testMethodArguments: null, settings) : new XunitSkippedDataRowTestCase(this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, "WinForms only exists on Windows."); } } diff --git a/src/Xunit.StaFact/Sdk.WindowsDesktop/WinFormsTheoryDiscoverer.cs b/src/Xunit.StaFact/Sdk.WindowsDesktop/WinFormsTheoryDiscoverer.cs index 28688281..a568b8fc 100644 --- a/src/Xunit.StaFact/Sdk.WindowsDesktop/WinFormsTheoryDiscoverer.cs +++ b/src/Xunit.StaFact/Sdk.WindowsDesktop/WinFormsTheoryDiscoverer.cs @@ -21,15 +21,16 @@ public WinFormsTheoryDiscoverer(IMessageSink diagnosticMessageSink) protected override IEnumerable CreateTestCasesForDataRow(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute, object[] dataRow) { + UISettingsKey settings = UIFactDiscoverer.GetSettings(testMethod, theoryAttribute); yield return RuntimeInformation.IsOSPlatform(OSPlatform.Windows) - ? (IXunitTestCase)new UITestCase(UITestCase.SyncContextType.WinForms, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, dataRow) + ? (IXunitTestCase)new UITestCase(UITestCase.SyncContextType.WinForms, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, dataRow, settings) : new XunitSkippedDataRowTestCase(this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, "WinForms only exists on Windows."); } protected override IEnumerable CreateTestCasesForTheory(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute) { yield return RuntimeInformation.IsOSPlatform(OSPlatform.Windows) - ? (IXunitTestCase)new UITheoryTestCase(UITestCase.SyncContextType.WinForms, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod) + ? (IXunitTestCase)new UITheoryTestCase(UITestCase.SyncContextType.WinForms, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, theoryAttribute) : new XunitSkippedDataRowTestCase(this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, "WinForms only exists on Windows."); } } diff --git a/src/Xunit.StaFact/Sdk.WindowsDesktop/WpfFactDiscoverer.cs b/src/Xunit.StaFact/Sdk.WindowsDesktop/WpfFactDiscoverer.cs index 7a2bb32b..acc4dbe7 100644 --- a/src/Xunit.StaFact/Sdk.WindowsDesktop/WpfFactDiscoverer.cs +++ b/src/Xunit.StaFact/Sdk.WindowsDesktop/WpfFactDiscoverer.cs @@ -36,8 +36,9 @@ protected override IXunitTestCase CreateTestCase(ITestFrameworkDiscoveryOptions return new ExecutionErrorTestCase(this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, "Async void methods are not supported."); } + UISettingsKey settings = UIFactDiscoverer.GetSettings(testMethod, factAttribute); return RuntimeInformation.IsOSPlatform(OSPlatform.Windows) - ? (IXunitTestCase)new UITestCase(UITestCase.SyncContextType.WPF, this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod) + ? (IXunitTestCase)new UITestCase(UITestCase.SyncContextType.WPF, this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, testMethodArguments: null, settings) : new XunitSkippedDataRowTestCase(this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, "WPF only exists on Windows."); } } diff --git a/src/Xunit.StaFact/Sdk.WindowsDesktop/WpfTheoryDiscoverer.cs b/src/Xunit.StaFact/Sdk.WindowsDesktop/WpfTheoryDiscoverer.cs index 5fee6765..26dd5afa 100644 --- a/src/Xunit.StaFact/Sdk.WindowsDesktop/WpfTheoryDiscoverer.cs +++ b/src/Xunit.StaFact/Sdk.WindowsDesktop/WpfTheoryDiscoverer.cs @@ -21,15 +21,16 @@ public WpfTheoryDiscoverer(IMessageSink diagnosticMessageSink) protected override IEnumerable CreateTestCasesForDataRow(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute, object[] dataRow) { + UISettingsKey settings = UIFactDiscoverer.GetSettings(testMethod, theoryAttribute); yield return RuntimeInformation.IsOSPlatform(OSPlatform.Windows) - ? (IXunitTestCase)new UITestCase(UITestCase.SyncContextType.WPF, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, dataRow) + ? (IXunitTestCase)new UITestCase(UITestCase.SyncContextType.WPF, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, dataRow, settings) : new XunitSkippedDataRowTestCase(this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, "WPF only exists on Windows."); } protected override IEnumerable CreateTestCasesForTheory(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute) { yield return RuntimeInformation.IsOSPlatform(OSPlatform.Windows) - ? (IXunitTestCase)new UITheoryTestCase(UITestCase.SyncContextType.WPF, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod) + ? (IXunitTestCase)new UITheoryTestCase(UITestCase.SyncContextType.WPF, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, theoryAttribute) : new XunitSkippedDataRowTestCase(this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, "WPF only exists on Windows."); } } diff --git a/src/Xunit.StaFact/Sdk/StaFactDiscoverer.cs b/src/Xunit.StaFact/Sdk/StaFactDiscoverer.cs index 129c3b74..9a07d878 100644 --- a/src/Xunit.StaFact/Sdk/StaFactDiscoverer.cs +++ b/src/Xunit.StaFact/Sdk/StaFactDiscoverer.cs @@ -37,8 +37,9 @@ protected override IXunitTestCase CreateTestCase(ITestFrameworkDiscoveryOptions return new ExecutionErrorTestCase(this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, "Async void methods are not supported."); } + UISettingsKey settings = UIFactDiscoverer.GetSettings(testMethod, factAttribute); return RuntimeInformation.IsOSPlatform(OSPlatform.Windows) - ? (IXunitTestCase)new UITestCase(UITestCase.SyncContextType.None, this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod) + ? (IXunitTestCase)new UITestCase(UITestCase.SyncContextType.None, this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, testMethodArguments: null, settings) : new XunitSkippedDataRowTestCase(this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, "STA threads only exist on Windows."); } } diff --git a/src/Xunit.StaFact/Sdk/StaTheoryDiscoverer.cs b/src/Xunit.StaFact/Sdk/StaTheoryDiscoverer.cs index 53ba8b7d..b74969c2 100644 --- a/src/Xunit.StaFact/Sdk/StaTheoryDiscoverer.cs +++ b/src/Xunit.StaFact/Sdk/StaTheoryDiscoverer.cs @@ -33,8 +33,9 @@ protected override IEnumerable CreateTestCasesForDataRow(ITestFr yield return new ExecutionErrorTestCase(this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, "Async void methods are not supported."); } + UISettingsKey settings = UIFactDiscoverer.GetSettings(testMethod, theoryAttribute); yield return RuntimeInformation.IsOSPlatform(OSPlatform.Windows) - ? (IXunitTestCase)new UITestCase(UITestCase.SyncContextType.None, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, dataRow) + ? (IXunitTestCase)new UITestCase(UITestCase.SyncContextType.None, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, dataRow, settings) : new XunitSkippedDataRowTestCase(this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, "STA threads only exist on Windows."); } @@ -51,7 +52,7 @@ protected override IEnumerable CreateTestCasesForTheory(ITestFra } yield return RuntimeInformation.IsOSPlatform(OSPlatform.Windows) - ? (IXunitTestCase)new UITheoryTestCase(UITestCase.SyncContextType.None, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod) + ? (IXunitTestCase)new UITheoryTestCase(UITestCase.SyncContextType.None, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, theoryAttribute) : new XunitSkippedDataRowTestCase(this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, "STA threads only exist on Windows."); } } diff --git a/src/Xunit.StaFact/Sdk/UIFactDiscoverer.cs b/src/Xunit.StaFact/Sdk/UIFactDiscoverer.cs index 48fc76b4..d7a6d455 100644 --- a/src/Xunit.StaFact/Sdk/UIFactDiscoverer.cs +++ b/src/Xunit.StaFact/Sdk/UIFactDiscoverer.cs @@ -1,6 +1,8 @@ // Copyright (c) Andrew Arnott. All rights reserved. // Licensed under the Ms-PL license. See LICENSE file in the project root for full license information. +using System.Runtime.CompilerServices; + namespace Xunit.Sdk; /// @@ -20,8 +22,103 @@ public UIFactDiscoverer(IMessageSink diagnosticMessageSink) this.diagnosticMessageSink = diagnosticMessageSink; } + internal static UISettingsKey GetSettings(ITestMethod testMethod, IAttributeInfo factAttribute) + { + var maxAttempts = GetMaxAttempts(testMethod, factAttribute); + return new UISettingsKey(maxAttempts); + } + protected override IXunitTestCase CreateTestCase(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { - return new UITestCase(UITestCase.SyncContextType.Portable, this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod); + var maxAttempts = GetMaxAttempts(testMethod, factAttribute); + var settings = new UISettingsKey(maxAttempts); + return new UITestCase(UITestCase.SyncContextType.Portable, this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, testMethodArguments: null, settings); + } + + private static int GetMaxAttempts(ITestMethod testMethod, IAttributeInfo factAttribute) + { + return GetMaxAttempts(factAttribute, GetSettingsAttributes(testMethod).ToArray()); + } + + private static IEnumerable GetSettingsAttributes(ITestMethod testMethod) + { + foreach (IAttributeInfo attributeData in testMethod.Method.GetCustomAttributes(typeof(UISettingsAttribute))) + { + yield return attributeData; + } + + foreach (IAttributeInfo attributeData in testMethod.TestClass.Class.GetCustomAttributes(typeof(UISettingsAttribute))) + { + yield return attributeData; + } + } + + private static int GetMaxAttempts(IAttributeInfo factAttribute, IAttributeInfo[] settingsAttributes) + { + return GetNamedArgument( + factAttribute, + settingsAttributes, + nameof(UISettingsAttribute.MaxAttempts), + static value => value > 0, + defaultValue: 1); + } + + private static TValue GetNamedArgument(IAttributeInfo factAttribute, IAttributeInfo[] settingsAttributes, string argumentName, Func isValidValue, TValue defaultValue) + { + return GetNamedArgument( + factAttribute, + settingsAttributes, + argumentName, + isValidValue, + merge: null, + defaultValue); + } + + private static TValue GetNamedArgument(IAttributeInfo factAttribute, IAttributeInfo[] settingsAttributes, string argumentName, Func isValidValue, Func? merge, TValue defaultValue) + { + StrongBox? result = null; + if (TryGetNamedArgument(factAttribute, argumentName, isValidValue, out TValue? value)) + { + if (merge is null) + { + return value; + } + + result = new StrongBox(value); + } + + foreach (IAttributeInfo attribute in settingsAttributes) + { + if (TryGetNamedArgument(attribute, argumentName, isValidValue, out value)) + { + if (merge is null) + { + return value; + } + else if (result is null) + { + result = new StrongBox(value); + } + else + { + result.Value = merge(value, result.Value!); + } + + return value; + } + } + + if (result is not null) + { + return result.Value!; + } + + return defaultValue; + + static bool TryGetNamedArgument(IAttributeInfo attribute, string argumentName, Func isValidValue, out TValue value) + { + value = attribute.GetNamedArgument(argumentName); + return isValidValue(value); + } } } diff --git a/src/Xunit.StaFact/Sdk/UITestCase.cs b/src/Xunit.StaFact/Sdk/UITestCase.cs index 43d144e0..0539b0f5 100644 --- a/src/Xunit.StaFact/Sdk/UITestCase.cs +++ b/src/Xunit.StaFact/Sdk/UITestCase.cs @@ -12,6 +12,7 @@ namespace Xunit.Sdk; [DebuggerDisplay(@"\{ class = {TestMethod.TestClass.Class.Name}, method = {TestMethod.Method.Name}, display = {DisplayName}, skip = {SkipReason} \}")] public class UITestCase : XunitTestCase { + private UISettingsKey settings; private SyncContextType synchronizationContextType; /// @@ -22,15 +23,15 @@ public class UITestCase : XunitTestCase /// Default method display to use (when not customized). /// The test method this test case belongs to. /// The arguments for the test method. + [Obsolete("Call the constructor which provides UISettingsKey.")] public UITestCase( SyncContextType synchronizationContextType, IMessageSink diagnosticMessageSink, TestMethodDisplay defaultMethodDisplay, ITestMethod testMethod, object?[]? testMethodArguments = null) - : base(diagnosticMessageSink, defaultMethodDisplay, TestMethodDisplayOptions.None, testMethod, testMethodArguments) + : this(synchronizationContextType, diagnosticMessageSink, defaultMethodDisplay, testMethod, testMethodArguments, UISettingsKey.Default) { - this.synchronizationContextType = synchronizationContextType; } /// @@ -43,6 +44,28 @@ public UITestCase() { } + /// + /// Initializes a new instance of the class. + /// + /// The type of to use. + /// The message sink used to send diagnostic messages. + /// Default method display to use (when not customized). + /// The test method this test case belongs to. + /// The test settings to apply. + /// The arguments for the test method. + internal UITestCase( + SyncContextType synchronizationContextType, + IMessageSink diagnosticMessageSink, + TestMethodDisplay defaultMethodDisplay, + ITestMethod testMethod, + object?[]? testMethodArguments, + UISettingsKey settings) + : base(diagnosticMessageSink, defaultMethodDisplay, TestMethodDisplayOptions.None, testMethod, testMethodArguments) + { + this.settings = settings; + this.synchronizationContextType = synchronizationContextType; + } + public enum SyncContextType { /// @@ -85,6 +108,7 @@ public override void Serialize(IXunitSerializationInfo data) } base.Serialize(data); + data.AddValue(nameof(UISettingsAttribute.MaxAttempts), this.settings.MaxAttempts); data.AddValue(nameof(this.synchronizationContextType), this.synchronizationContextType); } @@ -96,6 +120,7 @@ public override void Deserialize(IXunitSerializationInfo data) } base.Deserialize(data); + this.settings = new UISettingsKey(data.GetValue(nameof(UISettingsAttribute.MaxAttempts))); this.synchronizationContextType = (SyncContextType)data.GetValue(nameof(this.synchronizationContextType), typeof(SyncContextType)); } @@ -117,7 +142,7 @@ public override Task RunAsync( { using ThreadRental threadRental = await ThreadRental.CreateAsync(this.Adapter, this.TestMethod); await threadRental.SynchronizationContext; - var runner = new UITestCaseRunner(this, this.DisplayName, this.SkipReason, constructorArguments, this.TestMethodArguments, messageBus, aggregator, cancellationTokenSource, threadRental); + var runner = new UITestCaseRunner(this, this.DisplayName, this.SkipReason, constructorArguments, this.TestMethodArguments, messageBus, aggregator, cancellationTokenSource, this.settings, threadRental); return await runner.RunAsync(); }, cancellationTokenSource.Token); diff --git a/src/Xunit.StaFact/Sdk/UITestCaseRunner.cs b/src/Xunit.StaFact/Sdk/UITestCaseRunner.cs index 7b501769..f6730f4d 100644 --- a/src/Xunit.StaFact/Sdk/UITestCaseRunner.cs +++ b/src/Xunit.StaFact/Sdk/UITestCaseRunner.cs @@ -5,6 +5,7 @@ namespace Xunit.Sdk; public class UITestCaseRunner : XunitTestCaseRunner { + private UISettingsKey settings; private ThreadRental threadRental; /// @@ -18,6 +19,7 @@ public class UITestCaseRunner : XunitTestCaseRunner /// The message bus to report run status to. /// The exception aggregator used to run code and collect exceptions. /// The task cancellation token source, used to cancel the test run. + /// The settings to use for this test case. /// The instance to use. internal UITestCaseRunner( IXunitTestCase testCase, @@ -28,14 +30,30 @@ internal UITestCaseRunner( IMessageBus messageBus, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource, + UISettingsKey settings, ThreadRental threadRental) : base(testCase, displayName, skipReason, constructorArguments, testMethodArguments, messageBus, aggregator, cancellationTokenSource) { + this.settings = settings; this.threadRental = threadRental; } - protected override Task RunTestAsync() + protected override async Task RunTestAsync() { - return new UITestRunner(new XunitTest(this.TestCase, this.DisplayName), this.MessageBus, this.TestClass, this.ConstructorArguments, this.TestMethod, this.TestMethodArguments, this.SkipReason, this.BeforeAfterAttributes, this.Aggregator, this.CancellationTokenSource, this.threadRental).RunAsync(); + var result = new RunSummary(); + for (int i = 0; i < this.settings.MaxAttempts; i++) + { + bool finalAttempt = i == this.settings.MaxAttempts - 1; + RunSummary summary = await new UITestRunner(new XunitTest(this.TestCase, this.DisplayName), this.MessageBus, finalAttempt, this.TestClass, this.ConstructorArguments, this.TestMethod, this.TestMethodArguments, this.SkipReason, this.BeforeAfterAttributes, this.Aggregator, this.CancellationTokenSource, this.threadRental).RunAsync(); + result.Aggregate(summary); + if (summary.Skipped == 0) + { + // Failed tests prior to the final attempt are reported as skipped. If the current execution did not + // contain any skipped results, we know the summary is valid for the test as a whole. + break; + } + } + + return result; } } diff --git a/src/Xunit.StaFact/Sdk/UITestRunner.cs b/src/Xunit.StaFact/Sdk/UITestRunner.cs index 9424923f..e0252f02 100644 --- a/src/Xunit.StaFact/Sdk/UITestRunner.cs +++ b/src/Xunit.StaFact/Sdk/UITestRunner.cs @@ -9,8 +9,8 @@ public class UITestRunner : XunitTestRunner { private readonly ThreadRental threadRental; - internal UITestRunner(ITest test, IMessageBus messageBus, Type testClass, object[] constructorArguments, MethodInfo testMethod, object[] testMethodArguments, string skipReason, IReadOnlyList beforeAfterAttributes, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource, ThreadRental threadRental) - : base(test, messageBus, testClass, constructorArguments, testMethod, testMethodArguments, skipReason, beforeAfterAttributes, aggregator, cancellationTokenSource) + internal UITestRunner(ITest test, IMessageBus messageBus, bool finalAttempt, Type testClass, object[] constructorArguments, MethodInfo testMethod, object[] testMethodArguments, string skipReason, IReadOnlyList beforeAfterAttributes, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource, ThreadRental threadRental) + : base(test, finalAttempt ? messageBus : new FilteringMessageBus(messageBus), testClass, constructorArguments, testMethod, testMethodArguments, skipReason, beforeAfterAttributes, aggregator, cancellationTokenSource) { this.threadRental = threadRental; } @@ -19,4 +19,37 @@ protected override Task InvokeTestMethodAsync(ExceptionAggregator aggre { return new UITestInvoker(this.Test, this.MessageBus, this.TestClass, this.ConstructorArguments, this.TestMethod, this.TestMethodArguments, this.BeforeAfterAttributes, aggregator, this.CancellationTokenSource, this.threadRental).RunAsync(); } + + private sealed class FilteringMessageBus : IMessageBus + { + private readonly IMessageBus messageBus; + + public FilteringMessageBus(IMessageBus messageBus) + { + this.messageBus = messageBus; + } + + public void Dispose() + { + this.messageBus.Dispose(); + } + + public bool QueueMessage(IMessageSinkMessage message) + { + if (message is ITestFailed testFailed) + { + // This test will run again; report it as skipped instead of failed + // TODO: What kind of additional logs should we include? + message = new TestSkipped(testFailed.Test, "Test will automatically retry."); + } + else if (message is ITestCleanupFailure testCleanupFailure) + { + // This test will run again; report it as skipped instead of failed + // TODO: What kind of additional logs should we include? + message = new TestSkipped(testCleanupFailure.Test, "Test will automatically retry."); + } + + return this.messageBus.QueueMessage(message); + } + } } diff --git a/src/Xunit.StaFact/Sdk/UITheoryDiscoverer.cs b/src/Xunit.StaFact/Sdk/UITheoryDiscoverer.cs index fd3ebef6..18d2d1ba 100644 --- a/src/Xunit.StaFact/Sdk/UITheoryDiscoverer.cs +++ b/src/Xunit.StaFact/Sdk/UITheoryDiscoverer.cs @@ -19,11 +19,12 @@ public UITheoryDiscoverer(IMessageSink diagnosticMessageSink) protected override IEnumerable CreateTestCasesForDataRow(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute, object[] dataRow) { - yield return new UITestCase(UITestCase.SyncContextType.Portable, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, dataRow); + UISettingsKey settings = UIFactDiscoverer.GetSettings(testMethod, theoryAttribute); + yield return new UITestCase(UITestCase.SyncContextType.Portable, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, dataRow, settings); } protected override IEnumerable CreateTestCasesForTheory(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute) { - yield return new UITheoryTestCase(UITestCase.SyncContextType.Portable, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod); + yield return new UITheoryTestCase(UITestCase.SyncContextType.Portable, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, theoryAttribute); } } diff --git a/src/Xunit.StaFact/Sdk/UITheoryTestCase.cs b/src/Xunit.StaFact/Sdk/UITheoryTestCase.cs index 2a1b5326..f8ffe09e 100644 --- a/src/Xunit.StaFact/Sdk/UITheoryTestCase.cs +++ b/src/Xunit.StaFact/Sdk/UITheoryTestCase.cs @@ -13,19 +13,22 @@ public UITheoryTestCase() { } - public UITheoryTestCase(UITestCase.SyncContextType synchronizationContextType, IMessageSink diagnosticMessageSink, TestMethodDisplay defaultMethodDisplay, TestMethodDisplayOptions defaultMethodDisplayOptions, ITestMethod testMethod) + internal UITheoryTestCase(UITestCase.SyncContextType synchronizationContextType, IMessageSink diagnosticMessageSink, TestMethodDisplay defaultMethodDisplay, TestMethodDisplayOptions defaultMethodDisplayOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute) : base(diagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testMethod) { + this.Settings = UIFactDiscoverer.GetSettings(testMethod, theoryAttribute); this.SynchronizationContextType = synchronizationContextType; } + internal UISettingsKey Settings { get; private set; } + internal UITestCase.SyncContextType SynchronizationContextType { get; private set; } public override async Task RunAsync(IMessageSink diagnosticMessageSink, IMessageBus messageBus, object[] constructorArguments, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) { using ThreadRental threadRental = await ThreadRental.CreateAsync(UITestCase.GetAdapter(this.SynchronizationContextType), this.TestMethod); await threadRental.SynchronizationContext; - return await new UITheoryTestCaseRunner(this, this.DisplayName, this.SkipReason, constructorArguments, diagnosticMessageSink, messageBus, aggregator, cancellationTokenSource, threadRental).RunAsync(); + return await new UITheoryTestCaseRunner(this, this.DisplayName, this.SkipReason, constructorArguments, diagnosticMessageSink, messageBus, aggregator, cancellationTokenSource, this.Settings, threadRental).RunAsync(); } public override void Deserialize(IXunitSerializationInfo data) @@ -37,6 +40,7 @@ public override void Deserialize(IXunitSerializationInfo data) base.Deserialize(data); + this.Settings = new UISettingsKey(data.GetValue(nameof(UISettingsAttribute.MaxAttempts))); this.SynchronizationContextType = (UITestCase.SyncContextType)Enum.Parse(typeof(UITestCase.SyncContextType), data.GetValue("SyncContextType")); } @@ -49,6 +53,7 @@ public override void Serialize(IXunitSerializationInfo data) base.Serialize(data); + data.AddValue(nameof(UISettingsAttribute.MaxAttempts), this.Settings.MaxAttempts); data.AddValue("SyncContextType", this.SynchronizationContextType.ToString()); } } diff --git a/src/Xunit.StaFact/Sdk/UITheoryTestCaseRunner.cs b/src/Xunit.StaFact/Sdk/UITheoryTestCaseRunner.cs index cbfdde9b..71b84f48 100644 --- a/src/Xunit.StaFact/Sdk/UITheoryTestCaseRunner.cs +++ b/src/Xunit.StaFact/Sdk/UITheoryTestCaseRunner.cs @@ -7,16 +7,25 @@ namespace Xunit.Sdk; public class UITheoryTestCaseRunner : XunitTheoryTestCaseRunner { + private readonly UISettingsKey settings; private readonly ThreadRental threadRental; - internal UITheoryTestCaseRunner(UITheoryTestCase testCase, string displayName, string skipReason, object[] constructorArguments, IMessageSink diagnosticMessageSink, IMessageBus messageBus, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource, ThreadRental threadRental) + internal UITheoryTestCaseRunner(UITheoryTestCase testCase, string displayName, string skipReason, object[] constructorArguments, IMessageSink diagnosticMessageSink, IMessageBus messageBus, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource, UISettingsKey settings, ThreadRental threadRental) : base(testCase, displayName, skipReason, constructorArguments, diagnosticMessageSink, messageBus, aggregator, cancellationTokenSource) { + this.settings = settings; this.threadRental = threadRental; } internal new UITheoryTestCase TestCase => (UITheoryTestCase)base.TestCase; protected override XunitTestRunner CreateTestRunner(ITest test, IMessageBus messageBus, Type testClass, object[] constructorArguments, MethodInfo testMethod, object[] testMethodArguments, string skipReason, IReadOnlyList beforeAfterAttributes, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) - => new UITestRunner(test, messageBus, testClass, constructorArguments, testMethod, testMethodArguments, skipReason, beforeAfterAttributes, aggregator, cancellationTokenSource, this.threadRental); + { + if (this.settings.MaxAttempts != 1) + { + throw new NotSupportedException("MaxAttempts can only be set for pre-enumerated theories."); + } + + return new UITestRunner(test, messageBus, finalAttempt: true, testClass, constructorArguments, testMethod, testMethodArguments, skipReason, beforeAfterAttributes, aggregator, cancellationTokenSource, this.threadRental); + } } diff --git a/src/Xunit.StaFact/UISettingsAttribute.cs b/src/Xunit.StaFact/UISettingsAttribute.cs new file mode 100644 index 00000000..bbcf37dc --- /dev/null +++ b/src/Xunit.StaFact/UISettingsAttribute.cs @@ -0,0 +1,20 @@ +// Copyright (c) Andrew Arnott. All rights reserved. +// Licensed under the Ms-PL license. See LICENSE file in the project root for full license information. + +namespace Xunit; + +[AttributeUsage(AttributeTargets.Method | AttributeTargets.Class, AllowMultiple = false, Inherited = true)] +internal class UISettingsAttribute : Attribute +{ + /// + /// Gets or sets the maximum number of retry attempts for a test. + /// + /// + /// + /// 0 to inherit the value from an attribute applied to a containing type or member, or use the default value when no other value is specified (equivalent to 1; tests will not be automatically retried on failure) + /// 1 to not retry the test on failure + /// An explicit value greater than 1 to retry the test up to a total of this many attempts on failure + /// + /// + public int MaxAttempts { get; set; } +} diff --git a/src/Xunit.StaFact/UISettingsKey.cs b/src/Xunit.StaFact/UISettingsKey.cs new file mode 100644 index 00000000..be0b818e --- /dev/null +++ b/src/Xunit.StaFact/UISettingsKey.cs @@ -0,0 +1,9 @@ +// Copyright (c) Andrew Arnott. All rights reserved. +// Licensed under the Ms-PL license. See LICENSE file in the project root for full license information. + +namespace Xunit; + +public record struct UISettingsKey(int MaxAttempts) +{ + public static UISettingsKey Default { get; } = new(MaxAttempts: 1); +} diff --git a/src/Xunit.StaFact/net472/PublicAPI.Shipped.txt b/src/Xunit.StaFact/net472/PublicAPI.Shipped.txt index c4e3913a..60b120d5 100644 --- a/src/Xunit.StaFact/net472/PublicAPI.Shipped.txt +++ b/src/Xunit.StaFact/net472/PublicAPI.Shipped.txt @@ -39,7 +39,6 @@ Xunit.Sdk.UITheoryDiscoverer Xunit.Sdk.UITheoryDiscoverer.UITheoryDiscoverer(Xunit.Abstractions.IMessageSink! diagnosticMessageSink) -> void Xunit.Sdk.UITheoryTestCase Xunit.Sdk.UITheoryTestCase.UITheoryTestCase() -> void -Xunit.Sdk.UITheoryTestCase.UITheoryTestCase(Xunit.Sdk.UITestCase.SyncContextType synchronizationContextType, Xunit.Abstractions.IMessageSink! diagnosticMessageSink, Xunit.Sdk.TestMethodDisplay defaultMethodDisplay, Xunit.Sdk.TestMethodDisplayOptions defaultMethodDisplayOptions, Xunit.Abstractions.ITestMethod! testMethod) -> void Xunit.Sdk.UITheoryTestCaseRunner Xunit.Sdk.WinFormsFactDiscoverer Xunit.Sdk.WinFormsFactDiscoverer.WinFormsFactDiscoverer(Xunit.Abstractions.IMessageSink! diagnosticMessageSink) -> void diff --git a/src/Xunit.StaFact/net472/PublicAPI.Unshipped.txt b/src/Xunit.StaFact/net472/PublicAPI.Unshipped.txt index e69de29b..01e8dfae 100644 --- a/src/Xunit.StaFact/net472/PublicAPI.Unshipped.txt +++ b/src/Xunit.StaFact/net472/PublicAPI.Unshipped.txt @@ -0,0 +1,6 @@ +static Xunit.UISettingsKey.Default.get -> Xunit.UISettingsKey +Xunit.UISettingsKey +Xunit.UISettingsKey.MaxAttempts.get -> int +Xunit.UISettingsKey.MaxAttempts.set -> void +Xunit.UISettingsKey.UISettingsKey() -> void +Xunit.UISettingsKey.UISettingsKey(int MaxAttempts) -> void \ No newline at end of file diff --git a/src/Xunit.StaFact/net6.0-macos10.15/PublicAPI.Unshipped.txt b/src/Xunit.StaFact/net6.0-macos10.15/PublicAPI.Unshipped.txt index 5ce3afb1..75dfa62a 100644 --- a/src/Xunit.StaFact/net6.0-macos10.15/PublicAPI.Unshipped.txt +++ b/src/Xunit.StaFact/net6.0-macos10.15/PublicAPI.Unshipped.txt @@ -14,6 +14,7 @@ override Xunit.Sdk.UITheoryDiscoverer.CreateTestCasesForTheory(Xunit.Abstraction override Xunit.Sdk.UITheoryTestCase.Deserialize(Xunit.Abstractions.IXunitSerializationInfo! data) -> void override Xunit.Sdk.UITheoryTestCase.RunAsync(Xunit.Abstractions.IMessageSink! diagnosticMessageSink, Xunit.Sdk.IMessageBus! messageBus, object![]! constructorArguments, Xunit.Sdk.ExceptionAggregator! aggregator, System.Threading.CancellationTokenSource! cancellationTokenSource) -> System.Threading.Tasks.Task! override Xunit.Sdk.UITheoryTestCase.Serialize(Xunit.Abstractions.IXunitSerializationInfo! data) -> void +static Xunit.UISettingsKey.Default.get -> Xunit.UISettingsKey Xunit.CocoaFactAttribute Xunit.CocoaFactAttribute.CocoaFactAttribute() -> void Xunit.CocoaTheoryAttribute @@ -43,7 +44,6 @@ Xunit.Sdk.UITheoryDiscoverer Xunit.Sdk.UITheoryDiscoverer.UITheoryDiscoverer(Xunit.Abstractions.IMessageSink! diagnosticMessageSink) -> void Xunit.Sdk.UITheoryTestCase Xunit.Sdk.UITheoryTestCase.UITheoryTestCase() -> void -Xunit.Sdk.UITheoryTestCase.UITheoryTestCase(Xunit.Sdk.UITestCase.SyncContextType synchronizationContextType, Xunit.Abstractions.IMessageSink! diagnosticMessageSink, Xunit.Sdk.TestMethodDisplay defaultMethodDisplay, Xunit.Sdk.TestMethodDisplayOptions defaultMethodDisplayOptions, Xunit.Abstractions.ITestMethod! testMethod) -> void Xunit.Sdk.UITheoryTestCaseRunner Xunit.StaFactAttribute Xunit.StaFactAttribute.StaFactAttribute() -> void @@ -51,5 +51,10 @@ Xunit.StaTheoryAttribute Xunit.StaTheoryAttribute.StaTheoryAttribute() -> void Xunit.UIFactAttribute Xunit.UIFactAttribute.UIFactAttribute() -> void +Xunit.UISettingsKey +Xunit.UISettingsKey.MaxAttempts.get -> int +Xunit.UISettingsKey.MaxAttempts.set -> void +Xunit.UISettingsKey.UISettingsKey() -> void +Xunit.UISettingsKey.UISettingsKey(int MaxAttempts) -> void Xunit.UITheoryAttribute Xunit.UITheoryAttribute.UITheoryAttribute() -> void diff --git a/src/Xunit.StaFact/net6.0-windows/PublicAPI.Shipped.txt b/src/Xunit.StaFact/net6.0-windows/PublicAPI.Shipped.txt index c4e3913a..60b120d5 100644 --- a/src/Xunit.StaFact/net6.0-windows/PublicAPI.Shipped.txt +++ b/src/Xunit.StaFact/net6.0-windows/PublicAPI.Shipped.txt @@ -39,7 +39,6 @@ Xunit.Sdk.UITheoryDiscoverer Xunit.Sdk.UITheoryDiscoverer.UITheoryDiscoverer(Xunit.Abstractions.IMessageSink! diagnosticMessageSink) -> void Xunit.Sdk.UITheoryTestCase Xunit.Sdk.UITheoryTestCase.UITheoryTestCase() -> void -Xunit.Sdk.UITheoryTestCase.UITheoryTestCase(Xunit.Sdk.UITestCase.SyncContextType synchronizationContextType, Xunit.Abstractions.IMessageSink! diagnosticMessageSink, Xunit.Sdk.TestMethodDisplay defaultMethodDisplay, Xunit.Sdk.TestMethodDisplayOptions defaultMethodDisplayOptions, Xunit.Abstractions.ITestMethod! testMethod) -> void Xunit.Sdk.UITheoryTestCaseRunner Xunit.Sdk.WinFormsFactDiscoverer Xunit.Sdk.WinFormsFactDiscoverer.WinFormsFactDiscoverer(Xunit.Abstractions.IMessageSink! diagnosticMessageSink) -> void diff --git a/src/Xunit.StaFact/net6.0-windows/PublicAPI.Unshipped.txt b/src/Xunit.StaFact/net6.0-windows/PublicAPI.Unshipped.txt index e69de29b..01e8dfae 100644 --- a/src/Xunit.StaFact/net6.0-windows/PublicAPI.Unshipped.txt +++ b/src/Xunit.StaFact/net6.0-windows/PublicAPI.Unshipped.txt @@ -0,0 +1,6 @@ +static Xunit.UISettingsKey.Default.get -> Xunit.UISettingsKey +Xunit.UISettingsKey +Xunit.UISettingsKey.MaxAttempts.get -> int +Xunit.UISettingsKey.MaxAttempts.set -> void +Xunit.UISettingsKey.UISettingsKey() -> void +Xunit.UISettingsKey.UISettingsKey(int MaxAttempts) -> void \ No newline at end of file diff --git a/src/Xunit.StaFact/net6.0/PublicAPI.Shipped.txt b/src/Xunit.StaFact/net6.0/PublicAPI.Shipped.txt index 0abd6804..2eefdddd 100644 --- a/src/Xunit.StaFact/net6.0/PublicAPI.Shipped.txt +++ b/src/Xunit.StaFact/net6.0/PublicAPI.Shipped.txt @@ -31,7 +31,6 @@ Xunit.Sdk.UITheoryDiscoverer Xunit.Sdk.UITheoryDiscoverer.UITheoryDiscoverer(Xunit.Abstractions.IMessageSink! diagnosticMessageSink) -> void Xunit.Sdk.UITheoryTestCase Xunit.Sdk.UITheoryTestCase.UITheoryTestCase() -> void -Xunit.Sdk.UITheoryTestCase.UITheoryTestCase(Xunit.Sdk.UITestCase.SyncContextType synchronizationContextType, Xunit.Abstractions.IMessageSink! diagnosticMessageSink, Xunit.Sdk.TestMethodDisplay defaultMethodDisplay, Xunit.Sdk.TestMethodDisplayOptions defaultMethodDisplayOptions, Xunit.Abstractions.ITestMethod! testMethod) -> void Xunit.Sdk.UITheoryTestCaseRunner Xunit.StaFactAttribute Xunit.StaFactAttribute.StaFactAttribute() -> void diff --git a/src/Xunit.StaFact/net6.0/PublicAPI.Unshipped.txt b/src/Xunit.StaFact/net6.0/PublicAPI.Unshipped.txt index e69de29b..01e8dfae 100644 --- a/src/Xunit.StaFact/net6.0/PublicAPI.Unshipped.txt +++ b/src/Xunit.StaFact/net6.0/PublicAPI.Unshipped.txt @@ -0,0 +1,6 @@ +static Xunit.UISettingsKey.Default.get -> Xunit.UISettingsKey +Xunit.UISettingsKey +Xunit.UISettingsKey.MaxAttempts.get -> int +Xunit.UISettingsKey.MaxAttempts.set -> void +Xunit.UISettingsKey.UISettingsKey() -> void +Xunit.UISettingsKey.UISettingsKey(int MaxAttempts) -> void \ No newline at end of file diff --git a/src/Xunit.StaFact/netstandard2.0/PublicAPI.Shipped.txt b/src/Xunit.StaFact/netstandard2.0/PublicAPI.Shipped.txt index 0abd6804..2eefdddd 100644 --- a/src/Xunit.StaFact/netstandard2.0/PublicAPI.Shipped.txt +++ b/src/Xunit.StaFact/netstandard2.0/PublicAPI.Shipped.txt @@ -31,7 +31,6 @@ Xunit.Sdk.UITheoryDiscoverer Xunit.Sdk.UITheoryDiscoverer.UITheoryDiscoverer(Xunit.Abstractions.IMessageSink! diagnosticMessageSink) -> void Xunit.Sdk.UITheoryTestCase Xunit.Sdk.UITheoryTestCase.UITheoryTestCase() -> void -Xunit.Sdk.UITheoryTestCase.UITheoryTestCase(Xunit.Sdk.UITestCase.SyncContextType synchronizationContextType, Xunit.Abstractions.IMessageSink! diagnosticMessageSink, Xunit.Sdk.TestMethodDisplay defaultMethodDisplay, Xunit.Sdk.TestMethodDisplayOptions defaultMethodDisplayOptions, Xunit.Abstractions.ITestMethod! testMethod) -> void Xunit.Sdk.UITheoryTestCaseRunner Xunit.StaFactAttribute Xunit.StaFactAttribute.StaFactAttribute() -> void diff --git a/src/Xunit.StaFact/netstandard2.0/PublicAPI.Unshipped.txt b/src/Xunit.StaFact/netstandard2.0/PublicAPI.Unshipped.txt index e69de29b..01e8dfae 100644 --- a/src/Xunit.StaFact/netstandard2.0/PublicAPI.Unshipped.txt +++ b/src/Xunit.StaFact/netstandard2.0/PublicAPI.Unshipped.txt @@ -0,0 +1,6 @@ +static Xunit.UISettingsKey.Default.get -> Xunit.UISettingsKey +Xunit.UISettingsKey +Xunit.UISettingsKey.MaxAttempts.get -> int +Xunit.UISettingsKey.MaxAttempts.set -> void +Xunit.UISettingsKey.UISettingsKey() -> void +Xunit.UISettingsKey.UISettingsKey(int MaxAttempts) -> void \ No newline at end of file From 4788c2b7d967d7d3dd111ad156b105f76c217d48 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Mon, 5 Jun 2023 15:40:13 -0500 Subject: [PATCH 3/4] Add tests for UISettingsAttribute --- src/Xunit.StaFact/UISettingsAttribute.cs | 2 +- .../net472/PublicAPI.Unshipped.txt | 4 +++ .../net6.0-macos10.15/PublicAPI.Unshipped.txt | 4 +++ .../net6.0-windows/PublicAPI.Unshipped.txt | 4 +++ .../net6.0/PublicAPI.Unshipped.txt | 4 +++ .../netstandard2.0/PublicAPI.Unshipped.txt | 4 +++ test/Xunit.StaFact.Tests/MaxAttemptsHelper.cs | 30 +++++++++++++++++++ test/Xunit.StaFact.Tests/StaFactTests.cs | 20 +++++++++++++ test/Xunit.StaFact.Tests/StaTheoryTests.cs | 24 +++++++++++++++ test/Xunit.StaFact.Tests/UIFactTests.cs | 20 +++++++++++++ test/Xunit.StaFact.Tests/UITheoryTests.cs | 24 +++++++++++++++ .../WindowsDesktop/WinFormsFactTests.cs | 20 +++++++++++++ .../WindowsDesktop/WinFormsTheoryTests.cs | 24 +++++++++++++++ .../WindowsDesktop/WpfFactTests.cs | 20 +++++++++++++ .../WindowsDesktop/WpfTheoryTests.cs | 24 +++++++++++++++ 15 files changed, 227 insertions(+), 1 deletion(-) create mode 100644 test/Xunit.StaFact.Tests/MaxAttemptsHelper.cs diff --git a/src/Xunit.StaFact/UISettingsAttribute.cs b/src/Xunit.StaFact/UISettingsAttribute.cs index bbcf37dc..e55f4ec1 100644 --- a/src/Xunit.StaFact/UISettingsAttribute.cs +++ b/src/Xunit.StaFact/UISettingsAttribute.cs @@ -4,7 +4,7 @@ namespace Xunit; [AttributeUsage(AttributeTargets.Method | AttributeTargets.Class, AllowMultiple = false, Inherited = true)] -internal class UISettingsAttribute : Attribute +public sealed class UISettingsAttribute : Attribute { /// /// Gets or sets the maximum number of retry attempts for a test. diff --git a/src/Xunit.StaFact/net472/PublicAPI.Unshipped.txt b/src/Xunit.StaFact/net472/PublicAPI.Unshipped.txt index 01e8dfae..84be6f71 100644 --- a/src/Xunit.StaFact/net472/PublicAPI.Unshipped.txt +++ b/src/Xunit.StaFact/net472/PublicAPI.Unshipped.txt @@ -1,4 +1,8 @@ static Xunit.UISettingsKey.Default.get -> Xunit.UISettingsKey +Xunit.UISettingsAttribute +Xunit.UISettingsAttribute.MaxAttempts.get -> int +Xunit.UISettingsAttribute.MaxAttempts.set -> void +Xunit.UISettingsAttribute.UISettingsAttribute() -> void Xunit.UISettingsKey Xunit.UISettingsKey.MaxAttempts.get -> int Xunit.UISettingsKey.MaxAttempts.set -> void diff --git a/src/Xunit.StaFact/net6.0-macos10.15/PublicAPI.Unshipped.txt b/src/Xunit.StaFact/net6.0-macos10.15/PublicAPI.Unshipped.txt index 75dfa62a..dc5e0906 100644 --- a/src/Xunit.StaFact/net6.0-macos10.15/PublicAPI.Unshipped.txt +++ b/src/Xunit.StaFact/net6.0-macos10.15/PublicAPI.Unshipped.txt @@ -51,6 +51,10 @@ Xunit.StaTheoryAttribute Xunit.StaTheoryAttribute.StaTheoryAttribute() -> void Xunit.UIFactAttribute Xunit.UIFactAttribute.UIFactAttribute() -> void +Xunit.UISettingsAttribute +Xunit.UISettingsAttribute.MaxAttempts.get -> int +Xunit.UISettingsAttribute.MaxAttempts.set -> void +Xunit.UISettingsAttribute.UISettingsAttribute() -> void Xunit.UISettingsKey Xunit.UISettingsKey.MaxAttempts.get -> int Xunit.UISettingsKey.MaxAttempts.set -> void diff --git a/src/Xunit.StaFact/net6.0-windows/PublicAPI.Unshipped.txt b/src/Xunit.StaFact/net6.0-windows/PublicAPI.Unshipped.txt index 01e8dfae..84be6f71 100644 --- a/src/Xunit.StaFact/net6.0-windows/PublicAPI.Unshipped.txt +++ b/src/Xunit.StaFact/net6.0-windows/PublicAPI.Unshipped.txt @@ -1,4 +1,8 @@ static Xunit.UISettingsKey.Default.get -> Xunit.UISettingsKey +Xunit.UISettingsAttribute +Xunit.UISettingsAttribute.MaxAttempts.get -> int +Xunit.UISettingsAttribute.MaxAttempts.set -> void +Xunit.UISettingsAttribute.UISettingsAttribute() -> void Xunit.UISettingsKey Xunit.UISettingsKey.MaxAttempts.get -> int Xunit.UISettingsKey.MaxAttempts.set -> void diff --git a/src/Xunit.StaFact/net6.0/PublicAPI.Unshipped.txt b/src/Xunit.StaFact/net6.0/PublicAPI.Unshipped.txt index 01e8dfae..84be6f71 100644 --- a/src/Xunit.StaFact/net6.0/PublicAPI.Unshipped.txt +++ b/src/Xunit.StaFact/net6.0/PublicAPI.Unshipped.txt @@ -1,4 +1,8 @@ static Xunit.UISettingsKey.Default.get -> Xunit.UISettingsKey +Xunit.UISettingsAttribute +Xunit.UISettingsAttribute.MaxAttempts.get -> int +Xunit.UISettingsAttribute.MaxAttempts.set -> void +Xunit.UISettingsAttribute.UISettingsAttribute() -> void Xunit.UISettingsKey Xunit.UISettingsKey.MaxAttempts.get -> int Xunit.UISettingsKey.MaxAttempts.set -> void diff --git a/src/Xunit.StaFact/netstandard2.0/PublicAPI.Unshipped.txt b/src/Xunit.StaFact/netstandard2.0/PublicAPI.Unshipped.txt index 01e8dfae..84be6f71 100644 --- a/src/Xunit.StaFact/netstandard2.0/PublicAPI.Unshipped.txt +++ b/src/Xunit.StaFact/netstandard2.0/PublicAPI.Unshipped.txt @@ -1,4 +1,8 @@ static Xunit.UISettingsKey.Default.get -> Xunit.UISettingsKey +Xunit.UISettingsAttribute +Xunit.UISettingsAttribute.MaxAttempts.get -> int +Xunit.UISettingsAttribute.MaxAttempts.set -> void +Xunit.UISettingsAttribute.UISettingsAttribute() -> void Xunit.UISettingsKey Xunit.UISettingsKey.MaxAttempts.get -> int Xunit.UISettingsKey.MaxAttempts.set -> void diff --git a/test/Xunit.StaFact.Tests/MaxAttemptsHelper.cs b/test/Xunit.StaFact.Tests/MaxAttemptsHelper.cs new file mode 100644 index 00000000..b782dd8c --- /dev/null +++ b/test/Xunit.StaFact.Tests/MaxAttemptsHelper.cs @@ -0,0 +1,30 @@ +// Copyright (c) Andrew Arnott. All rights reserved. +// Licensed under the Ms-PL license. See LICENSE file in the project root for full license information. + +using System.Runtime.CompilerServices; + +internal static class MaxAttemptsHelper +{ + private static readonly Dictionary<(Type, string), StrongBox> AttemptNumber = new(); + + /// + /// Gets the current attempt number for a test involving automatic retries. + /// + /// The class defining the test. + /// The unique name of the test method within . + /// The 0-based attempt number. + public static int GetAndIncrementAttemptNumber(Type testClass, string testMethod) + { + StrongBox? attemptNumber; + lock (AttemptNumber) + { + if (!AttemptNumber.TryGetValue((testClass, testMethod), out attemptNumber)) + { + attemptNumber = new(); + AttemptNumber.Add((testClass, testMethod), attemptNumber); + } + } + + return Interlocked.Increment(ref attemptNumber.Value) - 1; + } +} diff --git a/test/Xunit.StaFact.Tests/StaFactTests.cs b/test/Xunit.StaFact.Tests/StaFactTests.cs index 5bee820c..b0968e17 100644 --- a/test/Xunit.StaFact.Tests/StaFactTests.cs +++ b/test/Xunit.StaFact.Tests/StaFactTests.cs @@ -38,4 +38,24 @@ public async void AsyncVoid_IsNotSupported() [StaFact, Trait("TestCategory", "FailureExpected")] public void JustFailVoid() => throw new InvalidOperationException("Expected failure."); + + [StaFact] + [UISettings(MaxAttempts = 2)] + public void StaFact_AutomaticRetryNeeded() + { + if (MaxAttemptsHelper.GetAndIncrementAttemptNumber(typeof(StaFactTests), nameof(this.StaFact_AutomaticRetryNeeded)) != 1) + { + Assert.Fail("The first attempt false, but a second attempt will pass."); + } + } + + [StaFact] + [UISettings(MaxAttempts = 2)] + public void StaFact_AutomaticRetryNotNeeded() + { + if (MaxAttemptsHelper.GetAndIncrementAttemptNumber(typeof(StaFactTests), nameof(this.StaFact_AutomaticRetryNeeded)) != 0) + { + Assert.Fail("This test should not have run a second time because the first run was successful."); + } + } } diff --git a/test/Xunit.StaFact.Tests/StaTheoryTests.cs b/test/Xunit.StaFact.Tests/StaTheoryTests.cs index 222ebda0..f816f3d2 100644 --- a/test/Xunit.StaFact.Tests/StaTheoryTests.cs +++ b/test/Xunit.StaFact.Tests/StaTheoryTests.cs @@ -69,4 +69,28 @@ public void ThreadAffinitizedDataObject(NonSerializableObject o) [StaTheory, Trait("TestCategory", "FailureExpected")] [InlineData(0)] public void JustFailVoid(int a) => throw new InvalidOperationException("Expected failure " + a); + + [StaTheory] + [InlineData(0)] + [InlineData(1)] + [UISettings(MaxAttempts = 2)] + public void StaTheory_AutomaticRetryNeeded(int arg) + { + if (MaxAttemptsHelper.GetAndIncrementAttemptNumber(typeof(StaTheoryTests), $"{nameof(this.StaTheory_AutomaticRetryNeeded)}_{arg}") != 1) + { + Assert.Fail("The first attempt false, but a second attempt will pass."); + } + } + + [StaTheory] + [InlineData(0)] + [InlineData(1)] + [UISettings(MaxAttempts = 2)] + public void StaTheory_AutomaticRetryNotNeeded(int arg) + { + if (MaxAttemptsHelper.GetAndIncrementAttemptNumber(typeof(StaTheoryTests), $"{nameof(this.StaTheory_AutomaticRetryNeeded)}_{arg}") != 0) + { + Assert.Fail("This test should not have run a second time because the first run was successful."); + } + } } diff --git a/test/Xunit.StaFact.Tests/UIFactTests.cs b/test/Xunit.StaFact.Tests/UIFactTests.cs index c6ee39dd..2877df82 100644 --- a/test/Xunit.StaFact.Tests/UIFactTests.cs +++ b/test/Xunit.StaFact.Tests/UIFactTests.cs @@ -141,4 +141,24 @@ await Task.Run(delegate [UIFact, Trait("TestCategory", "FailureExpected")] public void JustFailVoid() => throw new InvalidOperationException("Expected failure."); + + [UIFact] + [UISettings(MaxAttempts = 2)] + public void UIFact_AutomaticRetryNeeded() + { + if (MaxAttemptsHelper.GetAndIncrementAttemptNumber(typeof(UIFactTests), nameof(this.UIFact_AutomaticRetryNeeded)) != 1) + { + Assert.Fail("The first attempt false, but a second attempt will pass."); + } + } + + [UIFact] + [UISettings(MaxAttempts = 2)] + public void UIFact_AutomaticRetryNotNeeded() + { + if (MaxAttemptsHelper.GetAndIncrementAttemptNumber(typeof(UIFactTests), nameof(this.UIFact_AutomaticRetryNeeded)) != 0) + { + Assert.Fail("This test should not have run a second time because the first run was successful."); + } + } } diff --git a/test/Xunit.StaFact.Tests/UITheoryTests.cs b/test/Xunit.StaFact.Tests/UITheoryTests.cs index f1181c91..25e515ac 100644 --- a/test/Xunit.StaFact.Tests/UITheoryTests.cs +++ b/test/Xunit.StaFact.Tests/UITheoryTests.cs @@ -143,4 +143,28 @@ public void ThreadAffinitizedDataObject(NonSerializableObject o) [UITheory, Trait("TestCategory", "FailureExpected")] [InlineData(0)] public void JustFailVoid(int a) => throw new InvalidOperationException("Expected failure " + a); + + [UITheory] + [InlineData(0)] + [InlineData(1)] + [UISettings(MaxAttempts = 2)] + public void UITheory_AutomaticRetryNeeded(int arg) + { + if (MaxAttemptsHelper.GetAndIncrementAttemptNumber(typeof(UITheoryTests), $"{nameof(this.UITheory_AutomaticRetryNeeded)}_{arg}") != 1) + { + Assert.Fail("The first attempt false, but a second attempt will pass."); + } + } + + [UITheory] + [InlineData(0)] + [InlineData(1)] + [UISettings(MaxAttempts = 2)] + public void UITheory_AutomaticRetryNotNeeded(int arg) + { + if (MaxAttemptsHelper.GetAndIncrementAttemptNumber(typeof(UITheoryTests), $"{nameof(this.UITheory_AutomaticRetryNeeded)}_{arg}") != 0) + { + Assert.Fail("This test should not have run a second time because the first run was successful."); + } + } } diff --git a/test/Xunit.StaFact.Tests/WindowsDesktop/WinFormsFactTests.cs b/test/Xunit.StaFact.Tests/WindowsDesktop/WinFormsFactTests.cs index 732e2116..4d71c8db 100644 --- a/test/Xunit.StaFact.Tests/WindowsDesktop/WinFormsFactTests.cs +++ b/test/Xunit.StaFact.Tests/WindowsDesktop/WinFormsFactTests.cs @@ -69,6 +69,26 @@ public async Task OperationCanceledException_Thrown() [DesktopFact, Trait("TestCategory", "FailureExpected")] public void JustFailVoid() => throw new InvalidOperationException("Expected failure."); + [DesktopFact] + [UISettings(MaxAttempts = 2)] + public void WinFormsFact_AutomaticRetryNeeded() + { + if (MaxAttemptsHelper.GetAndIncrementAttemptNumber(typeof(WinFormsFactTests), nameof(this.WinFormsFact_AutomaticRetryNeeded)) != 1) + { + Assert.Fail("The first attempt false, but a second attempt will pass."); + } + } + + [DesktopFact] + [UISettings(MaxAttempts = 2)] + public void WinFormsFact_AutomaticRetryNotNeeded() + { + if (MaxAttemptsHelper.GetAndIncrementAttemptNumber(typeof(WinFormsFactTests), nameof(this.WinFormsFact_AutomaticRetryNeeded)) != 0) + { + Assert.Fail("This test should not have run a second time because the first run was successful."); + } + } + private void AssertThreadCharacteristics() { Assert.Same(this.ctorSyncContext, SynchronizationContext.Current); diff --git a/test/Xunit.StaFact.Tests/WindowsDesktop/WinFormsTheoryTests.cs b/test/Xunit.StaFact.Tests/WindowsDesktop/WinFormsTheoryTests.cs index 24f91a10..e381ca15 100644 --- a/test/Xunit.StaFact.Tests/WindowsDesktop/WinFormsTheoryTests.cs +++ b/test/Xunit.StaFact.Tests/WindowsDesktop/WinFormsTheoryTests.cs @@ -41,4 +41,28 @@ public void ThreadAffinitizedDataObject(NonSerializableObject o) [WinFormsTheory, Trait("TestCategory", "FailureExpected")] [InlineData(0)] public void JustFailVoid(int a) => throw new InvalidOperationException("Expected failure " + a); + + [WinFormsTheory] + [InlineData(0)] + [InlineData(1)] + [UISettings(MaxAttempts = 2)] + public void WinFormsTheory_AutomaticRetryNeeded(int arg) + { + if (MaxAttemptsHelper.GetAndIncrementAttemptNumber(typeof(WinFormsTheoryTests), $"{nameof(this.WinFormsTheory_AutomaticRetryNeeded)}_{arg}") != 1) + { + Assert.Fail("The first attempt false, but a second attempt will pass."); + } + } + + [WinFormsTheory] + [InlineData(0)] + [InlineData(1)] + [UISettings(MaxAttempts = 2)] + public void WinFormsTheory_AutomaticRetryNotNeeded(int arg) + { + if (MaxAttemptsHelper.GetAndIncrementAttemptNumber(typeof(WinFormsTheoryTests), $"{nameof(this.WinFormsTheory_AutomaticRetryNeeded)}_{arg}") != 0) + { + Assert.Fail("This test should not have run a second time because the first run was successful."); + } + } } diff --git a/test/Xunit.StaFact.Tests/WindowsDesktop/WpfFactTests.cs b/test/Xunit.StaFact.Tests/WindowsDesktop/WpfFactTests.cs index 0ca5cb19..2c1784b8 100644 --- a/test/Xunit.StaFact.Tests/WindowsDesktop/WpfFactTests.cs +++ b/test/Xunit.StaFact.Tests/WindowsDesktop/WpfFactTests.cs @@ -79,6 +79,26 @@ public void ShouldShowWindow() [DesktopFact, Trait("TestCategory", "FailureExpected")] public void JustFailVoid() => throw new InvalidOperationException("Expected failure."); + [DesktopFact] + [UISettings(MaxAttempts = 2)] + public void WpfFact_AutomaticRetryNeeded() + { + if (MaxAttemptsHelper.GetAndIncrementAttemptNumber(typeof(WpfFactTests), nameof(this.WpfFact_AutomaticRetryNeeded)) != 1) + { + Assert.Fail("The first attempt false, but a second attempt will pass."); + } + } + + [DesktopFact] + [UISettings(MaxAttempts = 2)] + public void WpfFact_AutomaticRetryNotNeeded() + { + if (MaxAttemptsHelper.GetAndIncrementAttemptNumber(typeof(WpfFactTests), nameof(this.WpfFact_AutomaticRetryNeeded)) != 0) + { + Assert.Fail("This test should not have run a second time because the first run was successful."); + } + } + private void AssertThreadCharacteristics() { Assert.IsType(SynchronizationContext.Current); diff --git a/test/Xunit.StaFact.Tests/WindowsDesktop/WpfTheoryTests.cs b/test/Xunit.StaFact.Tests/WindowsDesktop/WpfTheoryTests.cs index b41cda5a..b74276e2 100644 --- a/test/Xunit.StaFact.Tests/WindowsDesktop/WpfTheoryTests.cs +++ b/test/Xunit.StaFact.Tests/WindowsDesktop/WpfTheoryTests.cs @@ -65,4 +65,28 @@ public void ThreadAffinitizedDataObject(NonSerializableObject o) [WpfTheory, Trait("TestCategory", "FailureExpected")] [InlineData(0)] public void JustFailVoid(int a) => throw new InvalidOperationException("Expected failure " + a); + + [WpfTheory] + [InlineData(0)] + [InlineData(1)] + [UISettings(MaxAttempts = 2)] + public void WpfTheory_AutomaticRetryNeeded(int arg) + { + if (MaxAttemptsHelper.GetAndIncrementAttemptNumber(typeof(WpfTheoryTests), $"{nameof(this.WpfTheory_AutomaticRetryNeeded)}_{arg}") != 1) + { + Assert.Fail("The first attempt false, but a second attempt will pass."); + } + } + + [WpfTheory] + [InlineData(0)] + [InlineData(1)] + [UISettings(MaxAttempts = 2)] + public void WpfTheory_AutomaticRetryNotNeeded(int arg) + { + if (MaxAttemptsHelper.GetAndIncrementAttemptNumber(typeof(WpfTheoryTests), $"{nameof(this.WpfTheory_AutomaticRetryNeeded)}_{arg}") != 0) + { + Assert.Fail("This test should not have run a second time because the first run was successful."); + } + } } From 1fbb4625e402aa0b8c03ca8592025d7fef88b93b Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Mon, 5 Jun 2023 16:31:14 -0500 Subject: [PATCH 4/4] Simplify implementation and avoid discovery exceptions --- .../Sdk.Mac/CocoaTheoryDiscoverer.cs | 3 ++- .../WinFormsTheoryDiscoverer.cs | 3 ++- .../Sdk.WindowsDesktop/WpfTheoryDiscoverer.cs | 3 ++- src/Xunit.StaFact/Sdk/StaTheoryDiscoverer.cs | 3 ++- src/Xunit.StaFact/Sdk/UIFactDiscoverer.cs | 9 +++++++-- src/Xunit.StaFact/Sdk/UITestCase.cs | 19 ------------------- src/Xunit.StaFact/Sdk/UITheoryDiscoverer.cs | 3 ++- src/Xunit.StaFact/Sdk/UITheoryTestCase.cs | 8 +++++--- .../Sdk/UITheoryTestCaseRunner.cs | 13 ++----------- .../net472/PublicAPI.Shipped.txt | 1 - .../net6.0-macos10.15/PublicAPI.Unshipped.txt | 1 - .../net6.0-windows/PublicAPI.Shipped.txt | 1 - .../net6.0/PublicAPI.Shipped.txt | 1 - .../netstandard2.0/PublicAPI.Shipped.txt | 1 - 14 files changed, 24 insertions(+), 45 deletions(-) diff --git a/src/Xunit.StaFact/Sdk.Mac/CocoaTheoryDiscoverer.cs b/src/Xunit.StaFact/Sdk.Mac/CocoaTheoryDiscoverer.cs index a30c15af..b570521a 100644 --- a/src/Xunit.StaFact/Sdk.Mac/CocoaTheoryDiscoverer.cs +++ b/src/Xunit.StaFact/Sdk.Mac/CocoaTheoryDiscoverer.cs @@ -25,6 +25,7 @@ protected override IEnumerable CreateTestCasesForDataRow(ITestFr protected override IEnumerable CreateTestCasesForTheory(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute) { - yield return new UITheoryTestCase(UITestCase.SyncContextType.Cocoa, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, theoryAttribute); + UISettingsKey settings = UIFactDiscoverer.GetSettings(testMethod, theoryAttribute); + yield return new UITheoryTestCase(UITestCase.SyncContextType.Cocoa, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, settings); } } diff --git a/src/Xunit.StaFact/Sdk.WindowsDesktop/WinFormsTheoryDiscoverer.cs b/src/Xunit.StaFact/Sdk.WindowsDesktop/WinFormsTheoryDiscoverer.cs index a568b8fc..1c09dd88 100644 --- a/src/Xunit.StaFact/Sdk.WindowsDesktop/WinFormsTheoryDiscoverer.cs +++ b/src/Xunit.StaFact/Sdk.WindowsDesktop/WinFormsTheoryDiscoverer.cs @@ -29,8 +29,9 @@ protected override IEnumerable CreateTestCasesForDataRow(ITestFr protected override IEnumerable CreateTestCasesForTheory(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute) { + UISettingsKey settings = UIFactDiscoverer.GetSettings(testMethod, theoryAttribute); yield return RuntimeInformation.IsOSPlatform(OSPlatform.Windows) - ? (IXunitTestCase)new UITheoryTestCase(UITestCase.SyncContextType.WinForms, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, theoryAttribute) + ? (IXunitTestCase)new UITheoryTestCase(UITestCase.SyncContextType.WinForms, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, settings) : new XunitSkippedDataRowTestCase(this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, "WinForms only exists on Windows."); } } diff --git a/src/Xunit.StaFact/Sdk.WindowsDesktop/WpfTheoryDiscoverer.cs b/src/Xunit.StaFact/Sdk.WindowsDesktop/WpfTheoryDiscoverer.cs index 26dd5afa..fa6e6332 100644 --- a/src/Xunit.StaFact/Sdk.WindowsDesktop/WpfTheoryDiscoverer.cs +++ b/src/Xunit.StaFact/Sdk.WindowsDesktop/WpfTheoryDiscoverer.cs @@ -29,8 +29,9 @@ protected override IEnumerable CreateTestCasesForDataRow(ITestFr protected override IEnumerable CreateTestCasesForTheory(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute) { + UISettingsKey settings = UIFactDiscoverer.GetSettings(testMethod, theoryAttribute); yield return RuntimeInformation.IsOSPlatform(OSPlatform.Windows) - ? (IXunitTestCase)new UITheoryTestCase(UITestCase.SyncContextType.WPF, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, theoryAttribute) + ? (IXunitTestCase)new UITheoryTestCase(UITestCase.SyncContextType.WPF, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, settings) : new XunitSkippedDataRowTestCase(this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, "WPF only exists on Windows."); } } diff --git a/src/Xunit.StaFact/Sdk/StaTheoryDiscoverer.cs b/src/Xunit.StaFact/Sdk/StaTheoryDiscoverer.cs index b74969c2..7f3af59b 100644 --- a/src/Xunit.StaFact/Sdk/StaTheoryDiscoverer.cs +++ b/src/Xunit.StaFact/Sdk/StaTheoryDiscoverer.cs @@ -51,8 +51,9 @@ protected override IEnumerable CreateTestCasesForTheory(ITestFra yield return new ExecutionErrorTestCase(this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, "Async void methods are not supported."); } + UISettingsKey settings = UIFactDiscoverer.GetSettings(testMethod, theoryAttribute); yield return RuntimeInformation.IsOSPlatform(OSPlatform.Windows) - ? (IXunitTestCase)new UITheoryTestCase(UITestCase.SyncContextType.None, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, theoryAttribute) + ? (IXunitTestCase)new UITheoryTestCase(UITestCase.SyncContextType.None, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, settings) : new XunitSkippedDataRowTestCase(this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, "STA threads only exist on Windows."); } } diff --git a/src/Xunit.StaFact/Sdk/UIFactDiscoverer.cs b/src/Xunit.StaFact/Sdk/UIFactDiscoverer.cs index d7a6d455..8bd93aa2 100644 --- a/src/Xunit.StaFact/Sdk/UIFactDiscoverer.cs +++ b/src/Xunit.StaFact/Sdk/UIFactDiscoverer.cs @@ -1,6 +1,7 @@ // Copyright (c) Andrew Arnott. All rights reserved. // Licensed under the Ms-PL license. See LICENSE file in the project root for full license information. +using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; namespace Xunit.Sdk; @@ -77,7 +78,10 @@ private static TValue GetNamedArgument(IAttributeInfo factAttribute, IAt private static TValue GetNamedArgument(IAttributeInfo factAttribute, IAttributeInfo[] settingsAttributes, string argumentName, Func isValidValue, Func? merge, TValue defaultValue) { StrongBox? result = null; - if (TryGetNamedArgument(factAttribute, argumentName, isValidValue, out TValue? value)) + TValue? value; + +#if false // Currently the custom Fact and Theory attributes do not contain settings properties + if (TryGetNamedArgument(factAttribute, argumentName, isValidValue, out value)) { if (merge is null) { @@ -86,6 +90,7 @@ private static TValue GetNamedArgument(IAttributeInfo factAttribute, IAt result = new StrongBox(value); } +#endif foreach (IAttributeInfo attribute in settingsAttributes) { @@ -115,7 +120,7 @@ private static TValue GetNamedArgument(IAttributeInfo factAttribute, IAt return defaultValue; - static bool TryGetNamedArgument(IAttributeInfo attribute, string argumentName, Func isValidValue, out TValue value) + static bool TryGetNamedArgument(IAttributeInfo attribute, string argumentName, Func isValidValue, [MaybeNullWhen(false)] out TValue value) { value = attribute.GetNamedArgument(argumentName); return isValidValue(value); diff --git a/src/Xunit.StaFact/Sdk/UITestCase.cs b/src/Xunit.StaFact/Sdk/UITestCase.cs index 0539b0f5..baa90636 100644 --- a/src/Xunit.StaFact/Sdk/UITestCase.cs +++ b/src/Xunit.StaFact/Sdk/UITestCase.cs @@ -15,25 +15,6 @@ public class UITestCase : XunitTestCase private UISettingsKey settings; private SyncContextType synchronizationContextType; - /// - /// Initializes a new instance of the class. - /// - /// The type of to use. - /// The message sink used to send diagnostic messages. - /// Default method display to use (when not customized). - /// The test method this test case belongs to. - /// The arguments for the test method. - [Obsolete("Call the constructor which provides UISettingsKey.")] - public UITestCase( - SyncContextType synchronizationContextType, - IMessageSink diagnosticMessageSink, - TestMethodDisplay defaultMethodDisplay, - ITestMethod testMethod, - object?[]? testMethodArguments = null) - : this(synchronizationContextType, diagnosticMessageSink, defaultMethodDisplay, testMethod, testMethodArguments, UISettingsKey.Default) - { - } - /// /// Initializes a new instance of the class /// for deserialization. diff --git a/src/Xunit.StaFact/Sdk/UITheoryDiscoverer.cs b/src/Xunit.StaFact/Sdk/UITheoryDiscoverer.cs index 18d2d1ba..416dfa58 100644 --- a/src/Xunit.StaFact/Sdk/UITheoryDiscoverer.cs +++ b/src/Xunit.StaFact/Sdk/UITheoryDiscoverer.cs @@ -25,6 +25,7 @@ protected override IEnumerable CreateTestCasesForDataRow(ITestFr protected override IEnumerable CreateTestCasesForTheory(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute) { - yield return new UITheoryTestCase(UITestCase.SyncContextType.Portable, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, theoryAttribute); + UISettingsKey settings = UIFactDiscoverer.GetSettings(testMethod, theoryAttribute); + yield return new UITheoryTestCase(UITestCase.SyncContextType.Portable, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, settings); } } diff --git a/src/Xunit.StaFact/Sdk/UITheoryTestCase.cs b/src/Xunit.StaFact/Sdk/UITheoryTestCase.cs index f8ffe09e..b8edcd73 100644 --- a/src/Xunit.StaFact/Sdk/UITheoryTestCase.cs +++ b/src/Xunit.StaFact/Sdk/UITheoryTestCase.cs @@ -13,10 +13,10 @@ public UITheoryTestCase() { } - internal UITheoryTestCase(UITestCase.SyncContextType synchronizationContextType, IMessageSink diagnosticMessageSink, TestMethodDisplay defaultMethodDisplay, TestMethodDisplayOptions defaultMethodDisplayOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute) + internal UITheoryTestCase(UITestCase.SyncContextType synchronizationContextType, IMessageSink diagnosticMessageSink, TestMethodDisplay defaultMethodDisplay, TestMethodDisplayOptions defaultMethodDisplayOptions, ITestMethod testMethod, UISettingsKey settings) : base(diagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testMethod) { - this.Settings = UIFactDiscoverer.GetSettings(testMethod, theoryAttribute); + this.Settings = settings; this.SynchronizationContextType = synchronizationContextType; } @@ -28,7 +28,9 @@ public override async Task RunAsync(IMessageSink diagnosticMessageSi { using ThreadRental threadRental = await ThreadRental.CreateAsync(UITestCase.GetAdapter(this.SynchronizationContextType), this.TestMethod); await threadRental.SynchronizationContext; - return await new UITheoryTestCaseRunner(this, this.DisplayName, this.SkipReason, constructorArguments, diagnosticMessageSink, messageBus, aggregator, cancellationTokenSource, this.Settings, threadRental).RunAsync(); + + // TODO: retry here if any test cases failed + return await new UITheoryTestCaseRunner(this, this.DisplayName, this.SkipReason, constructorArguments, diagnosticMessageSink, messageBus, aggregator, cancellationTokenSource, threadRental).RunAsync(); } public override void Deserialize(IXunitSerializationInfo data) diff --git a/src/Xunit.StaFact/Sdk/UITheoryTestCaseRunner.cs b/src/Xunit.StaFact/Sdk/UITheoryTestCaseRunner.cs index 71b84f48..bfbe18d3 100644 --- a/src/Xunit.StaFact/Sdk/UITheoryTestCaseRunner.cs +++ b/src/Xunit.StaFact/Sdk/UITheoryTestCaseRunner.cs @@ -7,25 +7,16 @@ namespace Xunit.Sdk; public class UITheoryTestCaseRunner : XunitTheoryTestCaseRunner { - private readonly UISettingsKey settings; private readonly ThreadRental threadRental; - internal UITheoryTestCaseRunner(UITheoryTestCase testCase, string displayName, string skipReason, object[] constructorArguments, IMessageSink diagnosticMessageSink, IMessageBus messageBus, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource, UISettingsKey settings, ThreadRental threadRental) + internal UITheoryTestCaseRunner(UITheoryTestCase testCase, string displayName, string skipReason, object[] constructorArguments, IMessageSink diagnosticMessageSink, IMessageBus messageBus, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource, ThreadRental threadRental) : base(testCase, displayName, skipReason, constructorArguments, diagnosticMessageSink, messageBus, aggregator, cancellationTokenSource) { - this.settings = settings; this.threadRental = threadRental; } internal new UITheoryTestCase TestCase => (UITheoryTestCase)base.TestCase; protected override XunitTestRunner CreateTestRunner(ITest test, IMessageBus messageBus, Type testClass, object[] constructorArguments, MethodInfo testMethod, object[] testMethodArguments, string skipReason, IReadOnlyList beforeAfterAttributes, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) - { - if (this.settings.MaxAttempts != 1) - { - throw new NotSupportedException("MaxAttempts can only be set for pre-enumerated theories."); - } - - return new UITestRunner(test, messageBus, finalAttempt: true, testClass, constructorArguments, testMethod, testMethodArguments, skipReason, beforeAfterAttributes, aggregator, cancellationTokenSource, this.threadRental); - } + => new UITestRunner(test, messageBus, finalAttempt: true, testClass, constructorArguments, testMethod, testMethodArguments, skipReason, beforeAfterAttributes, aggregator, cancellationTokenSource, this.threadRental); } diff --git a/src/Xunit.StaFact/net472/PublicAPI.Shipped.txt b/src/Xunit.StaFact/net472/PublicAPI.Shipped.txt index 60b120d5..4f33aa3c 100644 --- a/src/Xunit.StaFact/net472/PublicAPI.Shipped.txt +++ b/src/Xunit.StaFact/net472/PublicAPI.Shipped.txt @@ -30,7 +30,6 @@ Xunit.Sdk.UITestCase.SyncContextType.Portable = 1 -> Xunit.Sdk.UITestCase.SyncCo Xunit.Sdk.UITestCase.SyncContextType.WinForms = 3 -> Xunit.Sdk.UITestCase.SyncContextType Xunit.Sdk.UITestCase.SyncContextType.WPF = 2 -> Xunit.Sdk.UITestCase.SyncContextType Xunit.Sdk.UITestCase.UITestCase() -> void -Xunit.Sdk.UITestCase.UITestCase(Xunit.Sdk.UITestCase.SyncContextType synchronizationContextType, Xunit.Abstractions.IMessageSink! diagnosticMessageSink, Xunit.Sdk.TestMethodDisplay defaultMethodDisplay, Xunit.Abstractions.ITestMethod! testMethod, object?[]? testMethodArguments = null) -> void Xunit.Sdk.UITestCaseRunner Xunit.Sdk.UITestInvoker Xunit.Sdk.UITestInvoker.RunAsync() -> System.Threading.Tasks.Task! diff --git a/src/Xunit.StaFact/net6.0-macos10.15/PublicAPI.Unshipped.txt b/src/Xunit.StaFact/net6.0-macos10.15/PublicAPI.Unshipped.txt index dc5e0906..58018dfb 100644 --- a/src/Xunit.StaFact/net6.0-macos10.15/PublicAPI.Unshipped.txt +++ b/src/Xunit.StaFact/net6.0-macos10.15/PublicAPI.Unshipped.txt @@ -35,7 +35,6 @@ Xunit.Sdk.UITestCase.SyncContextType.Cocoa = 2 -> Xunit.Sdk.UITestCase.SyncConte Xunit.Sdk.UITestCase.SyncContextType.None = 0 -> Xunit.Sdk.UITestCase.SyncContextType Xunit.Sdk.UITestCase.SyncContextType.Portable = 1 -> Xunit.Sdk.UITestCase.SyncContextType Xunit.Sdk.UITestCase.UITestCase() -> void -Xunit.Sdk.UITestCase.UITestCase(Xunit.Sdk.UITestCase.SyncContextType synchronizationContextType, Xunit.Abstractions.IMessageSink! diagnosticMessageSink, Xunit.Sdk.TestMethodDisplay defaultMethodDisplay, Xunit.Abstractions.ITestMethod! testMethod, object?[]? testMethodArguments = null) -> void Xunit.Sdk.UITestCaseRunner Xunit.Sdk.UITestInvoker Xunit.Sdk.UITestInvoker.RunAsync() -> System.Threading.Tasks.Task! diff --git a/src/Xunit.StaFact/net6.0-windows/PublicAPI.Shipped.txt b/src/Xunit.StaFact/net6.0-windows/PublicAPI.Shipped.txt index 60b120d5..4f33aa3c 100644 --- a/src/Xunit.StaFact/net6.0-windows/PublicAPI.Shipped.txt +++ b/src/Xunit.StaFact/net6.0-windows/PublicAPI.Shipped.txt @@ -30,7 +30,6 @@ Xunit.Sdk.UITestCase.SyncContextType.Portable = 1 -> Xunit.Sdk.UITestCase.SyncCo Xunit.Sdk.UITestCase.SyncContextType.WinForms = 3 -> Xunit.Sdk.UITestCase.SyncContextType Xunit.Sdk.UITestCase.SyncContextType.WPF = 2 -> Xunit.Sdk.UITestCase.SyncContextType Xunit.Sdk.UITestCase.UITestCase() -> void -Xunit.Sdk.UITestCase.UITestCase(Xunit.Sdk.UITestCase.SyncContextType synchronizationContextType, Xunit.Abstractions.IMessageSink! diagnosticMessageSink, Xunit.Sdk.TestMethodDisplay defaultMethodDisplay, Xunit.Abstractions.ITestMethod! testMethod, object?[]? testMethodArguments = null) -> void Xunit.Sdk.UITestCaseRunner Xunit.Sdk.UITestInvoker Xunit.Sdk.UITestInvoker.RunAsync() -> System.Threading.Tasks.Task! diff --git a/src/Xunit.StaFact/net6.0/PublicAPI.Shipped.txt b/src/Xunit.StaFact/net6.0/PublicAPI.Shipped.txt index 2eefdddd..2e7b86ff 100644 --- a/src/Xunit.StaFact/net6.0/PublicAPI.Shipped.txt +++ b/src/Xunit.StaFact/net6.0/PublicAPI.Shipped.txt @@ -22,7 +22,6 @@ Xunit.Sdk.UITestCase.SyncContextType Xunit.Sdk.UITestCase.SyncContextType.None = 0 -> Xunit.Sdk.UITestCase.SyncContextType Xunit.Sdk.UITestCase.SyncContextType.Portable = 1 -> Xunit.Sdk.UITestCase.SyncContextType Xunit.Sdk.UITestCase.UITestCase() -> void -Xunit.Sdk.UITestCase.UITestCase(Xunit.Sdk.UITestCase.SyncContextType synchronizationContextType, Xunit.Abstractions.IMessageSink! diagnosticMessageSink, Xunit.Sdk.TestMethodDisplay defaultMethodDisplay, Xunit.Abstractions.ITestMethod! testMethod, object?[]? testMethodArguments = null) -> void Xunit.Sdk.UITestCaseRunner Xunit.Sdk.UITestInvoker Xunit.Sdk.UITestInvoker.RunAsync() -> System.Threading.Tasks.Task! diff --git a/src/Xunit.StaFact/netstandard2.0/PublicAPI.Shipped.txt b/src/Xunit.StaFact/netstandard2.0/PublicAPI.Shipped.txt index 2eefdddd..2e7b86ff 100644 --- a/src/Xunit.StaFact/netstandard2.0/PublicAPI.Shipped.txt +++ b/src/Xunit.StaFact/netstandard2.0/PublicAPI.Shipped.txt @@ -22,7 +22,6 @@ Xunit.Sdk.UITestCase.SyncContextType Xunit.Sdk.UITestCase.SyncContextType.None = 0 -> Xunit.Sdk.UITestCase.SyncContextType Xunit.Sdk.UITestCase.SyncContextType.Portable = 1 -> Xunit.Sdk.UITestCase.SyncContextType Xunit.Sdk.UITestCase.UITestCase() -> void -Xunit.Sdk.UITestCase.UITestCase(Xunit.Sdk.UITestCase.SyncContextType synchronizationContextType, Xunit.Abstractions.IMessageSink! diagnosticMessageSink, Xunit.Sdk.TestMethodDisplay defaultMethodDisplay, Xunit.Abstractions.ITestMethod! testMethod, object?[]? testMethodArguments = null) -> void Xunit.Sdk.UITestCaseRunner Xunit.Sdk.UITestInvoker Xunit.Sdk.UITestInvoker.RunAsync() -> System.Threading.Tasks.Task!