Implement automatic retries using UISettingsAttribute

src/Xunit.StaFact/Sdk.Mac/CocoaFactDiscoverer.cs

@@ -35,6 +35,7 @@ protected override IXunitTestCase CreateTestCase(ITestFrameworkDiscoveryOptions
             return new ExecutionErrorTestCase(this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, "Async void methods are not supported.");
         }
 
-        return new UITestCase(UITestCase.SyncContextType.Cocoa, this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod);
+        UISettingsKey settings = UIFactDiscoverer.GetSettings(testMethod, factAttribute);
+        return new UITestCase(UITestCase.SyncContextType.Cocoa, this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, testMethodArguments: null, settings);
     }
 }

src/Xunit.StaFact/Sdk.Mac/CocoaTheoryDiscoverer.cs

@@ -19,11 +19,13 @@ public CocoaTheoryDiscoverer(IMessageSink diagnosticMessageSink)
 
     protected override IEnumerable<IXunitTestCase> CreateTestCasesForDataRow(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute, object[] dataRow)
     {
-        yield return new UITestCase(UITestCase.SyncContextType.Cocoa, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, dataRow);
+        UISettingsKey settings = UIFactDiscoverer.GetSettings(testMethod, theoryAttribute);
+        yield return new UITestCase(UITestCase.SyncContextType.Cocoa, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, dataRow, settings);
     }
 
     protected override IEnumerable<IXunitTestCase> CreateTestCasesForTheory(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute)
     {
-        yield return new UITheoryTestCase(UITestCase.SyncContextType.Cocoa, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod);
+        UISettingsKey settings = UIFactDiscoverer.GetSettings(testMethod, theoryAttribute);
+        yield return new UITheoryTestCase(UITestCase.SyncContextType.Cocoa, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, settings);
     }
 }

src/Xunit.StaFact/Sdk.WindowsDesktop/WinFormsFactDiscoverer.cs

@@ -36,8 +36,9 @@ protected override IXunitTestCase CreateTestCase(ITestFrameworkDiscoveryOptions
             return new ExecutionErrorTestCase(this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, "Async void methods are not supported.");
         }
 
+        UISettingsKey settings = UIFactDiscoverer.GetSettings(testMethod, factAttribute);
         return RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
-            ? (IXunitTestCase)new UITestCase(UITestCase.SyncContextType.WinForms, this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod)
+            ? (IXunitTestCase)new UITestCase(UITestCase.SyncContextType.WinForms, this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, testMethodArguments: null, settings)
             : new XunitSkippedDataRowTestCase(this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, "WinForms only exists on Windows.");
     }
 }

src/Xunit.StaFact/Sdk.WindowsDesktop/WinFormsTheoryDiscoverer.cs

@@ -21,15 +21,17 @@ public WinFormsTheoryDiscoverer(IMessageSink diagnosticMessageSink)
 
     protected override IEnumerable<IXunitTestCase> CreateTestCasesForDataRow(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute, object[] dataRow)
     {
+        UISettingsKey settings = UIFactDiscoverer.GetSettings(testMethod, theoryAttribute);
         yield return RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
-            ? (IXunitTestCase)new UITestCase(UITestCase.SyncContextType.WinForms, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, dataRow)
+            ? (IXunitTestCase)new UITestCase(UITestCase.SyncContextType.WinForms, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, dataRow, settings)
             : new XunitSkippedDataRowTestCase(this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, "WinForms only exists on Windows.");
     }
 
     protected override IEnumerable<IXunitTestCase> CreateTestCasesForTheory(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute)
     {
+        UISettingsKey settings = UIFactDiscoverer.GetSettings(testMethod, theoryAttribute);
         yield return RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
-            ? (IXunitTestCase)new UITheoryTestCase(UITestCase.SyncContextType.WinForms, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod)
+            ? (IXunitTestCase)new UITheoryTestCase(UITestCase.SyncContextType.WinForms, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, settings)
             : new XunitSkippedDataRowTestCase(this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, "WinForms only exists on Windows.");
     }
 }

src/Xunit.StaFact/Sdk.WindowsDesktop/WpfFactDiscoverer.cs

@@ -36,8 +36,9 @@ protected override IXunitTestCase CreateTestCase(ITestFrameworkDiscoveryOptions
             return new ExecutionErrorTestCase(this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, "Async void methods are not supported.");
         }
 
+        UISettingsKey settings = UIFactDiscoverer.GetSettings(testMethod, factAttribute);
         return RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
-            ? (IXunitTestCase)new UITestCase(UITestCase.SyncContextType.WPF, this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod)
+            ? (IXunitTestCase)new UITestCase(UITestCase.SyncContextType.WPF, this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, testMethodArguments: null, settings)
             : new XunitSkippedDataRowTestCase(this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, "WPF only exists on Windows.");
     }
 }

src/Xunit.StaFact/Sdk.WindowsDesktop/WpfTheoryDiscoverer.cs

@@ -21,15 +21,17 @@ public WpfTheoryDiscoverer(IMessageSink diagnosticMessageSink)
 
     protected override IEnumerable<IXunitTestCase> CreateTestCasesForDataRow(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute, object[] dataRow)
     {
+        UISettingsKey settings = UIFactDiscoverer.GetSettings(testMethod, theoryAttribute);
         yield return RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
-            ? (IXunitTestCase)new UITestCase(UITestCase.SyncContextType.WPF, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, dataRow)
+            ? (IXunitTestCase)new UITestCase(UITestCase.SyncContextType.WPF, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, dataRow, settings)
             : new XunitSkippedDataRowTestCase(this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, "WPF only exists on Windows.");
     }
 
     protected override IEnumerable<IXunitTestCase> CreateTestCasesForTheory(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute)
     {
+        UISettingsKey settings = UIFactDiscoverer.GetSettings(testMethod, theoryAttribute);
         yield return RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
-            ? (IXunitTestCase)new UITheoryTestCase(UITestCase.SyncContextType.WPF, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod)
+            ? (IXunitTestCase)new UITheoryTestCase(UITestCase.SyncContextType.WPF, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, settings)
             : new XunitSkippedDataRowTestCase(this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, "WPF only exists on Windows.");
     }
 }

src/Xunit.StaFact/Sdk/StaFactDiscoverer.cs

@@ -37,8 +37,9 @@ protected override IXunitTestCase CreateTestCase(ITestFrameworkDiscoveryOptions
             return new ExecutionErrorTestCase(this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, "Async void methods are not supported.");
         }
 
+        UISettingsKey settings = UIFactDiscoverer.GetSettings(testMethod, factAttribute);
         return RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
-            ? (IXunitTestCase)new UITestCase(UITestCase.SyncContextType.None, this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod)
+            ? (IXunitTestCase)new UITestCase(UITestCase.SyncContextType.None, this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, testMethodArguments: null, settings)
             : new XunitSkippedDataRowTestCase(this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, "STA threads only exist on Windows.");
     }
 }

src/Xunit.StaFact/Sdk/StaTheoryDiscoverer.cs

@@ -33,8 +33,9 @@ protected override IEnumerable<IXunitTestCase> CreateTestCasesForDataRow(ITestFr
             yield return new ExecutionErrorTestCase(this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, "Async void methods are not supported.");
         }
 
+        UISettingsKey settings = UIFactDiscoverer.GetSettings(testMethod, theoryAttribute);
         yield return RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
-            ? (IXunitTestCase)new UITestCase(UITestCase.SyncContextType.None, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, dataRow)
+            ? (IXunitTestCase)new UITestCase(UITestCase.SyncContextType.None, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, dataRow, settings)
             : new XunitSkippedDataRowTestCase(this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, "STA threads only exist on Windows.");
     }
 
@@ -50,8 +51,9 @@ protected override IEnumerable<IXunitTestCase> CreateTestCasesForTheory(ITestFra
             yield return new ExecutionErrorTestCase(this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, "Async void methods are not supported.");
         }
 
+        UISettingsKey settings = UIFactDiscoverer.GetSettings(testMethod, theoryAttribute);
         yield return RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
-            ? (IXunitTestCase)new UITheoryTestCase(UITestCase.SyncContextType.None, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod)
+            ? (IXunitTestCase)new UITheoryTestCase(UITestCase.SyncContextType.None, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, settings)
             : new XunitSkippedDataRowTestCase(this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, "STA threads only exist on Windows.");
     }
 }

src/Xunit.StaFact/Sdk/UIFactDiscoverer.cs

@@ -1,6 +1,9 @@
 // Copyright (c) Andrew Arnott. All rights reserved.
 // Licensed under the Ms-PL license. See LICENSE file in the project root for full license information.
 
+using System.Diagnostics.CodeAnalysis;
+using System.Runtime.CompilerServices;
+
 namespace Xunit.Sdk;
 
 /// <summary>
@@ -20,8 +23,107 @@ public UIFactDiscoverer(IMessageSink diagnosticMessageSink)
         this.diagnosticMessageSink = diagnosticMessageSink;
     }
 
+    internal static UISettingsKey GetSettings(ITestMethod testMethod, IAttributeInfo factAttribute)
+    {
+        var maxAttempts = GetMaxAttempts(testMethod, factAttribute);
+        return new UISettingsKey(maxAttempts);
+    }
+
     protected override IXunitTestCase CreateTestCase(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
     {
-        return new UITestCase(UITestCase.SyncContextType.Portable, this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod);
+        var maxAttempts = GetMaxAttempts(testMethod, factAttribute);
+        var settings = new UISettingsKey(maxAttempts);
+        return new UITestCase(UITestCase.SyncContextType.Portable, this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, testMethodArguments: null, settings);
+    }
+
+    private static int GetMaxAttempts(ITestMethod testMethod, IAttributeInfo factAttribute)
+    {
+        return GetMaxAttempts(factAttribute, GetSettingsAttributes(testMethod).ToArray());
+    }
+
+    private static IEnumerable<IAttributeInfo> GetSettingsAttributes(ITestMethod testMethod)
+    {
+        foreach (IAttributeInfo attributeData in testMethod.Method.GetCustomAttributes(typeof(UISettingsAttribute)))
+        {
+            yield return attributeData;
+        }
+
+        foreach (IAttributeInfo attributeData in testMethod.TestClass.Class.GetCustomAttributes(typeof(UISettingsAttribute)))
+        {
+            yield return attributeData;
+        }
+    }
+
+    private static int GetMaxAttempts(IAttributeInfo factAttribute, IAttributeInfo[] settingsAttributes)
+    {
+        return GetNamedArgument(
+            factAttribute,
+            settingsAttributes,
+            nameof(UISettingsAttribute.MaxAttempts),
+            static value => value > 0,
+            defaultValue: 1);
+    }
+
+    private static TValue GetNamedArgument<TValue>(IAttributeInfo factAttribute, IAttributeInfo[] settingsAttributes, string argumentName, Func<TValue, bool> isValidValue, TValue defaultValue)
+    {
+        return GetNamedArgument(
+            factAttribute,
+            settingsAttributes,
+            argumentName,
+            isValidValue,
+            merge: null,
+            defaultValue);
+    }
+
+    private static TValue GetNamedArgument<TValue>(IAttributeInfo factAttribute, IAttributeInfo[] settingsAttributes, string argumentName, Func<TValue, bool> isValidValue, Func<TValue, TValue, TValue>? merge, TValue defaultValue)
+    {
+        StrongBox<TValue>? result = null;
+        TValue? value;
+
+#if false // Currently the custom Fact and Theory attributes do not contain settings properties
+        if (TryGetNamedArgument(factAttribute, argumentName, isValidValue, out value))
+        {
+            if (merge is null)
+            {
+                return value;
+            }
+
+            result = new StrongBox<TValue>(value);
+        }
+#endif
+
+        foreach (IAttributeInfo attribute in settingsAttributes)
+        {
+            if (TryGetNamedArgument(attribute, argumentName, isValidValue, out value))
+            {
+                if (merge is null)
+                {
+                    return value;
+                }
+                else if (result is null)
+                {
+                    result = new StrongBox<TValue>(value);
+                }
+                else
+                {
+                    result.Value = merge(value, result.Value!);
+                }
+
+                return value;
+            }
+        }
+
+        if (result is not null)
+        {
+            return result.Value!;
+        }
+
+        return defaultValue;
+
+        static bool TryGetNamedArgument(IAttributeInfo attribute, string argumentName, Func<TValue, bool> isValidValue, [MaybeNullWhen(false)] out TValue value)
+        {
+            value = attribute.GetNamedArgument<TValue>(argumentName);
+            return isValidValue(value);
+        }
     }
 }

src/Xunit.StaFact/Sdk/UITestCase.cs

@@ -12,37 +12,41 @@ namespace Xunit.Sdk;
 [DebuggerDisplay(@"\{ class = {TestMethod.TestClass.Class.Name}, method = {TestMethod.Method.Name}, display = {DisplayName}, skip = {SkipReason} \}")]
 public class UITestCase : XunitTestCase
 {
+    private UISettingsKey settings;
     private SyncContextType synchronizationContextType;
 
+    /// <summary>
+    /// Initializes a new instance of the <see cref="UITestCase"/> class
+    /// for deserialization.
+    /// </summary>
+    [EditorBrowsable(EditorBrowsableState.Never)]
+    [Obsolete("Called by the de-serializer; should only be called by deriving classes for de-serialization purposes")]
+    public UITestCase()
+    {
+    }
+
     /// <summary>
     /// Initializes a new instance of the <see cref="UITestCase"/> class.
     /// </summary>
     /// <param name="synchronizationContextType">The type of <see cref="SynchronizationContext"/> to use.</param>
     /// <param name="diagnosticMessageSink">The message sink used to send diagnostic messages.</param>
     /// <param name="defaultMethodDisplay">Default method display to use (when not customized).</param>
     /// <param name="testMethod">The test method this test case belongs to.</param>
+    /// <param name="settings">The test settings to apply.</param>
     /// <param name="testMethodArguments">The arguments for the test method.</param>
-    public UITestCase(
+    internal UITestCase(
         SyncContextType synchronizationContextType,
         IMessageSink diagnosticMessageSink,
         TestMethodDisplay defaultMethodDisplay,
         ITestMethod testMethod,
-        object?[]? testMethodArguments = null)
+        object?[]? testMethodArguments,
+        UISettingsKey settings)
         : base(diagnosticMessageSink, defaultMethodDisplay, TestMethodDisplayOptions.None, testMethod, testMethodArguments)
     {
+        this.settings = settings;
         this.synchronizationContextType = synchronizationContextType;
     }
 
-    /// <summary>
-    /// Initializes a new instance of the <see cref="UITestCase"/> class
-    /// for deserialization.
-    /// </summary>
-    [EditorBrowsable(EditorBrowsableState.Never)]
-    [Obsolete("Called by the de-serializer; should only be called by deriving classes for de-serialization purposes")]
-    public UITestCase()
-    {
-    }
-
     public enum SyncContextType
     {
         /// <summary>
@@ -85,6 +89,7 @@ public override void Serialize(IXunitSerializationInfo data)
         }
 
         base.Serialize(data);
+        data.AddValue(nameof(UISettingsAttribute.MaxAttempts), this.settings.MaxAttempts);
         data.AddValue(nameof(this.synchronizationContextType), this.synchronizationContextType);
     }
 
@@ -96,6 +101,7 @@ public override void Deserialize(IXunitSerializationInfo data)
         }
 
         base.Deserialize(data);
+        this.settings = new UISettingsKey(data.GetValue<int>(nameof(UISettingsAttribute.MaxAttempts)));
         this.synchronizationContextType = (SyncContextType)data.GetValue(nameof(this.synchronizationContextType), typeof(SyncContextType));
     }
 
@@ -117,7 +123,7 @@ public override Task<RunSummary> RunAsync(
             {
                 using ThreadRental threadRental = await ThreadRental.CreateAsync(this.Adapter, this.TestMethod);
                 await threadRental.SynchronizationContext;
-                var runner = new UITestCaseRunner(this, this.DisplayName, this.SkipReason, constructorArguments, this.TestMethodArguments, messageBus, aggregator, cancellationTokenSource, threadRental);
+                var runner = new UITestCaseRunner(this, this.DisplayName, this.SkipReason, constructorArguments, this.TestMethodArguments, messageBus, aggregator, cancellationTokenSource, this.settings, threadRental);
                 return await runner.RunAsync();
             },
             cancellationTokenSource.Token);