Recovery hint attributes: declare which failure recovers how

src/CoreTests/Testing/RecoveryHintAttributeTests.cs

@@ -0,0 +1,127 @@
+using JasperFx.Testing;
+using Shouldly;
+using Xunit;
+
+namespace CoreTests.Testing;
+
+/// <summary>
+/// These attributes carry no behaviour, so what is worth testing is the contract a runner reads
+/// them through: the disposition each one declares, that resources parse, and that they can be
+/// discovered from every scope they claim to support.
+/// </summary>
+public class RecoveryHintAttributeTests
+{
+    [Fact]
+    public void each_hint_declares_the_disposition_its_name_promises()
+    {
+        new ClearsOnRetryAttribute(typeof(TimeoutException)).Kind
+            .ShouldBe(DispositionKind.RetryInProcess);
+
+        new ClearsInFreshProcessAttribute(typeof(BadImageFormatException)).Kind
+            .ShouldBe(DispositionKind.RetryInFreshProcess);
+
+        new ClearsOnRecycleAttribute("rabbit", typeof(TimeoutException)).Kind
+            .ShouldBe(DispositionKind.RetryAfterRecycle);
+
+        // The counterweight: a hint that spends no attempts at all.
+        new NeverRecoversAttribute(typeof(InvalidOperationException)).Kind
+            .ShouldBe(DispositionKind.FailAndContinue);
+    }
+
+    [Fact]
+    public void the_failure_type_and_reason_are_carried_verbatim()
+    {
+        var hint = new ClearsOnRetryAttribute(typeof(TimeoutException))
+        {
+            Because = "the broker is slow to warm up"
+        };
+
+        hint.FailureType.ShouldBe(typeof(TimeoutException));
+
+        // Reaches a run report unchanged, so it must not be normalised or truncated here.
+        hint.Because.ShouldBe("the broker is slow to warm up");
+    }
+
+    [Fact]
+    public void a_hint_with_no_reason_given_simply_has_none()
+    {
+        new ClearsOnRetryAttribute(typeof(TimeoutException)).Because.ShouldBeNull();
+    }
+
+    [Theory]
+    [InlineData("rabbit", new[] { "rabbit" })]
+    [InlineData("rabbit,kafka", new[] { "rabbit", "kafka" })]
+    [InlineData(" rabbit , kafka ", new[] { "rabbit", "kafka" })]
+    [InlineData("rabbit,,kafka", new[] { "rabbit", "kafka" })]
+    public void recycle_resources_are_split_trimmed_and_compacted(string declared, string[] expected)
+    {
+        // Comma-separated to match the recycle(rabbit,kafka) tag vocabulary. Whitespace and empty
+        // entries are the author's typos, not resource names — a runner asked to recycle "" would
+        // report a wiring mistake for something nobody meant to write.
+        new ClearsOnRecycleAttribute(declared, typeof(TimeoutException)).Resources.ShouldBe(expected);
+    }
+
+    [Fact]
+    public void an_empty_recycle_list_is_empty_rather_than_a_single_blank_resource()
+    {
+        new ClearsOnRecycleAttribute("", typeof(TimeoutException)).Resources.ShouldBeEmpty();
+        new ClearsOnRecycleAttribute("   ", typeof(TimeoutException)).Resources.ShouldBeEmpty();
+    }
+
+    [Fact]
+    public void hints_other_than_recycle_name_no_resources()
+    {
+        new ClearsOnRetryAttribute(typeof(TimeoutException)).Resources.ShouldBeEmpty();
+        new NeverRecoversAttribute(typeof(TimeoutException)).Resources.ShouldBeEmpty();
+    }
+
+    // ---------------------------------------------------------------- discovery
+
+    [ClearsOnRetry(typeof(TimeoutException), Because = "warm-up")]
+    [NeverRecovers(typeof(InvalidOperationException))]
+    private class Annotated
+    {
+        [ClearsInFreshProcess(typeof(BadImageFormatException))]
+        public void Method() { }
+    }
+
+    [Fact]
+    public void several_hints_can_be_declared_on_one_target()
+    {
+        // AllowMultiple: a class routinely has more than one failure worth describing, and the
+        // whole point is that each is described separately rather than lumped into "flaky".
+        var hints = typeof(Annotated)
+            .GetCustomAttributes(typeof(RecoveryHintAttribute), inherit: true)
+            .Cast<RecoveryHintAttribute>()
+            .ToList();
+
+        hints.Count.ShouldBe(2);
+        hints.Select(h => h.FailureType)
+            .ShouldBe([typeof(TimeoutException), typeof(InvalidOperationException)], ignoreOrder: true);
+    }
+
+    [Fact]
+    public void a_hint_is_discoverable_on_a_method_as_well_as_a_class()
+    {
+        // Scope is how a narrow declaration overrides a broad one, so a runner has to be able to
+        // find them at every level the attribute claims to support.
+        var hint = typeof(Annotated)
+            .GetMethod(nameof(Annotated.Method))!
+            .GetCustomAttributes(typeof(RecoveryHintAttribute), inherit: true)
+            .Cast<RecoveryHintAttribute>()
+            .ShouldHaveSingleItem();
+
+        hint.Kind.ShouldBe(DispositionKind.RetryInFreshProcess);
+    }
+
+    [Fact]
+    public void the_attribute_targets_cover_assembly_class_and_method()
+    {
+        var usage = (AttributeUsageAttribute)typeof(RecoveryHintAttribute)
+            .GetCustomAttributes(typeof(AttributeUsageAttribute), inherit: false)
+            .Single();
+
+        usage.ValidOn.ShouldBe(AttributeTargets.Class | AttributeTargets.Method | AttributeTargets.Assembly);
+        usage.AllowMultiple.ShouldBeTrue();
+    }
+}

src/JasperFx/Testing/DispositionKind.cs

@@ -0,0 +1,44 @@
+namespace JasperFx.Testing;
+
+/// <summary>
+/// What a test runner should do about an attempt. The vocabulary a
+/// <see cref="RecoveryHintAttribute"/> declares against.
+/// </summary>
+/// <remarks>
+/// <para>
+/// This lives in JasperFx rather than in a particular test runner so that a test project can
+/// declare what it knows about its own failures without taking a dependency on the runner. Any
+/// suite already referencing JasperFx — directly, or through Marten, Wolverine or Polecat — can
+/// annotate itself and have a runner that understands these attributes act on them.
+/// </para>
+/// <para>
+/// One enum rather than a runner-side copy mapped at a seam: two enums meaning the same thing is
+/// how a vocabulary starts drifting.
+/// </para>
+/// </remarks>
+public enum DispositionKind
+{
+    /// <summary>The attempt succeeded.</summary>
+    Pass,
+
+    /// <summary>An ordinary failure: record it and keep going.</summary>
+    FailAndContinue,
+
+    /// <summary>Try again in the same process, after resources are reset.</summary>
+    RetryInProcess,
+
+    /// <summary>
+    /// Try again in a brand-new process, with this test running alone. For the tests that only
+    /// pass when nothing else shares their process.
+    /// </summary>
+    RetryInFreshProcess,
+
+    /// <summary>
+    /// Throw the named resources away, stand fresh ones up, then try again. For brokers whose
+    /// in-flight state cannot be reliably drained the way a database is truncated.
+    /// </summary>
+    RetryAfterRecycle,
+
+    /// <summary>Stop the run now — nothing downstream can pass.</summary>
+    AbortRun
+}

src/JasperFx/Testing/RecoveryHintAttributes.cs

@@ -0,0 +1,103 @@
+namespace JasperFx.Testing;
+
+/// <summary>
+/// Declares that a named class of failure, on the tests in scope, recovers a particular way.
+/// </summary>
+/// <remarks>
+/// <para>
+/// A tag or trait says <em>this test</em> is unreliable. A hint says <em>which failure</em> is
+/// unreliable and what fixes it — so an assertion failure on the same test is still reported as
+/// the bug it is, rather than being retried away with everything else.
+/// </para>
+/// <para>
+/// These attributes are pure declarations. They carry no behaviour and start nothing: a test
+/// runner that understands them reads them and decides, and a runner that does not is unaffected.
+/// That is why they live here rather than in a runner — a suite already referencing JasperFx,
+/// directly or through Marten, Wolverine or Polecat, can write down what it knows about its own
+/// flakiness without taking a dependency on whatever ends up running it.
+/// </para>
+/// <para>
+/// <strong>A hint is not permission to retry.</strong> How much time a run may spend is the
+/// operator's decision, expressed by whatever budget the runner exposes; what recovers is the
+/// author's knowledge, expressed here. A runner honouring these must not let a hint widen its own
+/// ceiling, or a test author could escape a limit set by whoever runs the suite.
+/// </para>
+/// <para>
+/// Applicable to a class (every test it owns), a method, or a whole assembly. A runner should
+/// treat the narrowest declaration as the winner, so an assembly-wide default can be overridden
+/// per class without either knowing about the other.
+/// </para>
+/// </remarks>
+[AttributeUsage(
+    AttributeTargets.Class | AttributeTargets.Method | AttributeTargets.Assembly,
+    AllowMultiple = true)]
+public abstract class RecoveryHintAttribute : Attribute
+{
+    protected RecoveryHintAttribute(Type failureType) => FailureType = failureType;
+
+    /// <summary>The exception type this hint describes. Base types match derived failures.</summary>
+    public Type FailureType { get; }
+
+    /// <summary>
+    /// Why the author believes this. Intended to reach the run report verbatim, so it should read
+    /// as an explanation to whoever is looking at the retry six months from now.
+    /// </summary>
+    public string? Because { get; set; }
+
+    /// <summary>What to do about it.</summary>
+    public abstract DispositionKind Kind { get; }
+
+    /// <summary>
+    /// Resources to recycle. Only meaningful for <see cref="ClearsOnRecycleAttribute"/>.
+    /// </summary>
+    public virtual IReadOnlyList<string> Resources => [];
+}
+
+/// <summary>This failure clears by running the test again in the same process.</summary>
+/// <example><c>[ClearsOnRetry(typeof(TimeoutException), Because = "the broker is slow to warm up")]</c></example>
+public sealed class ClearsOnRetryAttribute(Type failureType) : RecoveryHintAttribute(failureType)
+{
+    public override DispositionKind Kind => DispositionKind.RetryInProcess;
+}
+
+/// <summary>
+/// This failure clears only in a brand-new process, with the test running alone — the shape of
+/// leak that a scope reset cannot undo, like a static cached the first time anything touched it.
+/// </summary>
+public sealed class ClearsInFreshProcessAttribute(Type failureType) : RecoveryHintAttribute(failureType)
+{
+    public override DispositionKind Kind => DispositionKind.RetryInFreshProcess;
+}
+
+/// <summary>
+/// This failure clears only after the named resources are thrown away and stood up fresh.
+/// </summary>
+/// <example><c>[ClearsOnRecycle("rabbit", typeof(BrokerUnavailableException))]</c></example>
+/// <remarks>
+/// <paramref name="resources"/> is comma-separated, matching the <c>recycle(rabbit,kafka)</c> tag
+/// vocabulary it is meant to share.
+/// </remarks>
+public sealed class ClearsOnRecycleAttribute(string resources, Type failureType)
+    : RecoveryHintAttribute(failureType)
+{
+    public override DispositionKind Kind => DispositionKind.RetryAfterRecycle;
+
+    public override IReadOnlyList<string> Resources { get; } =
+        string.IsNullOrWhiteSpace(resources)
+            ? []
+            : resources.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
+}
+
+/// <summary>
+/// This failure never clears, so do not spend attempts on it.
+/// </summary>
+/// <remarks>
+/// The counterweight to the rest of the file, and the reason the set is usable at all. Without it,
+/// the only way to stop a broad "retry everything three times" policy from re-running a
+/// deterministic bug is to take the retry off the test — which also stops the retries that were
+/// pulling their weight.
+/// </remarks>
+public sealed class NeverRecoversAttribute(Type failureType) : RecoveryHintAttribute(failureType)
+{
+    public override DispositionKind Kind => DispositionKind.FailAndContinue;
+}