diff --git a/docs/docs/writing-tests/data-driven-overview.md b/docs/docs/writing-tests/data-driven-overview.md index 7a9a2bf09e0..48f07ed4dfb 100644 --- a/docs/docs/writing-tests/data-driven-overview.md +++ b/docs/docs/writing-tests/data-driven-overview.md @@ -11,6 +11,7 @@ TUnit offers several ways to provide data to your tests. Use this guide to pick | Shared object with lifecycle | `[ClassDataSource]` | [Class Data Source](class-data-source.md) | | Reusable data rows | `[TestDataRow]` | [Test Data Row](test-data-row.md) | | All parameter combinations | `[MatrixDataSource]` | [Matrix Tests](matrix-tests.md) | +| Every pair of parameter values (fewer cases) | `[PairwiseDataSource]` | [Pairwise Tests](pairwise-tests.md) | | Multiple sources on one method | Combined attributes | [Combined Data Sources](combined-data-source.md) | | Hierarchical injection | Nested properties | [Nested Data Sources](nested-data-sources.md) | | Custom generic attributes | `[GenerateGenericTest(typeof(...))]` | [Generic Attributes](generic-attributes.md) | @@ -78,4 +79,5 @@ public async Task Multiply( - **`[MethodDataSource]`** is best for computed or complex data. - **`[ClassDataSource]`** manages object lifecycles (initialization, disposal, sharing across tests). - **`[MatrixDataSource]`** generates the Cartesian product of all `[Matrix]` parameter values. +- **`[PairwiseDataSource]`** uses the same `[Matrix]` parameters but only covers every pair of values, which is far fewer test cases when there are many parameters. - Attributes can be combined on a single method — see [Combined Data Sources](combined-data-source.md). diff --git a/docs/docs/writing-tests/matrix-tests.md b/docs/docs/writing-tests/matrix-tests.md index 5b9e1e1cbfc..c8eb8689642 100644 --- a/docs/docs/writing-tests/matrix-tests.md +++ b/docs/docs/writing-tests/matrix-tests.md @@ -6,6 +6,7 @@ The Matrix data source is a way to specify different arguments per parameter, an As your number of arguments and/or parameters increase, the number of test cases will grow exponentially. For example, 3 parameters with 10 values each will generate 1,000 test cases (10 × 10 × 10). Use with caution to avoid very large test suites. +If you don't need every combination, [Pairwise Tests](pairwise-tests.md) use the same `[Matrix]` parameter attributes but only generate enough test cases to cover every pair of values. ::: For our arguments, we'll add a `[Matrix]` attribute. Instead of this being added to the test method, it's added to the parameters themselves. diff --git a/docs/docs/writing-tests/pairwise-tests.md b/docs/docs/writing-tests/pairwise-tests.md new file mode 100644 index 00000000000..5d81d547e36 --- /dev/null +++ b/docs/docs/writing-tests/pairwise-tests.md @@ -0,0 +1,176 @@ +# Pairwise Tests + +[Matrix tests](matrix-tests.md) run every combination of parameter values, so the number of test cases grows exponentially with the number of parameters. +Pairwise testing (also known as *all-pairs* testing) is a well-known way to keep that under control: instead of every combination, it generates a much smaller set of test cases in which **every pair of values, across every pair of parameters, appears at least once**. + +Most bugs are triggered by a single value or by the interaction of two parameters, so pairwise testing catches a large share of them with far fewer test cases. + +To use it, swap `[MatrixDataSource]` for `[PairwiseDataSource]`. The parameters are declared exactly as for matrix tests: + +```csharp +using TUnit.Assertions; +using TUnit.Assertions.Extensions; +using TUnit.Core; + +namespace MyTestProject; + +public class MyTestClass +{ + [Test] + [PairwiseDataSource] + public async Task MyTest( + bool largeArrayPool, + bool excessSpan, + [Matrix(0, 99, 100, 101)] int sizeHint, + [Matrix(1, 2, 3, 100)] int stepCount) + { + var total = sizeHint + stepCount; + + await Assert.That(total).IsPositive(); + } +} +``` + +`[MatrixDataSource]` would generate 2 × 2 × 4 × 4 = 64 test cases for this test. `[PairwiseDataSource]` generates 16, and every combination of values for any two of the parameters (for example `excessSpan: true` with `sizeHint: 101`) is still tested. + +The savings grow quickly with more parameters: + +| Parameter value counts | `[MatrixDataSource]` | `[PairwiseDataSource]` | +|---|---:|---:| +| 2 × 2 × 4 × 4 | 64 | 16 | +| 2 × 2 × 3 × 3 × 2 | 72 | 10 | +| 3 × 3 × 3 × 3 | 81 | 9 | +| 3 × 4 × 5 × 6 | 360 | 30 | +| 10 × 10 × 10 | 1,000 | 106 | +| 2 (×10 parameters) | 1,024 | 9 | + +:::info +With two parameters, pairwise and matrix generate the same test cases, because every combination *is* a pair. +Pairwise starts to pay off from three parameters. +::: + +:::warning +Pairwise testing does not test every combination. If a bug only shows up for a specific combination of three or more parameter values, a pairwise test may not hit it. +Use `[MatrixDataSource]` when you need exhaustive coverage. +::: + +## Parameter values + +`[PairwiseDataSource]` reads parameter values the same way as `[MatrixDataSource]`, so everything described in [Matrix Tests](matrix-tests.md) works here too: + +- `[Matrix(...)]` and `[Matrix(...)]` with explicit values, including `Excluding = [...]` +- `[MatrixRange(...)]` for numeric ranges +- `[MatrixMethod(...)]` for values returned by a method +- `bool` and `enum` parameters (and their nullable forms) expand to all of their values automatically. For nullable types, `null` is added after the other values. + +```csharp +using TUnit.Assertions; +using TUnit.Assertions.Extensions; +using TUnit.Core; + +namespace MyTestProject; + +public enum Color +{ + Red, + Green, + Blue +} + +public class MyTestClass +{ + [Test] + [PairwiseDataSource] + public async Task MyTest( + Color color, + bool? flag, + [MatrixRange(1, 5)] int number, + [Matrix("x", "y")] string text, + [MatrixMethod(nameof(Sizes))] int size) + { + await Assert.That(number).IsBetween(1, 5); + } + + public static IEnumerable Sizes() + { + yield return 1; + yield return 10; + yield return 100; + } +} +``` + +This generates 17 test cases instead of 270. + +## Exclusions + +`[MatrixExclusion(...)]` works with `[PairwiseDataSource]` as well. Exclusions are applied *while* the test cases are generated, not by filtering afterwards, so every pair of values that can still appear in an allowed test case remains covered. Pairs that no allowed test case can contain (because your exclusions rule out every row with that pair) are skipped silently, so over-excluding produces fewer test cases rather than an error. + +```csharp +using TUnit.Assertions; +using TUnit.Assertions.Extensions; +using TUnit.Core; + +namespace MyTestProject; + +public class MyTestClass +{ + [Test] + [PairwiseDataSource] + [MatrixExclusion(1, 1, 1)] + [MatrixExclusion(2, 2, 2)] + [MatrixExclusion(3, 3, 3)] + public async Task MyTest( + [Matrix(1, 2, 3)] int value1, + [Matrix(1, 2, 3)] int value2, + [Matrix(1, 2, 3)] int value3) + { + var allEqual = value1 == value2 && value2 == value3; + + await Assert.That(allEqual).IsFalse(); + } +} +``` + +## Determinism and seeds + +The generated test cases are deterministic: the same parameter values always produce the same test cases, so test IDs are stable between runs and between source-generated and reflection mode. + +The algorithm uses a seeded pseudo-random number generator to pick test cases. You can change the seed with the `Seed` property to get a different, equally valid, set of test cases: + +```csharp +using TUnit.Assertions; +using TUnit.Assertions.Extensions; +using TUnit.Core; + +namespace MyTestProject; + +public class MyTestClass +{ + [Test] + [PairwiseDataSource(Seed = 42)] + public async Task MyTest( + [Matrix(1, 2, 3)] int a, + [Matrix(1, 2, 3, 4)] int b, + [Matrix(1, 2, 3, 4, 5)] int c) + { + await Assert.That(a + b + c).IsPositive(); + } +} +``` + +:::note +Changing the parameter values (or their order) can change which test cases are generated, not just add or remove the cases for the changed value. +::: + +## Limitations + +- Like `[MatrixDataSource]`, `[PairwiseDataSource]` only supports test method parameters. It can't be used for class constructor parameters. +- Pairwise coverage only guarantees that every *pair* of values is tested. It does not try to cover combinations of three or more values. +- When many values are combined with heavy `[MatrixExclusion]` sets, discovery may still probe a large search space for unreachable pairs. Prefer fewer values or fewer exclusions when generation is slow. +- Do not apply `[MatrixDataSource]` and `[PairwiseDataSource]` to the same method or class. Each attribute is its own data source, so you would get both the full Cartesian product and a pairwise covering set rather than one coherent strategy. + +## Algorithm + +The pairwise algorithm is based on Bob Jenkins' [jenny](https://burtleburtle.net/bob/math/jenny.html) tool, by way of NUnit's `PairwiseStrategy` and [Xunit.Combinatorial](https://github.com/AArnott/Xunit.Combinatorial)'s `[PairwiseData]`. +It uses the same default seed as Xunit.Combinatorial, so tests migrated from `[PairwiseData]` generate the same number of test cases. diff --git a/docs/sidebars.ts b/docs/sidebars.ts index 03307acf5cd..48d0df6d244 100644 --- a/docs/sidebars.ts +++ b/docs/sidebars.ts @@ -31,6 +31,7 @@ const sidebars: SidebarsConfig = { 'writing-tests/test-data-row', 'writing-tests/defer-enumeration', 'writing-tests/matrix-tests', + 'writing-tests/pairwise-tests', 'writing-tests/combined-data-source', 'writing-tests/nested-data-sources', 'writing-tests/generic-attributes', diff --git a/src/TUnit.Analyzers.CodeFixers/Base/TestAttributeEnsurer.cs b/src/TUnit.Analyzers.CodeFixers/Base/TestAttributeEnsurer.cs index ed802e01894..324063176dd 100644 --- a/src/TUnit.Analyzers.CodeFixers/Base/TestAttributeEnsurer.cs +++ b/src/TUnit.Analyzers.CodeFixers/Base/TestAttributeEnsurer.cs @@ -15,7 +15,8 @@ public class TestAttributeEnsurer : CSharpSyntaxRewriter "Arguments", "MethodDataSource", "ClassDataSource", - "MatrixDataSource" + "MatrixDataSource", + "PairwiseDataSource" ]; public override SyntaxNode? VisitMethodDeclaration(MethodDeclarationSyntax node) diff --git a/src/TUnit.Analyzers.CodeFixers/MatrixDataSourceCodeFixProvider.cs b/src/TUnit.Analyzers.CodeFixers/MatrixDataSourceCodeFixProvider.cs index eaed2b9d05c..0d79190f89e 100644 --- a/src/TUnit.Analyzers.CodeFixers/MatrixDataSourceCodeFixProvider.cs +++ b/src/TUnit.Analyzers.CodeFixers/MatrixDataSourceCodeFixProvider.cs @@ -12,7 +12,8 @@ namespace TUnit.Analyzers.CodeFixers; [ExportCodeFixProvider(LanguageNames.CSharp, Name = nameof(MatrixDataSourceCodeFixProvider)), Shared] public class MatrixDataSourceCodeFixProvider : CodeFixProvider { - private const string Title = "Add [MatrixDataSource]"; + private const string MatrixTitle = "Add [MatrixDataSource]"; + private const string PairwiseTitle = "Add [PairwiseDataSource]"; public sealed override ImmutableArray FixableDiagnosticIds { get; } = ImmutableArray.Create(DiagnosticIds.MatrixDataSourceAttributeRequired); @@ -39,19 +40,35 @@ public sealed override async Task RegisterCodeFixesAsync(CodeFixContext context) context.RegisterCodeFix( CodeAction.Create( - title: Title, - createChangedDocument: c => AddMatrixDataSourceAsync(context.Document, target, c), - equivalenceKey: Title), + title: MatrixTitle, + createChangedDocument: c => AddDataSourceAsync(context.Document, target, "MatrixDataSource", c), + equivalenceKey: MatrixTitle), diagnostic); + + // [PairwiseDataSource] implements IAccessesInstanceData, which TestBuilder rejects for + // class (constructor) data sources, so only offer it for test methods. + if (target is MethodDeclarationSyntax) + { + context.RegisterCodeFix( + CodeAction.Create( + title: PairwiseTitle, + createChangedDocument: c => AddDataSourceAsync(context.Document, target, "PairwiseDataSource", c), + equivalenceKey: PairwiseTitle), + diagnostic); + } } } - private static async Task AddMatrixDataSourceAsync(Document document, SyntaxNode target, CancellationToken cancellationToken) + private static async Task AddDataSourceAsync( + Document document, + SyntaxNode target, + string attributeName, + CancellationToken cancellationToken) { var editor = await DocumentEditor.CreateAsync(document, cancellationToken).ConfigureAwait(false); var attributeList = SyntaxFactory.AttributeList( SyntaxFactory.SingletonSeparatedList( - SyntaxFactory.Attribute(SyntaxFactory.IdentifierName("MatrixDataSource")))); + SyntaxFactory.Attribute(SyntaxFactory.IdentifierName(attributeName)))); SyntaxNode updated = target switch { diff --git a/src/TUnit.Analyzers/CombinedDataSourceAnalyzer.cs b/src/TUnit.Analyzers/CombinedDataSourceAnalyzer.cs index 3b33191b021..d469f5ac807 100644 --- a/src/TUnit.Analyzers/CombinedDataSourceAnalyzer.cs +++ b/src/TUnit.Analyzers/CombinedDataSourceAnalyzer.cs @@ -105,9 +105,9 @@ private void CheckCombinedDataSourceErrors(SymbolAnalysisContext context, } } - // Rule 3: Warn if mixing CombinedDataSources with MatrixDataSource - var hasMatrixDataSource = attributes.Any(x => x.IsMatrixDataSourceAttribute(context.Compilation)); - if (hasMatrixDataSource) + // Rule 3: Warn if mixing CombinedDataSources with MatrixDataSource or PairwiseDataSource + var hasMatrixValuesDataSource = attributes.Any(x => x.IsMatrixValuesDataSourceAttribute(context.Compilation)); + if (hasMatrixValuesDataSource) { context.ReportDiagnostic( Diagnostic.Create(Rules.CombinedDataSourceConflictWithMatrix, diff --git a/src/TUnit.Analyzers/Extensions/AttributeExtensions.cs b/src/TUnit.Analyzers/Extensions/AttributeExtensions.cs index 99e1c1ef46d..133f093ab20 100644 --- a/src/TUnit.Analyzers/Extensions/AttributeExtensions.cs +++ b/src/TUnit.Analyzers/Extensions/AttributeExtensions.cs @@ -97,6 +97,22 @@ public static bool IsMatrixDataSourceAttribute(this AttributeData attributeData, TUnitSymbols.For(compilation).MatrixDataSourceAttribute); } + public static bool IsPairwiseDataSourceAttribute(this AttributeData attributeData, Compilation compilation) + { + return SymbolEqualityComparer.Default.Equals(attributeData.AttributeClass, + TUnitSymbols.For(compilation).PairwiseDataSourceAttribute); + } + + /// + /// Returns true for data sources that generate test cases from the [Matrix] values on parameters + /// ([MatrixDataSource] and [PairwiseDataSource]). + /// + public static bool IsMatrixValuesDataSourceAttribute(this AttributeData attributeData, Compilation compilation) + { + return attributeData.IsMatrixDataSourceAttribute(compilation) + || attributeData.IsPairwiseDataSourceAttribute(compilation); + } + public static bool IsCombinedDataSourceAttribute(this AttributeData attributeData, Compilation compilation) { return SymbolEqualityComparer.Default.Equals(attributeData.AttributeClass, diff --git a/src/TUnit.Analyzers/Helpers/TUnitSymbols.cs b/src/TUnit.Analyzers/Helpers/TUnitSymbols.cs index 655e8950e4a..204ac76f1c4 100644 --- a/src/TUnit.Analyzers/Helpers/TUnitSymbols.cs +++ b/src/TUnit.Analyzers/Helpers/TUnitSymbols.cs @@ -38,6 +38,7 @@ private TUnitSymbols(Compilation compilation) BaseTestAttribute = compilation.GetTypeByMetadataName("TUnit.Core.BaseTestAttribute"); MatrixAttribute = compilation.GetTypeByMetadataName(WellKnown.AttributeFullyQualifiedClasses.Matrix.WithoutGlobalPrefix); MatrixDataSourceAttribute = compilation.GetTypeByMetadataName(WellKnown.AttributeFullyQualifiedClasses.MatrixDataSourceAttribute.WithoutGlobalPrefix); + PairwiseDataSourceAttribute = compilation.GetTypeByMetadataName(WellKnown.AttributeFullyQualifiedClasses.PairwiseDataSourceAttribute.WithoutGlobalPrefix); CombinedDataSourceAttribute = compilation.GetTypeByMetadataName(WellKnown.AttributeFullyQualifiedClasses.CombinedDataSourceAttribute.WithoutGlobalPrefix); DataSourceAttributeInterface = compilation.GetTypeByMetadataName(WellKnown.AttributeFullyQualifiedClasses.IDataSourceAttribute.WithoutGlobalPrefix); } @@ -60,6 +61,8 @@ private TUnitSymbols(Compilation compilation) public INamedTypeSymbol? MatrixDataSourceAttribute { get; } + public INamedTypeSymbol? PairwiseDataSourceAttribute { get; } + public INamedTypeSymbol? CombinedDataSourceAttribute { get; } public INamedTypeSymbol? DataSourceAttributeInterface { get; } diff --git a/src/TUnit.Analyzers/Helpers/WellKnown.cs b/src/TUnit.Analyzers/Helpers/WellKnown.cs index f4cd8bf98bd..5fb94078efa 100644 --- a/src/TUnit.Analyzers/Helpers/WellKnown.cs +++ b/src/TUnit.Analyzers/Helpers/WellKnown.cs @@ -8,6 +8,7 @@ public static class AttributeFullyQualifiedClasses public static readonly FullyQualifiedTypeName Explicit = GetTypeName("ExplicitAttribute"); public static readonly FullyQualifiedTypeName Matrix = GetTypeName("MatrixAttribute"); public static readonly FullyQualifiedTypeName MatrixDataSourceAttribute = GetTypeName("MatrixDataSourceAttribute"); + public static readonly FullyQualifiedTypeName PairwiseDataSourceAttribute = GetTypeName("PairwiseDataSourceAttribute"); public static readonly FullyQualifiedTypeName CombinedDataSourceAttribute = GetTypeName("CombinedDataSourcesAttribute"); public static readonly FullyQualifiedTypeName BeforeAttribute = GetTypeName("BeforeAttribute"); diff --git a/src/TUnit.Analyzers/MatrixAnalyzer.cs b/src/TUnit.Analyzers/MatrixAnalyzer.cs index 5430ddc8950..d83951a11a4 100644 --- a/src/TUnit.Analyzers/MatrixAnalyzer.cs +++ b/src/TUnit.Analyzers/MatrixAnalyzer.cs @@ -66,7 +66,7 @@ private void CheckMatrixErrors(SymbolAnalysisContext context, ImmutableArray x.IsMatrixDataSourceAttribute(context.Compilation))) + if (!attributes.Any(x => x.IsMatrixValuesDataSourceAttribute(context.Compilation))) { context.ReportDiagnostic( Diagnostic.Create(Rules.MatrixDataSourceAttributeRequired, diff --git a/src/TUnit.Analyzers/Resources.Designer.cs b/src/TUnit.Analyzers/Resources.Designer.cs index 7281858658e..a281039eb22 100644 --- a/src/TUnit.Analyzers/Resources.Designer.cs +++ b/src/TUnit.Analyzers/Resources.Designer.cs @@ -1326,7 +1326,7 @@ internal static string TUnit0048Title { } /// - /// Looks up a localized string similar to [MatrixDataSourceAttribute] is required if using [Matrix] values on your parameters.. + /// Looks up a localized string similar to [MatrixDataSource] or [PairwiseDataSource] is required if using [Matrix] values on your parameters.. /// internal static string TUnit0049Description { get { @@ -1335,7 +1335,7 @@ internal static string TUnit0049Description { } /// - /// Looks up a localized string similar to [MatrixDataSourceAttribute] is required if using [Matrix] values on your parameters.. + /// Looks up a localized string similar to [MatrixDataSource] or [PairwiseDataSource] is required if using [Matrix] values on your parameters.. /// internal static string TUnit0049MessageFormat { get { @@ -1344,7 +1344,7 @@ internal static string TUnit0049MessageFormat { } /// - /// Looks up a localized string similar to [MatrixDataSourceAttribute] is required. + /// Looks up a localized string similar to [MatrixDataSource] or [PairwiseDataSource] is required. /// internal static string TUnit0049Title { get { diff --git a/src/TUnit.Analyzers/Resources.resx b/src/TUnit.Analyzers/Resources.resx index 444162b8dca..48070608a30 100644 --- a/src/TUnit.Analyzers/Resources.resx +++ b/src/TUnit.Analyzers/Resources.resx @@ -346,13 +346,13 @@ Test methods must not be static - [MatrixDataSourceAttribute] is required if using [Matrix] values on your parameters. + [MatrixDataSource] or [PairwiseDataSource] is required if using [Matrix] values on your parameters. - [MatrixDataSourceAttribute] is required if using [Matrix] values on your parameters. + [MatrixDataSource] or [PairwiseDataSource] is required if using [Matrix] values on your parameters. - [MatrixDataSourceAttribute] is required + [MatrixDataSource] or [PairwiseDataSource] is required Too many arguments provided. @@ -481,10 +481,10 @@ Parameter missing data source attribute - Using [CombinedDataSources] together with [MatrixDataSource] is not recommended as they serve different purposes and may cause confusion. + Using [CombinedDataSources] together with [MatrixDataSource] or [PairwiseDataSource] is not recommended as they serve different purposes and may cause confusion. - [CombinedDataSources] should not be used with [MatrixDataSource]. Use one or the other. + [CombinedDataSources] should not be used with [MatrixDataSource] or [PairwiseDataSource]. Use one or the other. Conflicting data source attributes diff --git a/src/TUnit.Core/Attributes/TestData/MatrixDataSourceAttribute.cs b/src/TUnit.Core/Attributes/TestData/MatrixDataSourceAttribute.cs index 25e1684d843..8f9f150f44b 100644 --- a/src/TUnit.Core/Attributes/TestData/MatrixDataSourceAttribute.cs +++ b/src/TUnit.Core/Attributes/TestData/MatrixDataSourceAttribute.cs @@ -1,7 +1,4 @@ -using System.Diagnostics.CodeAnalysis; -using TUnit.Core.Enums; -using TUnit.Core.Extensions; -using TUnit.Core.Helpers; +using TUnit.Core.Helpers; namespace TUnit.Core; @@ -17,6 +14,10 @@ namespace TUnit.Core; /// /// Use on the test method to exclude specific combinations. /// +/// +/// When the full Cartesian product is too large, consider , +/// which uses the same parameter attributes but only guarantees that every pair of values is covered. +/// /// /// /// @@ -40,50 +41,13 @@ public sealed class MatrixDataSourceAttribute : UntypedDataSourceGeneratorAttrib { protected override IEnumerable> GenerateDataSources(DataGeneratorMetadata dataGeneratorMetadata) { - var parameterInformation = dataGeneratorMetadata - .MembersToGenerate - .OfType() - .ToArray(); - - if (parameterInformation.Length != dataGeneratorMetadata.MembersToGenerate.Length - || parameterInformation.Length is 0) - { - throw new Exception("[MatrixDataSource] only supports parameterised tests"); - } - - if (dataGeneratorMetadata.TestInformation == null) - { - throw new InvalidOperationException("MatrixDataSource requires test information but none is available. This may occur during static property initialization."); - } - - var testInformation = dataGeneratorMetadata.TestInformation; - - var classType = testInformation.Class.Type; - - - var exclusions = GetExclusions(dataGeneratorMetadata.Type == DataGeneratorType.TestParameters - ? dataGeneratorMetadata.TestInformation.GetCustomAttributes() - : classType.GetCustomAttributesSafe()); - - var valueSets = new IReadOnlyList[parameterInformation.Length]; - for (var i = 0; i < parameterInformation.Length; i++) - { - valueSets[i] = GetAllArguments(dataGeneratorMetadata, parameterInformation[i]); - } + var parameterInformation = MatrixParameterValues.GetParameters(dataGeneratorMetadata, "MatrixDataSource"); + var exclusions = MatrixParameterValues.GetExclusions(dataGeneratorMetadata); + var valueSets = MatrixParameterValues.GetValueSets(dataGeneratorMetadata, parameterInformation); foreach (var rowArray in CartesianProductHelper.GetCartesianProduct(valueSets)) { - var excluded = false; - foreach (var exclusion in exclusions) - { - if (IsExcluded(exclusion, rowArray)) - { - excluded = true; - break; - } - } - - if (excluded) + if (MatrixParameterValues.IsExcluded(exclusions, rowArray)) { continue; } @@ -91,160 +55,4 @@ public sealed class MatrixDataSourceAttribute : UntypedDataSourceGeneratorAttrib yield return () => rowArray; } } - - private bool IsExcluded(object?[] exclusion, object?[] rowArray) - { - if (exclusion.Length != rowArray.Length) - { - return false; - } - - for (var i = 0; i < exclusion.Length; i++) - { - var exclusionValue = exclusion[i]; - var rowValue = rowArray[i]; - - // Handle enum to underlying type conversion for both values - if (exclusionValue != null && exclusionValue.GetType().IsEnum) - { - exclusionValue = Convert.ChangeType(exclusionValue, Enum.GetUnderlyingType(exclusionValue.GetType())); - } - - if (rowValue != null && rowValue.GetType().IsEnum) - { - rowValue = Convert.ChangeType(rowValue, Enum.GetUnderlyingType(rowValue.GetType())); - } - - if (!Equals(exclusionValue, rowValue)) - { - return false; - } - } - - return true; - } - - private static object?[][] GetExclusions(IEnumerable attributes) - { - List? exclusions = null; - - foreach (var attribute in attributes) - { - if (attribute is MatrixExclusionAttribute exclusionAttribute) - { - (exclusions ??= []).Add(exclusionAttribute.Objects); - } - } - - return exclusions is null ? [] : [.. exclusions]; - } - - private IReadOnlyList GetAllArguments(DataGeneratorMetadata dataGeneratorMetadata, - ParameterMetadata sourceGeneratedParameterInformation) - { - // Get MatrixAttribute on this parameter - // Prefer cached attributes from source generator for AOT compatibility - MatrixAttribute? matrixAttribute; - - if (sourceGeneratedParameterInformation.CachedDataSourceAttributes != null) - { - // Source-generated mode: use cached attributes (no reflection!) - matrixAttribute = sourceGeneratedParameterInformation.CachedDataSourceAttributes - .OfType() - .FirstOrDefault(); - } - else - { - // Reflection mode: fall back to runtime attribute discovery - if (sourceGeneratedParameterInformation.ReflectionInfo == null) - { - throw new InvalidOperationException($"Parameter reflection information is not available for parameter '{sourceGeneratedParameterInformation.Name}'. This typically occurs when using instance method data sources which are not supported at compile time."); - } - - matrixAttribute = sourceGeneratedParameterInformation.ReflectionInfo.GetCustomAttributesSafe() - .OfType() - .FirstOrDefault(); - } - - // Check if this is an instance data attribute and we don't have an instance - if (matrixAttribute is IAccessesInstanceData && dataGeneratorMetadata.TestClassInstance == null) - { - var className = dataGeneratorMetadata.TestInformation?.Class.Type.Name ?? "Unknown"; - if (dataGeneratorMetadata.TestInformation?.Class.Type.IsGenericTypeDefinition ?? false) - { - throw new InvalidOperationException( - $"Cannot use MatrixInstanceMethod attribute in generic class '{className}' when the generic type parameters " + - $"must be inferred from the matrix values. This creates a circular dependency: " + - $"the instance is needed to get the matrix values, but the generic types (which come from the matrix values) " + - $"are needed to create the instance. Consider using static methods for matrix data sources in generic classes, " + - $"or provide the generic type arguments explicitly using [Arguments] or other data source attributes."); - } - - throw new InvalidOperationException( - $"Instance is required for MatrixInstanceMethod but no instance is available. " + - $"This typically happens when the test class requires data that hasn't been expanded yet."); - } - - var objects = matrixAttribute?.GetObjects(dataGeneratorMetadata); - - if (matrixAttribute is not null && objects is { Length: > 0 }) - { - return matrixAttribute.Excluding is not null - ? objects.Except(matrixAttribute.Excluding).ToArray() - : objects; - } - - var type = sourceGeneratedParameterInformation.Type; - - // Use the IsNullable property for AOT-safe nullable detection - Type? underlyingType = null; - var isNullable = sourceGeneratedParameterInformation.IsNullable; - - if (isNullable) - { - // Try to get underlying type, but if it fails in AOT, we'll handle it - underlyingType = Nullable.GetUnderlyingType(type); - - // If Nullable.GetUnderlyingType failed but we know it's nullable from metadata, - // check if it's a generic type with one type argument - if (underlyingType == null && type.IsGenericType && type.GetGenericArguments().Length == 1) - { - underlyingType = type.GetGenericArguments()[0]; - } - } - - var resolvedType = underlyingType ?? type; - - if (resolvedType == typeof(bool)) - { - if (matrixAttribute?.Excluding is not null) - { - throw new InvalidOperationException("Do not exclude values from a boolean."); - } - - return isNullable ? [true, false, null] : [true, false]; - } - - if (resolvedType.IsEnum) - { - var enumValues = Enum.GetValuesAsUnderlyingType(resolvedType) - .Cast(); - - if (isNullable) - { - enumValues = enumValues.Append(null); - if (matrixAttribute?.Excluding?.Any(x => x is null) ?? false) - { - throw new InvalidOperationException("Do not exclude null from a nullable enum - instead use the enum directly"); - } - } - - return enumValues - .Except(matrixAttribute?.Excluding?.Select(e => Convert.ChangeType(e, Enum.GetUnderlyingType(resolvedType))) ?? []) - .ToArray(); - } - - throw new ArgumentNullException($"No MatrixAttribute found for parameter '{sourceGeneratedParameterInformation.Name}' and the parameter type '{resolvedType.Name}' cannot be auto-generated. Only bool and enum types support auto-generation."); - } - } diff --git a/src/TUnit.Core/Attributes/TestData/PairwiseDataSourceAttribute.cs b/src/TUnit.Core/Attributes/TestData/PairwiseDataSourceAttribute.cs new file mode 100644 index 00000000000..82dd26cb0c9 --- /dev/null +++ b/src/TUnit.Core/Attributes/TestData/PairwiseDataSourceAttribute.cs @@ -0,0 +1,114 @@ +using TUnit.Core.Helpers; + +namespace TUnit.Core; + +/// +/// Generates test cases that cover every pair of parameter values at least once +/// (all-pairs / pairwise testing), instead of every combination. +/// +/// +/// +/// This is an alternative to for tests with many parameters, +/// where the full Cartesian product becomes too large. Most defects are triggered by the interaction of +/// at most two parameters, so covering every pair of values usually catches them with far fewer test cases. +/// For example, four parameters with 2, 2, 4 and 4 values produce 64 cases with +/// but only 16 with . +/// +/// +/// Parameter values are discovered exactly as for : +/// [Matrix(...)], [Matrix<T>], [MatrixRange<T>], [MatrixMethod<T>] +/// and [MatrixInstanceMethod<T>] on individual parameters, with honored. +/// , enum and nullable /enum parameters get all of their values automatically. +/// +/// +/// is applied as a constraint during generation, +/// so every pair of values that can appear in an allowed test case is still covered. +/// +/// +/// Generation is deterministic: the same parameter values and always produce the same test cases. +/// +/// +/// +/// +/// [Test, PairwiseDataSource] +/// public void TestPairs( +/// bool largeArrayPool, +/// bool excessSpan, +/// [Matrix(0, 99, 100, 101)] int sizeHint, +/// [Matrix(1, 2, 3, 100)] int stepCount) +/// { +/// // 16 test cases instead of the 64 that [MatrixDataSource] would generate. +/// } +/// +/// +[AttributeUsage(AttributeTargets.Class | AttributeTargets.Method)] +public sealed class PairwiseDataSourceAttribute : UntypedDataSourceGeneratorAttribute, IAccessesInstanceData +{ + /// + /// The default value of . + /// + public const int DefaultSeed = 15485863; + + /// + /// Gets or sets the seed for the pseudo-random number generator used to build the covering set. + /// The generated test cases are fully deterministic for a given seed and set of parameter values; + /// changing the seed produces a different (but equally valid) set of test cases. + /// + public int Seed { get; set; } = DefaultSeed; + + protected override IEnumerable> GenerateDataSources(DataGeneratorMetadata dataGeneratorMetadata) + { + var parameterInformation = MatrixParameterValues.GetParameters(dataGeneratorMetadata, "PairwiseDataSource"); + var exclusions = MatrixParameterValues.GetExclusions(dataGeneratorMetadata); + var valueSets = MatrixParameterValues.GetValueSets(dataGeneratorMetadata, parameterInformation); + + foreach (var row in GetTestCases(valueSets, exclusions, Seed)) + { + yield return () => row; + } + } + + internal static List GetTestCases(IReadOnlyList[] valueSets, object?[][] exclusions, int seed) + { + var dimensions = new int[valueSets.Length]; + for (var i = 0; i < valueSets.Length; i++) + { + dimensions[i] = valueSets[i].Count; + if (dimensions[i] == 0) + { + return []; + } + } + + Func? isAllowed = null; + if (exclusions.Length > 0) + { + // Shared across predicate invocations; generation is single-threaded, so this is safe. + var buffer = new object?[valueSets.Length]; + isAllowed = indices => + { + for (var i = 0; i < indices.Length; i++) + { + buffer[i] = valueSets[i][indices[i]]; + } + + return !MatrixParameterValues.IsExcluded(exclusions, buffer); + }; + } + + var testCases = PairwiseStrategy.GetTestCases(dimensions, isAllowed, seed); + var rows = new List(testCases.Length); + foreach (var testCase in testCases) + { + var row = new object?[testCase.Length]; + for (var i = 0; i < testCase.Length; i++) + { + row[i] = valueSets[i][testCase[i]]; + } + + rows.Add(row); + } + + return rows; + } +} diff --git a/src/TUnit.Core/Helpers/MatrixParameterValues.cs b/src/TUnit.Core/Helpers/MatrixParameterValues.cs new file mode 100644 index 00000000000..b84f8d40972 --- /dev/null +++ b/src/TUnit.Core/Helpers/MatrixParameterValues.cs @@ -0,0 +1,217 @@ +using TUnit.Core.Enums; +using TUnit.Core.Extensions; + +namespace TUnit.Core.Helpers; + +/// +/// Shared value discovery for the parameter-driven data sources +/// ( and ): +/// reads (and derived attributes) from each parameter, applies the +/// bool/enum/nullable defaults, and reads s. +/// +internal static class MatrixParameterValues +{ + public static ParameterMetadata[] GetParameters(DataGeneratorMetadata dataGeneratorMetadata, string attributeName) + { + var parameterInformation = dataGeneratorMetadata + .MembersToGenerate + .OfType() + .ToArray(); + + if (parameterInformation.Length != dataGeneratorMetadata.MembersToGenerate.Length + || parameterInformation.Length is 0) + { + throw new Exception($"[{attributeName}] only supports parameterised tests"); + } + + if (dataGeneratorMetadata.TestInformation == null) + { + throw new InvalidOperationException($"{attributeName} requires test information but none is available. This may occur during static property initialization."); + } + + return parameterInformation; + } + + public static IReadOnlyList[] GetValueSets(DataGeneratorMetadata dataGeneratorMetadata, ParameterMetadata[] parameterInformation) + { + var valueSets = new IReadOnlyList[parameterInformation.Length]; + for (var i = 0; i < parameterInformation.Length; i++) + { + valueSets[i] = GetAllArguments(dataGeneratorMetadata, parameterInformation[i]); + } + + return valueSets; + } + + public static object?[][] GetExclusions(DataGeneratorMetadata dataGeneratorMetadata) + { + var attributes = dataGeneratorMetadata.Type == DataGeneratorType.TestParameters + ? dataGeneratorMetadata.TestInformation!.GetCustomAttributes() + : dataGeneratorMetadata.TestInformation!.Class.Type.GetCustomAttributesSafe(); + + List? exclusions = null; + + foreach (var attribute in attributes) + { + if (attribute is MatrixExclusionAttribute exclusionAttribute) + { + (exclusions ??= []).Add(exclusionAttribute.Objects); + } + } + + return exclusions is null ? [] : [.. exclusions]; + } + + public static bool IsExcluded(object?[][] exclusions, object?[] rowArray) + { + foreach (var exclusion in exclusions) + { + if (IsExcluded(exclusion, rowArray)) + { + return true; + } + } + + return false; + } + + private static bool IsExcluded(object?[] exclusion, object?[] rowArray) + { + if (exclusion.Length != rowArray.Length) + { + return false; + } + + for (var i = 0; i < exclusion.Length; i++) + { + var exclusionValue = exclusion[i]; + var rowValue = rowArray[i]; + + // Handle enum to underlying type conversion for both values + if (exclusionValue != null && exclusionValue.GetType().IsEnum) + { + exclusionValue = Convert.ChangeType(exclusionValue, Enum.GetUnderlyingType(exclusionValue.GetType())); + } + + if (rowValue != null && rowValue.GetType().IsEnum) + { + rowValue = Convert.ChangeType(rowValue, Enum.GetUnderlyingType(rowValue.GetType())); + } + + if (!Equals(exclusionValue, rowValue)) + { + return false; + } + } + + return true; + } + + private static IReadOnlyList GetAllArguments(DataGeneratorMetadata dataGeneratorMetadata, + ParameterMetadata sourceGeneratedParameterInformation) + { + // Get MatrixAttribute on this parameter + // Prefer cached attributes from source generator for AOT compatibility + MatrixAttribute? matrixAttribute; + + if (sourceGeneratedParameterInformation.CachedDataSourceAttributes != null) + { + // Source-generated mode: use cached attributes (no reflection!) + matrixAttribute = sourceGeneratedParameterInformation.CachedDataSourceAttributes + .OfType() + .FirstOrDefault(); + } + else + { + // Reflection mode: fall back to runtime attribute discovery + if (sourceGeneratedParameterInformation.ReflectionInfo == null) + { + throw new InvalidOperationException($"Parameter reflection information is not available for parameter '{sourceGeneratedParameterInformation.Name}'. This typically occurs when using instance method data sources which are not supported at compile time."); + } + + matrixAttribute = sourceGeneratedParameterInformation.ReflectionInfo.GetCustomAttributesSafe() + .OfType() + .FirstOrDefault(); + } + + // Check if this is an instance data attribute and we don't have an instance + if (matrixAttribute is IAccessesInstanceData && dataGeneratorMetadata.TestClassInstance == null) + { + var className = dataGeneratorMetadata.TestInformation?.Class.Type.Name ?? "Unknown"; + if (dataGeneratorMetadata.TestInformation?.Class.Type.IsGenericTypeDefinition ?? false) + { + throw new InvalidOperationException( + $"Cannot use MatrixInstanceMethod attribute in generic class '{className}' when the generic type parameters " + + $"must be inferred from the matrix values. This creates a circular dependency: " + + $"the instance is needed to get the matrix values, but the generic types (which come from the matrix values) " + + $"are needed to create the instance. Consider using static methods for matrix data sources in generic classes, " + + $"or provide the generic type arguments explicitly using [Arguments] or other data source attributes."); + } + + throw new InvalidOperationException( + $"Instance is required for MatrixInstanceMethod but no instance is available. " + + $"This typically happens when the test class requires data that hasn't been expanded yet."); + } + + var objects = matrixAttribute?.GetObjects(dataGeneratorMetadata); + + if (matrixAttribute is not null && objects is { Length: > 0 }) + { + return matrixAttribute.Excluding is not null + ? objects.Except(matrixAttribute.Excluding).ToArray() + : objects; + } + + var type = sourceGeneratedParameterInformation.Type; + + // Use the IsNullable property for AOT-safe nullable detection + Type? underlyingType = null; + var isNullable = sourceGeneratedParameterInformation.IsNullable; + + if (isNullable) + { + // Try to get underlying type, but if it fails in AOT, we'll handle it + underlyingType = Nullable.GetUnderlyingType(type); + + // If Nullable.GetUnderlyingType failed but we know it's nullable from metadata, + // check if it's a generic type with one type argument + if (underlyingType == null && type.IsGenericType && type.GetGenericArguments().Length == 1) + { + underlyingType = type.GetGenericArguments()[0]; + } + } + + var resolvedType = underlyingType ?? type; + + if (resolvedType == typeof(bool)) + { + if (matrixAttribute?.Excluding is not null) + { + throw new InvalidOperationException("Do not exclude values from a boolean."); + } + + return isNullable ? [true, false, null] : [true, false]; + } + + if (resolvedType.IsEnum) + { + var enumValues = Enum.GetValuesAsUnderlyingType(resolvedType) + .Cast(); + + if (isNullable) + { + enumValues = enumValues.Append(null); + if (matrixAttribute?.Excluding?.Any(x => x is null) ?? false) + { + throw new InvalidOperationException("Do not exclude null from a nullable enum - instead use the enum directly"); + } + } + + return enumValues + .Except(matrixAttribute?.Excluding?.Select(e => Convert.ChangeType(e, Enum.GetUnderlyingType(resolvedType))) ?? []) + .ToArray(); + } + + throw new ArgumentNullException($"No MatrixAttribute found for parameter '{sourceGeneratedParameterInformation.Name}' and the parameter type '{resolvedType.Name}' cannot be auto-generated. Only bool and enum types support auto-generation."); + } +} diff --git a/src/TUnit.Core/Helpers/PairwiseStrategy.cs b/src/TUnit.Core/Helpers/PairwiseStrategy.cs new file mode 100644 index 00000000000..57842addc2f --- /dev/null +++ b/src/TUnit.Core/Helpers/PairwiseStrategy.cs @@ -0,0 +1,734 @@ +// *********************************************************************** +// Pairwise (all-pairs) test case generation. +// +// Provenance: +// - Originally written by Charlie Poole for NUnit's PairwiseStrategy (MIT), +// itself based on the "jenny" tool by Bob Jenkins. +// - Modified by Andrew Arnott in Xunit.Combinatorial (2015 onwards), including +// a configurable seed and constraint predicates that reject test cases +// during generation. +// - Contributed to TUnit under the MIT license by Andrew Arnott, who +// relicenses his modifications under the MIT license below. +// +// Copyright (c) 2008 Charlie Poole +// Copyright (c) 2015 Andrew Arnott (modified from Charlie's original) +// +// Permission is hereby granted, free of charge, to any person obtaining +// a copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to +// permit persons to whom the Software is furnished to do so, subject to +// the following conditions: +// +// The above copyright notice and this permission notice shall be +// included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// *********************************************************************** +namespace TUnit.Core.Helpers; + +/// +/// PairwiseStrategy creates test cases by combining the parameter +/// data so that all possible pairs of data items are used. +/// +/// +/// +/// The number of test cases that cover all possible pairs of test function +/// parameters values is significantly less than the number of test cases +/// that cover all possible combination of test function parameters values. +/// And because different studies show that most of software failures are +/// caused by combination of no more than two parameters, pairwise testing +/// can be an effective ways to test the system when it's impossible to test +/// all combinations of parameters. +/// +/// +/// The PairwiseStrategy code is based on "jenny" tool by Bob Jenkins: +/// . +/// +/// +internal static class PairwiseStrategy +{ + // NOTE: Terminology in this class is based on the literature + // relating to strategies for combining variable features when + // creating tests. This terminology is more closely related to + // higher level testing than it is to unit tests. See + // comments in the code for further explanations. + + /// + /// Creates a set of test cases for specified dimensions. + /// + /// + /// An array which contains information about dimensions. Each element of + /// this array represents a number of features in the specific dimension. + /// + /// A predicate that returns when a generated test case is allowed. + /// The seed for the pseudo-random number generator. + /// + /// A set of test cases. + /// + public static int[][] GetTestCases(int[] dimensions, Func? isTestCaseAllowed, int seed) + { + var testCases = new PairwiseTestCaseGenerator().GetTestCases(dimensions, isTestCaseAllowed, seed); + var result = new int[testCases.Count][]; + for (var i = 0; i < result.Length; i++) + { + result[i] = testCases[i].Features; + } + + return result; + } + + private static bool IsTupleCovered(this TestCaseInfo testCaseInfo, FeatureTuple tuple) + { + for (int i = 0; i < tuple.Length; i++) + { + if (testCaseInfo.Features[tuple[i].Dimension] != tuple[i].Feature) + { + return false; + } + } + + return true; + } + + /// + /// FleaRand is a pseudo-random number generator developed by Bob Jenkins: + /// . + /// + private class FleaRand + { + private uint _b; + private uint _c; + private uint _d; + private uint _z; + private uint[] _m; + private uint[] _r; + private uint _q; + + /// + /// Initializes a new instance of the class. + /// + /// The seed. + public FleaRand(uint seed) + { + _b = seed; + _c = seed; + _d = seed; + _z = seed; + _m = new uint[256]; + _r = new uint[256]; + + for (int i = 0; i < _m.Length; i++) + { + _m[i] = seed; + } + + for (int i = 0; i < 10; i++) + { + Batch(); + } + + _q = 0; + } + + public uint Next() + { + if (_q == 0) + { + Batch(); + _q = (uint)_r.Length - 1; + } + else + { + _q--; + } + + return _r[_q]; + } + + private void Batch() + { + uint a; + uint b = _b; + uint c = _c + (++_z); + uint d = _d; + + for (int i = 0; i < _r.Length; i++) + { + a = _m[b % _m.Length]; + _m[b % _m.Length] = d; + d = (c << 19) + (c >> 13) + b; + c = b ^ _m[i]; + b = a + d; + _r[i] = c; + } + + _b = b; + _c = c; + _d = d; + } + } + + /// + /// FeatureInfo represents coverage of a single value of test function + /// parameter, represented as a pair of indices, Dimension and Feature. In + /// terms of unit testing, Dimension is the index of the test parameter and + /// Feature is the index of the supplied value in that parameter's list of + /// sources. + /// + private class FeatureInfo + { + /// + /// Initializes a new instance of the class. + /// + /// Index of a dimension. + /// Index of a feature. + public FeatureInfo(int dimension, int feature) + { + Dimension = dimension; + Feature = feature; + } + + public int Dimension { get; } + + public int Feature { get; } + } + + /// + /// A FeatureTuple represents a combination of features, one per test + /// parameter, which should be covered by a test case. In the + /// PairwiseStrategy, we are only trying to cover pairs of features, so the + /// tuples actually may contain only single feature or pair of features, but + /// the algorithm itself works with triplets, quadruples and so on. + /// + private class FeatureTuple + { + private readonly FeatureInfo[] _features; + + /// + /// Initializes a new instance of the class + /// for a single feature. + /// + /// Single feature. + public FeatureTuple(FeatureInfo feature1) + { + _features = new FeatureInfo[] { feature1 }; + } + + /// + /// Initializes a new instance of the class + /// for a pair of features. + /// + /// First feature. + /// Second feature. + public FeatureTuple(FeatureInfo feature1, FeatureInfo feature2) + { + _features = new FeatureInfo[] { feature1, feature2 }; + } + + public int Length + { + get + { + return _features.Length; + } + } + + public FeatureInfo this[int index] + { + get + { + return _features[index]; + } + } + } + + /// + /// TestCase represents a single test case covering a list of features. + /// + private class TestCaseInfo + { + /// + /// Initializes a new instance of the class. + /// + /// A number of features in the test case. + public TestCaseInfo(int length) + { + Features = new int[length]; + } + + public int[] Features { get; } + } + + /// + /// PairwiseTestCaseGenerator class implements an algorithm which generates + /// a set of test cases which covers all pairs of possible values of test + /// function. + /// + /// + /// + /// The algorithm starts with creating a set of all feature tuples which we + /// will try to cover (see method). This set + /// includes every single feature and all possible pairs of features. We + /// store feature tuples in the 3-D collection (where axes are "dimension", + /// "feature", and "all combinations which includes this feature"), and for + /// every two feature (e.g. "A" and "B") we generate both ("A", "B") and + /// ("B", "A") pairs. This data structure extremely reduces the amount of + /// time needed to calculate coverage for a single test case (this + /// calculation is the most time-consuming part of the algorithm). + /// + /// + /// Then the algorithm picks one tuple from the uncovered tuple, creates a + /// test case that covers this tuple, and then removes this tuple and all + /// other tuples covered by this test case from the collection of uncovered + /// tuples. + /// + /// + /// Picking a tuple to cover. + /// + /// + /// There are no any special rules defined for picking tuples to cover. We + /// just pick them one by one, in the order they were generated. + /// + /// + /// Test generation. + /// + /// + /// Test generation starts from creating a completely random test case which + /// covers, nevertheless, previously selected tuple. Then the algorithm + /// tries to maximize number of tuples which this test covers. + /// + /// + /// Test generation and maximization process repeats seven times for every + /// selected tuple and then the algorithm picks the best test case ("seven" + /// is a magic number which provides good results in acceptable time). + /// + /// Maximizing test coverage. + /// + /// To maximize tests coverage, the algorithm walks thru the list of mutable + /// dimensions (mutable dimension is a dimension that are not included in + /// the previously selected tuple). Then for every dimension, the algorithm + /// walks thru the list of features and checks if this feature provides + /// better coverage than randomly selected feature, and if yes keeps this + /// feature. + /// + /// + /// This process repeats while it shows progress. If the last iteration + /// doesn't improve coverage, the process ends. + /// + /// + /// In addition, for better results, before start every iteration, the + /// algorithm "scrambles" dimensions - so for every iteration dimension + /// probes in a different order. + /// + /// + private class PairwiseTestCaseGenerator + { + private FleaRand? _prng; + + private int[]? _dimensions; + + private List[][]? _uncoveredTuples; + + /// + /// A predicate that determines whether a generated test case satisfies caller-supplied constraints. + /// + private Func? _isTestCaseAllowed; + + /// + /// Creates a set of test cases for specified dimensions. + /// + /// + /// An array which contains information about dimensions. Each element of + /// this array represents a number of features in the specific dimension. + /// + /// A predicate that returns when a generated test case is allowed. + /// The seed for the pseudo-random number generator. + /// + /// A set of test cases. + /// + public List GetTestCases(int[] dimensions, Func? isTestCaseAllowed, int seed) + { + _prng = new FleaRand(unchecked((uint)seed)); + _dimensions = dimensions; + _isTestCaseAllowed = isTestCaseAllowed; + + CreateAllTuples(); + + List testCases = new List(); + + while (true) + { + FeatureTuple? tuple = GetNextTuple(); + + if (tuple is null) + { + break; + } + + TestCaseInfo? testCase = CreateTestCase(tuple); + + if (testCase is null) + { + // No allowed row can cover this tuple, so remove it to avoid repeatedly trying to satisfy an impossible constraint. + RemoveTuple(tuple); + continue; + } + + RemoveTuplesCoveredByTest(testCase); + + testCases.Add(testCase); + } + + return testCases; + } + + private int GetNextRandomNumber() + { + return (int)(_prng!.Next() >> 1); + } + + private void CreateAllTuples() + { + _uncoveredTuples = new List[_dimensions!.Length][]; + + for (int d = 0; d < _dimensions.Length; d++) + { + _uncoveredTuples[d] = new List[_dimensions[d]]; + + for (int f = 0; f < _dimensions[d]; f++) + { + _uncoveredTuples[d][f] = CreateTuples(d, f); + } + } + } + + private List CreateTuples(int dimension, int feature) + { + List result = new List(); + + result.Add(new FeatureTuple(new FeatureInfo(dimension, feature))); + + for (int d = 0; d < _dimensions!.Length; d++) + { + if (d != dimension) + { + for (int f = 0; f < _dimensions[d]; f++) + { + result.Add(new FeatureTuple(new FeatureInfo(dimension, feature), new FeatureInfo(d, f))); + } + } + } + + return result; + } + + private FeatureTuple? GetNextTuple() + { + for (int d = 0; d < _uncoveredTuples!.Length; d++) + { + for (int f = 0; f < _uncoveredTuples[d].Length; f++) + { + List tuples = _uncoveredTuples[d][f]; + + if (tuples.Count > 0) + { + FeatureTuple tuple = tuples[0]; + tuples.RemoveAt(0); + return tuple; + } + } + } + + return null; + } + + private TestCaseInfo? CreateTestCase(FeatureTuple tuple) + { + TestCaseInfo? bestTestCase = null; + int bestCoverage = -1; + + for (int i = 0; i < 7; i++) + { + TestCaseInfo? testCase = CreateRandomTestCase(tuple); + if (testCase is null) + { + // FindAllowedTestCase already searched every completion of this tuple. + // Further random attempts cannot succeed, so stop immediately. + return null; + } + + int coverage = MaximizeCoverage(testCase, tuple); + + if (coverage > bestCoverage) + { + bestTestCase = testCase; + bestCoverage = coverage; + } + } + + return bestTestCase; + } + + private TestCaseInfo? CreateRandomTestCase(FeatureTuple tuple) + { + TestCaseInfo result = new TestCaseInfo(_dimensions!.Length); + + for (int d = 0; d < _dimensions.Length; d++) + { + result.Features[d] = GetNextRandomNumber() % _dimensions[d]; + } + + for (int i = 0; i < tuple.Length; i++) + { + result.Features[tuple[i].Dimension] = tuple[i].Feature; + } + + return IsTestCaseAllowed(result) ? result : FindAllowedTestCase(tuple); + } + + private int MaximizeCoverage(TestCaseInfo testCase, FeatureTuple tuple) + { + // It starts with one because we always have one tuple which is covered by the test. + int totalCoverage = 1; + int[] mutableDimensions = GetMutableDimensions(tuple); + + while (true) + { + bool progress = false; + + ScrambleDimensions(mutableDimensions); + + for (int i = 0; i < mutableDimensions.Length; i++) + { + int d = mutableDimensions[i]; + + int bestCoverage = CountTuplesCoveredByTest(testCase, d, testCase.Features[d]); + + int newCoverage = MaximizeCoverageForDimension(testCase, d, bestCoverage); + + totalCoverage += newCoverage; + + if (newCoverage > bestCoverage) + { + progress = true; + } + } + + if (!progress) + { + return totalCoverage; + } + } + } + + private int[] GetMutableDimensions(FeatureTuple tuple) + { + List result = new List(); + + bool[] immutableDimensions = new bool[_dimensions!.Length]; + + for (int i = 0; i < tuple.Length; i++) + { + immutableDimensions[tuple[i].Dimension] = true; + } + + for (int d = 0; d < _dimensions.Length; d++) + { + if (!immutableDimensions[d]) + { + result.Add(d); + } + } + + return result.ToArray(); + } + + private void ScrambleDimensions(int[] dimensions) + { + for (int i = 0; i < dimensions.Length; i++) + { + int j = GetNextRandomNumber() % dimensions.Length; + int t = dimensions[i]; + dimensions[i] = dimensions[j]; + dimensions[j] = t; + } + } + + private int MaximizeCoverageForDimension(TestCaseInfo testCase, int dimension, int bestCoverage) + { + List bestFeatures = new List(_dimensions![dimension]); + + for (int f = 0; f < _dimensions[dimension]; f++) + { + testCase.Features[dimension] = f; + + if (!IsTestCaseAllowed(testCase)) + { + continue; + } + + int coverage = CountTuplesCoveredByTest(testCase, dimension, f); + + if (coverage >= bestCoverage) + { + if (coverage > bestCoverage) + { + bestCoverage = coverage; + bestFeatures.Clear(); + } + + bestFeatures.Add(f); + } + } + + testCase.Features[dimension] = bestFeatures[GetNextRandomNumber() % bestFeatures.Count]; + + return bestCoverage; + } + + /// + /// Exhaustively searches the feature space to find any allowed test case that covers a tuple when a random test case is vetoed. + /// + /// The tuple that must be covered by the returned test case. + /// An allowed test case, or when none can cover . + private TestCaseInfo? FindAllowedTestCase(FeatureTuple tuple) + { + TestCaseInfo result = new TestCaseInfo(_dimensions!.Length); + bool[] initializedDimensions = new bool[_dimensions.Length]; + + for (int i = 0; i < tuple.Length; i++) + { + result.Features[tuple[i].Dimension] = tuple[i].Feature; + initializedDimensions[tuple[i].Dimension] = true; + } + + return FindAllowedTestCase(result, initializedDimensions, 0) ? result : null; + } + + /// + /// Recursively assigns features to uninitialized dimensions until it finds an allowed test case. + /// + /// The test case being initialized. + /// A value indicating whether each dimension has already been assigned. + /// The dimension to initialize. + /// A value indicating whether was assigned allowed features. + private bool FindAllowedTestCase(TestCaseInfo testCase, bool[] initializedDimensions, int dimension) + { + if (dimension == _dimensions!.Length) + { + return IsTestCaseAllowed(testCase); + } + + if (initializedDimensions[dimension]) + { + return FindAllowedTestCase(testCase, initializedDimensions, dimension + 1); + } + + for (int f = 0; f < _dimensions[dimension]; f++) + { + testCase.Features[dimension] = f; + if (FindAllowedTestCase(testCase, initializedDimensions, dimension + 1)) + { + return true; + } + } + + return false; + } + + /// + /// Checks whether a test case satisfies the caller-supplied constraints. + /// + /// The test case to validate. + /// if the test case is allowed; otherwise, . + private bool IsTestCaseAllowed(TestCaseInfo testCase) + { + return _isTestCaseAllowed?.Invoke(testCase.Features) ?? true; + } + + private int CountTuplesCoveredByTest(TestCaseInfo testCase, int dimension, int feature) + { + int result = 0; + + List tuples = _uncoveredTuples![dimension][feature]; + + for (int i = 0; i < tuples.Count; i++) + { + if (testCase.IsTupleCovered(tuples[i])) + { + result++; + } + } + + return result; + } + + private void RemoveTuplesCoveredByTest(TestCaseInfo testCase) + { + for (int d = 0; d < _uncoveredTuples!.Length; d++) + { + for (int f = 0; f < _uncoveredTuples[d].Length; f++) + { + List tuples = _uncoveredTuples[d][f]; + + for (int i = tuples.Count - 1; i >= 0; i--) + { + if (testCase.IsTupleCovered(tuples[i])) + { + tuples.RemoveAt(i); + } + } + } + } + } + + /// + /// Removes a tuple that cannot be covered by any allowed test case. + /// + /// The tuple to remove. + /// + /// stores both orientations of every pair, e.g. (A,B) and (B,A). + /// When one orientation is impossible under the caller's constraints, the other is too; dropping + /// only the instance we just failed would leave the mirror in its own bucket and force another + /// exhaustive search for the same dead end. + /// + private void RemoveTuple(FeatureTuple tuple) + { + for (int i = 0; i < tuple.Length; i++) + { + _uncoveredTuples![tuple[i].Dimension][tuple[i].Feature].Remove(tuple); + } + + if (tuple.Length != 2) + { + return; + } + + FeatureInfo first = tuple[0]; + FeatureInfo second = tuple[1]; + List mirrorBucket = _uncoveredTuples![second.Dimension][second.Feature]; + for (int i = mirrorBucket.Count - 1; i >= 0; i--) + { + FeatureTuple candidate = mirrorBucket[i]; + if (candidate.Length == 2 + && candidate[0].Dimension == second.Dimension + && candidate[0].Feature == second.Feature + && candidate[1].Dimension == first.Dimension + && candidate[1].Feature == first.Feature) + { + mirrorBucket.RemoveAt(i); + break; + } + } + } + } +} diff --git a/tests/TUnit.Analyzers.Tests/CombinedDataSourceAnalyzerTests.cs b/tests/TUnit.Analyzers.Tests/CombinedDataSourceAnalyzerTests.cs index 88f6285e0b3..20712157f14 100644 --- a/tests/TUnit.Analyzers.Tests/CombinedDataSourceAnalyzerTests.cs +++ b/tests/TUnit.Analyzers.Tests/CombinedDataSourceAnalyzerTests.cs @@ -103,6 +103,32 @@ public class MyClass ); } + [Test] + public async Task Method_With_CombinedDataSource_And_PairwiseDataSource_Has_Error() + { + await Verifier + .VerifyAnalyzerAsync( + """ + using TUnit.Core; + + public class MyClass + { + [CombinedDataSources] + [PairwiseDataSource] + [Test] + public void {|#0:MyTest|}( + [Arguments(1, 2, 3)] int value, + [Arguments("a", "b")] string text + ) + { + } + } + """, + Verifier.Diagnostic(Rules.CombinedDataSourceConflictWithMatrix) + .WithLocation(0) + ); + } + [Test] public async Task Class_With_CombinedDataSource_And_ParameterDataSources_No_Error() { diff --git a/tests/TUnit.Analyzers.Tests/MatrixDataSourceAnalyzerTests.cs b/tests/TUnit.Analyzers.Tests/MatrixDataSourceAnalyzerTests.cs index 5fa0e315a2a..cf107c2ceed 100644 --- a/tests/TUnit.Analyzers.Tests/MatrixDataSourceAnalyzerTests.cs +++ b/tests/TUnit.Analyzers.Tests/MatrixDataSourceAnalyzerTests.cs @@ -107,4 +107,79 @@ public class MyClass .WithLocation(0) ); } + + [Test] + public async Task Method_Pairwise_No_Error() + { + await Verifier + .VerifyAnalyzerAsync( + """ + using TUnit.Core; + + public class MyClass + { + [PairwiseDataSource] + [Test] + public void MyTest( + [Matrix(1, 2, 3)] int value, + [Matrix(true, false)] bool value2, + [Matrix("a", "b")] string value3 + ) + { + } + } + """ + ); + } + + [Test] + public async Task Method_Pairwise_With_Seed_No_Error() + { + await Verifier + .VerifyAnalyzerAsync( + """ + using TUnit.Core; + + public class MyClass + { + [PairwiseDataSource(Seed = 42)] + [MatrixExclusion(1, true)] + [Test] + public void MyTest( + [Matrix(1, 2, 3)] int value, + [Matrix(true, false)] bool value2 + ) + { + } + } + """ + ); + } + + [Test] + public async Task Class_Pairwise_No_Error() + { + await Verifier + .VerifyAnalyzerAsync( + """ + using TUnit.Core; + + [PairwiseDataSource] + public class MyClass + { + public MyClass( + [Matrix(1, 2, 3)] int value, + [Matrix(true, false)] bool value2 + ) + { + } + + [Test] + public void MyTest() + { + } + } + """ + ); + } } diff --git a/tests/TUnit.Analyzers.Tests/MatrixDataSourceCodeFixProviderTests.cs b/tests/TUnit.Analyzers.Tests/MatrixDataSourceCodeFixProviderTests.cs index abee8815b35..4532f8580c8 100644 --- a/tests/TUnit.Analyzers.Tests/MatrixDataSourceCodeFixProviderTests.cs +++ b/tests/TUnit.Analyzers.Tests/MatrixDataSourceCodeFixProviderTests.cs @@ -37,7 +37,44 @@ public void MyTest( { } } + """, + test => test.CodeActionEquivalenceKey = "Add [MatrixDataSource]" + ); + } + + [Test] + public async Task Adds_PairwiseDataSource_On_Method() + { + await Verifier.VerifyCodeFixAsync( """ + using TUnit.Core; + + public class MyClass + { + [Test] + public void {|#0:MyTest|}( + [Matrix(1, 2, 3)] int value, + [Matrix(true, false)] bool flag) + { + } + } + """, + Verifier.Diagnostic(Rules.MatrixDataSourceAttributeRequired).WithLocation(0), + """ + using TUnit.Core; + + public class MyClass + { + [Test] + [PairwiseDataSource] + public void MyTest( + [Matrix(1, 2, 3)] int value, + [Matrix(true, false)] bool flag) + { + } + } + """, + test => test.CodeActionEquivalenceKey = "Add [PairwiseDataSource]" ); } @@ -80,7 +117,38 @@ public void MyTest() { } } - """ + """, + test => test.CodeActionEquivalenceKey = "Add [MatrixDataSource]" + ); + } + + [Test] + public async Task Does_Not_Offer_PairwiseDataSource_On_Class() + { + const string source = """ + using TUnit.Core; + + public class {|#0:MyClass|} + { + public MyClass( + [Matrix(1, 2)] int value, + [Matrix(true, false)] bool flag) + { + } + + [Test] + public void MyTest() + { + } + } + """; + + // Fixed code identical to the source asserts that no matching code action is offered. + await Verifier.VerifyCodeFixAsync( + source, + Verifier.Diagnostic(Rules.MatrixDataSourceAttributeRequired).WithLocation(0), + source, + test => test.CodeActionEquivalenceKey = "Add [PairwiseDataSource]" ); } } diff --git a/tests/TUnit.Engine.Tests/PairwiseTests.cs b/tests/TUnit.Engine.Tests/PairwiseTests.cs new file mode 100644 index 00000000000..38c9acb661a --- /dev/null +++ b/tests/TUnit.Engine.Tests/PairwiseTests.cs @@ -0,0 +1,39 @@ +using Shouldly; +using TUnit.Engine.Tests.Enums; + +namespace TUnit.Engine.Tests; + +public class PairwiseTests(TestMode testMode) : InvokableTestBase(testMode) +{ + [Test] + public async Task Test() + { + // Pairwise_Range uses [MatrixRange], which is only compiled for .NET 8+. + var expectedCount = IsNetFramework ? 106 : 212; + + await RunTestsWithFilter( + "/*/*/PairwiseTests/*", + [ + result => result.ResultSummary.Outcome.ShouldBe("Completed"), + result => result.ResultSummary.Counters.Total.ShouldBe(expectedCount, "Total"), + result => result.ResultSummary.Counters.Passed.ShouldBe(expectedCount, "Passed"), + result => result.ResultSummary.Counters.Failed.ShouldBe(0, "Failed"), + result => result.ResultSummary.Counters.NotExecuted.ShouldBe(0, "Skipped"), + result => result.Results.Select(x => x.TestName).Distinct().Count().ShouldBe(expectedCount, "Distinct test names"), + result => CountOf(result, "Pairwise_Bools_And_Ints").ShouldBe(16, "Pairwise_Bools_And_Ints"), + result => CountOf(result, "Pairwise_Strings_And_Ints").ShouldBe(9, "Pairwise_Strings_And_Ints"), + result => CountOf(result, "Pairwise_Enum_And_Nullable").ShouldBe(17, "Pairwise_Enum_And_Nullable"), + result => CountOf(result, "Pairwise_Nullable_Enum").ShouldBe(8, "Pairwise_Nullable_Enum"), + result => CountOf(result, "Pairwise_Matrix_Excluding").ShouldBe(6, "Pairwise_Matrix_Excluding"), + result => CountOf(result, "Pairwise_Exclusion").ShouldBe(10, "Pairwise_Exclusion"), + result => CountOf(result, "Pairwise_Enum_Exclusion").ShouldBe(9, "Pairwise_Enum_Exclusion"), + result => CountOf(result, "Pairwise_Custom_Seed").ShouldBe(31, "Pairwise_Custom_Seed"), + result => CountOf(result, "Pairwise_Range").ShouldBe(IsNetFramework ? 0 : 106, "Pairwise_Range"), + ]); + } + + private static int CountOf(TrxTools.TrxParser.TestRun result, string testName) + { + return result.Results.Count(x => x.TestName!.StartsWith(testName + "(", StringComparison.Ordinal)); + } +} diff --git a/tests/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet10_0.verified.txt b/tests/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet10_0.verified.txt index 949cff5cd64..4473d4cea3d 100644 --- a/tests/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet10_0.verified.txt +++ b/tests/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet10_0.verified.txt @@ -1087,6 +1087,14 @@ namespace public . NotInParallelConstraintKeys { get; init; } public int Order { get; set; } } + [(.Class | .Method)] + public sealed class PairwiseDataSourceAttribute : .UntypedDataSourceGeneratorAttribute, .IAccessesInstanceData + { + public const int DefaultSeed = 15485863; + public PairwiseDataSourceAttribute() { } + public int Seed { get; set; } + protected override .<> GenerateDataSources(.DataGeneratorMetadata dataGeneratorMetadata) { } + } public class ParallelGroupAttribute : .TUnitAttribute, ., . { public ParallelGroupAttribute(string group) { } diff --git a/tests/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet8_0.verified.txt b/tests/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet8_0.verified.txt index 51b2ad3ffe9..16efc28093d 100644 --- a/tests/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet8_0.verified.txt +++ b/tests/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet8_0.verified.txt @@ -1087,6 +1087,14 @@ namespace public . NotInParallelConstraintKeys { get; init; } public int Order { get; set; } } + [(.Class | .Method)] + public sealed class PairwiseDataSourceAttribute : .UntypedDataSourceGeneratorAttribute, .IAccessesInstanceData + { + public const int DefaultSeed = 15485863; + public PairwiseDataSourceAttribute() { } + public int Seed { get; set; } + protected override .<> GenerateDataSources(.DataGeneratorMetadata dataGeneratorMetadata) { } + } public class ParallelGroupAttribute : .TUnitAttribute, ., . { public ParallelGroupAttribute(string group) { } diff --git a/tests/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet9_0.verified.txt b/tests/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet9_0.verified.txt index 20af9adc5f0..6820a65d09d 100644 --- a/tests/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet9_0.verified.txt +++ b/tests/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet9_0.verified.txt @@ -1087,6 +1087,14 @@ namespace public . NotInParallelConstraintKeys { get; init; } public int Order { get; set; } } + [(.Class | .Method)] + public sealed class PairwiseDataSourceAttribute : .UntypedDataSourceGeneratorAttribute, .IAccessesInstanceData + { + public const int DefaultSeed = 15485863; + public PairwiseDataSourceAttribute() { } + public int Seed { get; set; } + protected override .<> GenerateDataSources(.DataGeneratorMetadata dataGeneratorMetadata) { } + } public class ParallelGroupAttribute : .TUnitAttribute, ., . { public ParallelGroupAttribute(string group) { } diff --git a/tests/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.Net4_7.verified.txt b/tests/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.Net4_7.verified.txt index bbffd7c2230..377aec6bd68 100644 --- a/tests/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.Net4_7.verified.txt +++ b/tests/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.Net4_7.verified.txt @@ -1049,6 +1049,14 @@ namespace public . NotInParallelConstraintKeys { get; init; } public int Order { get; set; } } + [(.Class | .Method)] + public sealed class PairwiseDataSourceAttribute : .UntypedDataSourceGeneratorAttribute, .IAccessesInstanceData + { + public const int DefaultSeed = 15485863; + public PairwiseDataSourceAttribute() { } + public int Seed { get; set; } + protected override .<> GenerateDataSources(.DataGeneratorMetadata dataGeneratorMetadata) { } + } public class ParallelGroupAttribute : .TUnitAttribute, ., . { public ParallelGroupAttribute(string group) { } diff --git a/tests/TUnit.TestProject/PairwiseTests.cs b/tests/TUnit.TestProject/PairwiseTests.cs new file mode 100644 index 00000000000..98b40f930b6 --- /dev/null +++ b/tests/TUnit.TestProject/PairwiseTests.cs @@ -0,0 +1,130 @@ +using TUnit.TestProject.Attributes; + +namespace TUnit.TestProject; + +[EngineTest(ExpectedResult.Pass)] +public class PairwiseTests +{ + // 2 x 2 x 4 x 4 = 64 combinations; pairwise needs 16. + [Test] + [PairwiseDataSource] + public async Task Pairwise_Bools_And_Ints( + bool largeArrayPool, + bool excessSpan, + [Matrix(0, 99, 100, 101)] int sizeHint, + [Matrix(1, 2, 3, 100)] int stepCount) + { + await Task.CompletedTask; + } + + // 3 x 3 x 3 x 3 = 81 combinations; pairwise needs 9. + [Test] + [PairwiseDataSource] + public async Task Pairwise_Strings_And_Ints( + [Matrix("A", "B", "C")] string a, + [Matrix("D", "E", "F")] string b, + [Matrix(1, 2, 3)] int c, + [Matrix(4, 5, 6)] int d) + { + await Task.CompletedTask; + } + + // enum (3) x bool? (3) x 5 x 2 x 3 = 270 combinations; pairwise needs 17. + [Test] + [PairwiseDataSource] + public async Task Pairwise_Enum_And_Nullable( + PairwiseColor color, + bool? flag, + [Matrix(1, 2, 3, 4, 5)] int number, + [Matrix("x", "y")] string text, + [Matrix(1, 2, 3)] int other) + { + await Assert.That(Enum.IsDefined(typeof(PairwiseColor), color)).IsTrue(); + } + + // Nullable enum gets its values plus null, like [MatrixDataSource]: 4 x 2 x 2 = 16 combinations. + [Test] + [PairwiseDataSource] + public async Task Pairwise_Nullable_Enum( + PairwiseColor? color, + bool flag, + [Matrix("x", "y")] string text) + { + await Task.CompletedTask; + } + + [Test] + [PairwiseDataSource] + public async Task Pairwise_Matrix_Excluding( + [Matrix(1, 2, 3, 4, Excluding = [2])] int number, + [Matrix(Excluding = [PairwiseColor.Green])] PairwiseColor color, + bool flag) + { + await Assert.That(number).IsNotEqualTo(2); + await Assert.That(color).IsNotEqualTo(PairwiseColor.Green); + } + + [Test] + [PairwiseDataSource] + [MatrixExclusion(1, 1, 1)] + [MatrixExclusion(2, 2, 2)] + [MatrixExclusion(3, 3, 3)] + public async Task Pairwise_Exclusion( + [MatrixMethod(nameof(Values))] int a, + [MatrixMethod(nameof(Values))] int b, + [MatrixMethod(nameof(Values))] int c) + { + await Assert.That(a == b && b == c).IsFalse(); + } + + [Test] + [PairwiseDataSource] + [MatrixExclusion(PairwiseColor.Red, true, 1)] + [MatrixExclusion(PairwiseColor.Blue, false, 3)] + public async Task Pairwise_Enum_Exclusion( + PairwiseColor color, + bool flag, + [Matrix(1, 2, 3)] int number) + { + await Assert.That(color == PairwiseColor.Red && flag && number == 1).IsFalse(); + await Assert.That(color == PairwiseColor.Blue && !flag && number == 3).IsFalse(); + } + + [Test] + [PairwiseDataSource(Seed = 42)] + public async Task Pairwise_Custom_Seed( + [Matrix(1, 2, 3)] int a, + [Matrix(1, 2, 3, 4)] int b, + [Matrix(1, 2, 3, 4, 5)] int c, + [Matrix(1, 2, 3, 4, 5, 6)] int d) + { + await Task.CompletedTask; + } + +#if NET8_0_OR_GREATER + // 10 x 10 x 10 = 1000 combinations; pairwise needs 106. + [Test] + [PairwiseDataSource] + public async Task Pairwise_Range( + [MatrixRange(1, 10)] int a, + [MatrixRange(1, 10)] int b, + [MatrixRange(1, 10)] int c) + { + await Task.CompletedTask; + } +#endif + + public static IEnumerable Values() + { + yield return 1; + yield return 2; + yield return 3; + } + + public enum PairwiseColor + { + Red, + Green, + Blue + } +} diff --git a/tests/TUnit.UnitTests/PairwiseDataSourceTests.cs b/tests/TUnit.UnitTests/PairwiseDataSourceTests.cs new file mode 100644 index 00000000000..eabf8037ec0 --- /dev/null +++ b/tests/TUnit.UnitTests/PairwiseDataSourceTests.cs @@ -0,0 +1,271 @@ +using TUnit.Core.Helpers; + +namespace TUnit.UnitTests; + +public class PairwiseDataSourceTests +{ + [Test] + [Arguments("2,2,4,4", 16)] + [Arguments("2,2,3,3,2", 10)] + [Arguments("2,2,2,2,2", 6)] + [Arguments("3,3,3,3", 9)] + [Arguments("3,4,5,6", 30)] + [Arguments("10,10,10", 106)] + [Arguments("2,2,2,2,2,2,2,2,2,2", 9)] + [Arguments("3,3,5,2,3", 17)] + public async Task Covers_All_Pairs_With_Expected_Case_Count(string shape, int expectedCount) + { + var valueSets = CreateValueSets(shape); + + var rows = PairwiseDataSourceAttribute.GetTestCases(valueSets, [], PairwiseDataSourceAttribute.DefaultSeed); + + await Assert.That(rows.Count).IsEqualTo(expectedCount); + await Assert.That(GetUncoveredPairs(valueSets, rows, [])).IsEmpty(); + } + + [Test] + [Arguments("2,2,4,4")] + [Arguments("3,4,5,6")] + [Arguments("10,10,10")] + public async Task Is_Deterministic(string shape) + { + var valueSets = CreateValueSets(shape); + + var first = Format(PairwiseDataSourceAttribute.GetTestCases(valueSets, [], PairwiseDataSourceAttribute.DefaultSeed)); + var second = Format(PairwiseDataSourceAttribute.GetTestCases(valueSets, [], PairwiseDataSourceAttribute.DefaultSeed)); + + await Assert.That(second).IsEqualTo(first); + } + + [Test] + [Arguments(1)] + [Arguments(42)] + [Arguments(int.MaxValue)] + [Arguments(-7)] + public async Task Other_Seeds_Still_Cover_All_Pairs(int seed) + { + var valueSets = CreateValueSets("3,4,5,6"); + + var rows = PairwiseDataSourceAttribute.GetTestCases(valueSets, [], seed); + + await Assert.That(GetUncoveredPairs(valueSets, rows, [])).IsEmpty(); + await Assert.That(rows.Count).IsLessThan(3 * 4 * 5 * 6); + } + + [Test] + public async Task Different_Seeds_Can_Produce_Different_Cases() + { + var valueSets = CreateValueSets("3,4,5,6"); + + var first = Format(PairwiseDataSourceAttribute.GetTestCases(valueSets, [], PairwiseDataSourceAttribute.DefaultSeed)); + var second = Format(PairwiseDataSourceAttribute.GetTestCases(valueSets, [], 42)); + + await Assert.That(second).IsNotEqualTo(first); + } + + [Test] + public async Task Exclusions_Are_Never_Generated_And_Reachable_Pairs_Stay_Covered() + { + var valueSets = CreateValueSets("3,4,5,6"); + object?[][] exclusions = + [ + [0, 0, 0, 0], + [1, 1, 1, 1], + [2, 3, 4, 5], + [0, 1, 2, 3], + ]; + + // Excluding every case with (p0 = 0, p1 = 0) makes that pair unreachable. + var allExclusions = new List(exclusions); + for (var c = 0; c < 5; c++) + { + for (var d = 0; d < 6; d++) + { + allExclusions.Add([0, 0, c, d]); + } + } + + var rows = PairwiseDataSourceAttribute.GetTestCases(valueSets, [.. allExclusions], PairwiseDataSourceAttribute.DefaultSeed); + + foreach (var row in rows) + { + await Assert.That(MatrixParameterValues.IsExcluded([.. allExclusions], row)).IsFalse(); + } + + await Assert.That(GetUncoveredPairs(valueSets, rows, [.. allExclusions])).IsEmpty(); + } + + [Test] + public async Task Unreachable_Pair_With_Many_Parameters_Still_Covers_Reachable_Pairs() + { + // Five parameters of size 4 => 1024 Cartesian rows. Excluding every row that pairs + // p0=0 with p1=0 makes that pair impossible; generation must drop both mirrored + // tuple orientations without timing out, and still cover every reachable pair. + IReadOnlyList[] valueSets = + [ + [0, 1, 2, 3], + [0, 1, 2, 3], + [0, 1, 2, 3], + [0, 1, 2, 3], + [0, 1, 2, 3], + ]; + + var exclusions = new List(); + for (var c = 0; c < 4; c++) + for (var d = 0; d < 4; d++) + for (var e = 0; e < 4; e++) + { + exclusions.Add([0, 0, c, d, e]); + } + + var rows = PairwiseDataSourceAttribute.GetTestCases(valueSets, [.. exclusions], PairwiseDataSourceAttribute.DefaultSeed); + + await Assert.That(rows.Count).IsGreaterThan(0); + foreach (var row in rows) + { + await Assert.That(MatrixParameterValues.IsExcluded([.. exclusions], row)).IsFalse(); + } + + await Assert.That(GetUncoveredPairs(valueSets, rows, [.. exclusions])).IsEmpty(); + } + + [Test] + public async Task Enum_Exclusions_Match_Underlying_Values() + { + IReadOnlyList[] valueSets = + [ + [0, 1, 2], + [true, false], + ["a", "b", "c"], + ]; + + object?[][] exclusions = + [ + [DayOfWeek.Monday, true, "a"], + ]; + + var rows = PairwiseDataSourceAttribute.GetTestCases(valueSets, exclusions, PairwiseDataSourceAttribute.DefaultSeed); + + foreach (var row in rows) + { + var isExcludedRow = Equals(row[0], 1) && Equals(row[1], true) && Equals(row[2], "a"); + await Assert.That(isExcludedRow).IsFalse(); + } + + await Assert.That(GetUncoveredPairs(valueSets, rows, exclusions)).IsEmpty(); + } + + [Test] + public async Task Single_Parameter_Yields_Each_Value_Once() + { + IReadOnlyList[] valueSets = [[1, 2, 3, 4]]; + + var rows = PairwiseDataSourceAttribute.GetTestCases(valueSets, [], PairwiseDataSourceAttribute.DefaultSeed); + + var values = new List(); + foreach (var row in rows) + { + values.Add((int)row[0]!); + } + + values.Sort(); + await Assert.That(values).IsEquivalentTo([1, 2, 3, 4]); + await Assert.That(rows.Count).IsEqualTo(4); + } + + [Test] + public async Task Two_Parameters_Yield_Full_Product() + { + var valueSets = CreateValueSets("3,4"); + + var rows = PairwiseDataSourceAttribute.GetTestCases(valueSets, [], PairwiseDataSourceAttribute.DefaultSeed); + + await Assert.That(rows.Count).IsEqualTo(12); + await Assert.That(GetUncoveredPairs(valueSets, rows, [])).IsEmpty(); + } + + [Test] + public async Task Empty_Value_Set_Yields_No_Cases() + { + IReadOnlyList[] valueSets = [[1, 2], [], [true, false]]; + + var rows = PairwiseDataSourceAttribute.GetTestCases(valueSets, [], PairwiseDataSourceAttribute.DefaultSeed); + + await Assert.That(rows).IsEmpty(); + } + + [Test] + public async Task All_Cases_Excluded_Yields_No_Cases() + { + IReadOnlyList[] valueSets = [[1], [true]]; + + var rows = PairwiseDataSourceAttribute.GetTestCases(valueSets, [[1, true]], PairwiseDataSourceAttribute.DefaultSeed); + + await Assert.That(rows).IsEmpty(); + } + + private static IReadOnlyList[] CreateValueSets(string shape) + { + var sizes = shape.Split(','); + var valueSets = new IReadOnlyList[sizes.Length]; + for (var i = 0; i < sizes.Length; i++) + { + var values = new object?[int.Parse(sizes[i])]; + for (var v = 0; v < values.Length; v++) + { + values[v] = v; + } + + valueSets[i] = values; + } + + return valueSets; + } + + private static string Format(List rows) + { + var lines = new List(rows.Count); + foreach (var row in rows) + { + lines.Add(string.Join(",", row)); + } + + return string.Join("|", lines); + } + + /// + /// Returns every pair of values that appears in at least one allowed combination + /// of the full Cartesian product but in none of . + /// + private static List GetUncoveredPairs(IReadOnlyList[] valueSets, List rows, object?[][] exclusions) + { + var reachable = new HashSet(); + foreach (var combination in CartesianProductHelper.GetCartesianProduct(valueSets)) + { + if (!MatrixParameterValues.IsExcluded(exclusions, combination)) + { + AddPairs(reachable, combination); + } + } + + var covered = new HashSet(); + foreach (var row in rows) + { + AddPairs(covered, row); + } + + reachable.ExceptWith(covered); + return [.. reachable]; + } + + private static void AddPairs(HashSet pairs, object?[] row) + { + for (var i = 0; i < row.Length; i++) + { + for (var j = i + 1; j < row.Length; j++) + { + pairs.Add($"{i}={row[i]};{j}={row[j]}"); + } + } + } +}