Skip to content

Commit a740a6c

Browse files
vbreussvbreuss
authored
feat: enable/disable brittle tests via test settings (#514)
Some tests are marked as brittle. If possible, these should still be executed on the build pipeline. --------- Co-authored-by: Valentin <[email protected]>
1 parent cccf2b8 commit a740a6c

File tree

22 files changed

+226
-72
lines changed

22 files changed

+226
-72
lines changed

‎Directory.Packages.props‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,6 @@
3636
<ItemGroup>
3737
<PackageVersion Include="Testably.Abstractions" Version="2.6.1" />
3838
<PackageVersion Include="Testably.Abstractions.Testing" Version="2.6.1" />
39-
<PackageVersion Include="Testably.Abstractions.FluentAssertions" Version="1.0.0" />
39+
<PackageVersion Include="Testably.Abstractions.FluentAssertions" Version="1.0.1" />
4040
</ItemGroup>
4141
</Project>

‎Tests/Helpers/Testably.Abstractions.TestHelpers/FileSystemTestBase.cs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,15 @@ protected FileSystemTestBase()
3838
"The SourceGenerator didn't create the corresponding files!");
3939
}
4040

41+
/// <summary>
42+
/// Specifies, if brittle tests should be skipped on the real file system.
43+
/// </summary>
44+
/// <param name="condition">
45+
/// (optional) A condition that must be <see langword="true" /> for the test to be skipped on the
46+
/// real file system.
47+
/// </param>
48+
public abstract void SkipIfBrittleTestsShouldBeSkipped(bool condition = true);
49+
4150
/// <summary>
4251
/// Specifies, if long-running tests should be skipped on the real file system.
4352
/// </summary>

‎Tests/Helpers/Testably.Abstractions.TestHelpers/Settings/RealFileSystemFixture.cs‎

Lines changed: 0 additions & 30 deletions
This file was deleted.

‎Tests/Helpers/Testably.Abstractions.TestHelpers/Settings/TestEnvironment.cs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,13 @@
22

33
public class TestEnvironment
44
{
5+
/// <summary>
6+
/// Affects some tests, that are brittle on the real file system.
7+
/// </summary>
8+
/// <remarks>Per default, they are <see cref="TestSettingStatus.AlwaysDisabled" />.</remarks>
9+
public TestSettingStatus BrittleTests { get; set; }
10+
= TestSettingStatus.AlwaysDisabled;
11+
512
/// <summary>
613
/// Affects some tests, that take a long time to run against the real file system (e.g. timeout).
714
/// </summary>

‎Tests/Helpers/Testably.Abstractions.TestHelpers/Settings/TestSettingsFixture.cs‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
using Newtonsoft.Json;
2+
using System;
3+
using System.IO;
4+
5+
namespace Testably.Abstractions.TestHelpers.Settings;
6+
7+
public class TestSettingsFixture
8+
{
9+
public TestSettingStatus BrittleTests { get; }
10+
public TestSettingStatus LongRunningTests { get; }
11+
public TestSettingStatus RealFileSystemTests { get; }
12+
13+
public TestSettingsFixture()
14+
{
15+
TestEnvironment environment = LoadTestEnvironment();
16+
17+
RealFileSystemTests = environment.RealFileSystemTests;
18+
LongRunningTests = environment.LongRunningTests;
19+
BrittleTests = environment.BrittleTests;
20+
}
21+
22+
private static TestEnvironment LoadTestEnvironment()
23+
{
24+
try
25+
{
26+
string path = Path.GetFullPath(
27+
Path.Combine("..", "..", "..", "..", "test.settings.json"));
28+
if (File.Exists(path))
29+
{
30+
string content = File.ReadAllText(path);
31+
TestEnvironment? testEnvironment =
32+
JsonConvert.DeserializeObject<TestEnvironment>(content);
33+
if (testEnvironment != null)
34+
{
35+
return testEnvironment;
36+
}
37+
}
38+
}
39+
catch (Exception)
40+
{
41+
// Ignore all exceptions while reading the test.settings.json file and use the default settings as fallback.
42+
}
43+
44+
return new TestEnvironment();
45+
}
46+
}

‎Tests/Helpers/Testably.Abstractions.TestHelpers/Test.cs‎

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,4 @@
1-
using System.IO.Abstractions;
21
using System.Runtime.InteropServices;
3-
using Xunit;
42

53
namespace Testably.Abstractions.TestHelpers;
64

@@ -27,18 +25,4 @@ public Test()
2725
RunsOnWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
2826
IsNetFramework = RuntimeInformation.FrameworkDescription.StartsWith(".NET Framework");
2927
}
30-
31-
public static void SkipBrittleTestsOnRealFileSystem(
32-
IFileSystem fileSystem, bool condition = true)
33-
{
34-
Skip.If(fileSystem is RealFileSystem && condition,
35-
"Brittle tests are skipped on the real file system.");
36-
}
37-
38-
public static void SkipBrittleTestsOnRealTimeSystem(
39-
ITimeSystem timeSystem, bool condition = true)
40-
{
41-
Skip.If(timeSystem is RealTimeSystem && condition,
42-
"Brittle tests are skipped on the real time system.");
43-
}
4428
}

‎Tests/Helpers/Testably.Abstractions.TestHelpers/TimeSystemTestBase.cs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,4 +28,13 @@ protected TimeSystemTestBase()
2828
throw new NotSupportedException(
2929
"The SourceGenerator didn't create the corresponding files!");
3030
}
31+
32+
/// <summary>
33+
/// Specifies, if brittle tests should be skipped on the real time system.
34+
/// </summary>
35+
/// <param name="condition">
36+
/// (optional) A condition that must be <see langword="true" /> for the test to be skipped on the
37+
/// real time system.
38+
/// </param>
39+
public abstract void SkipIfBrittleTestsShouldBeSkipped(bool condition = true);
3140
}

‎Tests/Helpers/Testably.Abstractions.Tests.SourceGenerator/ClassGenerators/FileSystemClassGenerator.cs‎

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,12 @@ private MockFileSystemTests(MockFileSystem mockFileSystem) : base(
5656
public void Dispose()
5757
=> _directoryCleaner.Dispose();
5858
59+
/// <inheritdoc cref=""{@class.Name}{{TFileSystem}}.SkipIfBrittleTestsShouldBeSkipped(bool)"" />
60+
public override void SkipIfBrittleTestsShouldBeSkipped(bool condition = true)
61+
{{
62+
// Brittle tests are never skipped against the mock file system!
63+
}}
64+
5965
/// <inheritdoc cref=""{@class.Name}{{TFileSystem}}.LongRunningTestsShouldBeSkipped()"" />
6066
public override void SkipIfLongRunningTestsShouldBeSkipped()
6167
{{
@@ -74,20 +80,20 @@ public sealed class RealFileSystemTests : {@class.Name}<RealFileSystem>, IDispos
7480
public override string BasePath => _directoryCleaner.BasePath;
7581
7682
private readonly IDirectoryCleaner _directoryCleaner;
77-
private readonly RealFileSystemFixture _fixture;
83+
private readonly TestSettingsFixture _fixture;
7884
79-
public RealFileSystemTests(ITestOutputHelper testOutputHelper, RealFileSystemFixture fixture)
85+
public RealFileSystemTests(ITestOutputHelper testOutputHelper, TestSettingsFixture fixture)
8086
: base(new Test(), new RealFileSystem(), new RealTimeSystem())
8187
{{
8288
#if DEBUG
8389
if (fixture.RealFileSystemTests != TestSettingStatus.AlwaysEnabled)
8490
{{
85-
throw new SkipException($""RealFileSystemTests are {{fixture.RealFileSystemTests}}. You can enable them by executing the corresponding tests in Testably.Abstractions.TestSettings.RealFileSystemTests."");
91+
throw new SkipException($""Tests against the real file system are {{fixture.RealFileSystemTests}}. You can enable them by executing the corresponding tests in Testably.Abstractions.TestSettings.RealFileSystemTests."");
8692
}}
8793
#else
8894
if (fixture.RealFileSystemTests == TestSettingStatus.AlwaysDisabled)
8995
{{
90-
throw new SkipException($""RealFileSystemTests are {{fixture.RealFileSystemTests}}. You can enable them by executing the corresponding tests in Testably.Abstractions.TestSettings.RealFileSystemTests."");
96+
throw new SkipException($""Tests against the real file system are {{fixture.RealFileSystemTests}}. You can enable them by executing the corresponding tests in Testably.Abstractions.TestSettings.RealFileSystemTests."");
9197
}}
9298
#endif
9399
_fixture = fixture;
@@ -99,16 +105,28 @@ public RealFileSystemTests(ITestOutputHelper testOutputHelper, RealFileSystemFix
99105
public void Dispose()
100106
=> _directoryCleaner.Dispose();
101107
108+
#if DEBUG
109+
/// <inheritdoc cref=""{@class.Name}{{TFileSystem}}.SkipIfBrittleTestsShouldBeSkipped(bool)"" />
110+
public override void SkipIfBrittleTestsShouldBeSkipped(bool condition = true)
111+
=> Skip.If(condition && _fixture.BrittleTests != TestSettingStatus.AlwaysEnabled,
112+
$""Brittle tests are {{_fixture.BrittleTests}}. You can enable them by executing the corresponding tests in Testably.Abstractions.TestSettings.BrittleTests."");
113+
#else
114+
/// <inheritdoc cref=""{@class.Name}{{TFileSystem}}.SkipIfBrittleTestsShouldBeSkipped(bool)"" />
115+
public override void SkipIfBrittleTestsShouldBeSkipped(bool condition = true)
116+
=> Skip.If(condition && _fixture.BrittleTests == TestSettingStatus.AlwaysDisabled,
117+
$""Brittle tests are {{_fixture.BrittleTests}}. You can enable them by executing the corresponding tests in Testably.Abstractions.TestSettings.BrittleTests."");
118+
#endif
119+
102120
#if DEBUG
103121
/// <inheritdoc cref=""{@class.Name}{{TFileSystem}}.LongRunningTestsShouldBeSkipped()"" />
104122
public override void SkipIfLongRunningTestsShouldBeSkipped()
105123
=> Skip.If(_fixture.LongRunningTests != TestSettingStatus.AlwaysEnabled,
106-
$""LongRunningTests are {{_fixture.LongRunningTests}}. You can enable them by executing the corresponding tests in Testably.Abstractions.TestSettings.LongRunningTests."");
124+
$""Long-running tests are {{_fixture.LongRunningTests}}. You can enable them by executing the corresponding tests in Testably.Abstractions.TestSettings.LongRunningTests."");
107125
#else
108126
/// <inheritdoc cref=""{@class.Name}{{TFileSystem}}.LongRunningTestsShouldBeSkipped()"" />
109127
public override void SkipIfLongRunningTestsShouldBeSkipped()
110128
=> Skip.If(_fixture.LongRunningTests == TestSettingStatus.AlwaysDisabled,
111-
$""LongRunningTests are {{_fixture.LongRunningTests}}. You can enable them by executing the corresponding tests in Testably.Abstractions.TestSettings.LongRunningTests."");
129+
$""Long-running tests are {{_fixture.LongRunningTests}}. You can enable them by executing the corresponding tests in Testably.Abstractions.TestSettings.LongRunningTests."");
112130
#endif
113131
}}
114132
}}");

‎Tests/Helpers/Testably.Abstractions.Tests.SourceGenerator/ClassGenerators/TimeSystemClassGenerator.cs‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ public override string Marker
1414
protected override void GenerateSource(StringBuilder sourceBuilder, ClassModel @class)
1515
=> sourceBuilder.Append(@$"
1616
using Testably.Abstractions.TestHelpers;
17+
using Testably.Abstractions.TestHelpers.Settings;
1718
using Xunit.Abstractions;
1819
1920
namespace {@class.Namespace}
@@ -35,14 +36,36 @@ public sealed class MockTimeSystemTests : {@class.Name}<MockTimeSystem>
3536
public MockTimeSystemTests() : base(new MockTimeSystem(Testing.TimeProvider.Now()))
3637
{{
3738
}}
39+
40+
/// <inheritdoc cref=""{@class.Name}{{TTimeSystem}}.SkipIfBrittleTestsShouldBeSkipped(bool)"" />
41+
public override void SkipIfBrittleTestsShouldBeSkipped(bool condition = true)
42+
{{
43+
// Brittle tests are never skipped against the mock time system!
44+
}}
3845
}}
3946
4047
// ReSharper disable once UnusedMember.Global
48+
[Collection(nameof(RealTimeSystemTests))]
4149
public sealed class RealTimeSystemTests : {@class.Name}<RealTimeSystem>
4250
{{
43-
public RealTimeSystemTests() : base(new RealTimeSystem())
51+
private readonly TestSettingsFixture _fixture;
52+
53+
public RealTimeSystemTests(TestSettingsFixture fixture) : base(new RealTimeSystem())
4454
{{
55+
_fixture = fixture;
4556
}}
57+
58+
#if DEBUG
59+
/// <inheritdoc cref=""{@class.Name}{{TTimeSystem}}.SkipIfBrittleTestsShouldBeSkipped(bool)"" />
60+
public override void SkipIfBrittleTestsShouldBeSkipped(bool condition = true)
61+
=> Skip.If(condition && _fixture.BrittleTests != TestSettingStatus.AlwaysEnabled,
62+
$""Brittle tests are {{_fixture.BrittleTests}}. You can enable them by executing the corresponding tests in Testably.Abstractions.TestSettings.BrittleTests."");
63+
#else
64+
/// <inheritdoc cref=""{@class.Name}{{TTimeSystem}}.SkipIfBrittleTestsShouldBeSkipped(bool)"" />
65+
public override void SkipIfBrittleTestsShouldBeSkipped(bool condition = true)
66+
=> Skip.If(condition && _fixture.BrittleTests == TestSettingStatus.AlwaysDisabled,
67+
$""Brittle tests are {{_fixture.BrittleTests}}. You can enable them by executing the corresponding tests in Testably.Abstractions.TestSettings.BrittleTests."");
68+
#endif
4669
}}
4770
}}");
4871
}

‎Tests/Helpers/Testably.Abstractions.Tests.SourceGenerator/GlobalGenerator.cs‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ public void GenerateClass(GeneratorExecutionContext context)
1010
{
1111
StringBuilder sourceBuilder = GetSourceBuilder();
1212
GenerateSource(sourceBuilder);
13-
string fileName = "DatabaseCollection.cs";
13+
string fileName = "XunitCollectionFixtures.cs";
1414
context.AddSource(fileName,
1515
SourceText.From(sourceBuilder.ToString(), Encoding.UTF8));
1616
}
@@ -22,7 +22,15 @@ private void GenerateSource(StringBuilder sourceBuilder)
2222
namespace Testably.Abstractions.TestHelpers.Settings;
2323
2424
[CollectionDefinition(""RealFileSystemTests"")]
25-
public class DatabaseCollection : ICollectionFixture<RealFileSystemFixture>
25+
public class FileSystemTestSettingsFixture : ICollectionFixture<TestSettingsFixture>
26+
{
27+
// This class has no code, and is never created. Its purpose is simply
28+
// to be the place to apply [CollectionDefinition] and all the
29+
// ICollectionFixture<> interfaces.
30+
}
31+
32+
[CollectionDefinition(""RealTimeSystemTests"")]
33+
public class TimeSystemTestSettingsFixture : ICollectionFixture<TestSettingsFixture>
2634
{
2735
// This class has no code, and is never created. Its purpose is simply
2836
// to be the place to apply [CollectionDefinition] and all the

0 commit comments

Comments
 (0)