feat: include skip message for long running tests (#511)

Add a comment to skipped tests that indicate why they were skipped.

![image](https://github.com/Testably/Testably.Abstractions/assets/3438234/b2f889a5-1c0e-4261-9f0d-599bd2be1275)

Author: vbreuss

CONTRIBUTING.md

@@ -15,12 +15,12 @@ Additionally each push to the `main` branch checks the quality of the unit tests
 
 ## Tests
 On the build system, unit tests are executed both against the `MockFileSystem` and the `RealFileSystem`. This ensures that the tests verify correct assumptions.
-In order to simplify and speedup the development process, per default some tests are disabled in DEBUG mode.
+In order to simplify and speedup the development process, per default, some tests are disabled in DEBUG mode.
 These can be enabled by explicitely running the [`Testably.Abstractions.TestSettings`](https://github.com/Testably/Testably.Abstractions/tree/main/Tests/Settings/Testably.Abstractions.TestSettings) tests:
-- `LongRunningTestsAlsoInDebugMode` (`Include`/`Exclude`)
-  In order to increase the test execution speed during development, long-running tests are disabled per default in DEBUG mode. With this setting, the corresponding tests can be included.
-- `RealFileSystemTestsInDebugMode` (`Enable`/`Disable`)
-  If enabled, the classes for executing tests against the real file system also run in DEBUG mode.
+- `LongRunningTests` (`AlwaysEnabled` / `DisabledInDebugMode` (default) / `AlwaysDisabled`)
+  Some tests take a long time to run against the real file system (e.g. timeout). Per default, they are disabled in DEBUG mode.
+- `RealFileSystemTests` (`AlwaysEnabled` / `DisabledInDebugMode` (default) / `AlwaysDisabled`)
+  All tests against the real file system. Per default, they are disabled in DEBUG mode.
 
 *Note: These settings are stored locally in `test.settings.json` which is excluded in [`.gitignore`](https://github.com/Testably/Testably.Abstractions/blob/main/.gitignore) so that it only affects the individual developer!*
 

Tests/Helpers/Testably.Abstractions.TestHelpers/FileSystemTestBase.cs

@@ -41,5 +41,5 @@ protected FileSystemTestBase()
 	/// <summary>
 	///     Specifies, if long-running tests should be skipped on the real file system.
 	/// </summary>
-	public abstract bool LongRunningTestsShouldBeSkipped();
+	public abstract void SkipIfLongRunningTestsShouldBeSkipped();
 }

Tests/Helpers/Testably.Abstractions.TestHelpers/Settings/RealFileSystemFixture.cs

@@ -6,8 +6,8 @@ namespace Testably.Abstractions.TestHelpers.Settings;
 
 public class RealFileSystemFixture
 {
-	public bool EnableRealFileSystemTestsInDebugMode { get; }
-	public bool IncludeLongRunningTestsAlsoInDebugMode { get; }
+	public TestSettingStatus LongRunningTests { get; }
+	public TestSettingStatus RealFileSystemTests { get; }
 
 	public RealFileSystemFixture()
 	{
@@ -16,15 +16,15 @@ public RealFileSystemFixture()
 			string path = Path.GetFullPath(
 				Path.Combine("..", "..", "..", "..", "test.settings.json"));
 			string content = File.ReadAllText(path);
-			TestSettings settings = JsonConvert.DeserializeObject<TestSettings>(content)!;
-			EnableRealFileSystemTestsInDebugMode = settings.EnableRealFileSystemTestsInDebugMode;
-			IncludeLongRunningTestsAlsoInDebugMode =
-				settings.IncludeLongRunningTestsAlsoInDebugMode;
+			TestEnvironment environment = JsonConvert.DeserializeObject<TestEnvironment>(content)!;
+
+			RealFileSystemTests = environment.RealFileSystemTests;
+			LongRunningTests = environment.LongRunningTests;
 		}
 		catch (Exception)
 		{
-			EnableRealFileSystemTestsInDebugMode = false;
-			IncludeLongRunningTestsAlsoInDebugMode = false;
+			RealFileSystemTests = TestSettingStatus.DisabledInDebugMode;
+			LongRunningTests = TestSettingStatus.DisabledInDebugMode;
 		}
 	}
 }

Tests/Helpers/Testably.Abstractions.TestHelpers/Settings/TestEnvironment.cs

@@ -0,0 +1,18 @@
+namespace Testably.Abstractions.TestHelpers.Settings;
+
+public class TestEnvironment
+{
+	/// <summary>
+	///     Affects some tests, that take a long time to run against the real file system (e.g. timeout).
+	/// </summary>
+	/// <remarks>Per default, they are <see cref="TestSettingStatus.DisabledInDebugMode" />.</remarks>
+	public TestSettingStatus LongRunningTests { get; set; }
+		= TestSettingStatus.DisabledInDebugMode;
+
+	/// <summary>
+	///     Affects all tests against the real file system.
+	/// </summary>
+	/// <remarks>Per default, they are <see cref="TestSettingStatus.DisabledInDebugMode" />.</remarks>
+	public TestSettingStatus RealFileSystemTests { get; set; }
+		= TestSettingStatus.DisabledInDebugMode;
+}

Tests/Helpers/Testably.Abstractions.TestHelpers/Settings/TestSettingStatus.cs

@@ -0,0 +1,19 @@
+namespace Testably.Abstractions.TestHelpers.Settings;
+
+public enum TestSettingStatus
+{
+	/// <summary>
+	///     The tests are always enabled.
+	/// </summary>
+	AlwaysEnabled,
+
+	/// <summary>
+	///     The tests are only disabled in DEBUG mode.
+	/// </summary>
+	DisabledInDebugMode,
+
+	/// <summary>
+	///     The tests are always disabled.
+	/// </summary>
+	AlwaysDisabled
+}

Tests/Helpers/Testably.Abstractions.TestHelpers/Settings/TestSettings.cs

@@ -57,7 +57,10 @@ public void Dispose()
 			=> _directoryCleaner.Dispose();
 
 		/// <inheritdoc cref=""{@class.Name}{{TFileSystem}}.LongRunningTestsShouldBeSkipped()"" />
-		public override bool LongRunningTestsShouldBeSkipped() => false;
+		public override void SkipIfLongRunningTestsShouldBeSkipped()
+		{{
+			// Long-running tests are never skipped against the mock file system!
+		}}
 	}}
 }}
 
@@ -77,9 +80,14 @@ public RealFileSystemTests(ITestOutputHelper testOutputHelper, RealFileSystemFix
 			: base(new Test(), new RealFileSystem(), new RealTimeSystem())
 		{{
 #if DEBUG
-			if (!fixture.EnableRealFileSystemTestsInDebugMode)
+			if (fixture.RealFileSystemTests != TestSettingStatus.AlwaysEnabled)
+			{{
+				throw new SkipException($""RealFileSystemTests are {{fixture.RealFileSystemTests}}. You can enable them by executing the corresponding tests in Testably.Abstractions.TestSettings.RealFileSystemTests."");
+			}}
+#else
+			if (fixture.RealFileSystemTests == TestSettingStatus.AlwaysDisabled)
 			{{
-				throw new SkipException(""EnableRealFileSystemTestsInDebugMode is not set in test.settings.json"");
+				throw new SkipException($""RealFileSystemTests are {{fixture.RealFileSystemTests}}. You can enable them by executing the corresponding tests in Testably.Abstractions.TestSettings.RealFileSystemTests."");
 			}}
 #endif
 			_fixture = fixture;
@@ -91,8 +99,17 @@ public RealFileSystemTests(ITestOutputHelper testOutputHelper, RealFileSystemFix
 		public void Dispose()
 			=> _directoryCleaner.Dispose();
 
+#if DEBUG
+		/// <inheritdoc cref=""{@class.Name}{{TFileSystem}}.LongRunningTestsShouldBeSkipped()"" />
+		public override void SkipIfLongRunningTestsShouldBeSkipped()
+			=> Skip.If(_fixture.LongRunningTests != TestSettingStatus.AlwaysEnabled,
+				$""LongRunningTests are {{_fixture.LongRunningTests}}. You can enable them by executing the corresponding tests in Testably.Abstractions.TestSettings.LongRunningTests."");
+#else
 		/// <inheritdoc cref=""{@class.Name}{{TFileSystem}}.LongRunningTestsShouldBeSkipped()"" />
-		public override bool LongRunningTestsShouldBeSkipped() => !_fixture.IncludeLongRunningTestsAlsoInDebugMode;
+		public override void SkipIfLongRunningTestsShouldBeSkipped()
+			=> Skip.If(_fixture.LongRunningTests == TestSettingStatus.AlwaysDisabled,
+				$""LongRunningTests are {{_fixture.LongRunningTests}}. You can enable them by executing the corresponding tests in Testably.Abstractions.TestSettings.LongRunningTests."");
+#endif
 	}}
 }}");
 }

Tests/Helpers/Testably.Abstractions.Tests.SourceGenerator/ClassGenerators/FileSystemClassGenerator.cs

@@ -1,43 +1,49 @@
 using System;
 using System.IO;
 using System.Text.Json;
+using System.Text.Json.Serialization;
+using Testably.Abstractions.TestHelpers.Settings;
 
 namespace Testably.Abstractions.TestSettings;
 
 internal static class Helper
 {
-	public static TestHelpers.Settings.TestSettings ChangeTestSettings(
-		Action<TestHelpers.Settings.TestSettings> change)
+	public static TestEnvironment ChangeTestSettings(
+		Action<TestEnvironment> change)
 	{
-		TestHelpers.Settings.TestSettings settings = ReadTestSettings();
-		change(settings);
-		WriteTestSettings(settings);
-		return settings;
+		TestEnvironment environment = ReadTestSettings();
+		change(environment);
+		WriteTestSettings(environment);
+		return environment;
 	}
 
 	private static string GetTestSettingsPath() =>
 		Path.GetFullPath(Path.Combine("..", "..", "..", "..", "test.settings.json"));
 
-	private static TestHelpers.Settings.TestSettings ReadTestSettings()
+	private static TestEnvironment ReadTestSettings()
 	{
 		try
 		{
 			string path = GetTestSettingsPath();
 			string content = File.ReadAllText(path);
-			return JsonSerializer.Deserialize<TestHelpers.Settings.TestSettings>(content)
+			return JsonSerializer.Deserialize<TestEnvironment>(content)
 			       ?? throw new NotSupportedException("The file has an invalid syntax!");
 		}
 		catch (Exception)
 		{
-			return new TestHelpers.Settings.TestSettings();
+			return new TestEnvironment();
 		}
 	}
 
-	private static void WriteTestSettings(TestHelpers.Settings.TestSettings settings)
+	private static void WriteTestSettings(TestEnvironment environment)
 	{
-		string content = JsonSerializer.Serialize(settings, new JsonSerializerOptions
+		string content = JsonSerializer.Serialize(environment, new JsonSerializerOptions
 		{
-			WriteIndented = true
+			WriteIndented = true,
+			Converters =
+			{
+				new JsonStringEnumConverter()
+			}
 		});
 		string path = GetTestSettingsPath();
 		File.WriteAllText(path, content);

Tests/Settings/Testably.Abstractions.TestSettings/Helper.cs

@@ -0,0 +1,78 @@
+using NUnit.Framework;
+using Testably.Abstractions.TestHelpers.Settings;
+
+namespace Testably.Abstractions.TestSettings;
+
+public sealed class LongRunningTests
+{
+	/// <summary>
+	///     Some tests take a long time to run against the real file system (e.g. timeout).
+	///     <para />
+	///     Always disable these tests!
+	/// </summary>
+	[TestCase]
+	[Explicit]
+	public void DisableAlways()
+	{
+		_ = Helper.ChangeTestSettings(s =>
+			s.LongRunningTests = TestSettingStatus.AlwaysDisabled);
+
+		Assert.Pass("Long-running tests are always disabled.");
+	}
+
+	/// <summary>
+	///     Some tests take a long time to run against the real file system (e.g. timeout).
+	///     <para />
+	///     Disable these tests in DEBUG mode!
+	/// </summary>
+	[TestCase]
+	[Explicit]
+	public void DisableInDebugMode()
+	{
+		TestEnvironment result = Helper.ChangeTestSettings(s =>
+			s.LongRunningTests = TestSettingStatus.DisabledInDebugMode);
+
+		if (result.RealFileSystemTests == TestSettingStatus.AlwaysDisabled)
+		{
+			Assert.Warn("""
+			            Long-running tests are always enabled.
+			            But the tests against the real file system are always disabled.
+			            """);
+		}
+		else
+		{
+			Assert.Pass("Long-running tests are disabled in DEBUG mode.");
+		}
+	}
+
+	/// <summary>
+	///     Some tests take a long time to run against the real file system (e.g. timeout).
+	///     <para />
+	///     Always enable these tests!
+	/// </summary>
+	[TestCase]
+	[Explicit]
+	public void EnableAlways()
+	{
+		TestEnvironment result = Helper.ChangeTestSettings(s =>
+			s.LongRunningTests = TestSettingStatus.AlwaysEnabled);
+
+		if (result.RealFileSystemTests != TestSettingStatus.AlwaysEnabled)
+		{
+			Assert.Warn("""
+			            Long-running tests are always enabled.
+			            But the tests against the real file system are not always enabled.
+			            """);
+		}
+		else
+		{
+			Assert.Pass("Long-running tests are always enabled.");
+		}
+	}
+
+	[TestCase]
+	public void TestDummy_SoThatExplicitTestsAreIgnored()
+	{
+		Assert.Pass();
+	}
+}