diff --git a/samples/BenchmarkDotNet.Samples.FSharp/BenchmarkDotNet.Samples.FSharp.fsproj b/samples/BenchmarkDotNet.Samples.FSharp/BenchmarkDotNet.Samples.FSharp.fsproj index ee4eb0c8b8..6359340234 100644 --- a/samples/BenchmarkDotNet.Samples.FSharp/BenchmarkDotNet.Samples.FSharp.fsproj +++ b/samples/BenchmarkDotNet.Samples.FSharp/BenchmarkDotNet.Samples.FSharp.fsproj @@ -7,6 +7,8 @@ Exe net462;net8.0 false + + false diff --git a/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj b/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj index 6c861ed1f6..33275eb7ed 100644 --- a/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj +++ b/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj @@ -13,6 +13,8 @@ $(NoWarn);CA1018;CA5351;CA1825 false + + false diff --git a/src/BenchmarkDotNet.TestAdapter/Utility/LoggerHelper.cs b/src/BenchmarkDotNet.TestAdapter/Utility/LoggerHelper.cs new file mode 100644 index 0000000000..08238c1927 --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/Utility/LoggerHelper.cs @@ -0,0 +1,57 @@ +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; +using System.Diagnostics; +using System.IO; + +namespace BenchmarkDotNet.TestAdapter; + +internal class LoggerHelper +{ + public LoggerHelper(IMessageLogger logger, Stopwatch stopwatch) + { + InnerLogger = logger; + Stopwatch = stopwatch; + } + + public IMessageLogger InnerLogger { get; private set; } + + public Stopwatch Stopwatch { get; private set; } + + public void Log(string format, params object[] args) + { + SendMessage(TestMessageLevel.Informational, null, string.Format(format, args)); + } + + public void LogWithSource(string source, string format, params object[] args) + { + SendMessage(TestMessageLevel.Informational, source, string.Format(format, args)); + } + + public void LogError(string format, params object[] args) + { + SendMessage(TestMessageLevel.Error, null, string.Format(format, args)); + } + + public void LogErrorWithSource(string source, string format, params object[] args) + { + SendMessage(TestMessageLevel.Error, source, string.Format(format, args)); + } + + public void LogWarning(string format, params object[] args) + { + SendMessage(TestMessageLevel.Warning, null, string.Format(format, args)); + } + + public void LogWarningWithSource(string source, string format, params object[] args) + { + SendMessage(TestMessageLevel.Warning, source, string.Format(format, args)); + } + + private void SendMessage(TestMessageLevel level, string? assemblyName, string message) + { + var assemblyText = assemblyName == null + ? "" : + $"{Path.GetFileNameWithoutExtension(assemblyName)}: "; + + InnerLogger.SendMessage(level, $"[BenchmarkDotNet {Stopwatch.Elapsed:hh\\:mm\\:ss\\.ff}] {assemblyText}{message}"); + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/Utility/TestCaseFilter.cs b/src/BenchmarkDotNet.TestAdapter/Utility/TestCaseFilter.cs new file mode 100644 index 0000000000..ffd891bf61 --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/Utility/TestCaseFilter.cs @@ -0,0 +1,166 @@ +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; + +namespace BenchmarkDotNet.TestAdapter; + +internal class TestCaseFilter +{ + private const string DisplayNameString = "DisplayName"; + private const string FullyQualifiedNameString = "FullyQualifiedName"; + + private readonly HashSet knownTraits; + private List supportedPropertyNames; + private readonly ITestCaseFilterExpression? filterExpression; + private readonly bool successfullyGotFilter; + private readonly bool isDiscovery; + + public TestCaseFilter(IDiscoveryContext discoveryContext, LoggerHelper logger) + { + // Traits are not known at discovery time because we load them from benchmarks + isDiscovery = true; + knownTraits = []; + supportedPropertyNames = GetSupportedPropertyNames(); + successfullyGotFilter = GetTestCaseFilterExpressionFromDiscoveryContext(discoveryContext, logger, out filterExpression); + } + + public TestCaseFilter(IRunContext runContext, LoggerHelper logger, string assemblyFileName, HashSet knownTraits) + { + this.knownTraits = knownTraits; + supportedPropertyNames = GetSupportedPropertyNames(); + successfullyGotFilter = GetTestCaseFilterExpression(runContext, logger, assemblyFileName, out filterExpression); + } + + public string GetTestCaseFilterValue() + { + return successfullyGotFilter + ? filterExpression?.TestCaseFilterValue ?? "" + : ""; + } + + public bool MatchTestCase(TestCase testCase) + { + if (!successfullyGotFilter) + { + // Had an error while getting filter, match no testcase to ensure discovered test list is empty + return false; + } + else if (filterExpression == null) + { + // No filter specified, keep every testcase + return true; + } + + return filterExpression.MatchTestCase(testCase, p => PropertyProvider(testCase, p)); + } + + public object? PropertyProvider(TestCase testCase, string name) + { + // Traits filtering + if (isDiscovery || knownTraits.Contains(name)) + { + var result = new List(); + + foreach (var trait in GetTraits(testCase)) + if (string.Equals(trait.Key, name, StringComparison.OrdinalIgnoreCase)) + result.Add(trait.Value); + + if (result.Count > 0) + return result.ToArray(); + } + + // Property filtering + switch (name.ToLowerInvariant()) + { + // FullyQualifiedName + case "fullyqualifiedname": + return testCase.FullyQualifiedName; + // DisplayName + case "displayname": + return testCase.DisplayName; + default: + return null; + } + } + + private bool GetTestCaseFilterExpression(IRunContext runContext, LoggerHelper logger, string assemblyFileName, out ITestCaseFilterExpression? filter) + { + filter = null; + + try + { + filter = runContext.GetTestCaseFilter(supportedPropertyNames, null!); + return true; + } + catch (TestPlatformFormatException e) + { + logger.LogWarning("{0}: Exception filtering tests: {1}", Path.GetFileNameWithoutExtension(assemblyFileName), e.Message); + return false; + } + } + + private bool GetTestCaseFilterExpressionFromDiscoveryContext(IDiscoveryContext discoveryContext, LoggerHelper logger, out ITestCaseFilterExpression? filter) + { + filter = null; + + if (discoveryContext is IRunContext runContext) + { + try + { + filter = runContext.GetTestCaseFilter(supportedPropertyNames, null!); + return true; + } + catch (TestPlatformException e) + { + logger.LogWarning("Exception filtering tests: {0}", e.Message); + return false; + } + } + else + { + try + { + // GetTestCaseFilter is present on DiscoveryContext but not in IDiscoveryContext interface + var method = discoveryContext.GetType().GetRuntimeMethod("GetTestCaseFilter", [typeof(IEnumerable), typeof(Func)]); + filter = (ITestCaseFilterExpression)method?.Invoke(discoveryContext, [supportedPropertyNames, null])!; + + return true; + } + catch (TargetInvocationException e) + { + if (e?.InnerException is TestPlatformException ex) + { + logger.LogWarning("Exception filtering tests: {0}", ex.InnerException.Message ?? ""); + return false; + } + + throw e!.InnerException; + } + } + } + + private List GetSupportedPropertyNames() + { + // Returns the set of well-known property names usually used with the Test Plugins (Used Test Traits + DisplayName + FullyQualifiedName) + if (supportedPropertyNames == null) + { + supportedPropertyNames = knownTraits.ToList(); + supportedPropertyNames.Add(DisplayNameString); + supportedPropertyNames.Add(FullyQualifiedNameString); + } + + return supportedPropertyNames; + } + + private static IEnumerable> GetTraits(TestCase testCase) + { + var traitProperty = TestProperty.Find("TestObject.Traits"); + return traitProperty != null + ? testCase.GetPropertyValue(traitProperty, Array.Empty>()) + : []; + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/VSTestAdapter.cs b/src/BenchmarkDotNet.TestAdapter/VSTestAdapter.cs index eb6695de1b..3a3e155bf4 100644 --- a/src/BenchmarkDotNet.TestAdapter/VSTestAdapter.cs +++ b/src/BenchmarkDotNet.TestAdapter/VSTestAdapter.cs @@ -4,6 +4,7 @@ using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Linq; using System.Reflection; @@ -42,11 +43,18 @@ public void DiscoverTests( IMessageLogger logger, ITestCaseDiscoverySink discoverySink) { + var stopwatch = Stopwatch.StartNew(); + var loggerHelper = new LoggerHelper(logger, stopwatch); + var testCaseFilter = new TestCaseFilter(discoveryContext, loggerHelper); + foreach (var source in sources) { ValidateSourceIsAssemblyOrThrow(source); foreach (var testCase in GetVsTestCasesFromAssembly(source, logger)) { + if (!testCaseFilter.MatchTestCase(testCase)) + continue; + discoverySink.SendTestCase(testCase); } } @@ -67,14 +75,19 @@ public void RunTests(IEnumerable? tests, IRunContext? runContext, IFra cts ??= new CancellationTokenSource(); + var stopwatch = Stopwatch.StartNew(); + var logger = new LoggerHelper(frameworkHandle, stopwatch); + foreach (var testsPerAssembly in tests.GroupBy(t => t.Source)) + { RunBenchmarks(testsPerAssembly.Key, frameworkHandle, testsPerAssembly); + } cts = null; } /// - /// Runs all benchmarks in the given set of sources (assemblies). + /// Runs all/filtered benchmarks in the given set of sources (assemblies). /// /// The assemblies to run. /// A context that the run is performed in. @@ -88,8 +101,31 @@ public void RunTests(IEnumerable? sources, IRunContext? runContext, IFra cts ??= new CancellationTokenSource(); + var stopwatch = Stopwatch.StartNew(); + var logger = new LoggerHelper(frameworkHandle, stopwatch); + foreach (var source in sources) - RunBenchmarks(source, frameworkHandle); + { + var filter = new TestCaseFilter(runContext!, logger, source, ["Category"]); + if (filter.GetTestCaseFilterValue() != "") + { + var discoveredBenchmarks = GetVsTestCasesFromAssembly(source, frameworkHandle); + var filteredTestCases = discoveredBenchmarks.Where(x => filter.MatchTestCase(x)) + .ToArray(); + + if (filteredTestCases.Length == 0) + continue; + + // Run filtered tests. + RunBenchmarks(source, frameworkHandle, filteredTestCases); + } + else + { + // Run all benchmarks + RunBenchmarks(source, frameworkHandle); + } + } + cts = null; } diff --git a/src/BenchmarkDotNet.TestAdapter/build/BenchmarkDotNet.TestAdapter.props b/src/BenchmarkDotNet.TestAdapter/build/BenchmarkDotNet.TestAdapter.props index 8c716bb3bc..b672c62139 100644 --- a/src/BenchmarkDotNet.TestAdapter/build/BenchmarkDotNet.TestAdapter.props +++ b/src/BenchmarkDotNet.TestAdapter/build/BenchmarkDotNet.TestAdapter.props @@ -1,6 +1,8 @@  $(MSBuildThisFileDirectory)..\entrypoints\ + + false