Log a warning when an empty benchmark is run

src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs

@@ -19,7 +19,6 @@
 using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Loggers;
 using BenchmarkDotNet.Mathematics;
-using BenchmarkDotNet.Portability;
 using BenchmarkDotNet.Reports;
 using BenchmarkDotNet.Toolchains;
 using BenchmarkDotNet.Toolchains.Parameters;
@@ -73,9 +72,6 @@ internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos)
                 if (validationErrors.Any(validationError => validationError.IsCritical))
                     return new[] { Summary.ValidationFailed(title, resultsFolderPath, logFilePath, validationErrors.ToImmutableArray()) };
 
-                if (!supportedBenchmarks.Any(benchmarks => benchmarks.BenchmarksCases.Any()))
-                    return new[] { Summary.ValidationFailed(title, resultsFolderPath, logFilePath) };
-
                 eventProcessor.OnEndValidationStage();
 
                 int totalBenchmarkCount = supportedBenchmarks.Sum(benchmarkInfo => benchmarkInfo.BenchmarksCases.Length);
@@ -578,20 +574,41 @@ private static void LogTotalTime(ILogger logger, TimeSpan time, int executedBenc
         private static (BenchmarkRunInfo[], List<ValidationError>) GetSupportedBenchmarks(BenchmarkRunInfo[] benchmarkRunInfos, IResolver resolver)
         {
             List<ValidationError> validationErrors = new ();
+            List<BenchmarkRunInfo> runInfos = new (benchmarkRunInfos.Length);
+
+            foreach (var benchmarkRunInfo in benchmarkRunInfos)
+            {
+                if (benchmarkRunInfo.BenchmarksCases.Length == 0)
+                {
+                    validationErrors.Add(new ValidationError(true, $"No [Benchmark] attribute found on '{benchmarkRunInfo.Type.Name}' benchmark case."));
+                    continue;
+                }
 
-            var runInfos = benchmarkRunInfos.Select(info => new BenchmarkRunInfo(
-                    info.BenchmarksCases.Where(benchmark =>
+                var validBenchmarks = benchmarkRunInfo.BenchmarksCases
+                    .Where(benchmark =>
                     {
-                        var errors = benchmark.GetToolchain().Validate(benchmark, resolver).ToArray();
+
+                        var errors = benchmark.GetToolchain()
+                            .Validate(benchmark, resolver)
+                            .ToArray();
+
                         validationErrors.AddRange(errors);
-                        return !errors.Any();
-                    }).ToArray(),
-                    info.Type,
-                    info.Config))
-                .Where(infos => infos.BenchmarksCases.Any())
-                .ToArray();
-
-            return (runInfos, validationErrors);
+
+                        return errors.Length == 0;
+                    })
+                    .ToArray();
+
+                runInfos.Add(
+                    new BenchmarkRunInfo(
+                        validBenchmarks,
+                        benchmarkRunInfo.Type,
+                        benchmarkRunInfo.Config
+
+                    ));
+
+
+            }
+            return (runInfos.ToArray(), validationErrors);
         }
 
         private static string GetRootArtifactsFolderPath(BenchmarkRunInfo[] benchmarkRunInfos)

tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs

@@ -0,0 +1,70 @@
+using System;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Diagnosers;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Running;
+using BenchmarkDotNet.Tests.Loggers;
+using BenchmarkDotNet.Parameters;
+using System.Reflection;
+using Xunit;
+using Xunit.Abstractions;
+using BenchmarkDotNet.Tests.XUnit;
+using BenchmarkDotNet.Loggers;
+using System.Linq;
+
+namespace BenchmarkDotNet.Tests.Running
+{
+    public class RunningEmptyBenchmarkTests
+    {
+        [Fact]
+        public void WhenRunningSingleEmptyBenchmark_ValidationErrorIsThrown()
+        {
+            var summaries = BenchmarkRunnerClean.Run(new[] { BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmark), null) });
+            Assert.Single(summaries);
+            var summary = summaries[0];
+            Assert.True(summary.HasCriticalValidationErrors);
+            Assert.Single(summary.ValidationErrors);
+            Assert.Equal($"No [Benchmark] attribute found on '{typeof(EmptyBenchmark).Name}' benchmark case.", summary.ValidationErrors[0].Message);
+        }
+
+        [Fact]
+        public void WhenRunningMultipleEmptyBenchmarks_ValidationErrorIsThrown()
+        {
+            var summaries = BenchmarkRunnerClean.Run(new[] { BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmark), null), BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmark), null) });
+            Assert.Single(summaries);
+            var summary = summaries[0];
+            Assert.True(summary.HasCriticalValidationErrors);
+            Assert.Equal(2, summary.ValidationErrors.Count());
+            Assert.Equal($"No [Benchmark] attribute found on '{typeof(EmptyBenchmark).Name}' benchmark case.", summary.ValidationErrors[0].Message);
+            Assert.Equal($"No [Benchmark] attribute found on '{typeof(EmptyBenchmark).Name}' benchmark case.", summary.ValidationErrors[1].Message);
+        }
+
+        [Fact]
+        public void WhenRunningMultipleBenchmarksOneOfWhichIsEmpty_ValidationErrorIsThrown()
+        {
+            var summaries = BenchmarkRunnerClean.Run(new[] { BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmark), null), BenchmarkConverter.TypeToBenchmarks(typeof(NotEmptyBenchmark), null) });
+            Assert.Single(summaries);
+            var summary = summaries[0];
+            Assert.True(summary.HasCriticalValidationErrors);
+            Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == $"No [Benchmark] attribute found on '{typeof(EmptyBenchmark).Name}' benchmark case.");
+        }
+
+        public class EmptyBenchmark
+        {
+        }
+
+        public class NotEmptyBenchmark
+        {
+            [Benchmark]
+            public void Benchmark()
+            {
+                var sum = 0;
+                for (int i = 0; i < 1; i++)
+                {
+                    sum += i;
+                }
+            }
+        }
+    }
+}