diff --git a/src/BenchmarkDotNet/Configs/DefaultConfig.cs b/src/BenchmarkDotNet/Configs/DefaultConfig.cs index 8585a77797..d188b67415 100644 --- a/src/BenchmarkDotNet/Configs/DefaultConfig.cs +++ b/src/BenchmarkDotNet/Configs/DefaultConfig.cs @@ -72,6 +72,7 @@ public IEnumerable GetValidators() yield return DeferredExecutionValidator.FailOnError; yield return ParamsAllValuesValidator.FailOnError; yield return ParamsValidator.FailOnError; + yield return RuntimeValidator.DontFailOnError; } public IOrderer Orderer => null; diff --git a/src/BenchmarkDotNet/Validators/RuntimeValidator.cs b/src/BenchmarkDotNet/Validators/RuntimeValidator.cs new file mode 100644 index 0000000000..7be547890c --- /dev/null +++ b/src/BenchmarkDotNet/Validators/RuntimeValidator.cs @@ -0,0 +1,43 @@ +using System.Collections.Generic; +using System.Linq; +using BenchmarkDotNet.Characteristics; + +namespace BenchmarkDotNet.Validators; + +/// +/// Validator for runtime characteristic. +/// +/// +public class RuntimeValidator : IValidator +{ + public static readonly IValidator DontFailOnError = new RuntimeValidator(); + + private RuntimeValidator() { } + + public bool TreatsWarningsAsErrors => false; + + public IEnumerable Validate(ValidationParameters input) + { + var allBenchmarks = input.Benchmarks.ToArray(); + var nullRuntimeBenchmarks = allBenchmarks.Where(x => x.Job.Environment.Runtime == null).ToArray(); + + // There is no validation error if all the runtimes are set or if all the runtimes are null. + if (allBenchmarks.Length == nullRuntimeBenchmarks.Length) + { + return []; + } + + var errors = new List(); + foreach (var benchmark in nullRuntimeBenchmarks) + { + var job = benchmark.Job; + var jobText = job.HasValue(CharacteristicObject.IdCharacteristic) + ? job.Id + : CharacteristicSetPresenter.Display.ToPresentation(job); // Use job text representation instead for auto generated JobId. + + var message = $"Job({jobText}) doesn't have a Runtime characteristic. It's recommended to specify runtime by using WithRuntime explicitly."; + errors.Add(new ValidationError(false, message)); + } + return errors; + } +} diff --git a/tests/BenchmarkDotNet.Tests/Validators/RuntimeValidatorTests.cs b/tests/BenchmarkDotNet.Tests/Validators/RuntimeValidatorTests.cs new file mode 100644 index 0000000000..9eac2dd6f2 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Validators/RuntimeValidatorTests.cs @@ -0,0 +1,125 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains.CsProj; +using BenchmarkDotNet.Validators; +using System.Linq; +using Xunit; + +namespace BenchmarkDotNet.Tests.Validators; + +public class RuntimeValidatorTests +{ + [Fact] + public void SameRuntime_Should_Success() + { + // Arrange + var config = new TestConfig1().CreateImmutableConfig(); + var runInfo = BenchmarkConverter.TypeToBenchmarks(typeof(DummyBenchmark), config); + var parameters = new ValidationParameters(runInfo.BenchmarksCases, config); + + // Act + var errors = RuntimeValidator.DontFailOnError.Validate(parameters).Select(e => e.Message).ToArray(); + + // Assert + Assert.Empty(errors); + } + + [Fact] + public void NullRuntimeMixed_Should_Failed() + { + // Arrange + var config = new TestConfig2().CreateImmutableConfig(); + var runInfo = BenchmarkConverter.TypeToBenchmarks(typeof(DummyBenchmark), config); + var parameters = new ValidationParameters(runInfo.BenchmarksCases, config); + + // Act + var errors = RuntimeValidator.DontFailOnError.Validate(parameters).Select(e => e.Message).ToArray(); + + // Assert + { + var expectedMessage = "Job(Dry) doesn't have a Runtime characteristic. It's recommended to specify runtime by using WithRuntime explicitly."; + Assert.Contains(expectedMessage, errors); + } + { + var expectedMessage = "Job(Toolchain=.NET 10.0) doesn't have a Runtime characteristic. It's recommended to specify runtime by using WithRuntime explicitly."; + Assert.Contains(expectedMessage, errors); + } + } + + [Fact] + public void NotNullRuntimeOnly_Should_Success() + { + // Arrange + var config = new TestConfig3().CreateImmutableConfig(); + var runInfo = BenchmarkConverter.TypeToBenchmarks(typeof(DummyBenchmark), config); + var parameters = new ValidationParameters(runInfo.BenchmarksCases, config); + + // Act + var errors = RuntimeValidator.DontFailOnError.Validate(parameters).Select(e => e.Message).ToArray(); + + // Assert + Assert.Empty(errors); + } + + public class DummyBenchmark + { + [Benchmark] + public void Benchmark() + { + } + } + + // TestConfig that expicitly specify runtime. + private class TestConfig1 : ManualConfig + { + public TestConfig1() + { + var baseJob = Job.Dry; + + WithOption(ConfigOptions.DisableOptimizationsValidator, true); + AddColumnProvider(DefaultConfig.Instance.GetColumnProviders().ToArray()); + + AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp80)); + AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp90)); + } + } + + // TestConfig that contains job that don't specify runtime. + private class TestConfig2 : ManualConfig + { + public TestConfig2() + { + var baseJob = Job.Dry; + + WithOption(ConfigOptions.DisableOptimizationsValidator, true); + AddColumnProvider(DefaultConfig.Instance.GetColumnProviders().ToArray()); + + AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp80)); + AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp90) + .WithRuntime(CoreRuntime.Core90)); + + // Validate error message for auto generated jobid. + AddJob(Job.Default.WithToolchain(CsProjCoreToolchain.NetCoreApp10_0)); + } + } + + // TestConfig that expicitly specify runtime. + private class TestConfig3 : ManualConfig + { + public TestConfig3() + { + var baseJob = Job.Dry; + + WithOption(ConfigOptions.DisableOptimizationsValidator, true); + AddColumnProvider(DefaultConfig.Instance.GetColumnProviders().ToArray()); + + AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp80) + .WithRuntime(CoreRuntime.Core80)); ; + AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp90) + .WithRuntime(CoreRuntime.Core90)); + } + } +}