From 0cee16960f6f5f757158a897e762917163a43ada Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Wed, 21 Sep 2022 03:01:10 -0400 Subject: [PATCH 01/58] Disambiguate NativeAOT, and Wasm identification in BenchmarkDotNet.Portability.RuntimeInformation (#2112) Fixes https://github.com/dotnet/BenchmarkDotNet/issues/2099 . --- src/BenchmarkDotNet/Portability/RuntimeInformation.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/BenchmarkDotNet/Portability/RuntimeInformation.cs b/src/BenchmarkDotNet/Portability/RuntimeInformation.cs index 5b743a3129..1e0d88c67d 100644 --- a/src/BenchmarkDotNet/Portability/RuntimeInformation.cs +++ b/src/BenchmarkDotNet/Portability/RuntimeInformation.cs @@ -43,7 +43,8 @@ public static bool IsNetCore public static bool IsNativeAOT => Environment.Version.Major >= 5 - && string.IsNullOrEmpty(typeof(object).Assembly.Location); // it's merged to a single .exe and .Location returns null + && string.IsNullOrEmpty(typeof(object).Assembly.Location) // it's merged to a single .exe and .Location returns null + && !IsWasm; // Wasm also returns "" for assembly locations public static bool IsWasm => IsOSPlatform(OSPlatform.Create("BROWSER")); From 21a29406406aeca1cdfa195fa86ac28d04ba8e33 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 22 Sep 2022 09:35:43 +0200 Subject: [PATCH 02/58] Implement apples to apples comparison mode (#2116) * InvocationCount should be represented with Int64, not Int32 * implement apples-to-apples comparison --- src/BenchmarkDotNet/Configs/ConfigOptions.cs | 6 +- src/BenchmarkDotNet/Configs/ManualConfig.cs | 4 + .../ConsoleArguments/CommandLineOptions.cs | 5 +- .../ConsoleArguments/ConfigParser.cs | 1 + src/BenchmarkDotNet/Jobs/JobExtensions.cs | 2 +- src/BenchmarkDotNet/Jobs/RunMode.cs | 4 +- .../Parameters/ParameterInstances.cs | 24 +++++- .../Running/BenchmarkSwitcher.cs | 77 +++++++++++++++++++ src/BenchmarkDotNet/Running/Descriptor.cs | 8 +- .../Validators/RunModeValidator.cs | 2 +- 10 files changed, 125 insertions(+), 8 deletions(-) diff --git a/src/BenchmarkDotNet/Configs/ConfigOptions.cs b/src/BenchmarkDotNet/Configs/ConfigOptions.cs index 40fc189d42..29e9e1af6f 100644 --- a/src/BenchmarkDotNet/Configs/ConfigOptions.cs +++ b/src/BenchmarkDotNet/Configs/ConfigOptions.cs @@ -40,7 +40,11 @@ public enum ConfigOptions /// /// Determines whether to generate msbuild binlogs /// - GenerateMSBuildBinLog = 1 << 7 + GenerateMSBuildBinLog = 1 << 7, + /// + /// Performs apples-to-apples comparison for provided benchmarks and jobs. Experimental, will change in the near future! + /// + ApplesToApples = 1 << 8 } internal static class ConfigOptionsExtensions diff --git a/src/BenchmarkDotNet/Configs/ManualConfig.cs b/src/BenchmarkDotNet/Configs/ManualConfig.cs index e0d72e5558..9f632c03d0 100644 --- a/src/BenchmarkDotNet/Configs/ManualConfig.cs +++ b/src/BenchmarkDotNet/Configs/ManualConfig.cs @@ -300,6 +300,10 @@ public static ManualConfig Union(IConfig globalConfig, IConfig localConfig) return manualConfig; } + internal void RemoveAllJobs() => jobs.Clear(); + + internal void RemoveAllDiagnosers() => diagnosers.Clear(); + private static TimeSpan GetBuildTimeout(TimeSpan current, TimeSpan other) => current == DefaultConfig.Instance.BuildTimeout ? other diff --git a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs index dc3b0e2af6..8a52323ad2 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs @@ -135,7 +135,7 @@ public bool UseDisassemblyDiagnoser public int? MaxIterationCount { get; set; } [Option("invocationCount", Required = false, HelpText = "Invocation count in a single iteration. By default calculated by the heuristic.")] - public int? InvocationCount { get; set; } + public long? InvocationCount { get; set; } [Option("unrollFactor", Required = false, HelpText = "How many times the benchmark method will be invoked per one iteration of a generated loop. 16 by default")] public int? UnrollFactor { get; set; } @@ -152,6 +152,9 @@ public bool UseDisassemblyDiagnoser [Option("info", Required = false, Default = false, HelpText = "Print environment information.")] public bool PrintInformation { get; set; } + [Option("apples", Required = false, Default = false, HelpText = "Runs apples-to-apples comparison for specified Jobs.")] + public bool ApplesToApples { get; set; } + [Option("list", Required = false, Default = ListBenchmarkCaseMode.Disabled, HelpText = "Prints all of the available benchmark names. Flat/Tree")] public ListBenchmarkCaseMode ListBenchmarkCaseMode { get; set; } diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index f6d64920b3..5f5203cc30 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -244,6 +244,7 @@ private static IConfig CreateConfig(CommandLineOptions options, IConfig globalCo config.WithOption(ConfigOptions.DisableLogFile, options.DisableLogFile); config.WithOption(ConfigOptions.LogBuildOutput, options.LogBuildOutput); config.WithOption(ConfigOptions.GenerateMSBuildBinLog, options.GenerateMSBuildBinLog); + config.WithOption(ConfigOptions.ApplesToApples, options.ApplesToApples); if (options.MaxParameterColumnWidth.HasValue) config.WithSummaryStyle(SummaryStyle.Default.WithMaxParameterColumnWidth(options.MaxParameterColumnWidth.Value)); diff --git a/src/BenchmarkDotNet/Jobs/JobExtensions.cs b/src/BenchmarkDotNet/Jobs/JobExtensions.cs index 98dc0a5ed9..34605e273d 100644 --- a/src/BenchmarkDotNet/Jobs/JobExtensions.cs +++ b/src/BenchmarkDotNet/Jobs/JobExtensions.cs @@ -170,7 +170,7 @@ public static Job WithHeapAffinitizeMask(this Job job, int heapAffinitizeMask) = /// If specified, will be ignored. /// If specified, it must be a multiple of . /// - public static Job WithInvocationCount(this Job job, int count) => job.WithCore(j => j.Run.InvocationCount = count); + public static Job WithInvocationCount(this Job job, long count) => job.WithCore(j => j.Run.InvocationCount = count); /// /// How many times the benchmark method will be invoked per one iteration of a generated loop. diff --git a/src/BenchmarkDotNet/Jobs/RunMode.cs b/src/BenchmarkDotNet/Jobs/RunMode.cs index d3248b0921..a76b5dc832 100644 --- a/src/BenchmarkDotNet/Jobs/RunMode.cs +++ b/src/BenchmarkDotNet/Jobs/RunMode.cs @@ -12,7 +12,7 @@ public sealed class RunMode : JobMode public static readonly Characteristic RunStrategyCharacteristic = Characteristic.Create(nameof(RunStrategy), RunStrategy.Throughput); public static readonly Characteristic LaunchCountCharacteristic = CreateCharacteristic(nameof(LaunchCount)); - public static readonly Characteristic InvocationCountCharacteristic = CreateCharacteristic(nameof(InvocationCount)); + public static readonly Characteristic InvocationCountCharacteristic = CreateCharacteristic(nameof(InvocationCount)); public static readonly Characteristic UnrollFactorCharacteristic = CreateCharacteristic(nameof(UnrollFactor)); public static readonly Characteristic IterationCountCharacteristic = CreateCharacteristic(nameof(IterationCount)); public static readonly Characteristic MinIterationCountCharacteristic = CreateCharacteristic(nameof(MinIterationCount)); @@ -125,7 +125,7 @@ public TimeInterval IterationTime /// If specified, will be ignored. /// If specified, it must be a multiple of . /// - public int InvocationCount + public long InvocationCount { get { return InvocationCountCharacteristic[this]; } set { InvocationCountCharacteristic[this] = value; } diff --git a/src/BenchmarkDotNet/Parameters/ParameterInstances.cs b/src/BenchmarkDotNet/Parameters/ParameterInstances.cs index 1e1bed2cbf..1f83b59310 100644 --- a/src/BenchmarkDotNet/Parameters/ParameterInstances.cs +++ b/src/BenchmarkDotNet/Parameters/ParameterInstances.cs @@ -5,7 +5,7 @@ namespace BenchmarkDotNet.Parameters { - public class ParameterInstances : IDisposable + public class ParameterInstances : IEquatable, IDisposable { public IReadOnlyList Items { get; } public int Count => Items.Count; @@ -36,5 +36,27 @@ public void Dispose() public string PrintInfo => printInfo ?? (printInfo = string.Join("&", Items.Select(p => $"{p.Name}={p.ToDisplayText()}"))); public ParameterInstance GetArgument(string name) => Items.Single(parameter => parameter.IsArgument && parameter.Name == name); + + public bool Equals(ParameterInstances other) + { + if (other.Count != Count) + { + return false; + } + + for (int i = 0; i < Count; i++) + { + if (!Items[i].Value.Equals(other[i].Value)) + { + return false; + } + } + + return true; + } + + public override bool Equals(object obj) => obj is ParameterInstances other && Equals(other); + + public override int GetHashCode() => FolderInfo.GetHashCode(); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Running/BenchmarkSwitcher.cs b/src/BenchmarkDotNet/Running/BenchmarkSwitcher.cs index d265cc7477..d236491016 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkSwitcher.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkSwitcher.cs @@ -9,9 +9,12 @@ using BenchmarkDotNet.ConsoleArguments.ListBenchmarks; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Parameters; using BenchmarkDotNet.Reports; using JetBrains.Annotations; +using Perfolizer.Mathematics.OutlierDetection; namespace BenchmarkDotNet.Running { @@ -109,6 +112,11 @@ internal IEnumerable RunWithDirtyAssemblyResolveHelper(string[] args, I ? allAvailableTypesWithRunnableBenchmarks : userInteraction.AskUser(allAvailableTypesWithRunnableBenchmarks, logger); + if (effectiveConfig.Options.HasFlag(ConfigOptions.ApplesToApples)) + { + return ApplesToApples(effectiveConfig, benchmarksToFilter, logger, options); + } + var filteredBenchmarks = TypeFilter.Filter(effectiveConfig, benchmarksToFilter); if (filteredBenchmarks.IsEmpty()) @@ -131,5 +139,74 @@ private static void PrintList(ILogger nonNullLogger, IConfig effectiveConfig, IR printer.Print(testNames, nonNullLogger); } + + private IEnumerable ApplesToApples(ManualConfig effectiveConfig, IReadOnlyList benchmarksToFilter, ILogger logger, CommandLineOptions options) + { + var jobs = effectiveConfig.GetJobs().ToArray(); + if (jobs.Length <= 1) + { + logger.WriteError("To use apples-to-apples comparison you must specify at least two Job objects."); + return Array.Empty(); + } + var baselineJob = jobs.SingleOrDefault(job => job.Meta.Baseline); + if (baselineJob == default) + { + logger.WriteError("To use apples-to-apples comparison you must specify exactly ONE baseline Job object."); + return Array.Empty(); + } + else if (jobs.Any(job => !job.Run.HasValue(RunMode.IterationCountCharacteristic))) + { + logger.WriteError("To use apples-to-apples comparison you must specify the number of iterations in explicit way."); + return Array.Empty(); + } + + Job invocationCountJob = baselineJob + .WithWarmupCount(1) + .WithIterationCount(1) + .WithEvaluateOverhead(false); + + ManualConfig invocationCountConfig = ManualConfig.Create(effectiveConfig); + invocationCountConfig.RemoveAllJobs(); + invocationCountConfig.RemoveAllDiagnosers(); + invocationCountConfig.AddJob(invocationCountJob); + + var invocationCountBenchmarks = TypeFilter.Filter(invocationCountConfig, benchmarksToFilter); + if (invocationCountBenchmarks.IsEmpty()) + { + userInteraction.PrintWrongFilterInfo(benchmarksToFilter, logger, options.Filters.ToArray()); + return Array.Empty(); + } + + logger.WriteLineHeader("Each benchmark is going to be executed just once to get invocation counts."); + Summary[] invocationCountSummaries = BenchmarkRunnerClean.Run(invocationCountBenchmarks); + + Dictionary<(Descriptor Descriptor, ParameterInstances Parameters), Measurement> dictionary = invocationCountSummaries + .SelectMany(summary => summary.Reports) + .ToDictionary( + report => (report.BenchmarkCase.Descriptor, report.BenchmarkCase.Parameters), + report => report.AllMeasurements.Single(measurement => measurement.IsWorkload() && measurement.IterationStage == Engines.IterationStage.Actual)); + + int iterationCount = baselineJob.Run.IterationCount; + BenchmarkRunInfo[] benchmarksWithoutInvocationCount = TypeFilter.Filter(effectiveConfig, benchmarksToFilter); + BenchmarkRunInfo[] benchmarksWithInvocationCount = benchmarksWithoutInvocationCount + .Select(benchmarkInfo => new BenchmarkRunInfo( + benchmarkInfo.BenchmarksCases.Select(benchmark => + new BenchmarkCase( + benchmark.Descriptor, + benchmark.Job + .WithIterationCount(iterationCount) + .WithEvaluateOverhead(false) + .WithWarmupCount(1) + .WithOutlierMode(OutlierMode.DontRemove) + .WithInvocationCount(dictionary[(benchmark.Descriptor, benchmark.Parameters)].Operations) + .WithUnrollFactor(dictionary[(benchmark.Descriptor, benchmark.Parameters)].Operations % 16 == 0 ? 16 : 1), + benchmark.Parameters, + benchmark.Config)).ToArray(), + benchmarkInfo.Type, benchmarkInfo.Config)) + .ToArray(); + + logger.WriteLineHeader("Actual benchmarking is going to happen now!"); + return BenchmarkRunnerClean.Run(benchmarksWithInvocationCount); + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Running/Descriptor.cs b/src/BenchmarkDotNet/Running/Descriptor.cs index 85fb3e3068..b66e63872b 100644 --- a/src/BenchmarkDotNet/Running/Descriptor.cs +++ b/src/BenchmarkDotNet/Running/Descriptor.cs @@ -8,7 +8,7 @@ namespace BenchmarkDotNet.Running { - public class Descriptor + public class Descriptor : IEquatable { public Type Type { get; } public MethodInfo WorkloadMethod { get; } @@ -70,5 +70,11 @@ private static string FormatDescription([CanBeNull] string description) public bool HasCategory(string category) => Categories.Any(c => c.EqualsWithIgnoreCase(category)); public string GetFilterName() => $"{Type.GetCorrectCSharpTypeName(includeGenericArgumentsNamespace: false)}.{WorkloadMethod.Name}"; + + public bool Equals(Descriptor other) => GetFilterName().Equals(other.GetFilterName()); + + public override bool Equals(object obj) => obj is Descriptor && Equals((Descriptor)obj); + + public override int GetHashCode() => GetFilterName().GetHashCode(); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Validators/RunModeValidator.cs b/src/BenchmarkDotNet/Validators/RunModeValidator.cs index 7a7904c108..ca11c9732c 100644 --- a/src/BenchmarkDotNet/Validators/RunModeValidator.cs +++ b/src/BenchmarkDotNet/Validators/RunModeValidator.cs @@ -28,7 +28,7 @@ public IEnumerable Validate(ValidationParameters validationPara } else if (run.HasValue(RunMode.InvocationCountCharacteristic)) { - int invocationCount = run.InvocationCount; + long invocationCount = run.InvocationCount; if (invocationCount % unrollFactor != 0) { string message = $"Specified InvocationCount ({invocationCount}) must be a multiple of UnrollFactor ({unrollFactor})"; From 37d0cf5d36bc2c99822084c19e53dd4c56f28715 Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Mon, 26 Sep 2022 00:48:27 -0700 Subject: [PATCH 03/58] Roslyn Toolchain does not support .NET Core (#2120) --- src/BenchmarkDotNet/Toolchains/Roslyn/RoslynToolchain.cs | 7 +++++++ .../NugetReferenceTests.cs | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/BenchmarkDotNet/Toolchains/Roslyn/RoslynToolchain.cs b/src/BenchmarkDotNet/Toolchains/Roslyn/RoslynToolchain.cs index ed388b26ba..df40c09139 100644 --- a/src/BenchmarkDotNet/Toolchains/Roslyn/RoslynToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/Roslyn/RoslynToolchain.cs @@ -1,6 +1,7 @@ using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Portability; using BenchmarkDotNet.Running; using JetBrains.Annotations; @@ -27,6 +28,12 @@ public override bool IsSupported(BenchmarkCase benchmarkCase, ILogger logger, IR return false; } + if (!RuntimeInformation.IsFullFramework) + { + logger.WriteLineError("The Roslyn toolchain is only supported on .NET Framework"); + return false; + } + if (benchmarkCase.Job.ResolveValue(GcMode.RetainVmCharacteristic, resolver)) { logger.WriteLineError($"Currently App.config does not support RetainVM option, benchmark '{benchmarkCase.DisplayInfo}' will not be executed"); diff --git a/tests/BenchmarkDotNet.IntegrationTests/NugetReferenceTests.cs b/tests/BenchmarkDotNet.IntegrationTests/NugetReferenceTests.cs index 0f7cfec3ed..bff8b60a74 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/NugetReferenceTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/NugetReferenceTests.cs @@ -29,7 +29,7 @@ public void UserCanSpecifyCustomNuGetPackageDependency() CanExecute(config); } - [Fact] + [FactClassicDotNetOnly("Roslyn toolchain does not support .NET Core")] public void RoslynToolchainDoesNotSupportNuGetPackageDependency() { var toolchain = RoslynToolchain.Instance; From dbccef2dc2879a44dee983f843e991d761f6462c Mon Sep 17 00:00:00 2001 From: Andrey Akinshin Date: Mon, 26 Sep 2022 19:35:43 +0300 Subject: [PATCH 04/58] Add workflows/docs-stable.yaml --- .github/workflows/docs-stable.yaml | 56 ++++++++++++++++++++++++++++++ build/Program.cs | 16 ++++++--- 2 files changed, 68 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/docs-stable.yaml diff --git a/.github/workflows/docs-stable.yaml b/.github/workflows/docs-stable.yaml new file mode 100644 index 0000000000..8129ff2dd3 --- /dev/null +++ b/.github/workflows/docs-stable.yaml @@ -0,0 +1,56 @@ +name: docs-stable + +on: + push: + branches: + - docs-stable + workflow_dispatch: + +jobs: + build: + runs-on: windows-latest + steps: + + - name: Checkout + uses: actions/checkout@v3 + with: + ref: docs-stable + + - name: Build BenchmarkDotNet + run: ./build.bat --target Build + + - name: Download changelog + run: ./build.bat --target DocFX_Changelog_Download --LatestVersions true --StableVersions true + env: + GITHUB_PRODUCT: ChangelogBuilder + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Build documentation + run: ./build.bat --target DocFX_Build + + - name: Upload Artifacts + uses: actions/upload-artifact@v1 + with: + name: site + path: docs/_site + + deploy: + concurrency: ci-${{ github.ref }} + needs: [build] + runs-on: ubuntu-latest + steps: + + - name: Download Artifacts + uses: actions/download-artifact@v1 + with: + name: site + + - name: Deploy documentation + uses: JamesIves/github-pages-deploy-action@3.7.1 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BRANCH: gh-pages + FOLDER: site + GIT_CONFIG_NAME: Andrey Akinshin + GIT_CONFIG_EMAIL: andrey.akinshin@gmail.com + CLEAN: true \ No newline at end of file diff --git a/build/Program.cs b/build/Program.cs index 96e231dfdf..6182dfbb8b 100644 --- a/build/Program.cs +++ b/build/Program.cs @@ -1,3 +1,4 @@ +using System; using System.IO; using System.Linq; using System.Text; @@ -428,12 +429,19 @@ public override void Run(BuildContext context) context.DocfxChangelogDownload( DocumentationHelper.BdnAllVersions[i], DocumentationHelper.BdnAllVersions[i - 1]); + } else if (context.Argument("LatestVersions", false)) + { + for (int i = DocumentationHelper.BdnAllVersions.Length - 2; i < DocumentationHelper.BdnAllVersions.Length; i++) + context.DocfxChangelogDownload( + DocumentationHelper.BdnAllVersions[i], + DocumentationHelper.BdnAllVersions[i - 1]); } - context.DocfxChangelogDownload( - DocumentationHelper.BdnNextVersion, - DocumentationHelper.BdnAllVersions.Last(), - "HEAD"); + if (!context.Argument("StableVersions", false)) + context.DocfxChangelogDownload( + DocumentationHelper.BdnNextVersion, + DocumentationHelper.BdnAllVersions.Last(), + "HEAD"); } } From d03287b019f0443171de931b1330d2a129d3d91e Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 29 Sep 2022 16:42:31 +0200 Subject: [PATCH 05/58] PerfCollect diagnoser (#2117) --- .../IntroPerfCollectProfiler.cs | 18 + .../PerfCollectProfilerAttribute.cs | 19 + src/BenchmarkDotNet/BenchmarkDotNet.csproj | 1 + .../Configs/ImmutableConfig.cs | 2 + .../Diagnosers/DiagnosersLoader.cs | 5 + .../Diagnosers/PerfCollectProfiler.cs | 252 ++ .../Diagnosers/PerfCollectProfilerConfig.cs | 19 + .../Extensions/CommonExtensions.cs | 8 + .../Extensions/ProcessExtensions.cs | 12 + .../Helpers/ArtifactFileNameHelper.cs | 5 +- src/BenchmarkDotNet/Templates/perfcollect | 2202 +++++++++++++++++ .../Toolchains/DotNetCli/DotNetCliCommand.cs | 5 +- .../DotNetCli/DotNetCliCommandExecutor.cs | 26 +- .../NativeAot/NativeAotToolchain.cs | 3 + 14 files changed, 2573 insertions(+), 4 deletions(-) create mode 100644 samples/BenchmarkDotNet.Samples/IntroPerfCollectProfiler.cs create mode 100644 src/BenchmarkDotNet/Attributes/PerfCollectProfilerAttribute.cs create mode 100644 src/BenchmarkDotNet/Diagnosers/PerfCollectProfiler.cs create mode 100644 src/BenchmarkDotNet/Diagnosers/PerfCollectProfilerConfig.cs create mode 100644 src/BenchmarkDotNet/Templates/perfcollect diff --git a/samples/BenchmarkDotNet.Samples/IntroPerfCollectProfiler.cs b/samples/BenchmarkDotNet.Samples/IntroPerfCollectProfiler.cs new file mode 100644 index 0000000000..9e6a7fa5ae --- /dev/null +++ b/samples/BenchmarkDotNet.Samples/IntroPerfCollectProfiler.cs @@ -0,0 +1,18 @@ +using System.IO; +using BenchmarkDotNet.Attributes; + +namespace BenchmarkDotNet.Samples +{ + [PerfCollectProfiler(performExtraBenchmarksRun: false)] + public class IntroPerfCollectProfiler + { + private readonly string path = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + private readonly string content = new string('a', 100_000); + + [Benchmark] + public void WriteAllText() => File.WriteAllText(path, content); + + [GlobalCleanup] + public void Delete() => File.Delete(path); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Attributes/PerfCollectProfilerAttribute.cs b/src/BenchmarkDotNet/Attributes/PerfCollectProfilerAttribute.cs new file mode 100644 index 0000000000..8b3e0bb1d2 --- /dev/null +++ b/src/BenchmarkDotNet/Attributes/PerfCollectProfilerAttribute.cs @@ -0,0 +1,19 @@ +using System; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Diagnosers; + +namespace BenchmarkDotNet.Attributes +{ + [AttributeUsage(AttributeTargets.Class | AttributeTargets.Assembly)] + public class PerfCollectProfilerAttribute : Attribute, IConfigSource + { + /// When set to true, benchmarks will be executed one more time with the profiler attached. If set to false, there will be no extra run but the results will contain overhead. False by default. + /// How long should we wait for the perfcollect script to finish processing the trace. 120s by default. + public PerfCollectProfilerAttribute(bool performExtraBenchmarksRun = false, int timeoutInSeconds = 120) + { + Config = ManualConfig.CreateEmpty().AddDiagnoser(new PerfCollectProfiler(new PerfCollectProfilerConfig(performExtraBenchmarksRun, timeoutInSeconds))); + } + + public IConfig Config { get; } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/BenchmarkDotNet.csproj b/src/BenchmarkDotNet/BenchmarkDotNet.csproj index d2b55cdec8..ca2a18450d 100644 --- a/src/BenchmarkDotNet/BenchmarkDotNet.csproj +++ b/src/BenchmarkDotNet/BenchmarkDotNet.csproj @@ -20,6 +20,7 @@ + diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfig.cs b/src/BenchmarkDotNet/Configs/ImmutableConfig.cs index fadd50ff91..83b7b9374a 100644 --- a/src/BenchmarkDotNet/Configs/ImmutableConfig.cs +++ b/src/BenchmarkDotNet/Configs/ImmutableConfig.cs @@ -106,6 +106,8 @@ internal ImmutableConfig( public bool HasThreadingDiagnoser() => diagnosers.Contains(ThreadingDiagnoser.Default); + internal bool HasPerfCollectProfiler() => diagnosers.OfType().Any(); + public bool HasExtraStatsDiagnoser() => HasMemoryDiagnoser() || HasThreadingDiagnoser(); public IDiagnoser GetCompositeDiagnoser(BenchmarkCase benchmarkCase, RunMode runMode) diff --git a/src/BenchmarkDotNet/Diagnosers/DiagnosersLoader.cs b/src/BenchmarkDotNet/Diagnosers/DiagnosersLoader.cs index 4012015435..57bb5ee30e 100644 --- a/src/BenchmarkDotNet/Diagnosers/DiagnosersLoader.cs +++ b/src/BenchmarkDotNet/Diagnosers/DiagnosersLoader.cs @@ -37,8 +37,13 @@ private static IEnumerable LoadDiagnosers() yield return new DisassemblyDiagnoser(new DisassemblyDiagnoserConfig()); if (RuntimeInformation.IsNetCore) + { yield return EventPipeProfiler.Default; + if (RuntimeInformation.IsLinux()) + yield return PerfCollectProfiler.Default; + } + if (!RuntimeInformation.IsWindows()) yield break; diff --git a/src/BenchmarkDotNet/Diagnosers/PerfCollectProfiler.cs b/src/BenchmarkDotNet/Diagnosers/PerfCollectProfiler.cs new file mode 100644 index 0000000000..a050beb5dd --- /dev/null +++ b/src/BenchmarkDotNet/Diagnosers/PerfCollectProfiler.cs @@ -0,0 +1,252 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using BenchmarkDotNet.Analysers; +using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Helpers; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Portability; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains; +using BenchmarkDotNet.Toolchains.CoreRun; +using BenchmarkDotNet.Toolchains.CsProj; +using BenchmarkDotNet.Toolchains.DotNetCli; +using BenchmarkDotNet.Toolchains.NativeAot; +using BenchmarkDotNet.Validators; +using JetBrains.Annotations; +using Mono.Unix.Native; + +namespace BenchmarkDotNet.Diagnosers +{ + public class PerfCollectProfiler : IProfiler + { + public static readonly IDiagnoser Default = new PerfCollectProfiler(new PerfCollectProfilerConfig(performExtraBenchmarksRun: false)); + + private readonly PerfCollectProfilerConfig config; + private readonly DateTime creationTime = DateTime.Now; + private readonly Dictionary benchmarkToTraceFile = new (); + private readonly HashSet cliPathWithSymbolsInstalled = new (); + private FileInfo perfCollectFile; + private Process perfCollectProcess; + + [PublicAPI] + public PerfCollectProfiler(PerfCollectProfilerConfig config) => this.config = config; + + public string ShortName => "perf"; + + public IEnumerable Ids => new[] { nameof(PerfCollectProfiler) }; + + public IEnumerable Exporters => Array.Empty(); + + public IEnumerable Analysers => Array.Empty(); + + public IEnumerable ProcessResults(DiagnoserResults results) => Array.Empty(); + + public RunMode GetRunMode(BenchmarkCase benchmarkCase) => config.RunMode; + + public IEnumerable Validate(ValidationParameters validationParameters) + { + if (!RuntimeInformation.IsLinux()) + { + yield return new ValidationError(true, "The PerfCollectProfiler works only on Linux!"); + yield break; + } + + if (Syscall.getuid() != 0) + { + yield return new ValidationError(true, "You must run as root to use PerfCollectProfiler."); + yield break; + } + + if (validationParameters.Benchmarks.Any() && !TryInstallPerfCollect(validationParameters)) + { + yield return new ValidationError(true, "Failed to install perfcollect script. Please follow the instructions from https://github.com/dotnet/runtime/blob/main/docs/project/linux-performance-tracing.md"); + } + } + + public void DisplayResults(ILogger logger) + { + if (!benchmarkToTraceFile.Any()) + return; + + logger.WriteLineInfo($"Exported {benchmarkToTraceFile.Count} trace file(s). Example:"); + logger.WriteLineInfo(benchmarkToTraceFile.Values.First().FullName); + } + + public void Handle(HostSignal signal, DiagnoserActionParameters parameters) + { + if (signal == HostSignal.BeforeProcessStart) + perfCollectProcess = StartCollection(parameters); + else if (signal == HostSignal.AfterProcessExit) + StopCollection(parameters); + } + + private bool TryInstallPerfCollect(ValidationParameters validationParameters) + { + var scriptInstallationDirectory = new DirectoryInfo(validationParameters.Config.ArtifactsPath).CreateIfNotExists(); + + perfCollectFile = new FileInfo(Path.Combine(scriptInstallationDirectory.FullName, "perfcollect")); + if (perfCollectFile.Exists) + { + return true; + } + + var logger = validationParameters.Config.GetCompositeLogger(); + + string script = ResourceHelper.LoadTemplate(perfCollectFile.Name); + File.WriteAllText(perfCollectFile.FullName, script); + + if (Syscall.chmod(perfCollectFile.FullName, FilePermissions.S_IXUSR) != 0) + { + logger.WriteError($"Unable to make perfcollect script an executable, the last error was: {Syscall.GetLastError()}"); + } + else + { + (int exitCode, var output) = ProcessHelper.RunAndReadOutputLineByLine(perfCollectFile.FullName, "install -force", perfCollectFile.Directory.FullName, null, includeErrors: true, logger); + + if (exitCode == 0) + { + logger.WriteLine("Successfully installed perfcollect"); + return true; + } + + logger.WriteLineError("Failed to install perfcollect"); + foreach (var outputLine in output) + { + logger.WriteLine(outputLine); + } + } + + if (perfCollectFile.Exists) + { + perfCollectFile.Delete(); // if the file exists it means that perfcollect is installed + } + + return false; + } + + private Process StartCollection(DiagnoserActionParameters parameters) + { + EnsureSymbolsForNativeRuntime(parameters); + + var traceName = GetTraceFile(parameters, extension: null).Name; + + var start = new ProcessStartInfo + { + FileName = perfCollectFile.FullName, + Arguments = $"collect \"{traceName}\"", + UseShellExecute = false, + RedirectStandardOutput = true, + CreateNoWindow = true, + WorkingDirectory = perfCollectFile.Directory.FullName + }; + + return Process.Start(start); + } + + private void StopCollection(DiagnoserActionParameters parameters) + { + var logger = parameters.Config.GetCompositeLogger(); + + try + { + if (!perfCollectProcess.HasExited) + { + if (Syscall.kill(perfCollectProcess.Id, Signum.SIGINT) != 0) + { + var lastError = Stdlib.GetLastError(); + logger.WriteLineError($"kill(perfcollect, SIGINT) failed with {lastError}"); + } + + if (!perfCollectProcess.WaitForExit((int)config.Timeout.TotalMilliseconds)) + { + logger.WriteLineError($"The perfcollect script did not stop in {config.Timeout.TotalSeconds}s. It's going to be force killed now."); + logger.WriteLineInfo("You can create PerfCollectProfiler providing PerfCollectProfilerConfig with custom timeout value."); + + perfCollectProcess.KillTree(); // kill the entire process tree + } + + FileInfo traceFile = GetTraceFile(parameters, "trace.zip"); + if (traceFile.Exists) + { + benchmarkToTraceFile[parameters.BenchmarkCase] = traceFile; + } + } + else + { + logger.WriteLineError("For some reason the perfcollect script has finished sooner than expected."); + logger.WriteLineInfo($"Please run '{perfCollectFile.FullName} install' as root and re-try."); + } + } + finally + { + perfCollectProcess.Dispose(); + } + } + + private void EnsureSymbolsForNativeRuntime(DiagnoserActionParameters parameters) + { + string cliPath = parameters.BenchmarkCase.GetToolchain() switch + { + CsProjCoreToolchain core => core.CustomDotNetCliPath, + CoreRunToolchain coreRun => coreRun.CustomDotNetCliPath.FullName, + NativeAotToolchain nativeAot => nativeAot.CustomDotNetCliPath, + _ => DotNetCliCommandExecutor.DefaultDotNetCliPath.Value + }; + + if (!cliPathWithSymbolsInstalled.Add(cliPath)) + { + return; + } + + string sdkPath = DotNetCliCommandExecutor.GetSdkPath(cliPath); // /usr/share/dotnet/sdk/ + string dotnetPath = Path.GetDirectoryName(sdkPath); // /usr/share/dotnet/ + string[] missingSymbols = Directory.GetFiles(dotnetPath, "lib*.so", SearchOption.AllDirectories) + .Where(nativeLibPath => !nativeLibPath.Contains("FallbackFolder") && !File.Exists(Path.ChangeExtension(nativeLibPath, "so.dbg"))) + .Select(Path.GetDirectoryName) + .Distinct() + .ToArray(); + + if (!missingSymbols.Any()) + { + return; // the symbol files are already where we need them! + } + + ILogger logger = parameters.Config.GetCompositeLogger(); + // We install the tool in a dedicated directory in order to always use latest version and avoid issues with broken existing configs. + string toolPath = Path.Combine(Path.GetTempPath(), "BenchmarkDotNet", "symbols"); + DotNetCliCommand cliCommand = new ( + cliPath: cliPath, + arguments: $"tool install dotnet-symbol --tool-path \"{toolPath}\"", + generateResult: null, + logger: logger, + buildPartition: null, + environmentVariables: Array.Empty(), + timeout: TimeSpan.FromMinutes(3), + logOutput: true); // the following commands might take a while and fail, let's log them + + var installResult = DotNetCliCommandExecutor.Execute(cliCommand); + if (!installResult.IsSuccess) + { + logger.WriteError("Unable to install dotnet symbol."); + return; + } + + DotNetCliCommandExecutor.Execute(cliCommand + .WithCliPath(Path.Combine(toolPath, "dotnet-symbol")) + .WithArguments($"--recurse-subdirectories --symbols \"{dotnetPath}/dotnet\" \"{dotnetPath}/lib*.so\"")); + + DotNetCliCommandExecutor.Execute(cliCommand.WithArguments($"tool uninstall dotnet-symbol --tool-path \"{toolPath}\"")); + } + + private FileInfo GetTraceFile(DiagnoserActionParameters parameters, string extension) + => new (ArtifactFileNameHelper.GetTraceFilePath(parameters, creationTime, extension) + .Replace(" ", "_")); // perfcollect does not allow for spaces in the trace file name + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Diagnosers/PerfCollectProfilerConfig.cs b/src/BenchmarkDotNet/Diagnosers/PerfCollectProfilerConfig.cs new file mode 100644 index 0000000000..8d025c3366 --- /dev/null +++ b/src/BenchmarkDotNet/Diagnosers/PerfCollectProfilerConfig.cs @@ -0,0 +1,19 @@ +using System; + +namespace BenchmarkDotNet.Diagnosers +{ + public class PerfCollectProfilerConfig + { + /// When set to true, benchmarks will be executed one more time with the profiler attached. If set to false, there will be no extra run but the results will contain overhead. False by default. + /// How long should we wait for the perfcollect script to finish processing the trace. 120s by default. + public PerfCollectProfilerConfig(bool performExtraBenchmarksRun = false, int timeoutInSeconds = 120) + { + RunMode = performExtraBenchmarksRun ? RunMode.ExtraRun : RunMode.NoOverhead; + Timeout = TimeSpan.FromSeconds(timeoutInSeconds); + } + + public TimeSpan Timeout { get; } + + public RunMode RunMode { get; } + } +} diff --git a/src/BenchmarkDotNet/Extensions/CommonExtensions.cs b/src/BenchmarkDotNet/Extensions/CommonExtensions.cs index 29b2caac6c..72a423c016 100644 --- a/src/BenchmarkDotNet/Extensions/CommonExtensions.cs +++ b/src/BenchmarkDotNet/Extensions/CommonExtensions.cs @@ -79,6 +79,14 @@ internal static string CreateIfNotExists(this string directoryPath) return directoryPath; } + internal static DirectoryInfo CreateIfNotExists(this DirectoryInfo directory) + { + if (!directory.Exists) + directory.Create(); + + return directory; + } + internal static string DeleteFileIfExists(this string filePath) { if (File.Exists(filePath)) diff --git a/src/BenchmarkDotNet/Extensions/ProcessExtensions.cs b/src/BenchmarkDotNet/Extensions/ProcessExtensions.cs index 729f239709..9dffbf5eff 100644 --- a/src/BenchmarkDotNet/Extensions/ProcessExtensions.cs +++ b/src/BenchmarkDotNet/Extensions/ProcessExtensions.cs @@ -4,6 +4,7 @@ using System.Diagnostics; using System.IO; using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; @@ -128,6 +129,17 @@ internal static void SetEnvironmentVariables(this ProcessStartInfo start, Benchm if (benchmarkCase.Job.Environment.Runtime is MonoRuntime monoRuntime && !string.IsNullOrEmpty(monoRuntime.MonoBclPath)) start.EnvironmentVariables["MONO_PATH"] = monoRuntime.MonoBclPath; + if (benchmarkCase.Config.HasPerfCollectProfiler()) + { + // enable tracing configuration inside of CoreCLR (https://github.com/dotnet/coreclr/blob/master/Documentation/project-docs/linux-performance-tracing.md#collecting-a-trace) + start.EnvironmentVariables["COMPlus_PerfMapEnabled"] = "1"; + start.EnvironmentVariables["COMPlus_EnableEventLog"] = "1"; + // enable BDN Event Source (https://github.com/dotnet/coreclr/blob/master/Documentation/project-docs/linux-performance-tracing.md#filtering) + start.EnvironmentVariables["COMPlus_EventSourceFilter"] = EngineEventSource.SourceName; + // workaround for https://github.com/dotnet/runtime/issues/71786, will be solved by next perf version + start.EnvironmentVariables["DOTNET_EnableWriteXorExecute"] = "0"; + } + // corerun does not understand runtimeconfig.json files; // we have to set "COMPlus_GC*" environment variables as documented in // https://docs.microsoft.com/en-us/dotnet/core/run-time-config/garbage-collector diff --git a/src/BenchmarkDotNet/Helpers/ArtifactFileNameHelper.cs b/src/BenchmarkDotNet/Helpers/ArtifactFileNameHelper.cs index 20de492a82..490d47c2ea 100644 --- a/src/BenchmarkDotNet/Helpers/ArtifactFileNameHelper.cs +++ b/src/BenchmarkDotNet/Helpers/ArtifactFileNameHelper.cs @@ -71,7 +71,10 @@ private static string GetFilePath(string fileName, DiagnoserActionParameters det fileName = FolderNameHelper.ToFolderName(fileName); - return Path.Combine(details.Config.ArtifactsPath, $"{fileName}.{fileExtension}"); + if (!string.IsNullOrEmpty(fileExtension)) + fileName = $"{fileName}.{fileExtension}"; + + return Path.Combine(details.Config.ArtifactsPath, fileName); } } } diff --git a/src/BenchmarkDotNet/Templates/perfcollect b/src/BenchmarkDotNet/Templates/perfcollect new file mode 100644 index 0000000000..c266c9c06a --- /dev/null +++ b/src/BenchmarkDotNet/Templates/perfcollect @@ -0,0 +1,2202 @@ +#!/bin/bash + +############################################################################################################# +# .NET Performance Data Collection Script +############################################################################################################# + +############################################################################################################# +# +# ***** HOW TO USE THIS SCRIPT ***** +# +# This script can be used to collect and view performance data collected with perf_event on Linux. + +# It's job is to make it simple to collect performance traces. +# +# How to collect a performance trace: +# 1. Prior to starting the .NET process, set the environment variable COMPlus_PerfMapEnabled=1. +# This tells the runtime to emit information that enables perf_event to resolve JIT-compiled code symbols. +# +# 2. Setup your system to reproduce the performance issue you'd like to capture. Data collection can be +# started on already running processes. +# +# 3. [Usage #1] use "collect" command +# - Run this script: sudo ./perfcollect collect samplePerfTrace. This will start data collection. +# - Let the repro run as long as you need to capture the performance problem. +# - Hit CTRL+C to stop collection. +# +# [Usage #2] use "start" and "stop" command +# - Run this script: sudo ./perfcollect start samplePerfTrace. This will start data colletion. +# - Let the repro run as long as you need to capture the performance problem. +# - Run: sudo ./perfcollect stop samplePerfTrace. This will stop collection. +# +# 4. When collection is stopped, the script will create a trace.zip file matching the name specified on the +# command line. This file will contain the trace, JIT-compiled symbol information, and all debugging +# symbols for binaries referenced by this trace that were available on the machine at collection time. +# +# How to view a performance trace: +# 1. Run this script: ./perfcollect view samplePerfTrace.trace.zip +# This will extract the trace, place and register all symbol files and JIT-compiled symbol information +# and start the perf_event viewer. By default, you will be looking at a callee view - stacks are ordered +# top down. For a caller or bottom up view, specify '-graphtype caller'. +############################################################################################################# + +###################################### +## FOR DEBUGGING ONLY +###################################### +# set -x + +###################################### +## Collection Options +## NOTE: These values represent the collection defaults. +###################################### + +# Set when we parse command line arguments to determine if we should enable specific collection options. +collect_cpu=1 +collect_threadTime=0 +collect_offcpu=0 + +###################################### +## .NET Event Categories +###################################### + +# Separate GCCollectOnly list because our LTTng implementation doesn't set tracepoint verbosity. +# Once tracepoint verbosity is set, we can set verbosity and collapse this with DotNETRuntime_GCKeyword. +declare -a DotNETRuntime_GCKeyword_GCCollectOnly=( + DotNETRuntime:GCStart + DotNETRuntime:GCStart_V1 + DotNETRuntime:GCStart_V2 + DotNETRuntime:GCEnd + DotNETRuntime:GCEnd_V1 + DotNETRuntime:GCRestartEEEnd + DotNETRuntime:GCRestartEEEnd_V1 + DotNETRuntime:GCHeapStats + DotNETRuntime:GCHeapStats_V1 + DotNETRuntime:GCCreateSegment + DotNETRuntime:GCCreateSegment_V1 + DotNETRuntime:GCFreeSegment + DotNETRuntime:GCFreeSegment_V1 + DotNETRuntime:GCRestartEEBegin + DotNETRuntime:GCRestartEEBegin_V1 + DotNETRuntime:GCSuspendEEEnd + DotNETRuntime:GCSuspendEEEnd_V1 + DotNETRuntime:GCSuspendEEBegin + DotNETRuntime:GCSuspendEEBegin_V1 + DotNETRuntime:GCCreateConcurrentThread + DotNETRuntime:GCTerminateConcurrentThread + DotNETRuntime:GCFinalizersEnd + DotNETRuntime:GCFinalizersEnd_V1 + DotNETRuntime:GCFinalizersBegin + DotNETRuntime:GCFinalizersBegin_V1 + DotNETRuntime:GCMarkStackRoots + DotNETRuntime:GCMarkFinalizeQueueRoots + DotNETRuntime:GCMarkHandles + DotNETRuntime:GCMarkOlderGenerationRoots + DotNETRuntime:FinalizeObject + DotNETRuntime:GCTriggered + DotNETRuntime:IncreaseMemoryPressure + DotNETRuntime:DecreaseMemoryPressure + DotNETRuntime:GCMarkWithType + DotNETRuntime:GCPerHeapHistory_V3 + DotNETRuntime:GCGlobalHeapHistory_V2 + DotNETRuntime:GCCreateConcurrentThread_V1 + DotNETRuntime:GCTerminateConcurrentThread_V1 +) + + +declare -a DotNETRuntime_GCKeyword=( + DotNETRuntime:GCStart + DotNETRuntime:GCStart_V1 + DotNETRuntime:GCStart_V2 + DotNETRuntime:GCEnd + DotNETRuntime:GCEnd_V1 + DotNETRuntime:GCRestartEEEnd + DotNETRuntime:GCRestartEEEnd_V1 + DotNETRuntime:GCHeapStats + DotNETRuntime:GCHeapStats_V1 + DotNETRuntime:GCCreateSegment + DotNETRuntime:GCCreateSegment_V1 + DotNETRuntime:GCFreeSegment + DotNETRuntime:GCFreeSegment_V1 + DotNETRuntime:GCRestartEEBegin + DotNETRuntime:GCRestartEEBegin_V1 + DotNETRuntime:GCSuspendEEEnd + DotNETRuntime:GCSuspendEEEnd_V1 + DotNETRuntime:GCSuspendEEBegin + DotNETRuntime:GCSuspendEEBegin_V1 + DotNETRuntime:GCAllocationTick + DotNETRuntime:GCAllocationTick_V1 + DotNETRuntime:GCAllocationTick_V2 + DotNETRuntime:GCAllocationTick_V3 + DotNETRuntime:GCCreateConcurrentThread + DotNETRuntime:GCTerminateConcurrentThread + DotNETRuntime:GCFinalizersEnd + DotNETRuntime:GCFinalizersEnd_V1 + DotNETRuntime:GCFinalizersBegin + DotNETRuntime:GCFinalizersBegin_V1 + DotNETRuntime:GCMarkStackRoots + DotNETRuntime:GCMarkFinalizeQueueRoots + DotNETRuntime:GCMarkHandles + DotNETRuntime:GCMarkOlderGenerationRoots + DotNETRuntime:FinalizeObject + DotNETRuntime:PinObjectAtGCTime + DotNETRuntime:GCTriggered + DotNETRuntime:IncreaseMemoryPressure + DotNETRuntime:DecreaseMemoryPressure + DotNETRuntime:GCMarkWithType + DotNETRuntime:GCJoin_V2 + DotNETRuntime:GCPerHeapHistory_V3 + DotNETRuntime:GCGlobalHeapHistory_V2 + DotNETRuntime:GCCreateConcurrentThread_V1 + DotNETRuntime:GCTerminateConcurrentThread_V1 +) + +declare -a DotNETRuntime_TypeKeyword=( + DotNETRuntime:BulkType +) + +declare -a DotNETRuntime_GCHeapDumpKeyword=( + DotNETRuntime:GCBulkRootEdge + DotNETRuntime:GCBulkRootConditionalWeakTableElementEdge + DotNETRuntime:GCBulkNode + DotNETRuntime:GCBulkEdge + DotNETRuntime:GCBulkRootCCW + DotNETRuntime:GCBulkRCW + DotNETRuntime:GCBulkRootStaticVar +) + +declare -a DotNETRuntime_GCSampledObjectAllocationHighKeyword=( + DotNETRuntime:GCSampledObjectAllocationHigh +) + +declare -a DotNETRuntime_GCHeapSurvivalAndMovementKeyword=( + DotNETRuntime:GCBulkSurvivingObjectRanges + DotNETRuntime:GCBulkMovedObjectRanges + DotNETRuntime:GCGenerationRange +) + +declare -a DotNETRuntime_GCHandleKeyword=( + DotNETRuntime:SetGCHandle + DotNETRuntime:DestroyGCHandle +) + +declare -a DotNETRuntime_GCSampledObjectAllocationLowKeyword=( + DotNETRuntime:GCSampledObjectAllocationLow +) + +declare -a DotNETRuntime_ThreadingKeyword=( + DotNETRuntime:WorkerThreadCreate + DotNETRuntime:WorkerThreadTerminate + DotNETRuntime:WorkerThreadRetire + DotNETRuntime:WorkerThreadUnretire + DotNETRuntime:IOThreadCreate + DotNETRuntime:IOThreadCreate_V1 + DotNETRuntime:IOThreadTerminate + DotNETRuntime:IOThreadTerminate_V1 + DotNETRuntime:IOThreadRetire + DotNETRuntime:IOThreadRetire_V1 + DotNETRuntime:IOThreadUnretire + DotNETRuntime:IOThreadUnretire_V1 + DotNETRuntime:ThreadpoolSuspensionSuspendThread + DotNETRuntime:ThreadpoolSuspensionResumeThread + DotNETRuntime:ThreadPoolWorkerThreadStart + DotNETRuntime:ThreadPoolWorkerThreadStop + DotNETRuntime:ThreadPoolWorkerThreadRetirementStart + DotNETRuntime:ThreadPoolWorkerThreadRetirementStop + DotNETRuntime:ThreadPoolWorkerThreadAdjustmentSample + DotNETRuntime:ThreadPoolWorkerThreadAdjustmentAdjustment + DotNETRuntime:ThreadPoolWorkerThreadAdjustmentStats + DotNETRuntime:ThreadPoolWorkerThreadWait + DotNETRuntime:ThreadPoolWorkingThreadCount + DotNETRuntime:ThreadPoolIOPack + DotNETRuntime:GCCreateConcurrentThread_V1 + DotNETRuntime:GCTerminateConcurrentThread_V1 +) + +declare -a DotNETRuntime_ThreadingKeyword_ThreadTransferKeyword=( + DotNETRuntime:ThreadPoolEnqueue + DotNETRuntime:ThreadPoolDequeue + DotNETRuntime:ThreadPoolIOEnqueue + DotNETRuntime:ThreadPoolIODequeue + DotNETRuntime:ThreadCreating + DotNETRuntime:ThreadRunning +) + +declare -a DotNETRuntime_NoKeyword=( + DotNETRuntime:ExceptionThrown + DotNETRuntime:Contention + DotNETRuntime:RuntimeInformationStart + DotNETRuntime:EventSource +) + +declare -a DotNETRuntime_ExceptionKeyword=( + DotNETRuntime:ExceptionThrown_V1 + DotNETRuntime:ExceptionCatchStart + DotNETRuntime:ExceptionCatchStop + DotNETRuntime:ExceptionFinallyStart + DotNETRuntime:ExceptionFinallyStop + DotNETRuntime:ExceptionFilterStart + DotNETRuntime:ExceptionFilterStop + DotNETRuntime:ExceptionThrownStop +) + +declare -a DotNETRuntime_ContentionKeyword=( + DotNETRuntime:ContentionStart_V1 + DotNETRuntime:ContentionStop + DotNETRuntime:ContentionStop_V1 +) + +declare -a DotNETRuntime_StackKeyword=( + DotNETRuntime:CLRStackWalk +) + +declare -a DotNETRuntime_AppDomainResourceManagementKeyword=( + DotNETRuntime:AppDomainMemAllocated + DotNETRuntime:AppDomainMemSurvived +) + +declare -a DotNETRuntime_AppDomainResourceManagementKeyword_ThreadingKeyword=( + DotNETRuntime:ThreadCreated + DotNETRuntime:ThreadTerminated + DotNETRuntime:ThreadDomainEnter +) + +declare -a DotNETRuntime_InteropKeyword=( + DotNETRuntime:ILStubGenerated + DotNETRuntime:ILStubCacheHit +) + +declare -a DotNETRuntime_JitKeyword_NGenKeyword=( + DotNETRuntime:DCStartCompleteV2 + DotNETRuntime:DCEndCompleteV2 + DotNETRuntime:MethodDCStartV2 + DotNETRuntime:MethodDCEndV2 + DotNETRuntime:MethodDCStartVerboseV2 + DotNETRuntime:MethodDCEndVerboseV2 + DotNETRuntime:MethodLoad + DotNETRuntime:MethodLoad_V1 + DotNETRuntime:MethodLoad_V2 + DotNETRuntime:MethodUnload + DotNETRuntime:MethodUnload_V1 + DotNETRuntime:MethodUnload_V2 + DotNETRuntime:MethodLoadVerbose + DotNETRuntime:MethodLoadVerbose_V1 + DotNETRuntime:MethodLoadVerbose_V2 + DotNETRuntime:MethodUnloadVerbose + DotNETRuntime:MethodUnloadVerbose_V1 + DotNETRuntime:MethodUnloadVerbose_V2 +) + +declare -a DotNETRuntime_JitKeyword=( + DotNETRuntime:MethodJittingStarted + DotNETRuntime:MethodJittingStarted_V1 +) + +declare -a DotNETRuntime_JitTracingKeyword=( + DotNETRuntime:MethodJitInliningSucceeded + DotNETRuntime:MethodJitInliningFailed + DotNETRuntime:MethodJitTailCallSucceeded + DotNETRuntime:MethodJitTailCallFailed +) + +declare -a DotNETRuntime_JittedMethodILToNativeMapKeyword=( + DotNETRuntime:MethodILToNativeMap +) + +declare -a DotNETRuntime_LoaderKeyword=( + DotNETRuntime:ModuleDCStartV2 + DotNETRuntime:ModuleDCEndV2 + DotNETRuntime:DomainModuleLoad + DotNETRuntime:DomainModuleLoad_V1 + DotNETRuntime:ModuleLoad + DotNETRuntime:ModuleUnload + DotNETRuntime:AssemblyLoad + DotNETRuntime:AssemblyLoad_V1 + DotNETRuntime:AssemblyUnload + DotNETRuntime:AssemblyUnload_V1 + DotNETRuntime:AppDomainLoad + DotNETRuntime:AppDomainLoad_V1 + DotNETRuntime:AppDomainUnload + DotNETRuntime:AppDomainUnload_V1 +) + +declare -a DotNETRuntime_LoaderKeyword=( + DotNETRuntime:ModuleLoad_V1 + DotNETRuntime:ModuleLoad_V2 + DotNETRuntime:ModuleUnload_V1 + DotNETRuntime:ModuleUnload_V2 +) + +declare -a DotNETRuntime_SecurityKeyword=( + DotNETRuntime:StrongNameVerificationStart + DotNETRuntime:StrongNameVerificationStart_V1 + DotNETRuntime:StrongNameVerificationStop + DotNETRuntime:StrongNameVerificationStop_V1 + DotNETRuntime:AuthenticodeVerificationStart + DotNETRuntime:AuthenticodeVerificationStart_V1 + DotNETRuntime:AuthenticodeVerificationStop + DotNETRuntime:AuthenticodeVerificationStop_V1 +) + +declare -a DotNETRuntime_DebuggerKeyword=( + DotNETRuntime:DebugIPCEventStart + DotNETRuntime:DebugIPCEventEnd + DotNETRuntime:DebugExceptionProcessingStart + DotNETRuntime:DebugExceptionProcessingEnd +) + +declare -a DotNETRuntime_CodeSymbolsKeyword=( + DotNETRuntime:CodeSymbols +) + +declare -a DotNETRuntime_CompilationKeyword=( + DotNETRuntime:TieredCompilationSettings + DotNETRuntime:TieredCompilationPause + DotNETRuntime:TieredCompilationResume + DotNETRuntime:TieredCompilationBackgroundJitStart + DotNETRuntime:TieredCompilationBackgroundJitStop + DotNETRuntimeRundown:TieredCompilationSettingsDCStart +) + +# Separate GCCollectOnly list because our LTTng implementation doesn't set tracepoint verbosity. +# Once tracepoint verbosity is set, we can set verbosity and collapse this with DotNETRuntimePrivate_GCPrivateKeyword. +declare -a DotNETRuntimePrivate_GCPrivateKeyword_GCCollectOnly=( + DotNETRuntimePrivate:GCDecision + DotNETRuntimePrivate:GCDecision_V1 + DotNETRuntimePrivate:GCSettings + DotNETRuntimePrivate:GCSettings_V1 + DotNETRuntimePrivate:GCPerHeapHistory + DotNETRuntimePrivate:GCPerHeapHistory_V1 + DotNETRuntimePrivate:GCGlobalHeapHistory + DotNETRuntimePrivate:GCGlobalHeapHistory_V1 + DotNETRuntimePrivate:PrvGCMarkStackRoots + DotNETRuntimePrivate:PrvGCMarkStackRoots_V1 + DotNETRuntimePrivate:PrvGCMarkFinalizeQueueRoots + DotNETRuntimePrivate:PrvGCMarkFinalizeQueueRoots_V1 + DotNETRuntimePrivate:PrvGCMarkHandles + DotNETRuntimePrivate:PrvGCMarkHandles_V1 + DotNETRuntimePrivate:PrvGCMarkCards + DotNETRuntimePrivate:PrvGCMarkCards_V1 + DotNETRuntimePrivate:BGCBegin + DotNETRuntimePrivate:BGC1stNonConEnd + DotNETRuntimePrivate:BGC1stConEnd + DotNETRuntimePrivate:BGC2ndNonConBegin + DotNETRuntimePrivate:BGC2ndNonConEnd + DotNETRuntimePrivate:BGC2ndConBegin + DotNETRuntimePrivate:BGC2ndConEnd + DotNETRuntimePrivate:BGCPlanEnd + DotNETRuntimePrivate:BGCSweepEnd + DotNETRuntimePrivate:BGCDrainMark + DotNETRuntimePrivate:BGCRevisit + DotNETRuntimePrivate:BGCOverflow + DotNETRuntimePrivate:BGCAllocWaitBegin + DotNETRuntimePrivate:BGCAllocWaitEnd + DotNETRuntimePrivate:GCFullNotify + DotNETRuntimePrivate:GCFullNotify_V1 + DotNETRuntimePrivate:PrvFinalizeObject + DotNETRuntimePrivate:PinPlugAtGCTime +) + +declare -a DotNETRuntimePrivate_GCPrivateKeyword=( + DotNETRuntimePrivate:GCDecision + DotNETRuntimePrivate:GCDecision_V1 + DotNETRuntimePrivate:GCSettings + DotNETRuntimePrivate:GCSettings_V1 + DotNETRuntimePrivate:GCOptimized + DotNETRuntimePrivate:GCOptimized_V1 + DotNETRuntimePrivate:GCPerHeapHistory + DotNETRuntimePrivate:GCPerHeapHistory_V1 + DotNETRuntimePrivate:GCGlobalHeapHistory + DotNETRuntimePrivate:GCGlobalHeapHistory_V1 + DotNETRuntimePrivate:GCJoin + DotNETRuntimePrivate:GCJoin_V1 + DotNETRuntimePrivate:PrvGCMarkStackRoots + DotNETRuntimePrivate:PrvGCMarkStackRoots_V1 + DotNETRuntimePrivate:PrvGCMarkFinalizeQueueRoots + DotNETRuntimePrivate:PrvGCMarkFinalizeQueueRoots_V1 + DotNETRuntimePrivate:PrvGCMarkHandles + DotNETRuntimePrivate:PrvGCMarkHandles_V1 + DotNETRuntimePrivate:PrvGCMarkCards + DotNETRuntimePrivate:PrvGCMarkCards_V1 + DotNETRuntimePrivate:BGCBegin + DotNETRuntimePrivate:BGC1stNonConEnd + DotNETRuntimePrivate:BGC1stConEnd + DotNETRuntimePrivate:BGC2ndNonConBegin + DotNETRuntimePrivate:BGC2ndNonConEnd + DotNETRuntimePrivate:BGC2ndConBegin + DotNETRuntimePrivate:BGC2ndConEnd + DotNETRuntimePrivate:BGCPlanEnd + DotNETRuntimePrivate:BGCSweepEnd + DotNETRuntimePrivate:BGCDrainMark + DotNETRuntimePrivate:BGCRevisit + DotNETRuntimePrivate:BGCOverflow + DotNETRuntimePrivate:BGCAllocWaitBegin + DotNETRuntimePrivate:BGCAllocWaitEnd + DotNETRuntimePrivate:GCFullNotify + DotNETRuntimePrivate:GCFullNotify_V1 + DotNETRuntimePrivate:PrvFinalizeObject + DotNETRuntimePrivate:PinPlugAtGCTime +) + +declare -a DotNETRuntimePrivate_StartupKeyword=( + DotNETRuntimePrivate:EEStartupStart + DotNETRuntimePrivate:EEStartupStart_V1 + DotNETRuntimePrivate:EEStartupEnd + DotNETRuntimePrivate:EEStartupEnd_V1 + DotNETRuntimePrivate:EEConfigSetup + DotNETRuntimePrivate:EEConfigSetup_V1 + DotNETRuntimePrivate:EEConfigSetupEnd + DotNETRuntimePrivate:EEConfigSetupEnd_V1 + DotNETRuntimePrivate:LdSysBases + DotNETRuntimePrivate:LdSysBases_V1 + DotNETRuntimePrivate:LdSysBasesEnd + DotNETRuntimePrivate:LdSysBasesEnd_V1 + DotNETRuntimePrivate:ExecExe + DotNETRuntimePrivate:ExecExe_V1 + DotNETRuntimePrivate:ExecExeEnd + DotNETRuntimePrivate:ExecExeEnd_V1 + DotNETRuntimePrivate:Main + DotNETRuntimePrivate:Main_V1 + DotNETRuntimePrivate:MainEnd + DotNETRuntimePrivate:MainEnd_V1 + DotNETRuntimePrivate:ApplyPolicyStart + DotNETRuntimePrivate:ApplyPolicyStart_V1 + DotNETRuntimePrivate:ApplyPolicyEnd + DotNETRuntimePrivate:ApplyPolicyEnd_V1 + DotNETRuntimePrivate:LdLibShFolder + DotNETRuntimePrivate:LdLibShFolder_V1 + DotNETRuntimePrivate:LdLibShFolderEnd + DotNETRuntimePrivate:LdLibShFolderEnd_V1 + DotNETRuntimePrivate:PrestubWorker + DotNETRuntimePrivate:PrestubWorker_V1 + DotNETRuntimePrivate:PrestubWorkerEnd + DotNETRuntimePrivate:PrestubWorkerEnd_V1 + DotNETRuntimePrivate:GetInstallationStart + DotNETRuntimePrivate:GetInstallationStart_V1 + DotNETRuntimePrivate:GetInstallationEnd + DotNETRuntimePrivate:GetInstallationEnd_V1 + DotNETRuntimePrivate:OpenHModule + DotNETRuntimePrivate:OpenHModule_V1 + DotNETRuntimePrivate:OpenHModuleEnd + DotNETRuntimePrivate:OpenHModuleEnd_V1 + DotNETRuntimePrivate:ExplicitBindStart + DotNETRuntimePrivate:ExplicitBindStart_V1 + DotNETRuntimePrivate:ExplicitBindEnd + DotNETRuntimePrivate:ExplicitBindEnd_V1 + DotNETRuntimePrivate:ParseXml + DotNETRuntimePrivate:ParseXml_V1 + DotNETRuntimePrivate:ParseXmlEnd + DotNETRuntimePrivate:ParseXmlEnd_V1 + DotNETRuntimePrivate:InitDefaultDomain + DotNETRuntimePrivate:InitDefaultDomain_V1 + DotNETRuntimePrivate:InitDefaultDomainEnd + DotNETRuntimePrivate:InitDefaultDomainEnd_V1 + DotNETRuntimePrivate:InitSecurity + DotNETRuntimePrivate:InitSecurity_V1 + DotNETRuntimePrivate:InitSecurityEnd + DotNETRuntimePrivate:InitSecurityEnd_V1 + DotNETRuntimePrivate:AllowBindingRedirs + DotNETRuntimePrivate:AllowBindingRedirs_V1 + DotNETRuntimePrivate:AllowBindingRedirsEnd + DotNETRuntimePrivate:AllowBindingRedirsEnd_V1 + DotNETRuntimePrivate:EEConfigSync + DotNETRuntimePrivate:EEConfigSync_V1 + DotNETRuntimePrivate:EEConfigSyncEnd + DotNETRuntimePrivate:EEConfigSyncEnd_V1 + DotNETRuntimePrivate:FusionBinding + DotNETRuntimePrivate:FusionBinding_V1 + DotNETRuntimePrivate:FusionBindingEnd + DotNETRuntimePrivate:FusionBindingEnd_V1 + DotNETRuntimePrivate:LoaderCatchCall + DotNETRuntimePrivate:LoaderCatchCall_V1 + DotNETRuntimePrivate:LoaderCatchCallEnd + DotNETRuntimePrivate:LoaderCatchCallEnd_V1 + DotNETRuntimePrivate:FusionInit + DotNETRuntimePrivate:FusionInit_V1 + DotNETRuntimePrivate:FusionInitEnd + DotNETRuntimePrivate:FusionInitEnd_V1 + DotNETRuntimePrivate:FusionAppCtx + DotNETRuntimePrivate:FusionAppCtx_V1 + DotNETRuntimePrivate:FusionAppCtxEnd + DotNETRuntimePrivate:FusionAppCtxEnd_V1 + DotNETRuntimePrivate:Fusion2EE + DotNETRuntimePrivate:Fusion2EE_V1 + DotNETRuntimePrivate:Fusion2EEEnd + DotNETRuntimePrivate:Fusion2EEEnd_V1 + DotNETRuntimePrivate:SecurityCatchCall + DotNETRuntimePrivate:SecurityCatchCall_V1 + DotNETRuntimePrivate:SecurityCatchCallEnd + DotNETRuntimePrivate:SecurityCatchCallEnd_V1 +) + +declare -a DotNETRuntimePrivate_StackKeyword=( + DotNETRuntimePrivate:CLRStackWalkPrivate +) + +declare -a DotNETRuntimePrivate_PerfTrackPrivateKeyword=( + DotNETRuntimePrivate:ModuleRangeLoadPrivate +) + +declare -a DotNETRuntimePrivate_BindingKeyword=( + DotNETRuntimePrivate:BindingPolicyPhaseStart + DotNETRuntimePrivate:BindingPolicyPhaseEnd + DotNETRuntimePrivate:BindingNgenPhaseStart + DotNETRuntimePrivate:BindingNgenPhaseEnd + DotNETRuntimePrivate:BindingLookupAndProbingPhaseStart + DotNETRuntimePrivate:BindingLookupAndProbingPhaseEnd + DotNETRuntimePrivate:LoaderPhaseStart + DotNETRuntimePrivate:LoaderPhaseEnd + DotNETRuntimePrivate:BindingPhaseStart + DotNETRuntimePrivate:BindingPhaseEnd + DotNETRuntimePrivate:BindingDownloadPhaseStart + DotNETRuntimePrivate:BindingDownloadPhaseEnd + DotNETRuntimePrivate:LoaderAssemblyInitPhaseStart + DotNETRuntimePrivate:LoaderAssemblyInitPhaseEnd + DotNETRuntimePrivate:LoaderMappingPhaseStart + DotNETRuntimePrivate:LoaderMappingPhaseEnd + DotNETRuntimePrivate:LoaderDeliverEventsPhaseStart + DotNETRuntimePrivate:LoaderDeliverEventsPhaseEnd + DotNETRuntimePrivate:FusionMessageEvent + DotNETRuntimePrivate:FusionErrorCodeEvent +) + +declare -a DotNETRuntimePrivate_SecurityPrivateKeyword=( + DotNETRuntimePrivate:EvidenceGenerated + DotNETRuntimePrivate:ModuleTransparencyComputationStart + DotNETRuntimePrivate:ModuleTransparencyComputationEnd + DotNETRuntimePrivate:TypeTransparencyComputationStart + DotNETRuntimePrivate:TypeTransparencyComputationEnd + DotNETRuntimePrivate:MethodTransparencyComputationStart + DotNETRuntimePrivate:MethodTransparencyComputationEnd + DotNETRuntimePrivate:FieldTransparencyComputationStart + DotNETRuntimePrivate:FieldTransparencyComputationEnd + DotNETRuntimePrivate:TokenTransparencyComputationStart + DotNETRuntimePrivate:TokenTransparencyComputationEnd +) + +declare -a DotNETRuntimePrivate_PrivateFusionKeyword=( + DotNETRuntimePrivate:NgenBindEvent +) + +declare -a DotNETRuntimePrivate_NoKeyword=( + DotNETRuntimePrivate:FailFast +) + +declare -a DotNETRuntimePrivate_InteropPrivateKeyword=( + DotNETRuntimePrivate:CCWRefCountChange +) + +declare -a DotNETRuntimePrivate_GCHandlePrivateKeyword=( + DotNETRuntimePrivate:PrvSetGCHandle + DotNETRuntimePrivate:PrvDestroyGCHandle +) + +declare -a DotNETRuntimePrivate_LoaderHeapPrivateKeyword=( + DotNETRuntimePrivate:AllocRequest +) + +declare -a DotNETRuntimePrivate_MulticoreJitPrivateKeyword=( + DotNETRuntimePrivate:MulticoreJit + DotNETRuntimePrivate:MulticoreJitMethodCodeReturned +) + +declare -a DotNETRuntimePrivate_DynamicTypeUsageKeyword=( + DotNETRuntimePrivate:IInspectableRuntimeClassName + DotNETRuntimePrivate:WinRTUnbox + DotNETRuntimePrivate:CreateRCW + DotNETRuntimePrivate:RCWVariance + DotNETRuntimePrivate:RCWIEnumerableCasting + DotNETRuntimePrivate:CreateCCW + DotNETRuntimePrivate:CCWVariance + DotNETRuntimePrivate:ObjectVariantMarshallingToNative + DotNETRuntimePrivate:GetTypeFromGUID + DotNETRuntimePrivate:GetTypeFromProgID + DotNETRuntimePrivate:ConvertToCallbackEtw + DotNETRuntimePrivate:BeginCreateManagedReference + DotNETRuntimePrivate:EndCreateManagedReference + DotNETRuntimePrivate:ObjectVariantMarshallingToManaged +) + +declare -a EventSource=( + DotNETRuntime:EventSource +) + +declare -a LTTng_Kernel_ProcessLifetimeKeyword=( + sched_process_exec + sched_process_exit +) + + +###################################### +## Global Variables +###################################### + +# Install without waiting for standard input +forceInstall=0 + +# Declare an array of events to collect. +declare -a eventsToCollect + +# Use Perf_Event +usePerf=1 + +# Use LTTng +useLTTng=1 + +# LTTng Installed +lttngInstalled=0 + +# Collect hardware events +collect_HWevents=0 + +# Set to 1 when the CTRLC_Handler gets invoked. +handlerInvoked=0 + +# Log file +declare logFile +logFilePrefix='/tmp/perfcollect' +logEnabled=0 + +# Collect info to pass between processes +collectInfoFile=$(dirname `mktemp -u`)/'perfcollect.sessioninfo' + +###################################### +## Logging Functions +###################################### +LogAppend() +{ + if (( $logEnabled == 1 )) + then + echo $* >> $logFile + fi +} + +RunSilent() +{ + if (( $logEnabled == 1 )) + then + echo "Running \"$*\"" >> $logFile + $* >> $logFile 2>&1 + echo "" >> $logFile + else + $* > /dev/null 2>&1 + fi +} + +RunSilentBackground() +{ + if (( $logEnabled == 1 )) + then + echo "Running \"$*\"" >> $logFile + $* >> $logFile 2>&1 & sleep 1 + echo "" >> $logFile + else + $* > /dev/null 2>&1 & sleep 1 + fi + + # When handler is invoked, kill the process. + pid=$! + for (( ; ; )) + do + if [ "$handlerInvoked" == "1" ] + then + kill -INT $pid + break; + else + sleep 1 + fi + done + + # Wait for the process to exit. + wait $pid +} + +InitializeLog() +{ + # Pick the log file name. + logFile="$logFilePrefix.log" + while [ -f $logFile ]; + do + logFile="$logFilePrefix.$RANDOM.log" + done + + # Mark the log as enabled. + logEnabled=1 + + # Start the log + date=`date` + echo "Log started at ${date}" > $logFile + echo '' >> $logFile + + # The system information. + LogAppend 'Machine info: ' `uname -a` + if [ "$perfcmd" != "" ] + then + LogAppend 'perf version:' `$perfcmd --version 2>&1` + fi + if [ "$lttngcmd" != "" ] + then + LogAppend 'LTTng version: ' `$lttngcmd --version` + fi + LogAppend +} + +CloseLog() +{ + LogAppend "END LOG FILE" + LogAppend "NOTE: It is normal for the log file to end right before the trace is actually compressed. This occurs because the log file is part of the archive, and thus can't be written anymore." + + # The log itself doesn't need to be closed, + # but we need to tell the script not to log anymore. + logEnabled=0 +} + + +###################################### +## Helper Functions +###################################### + +## +# Console text color modification helpers. +## +RedText() +{ + tput=`GetCommandFullPath "tput"` + if [ "$tput" != "" ] + then + $tput setaf 1 + fi +} + +GreenText() +{ + tput=`GetCommandFullPath "tput"` + if [ "$tput" != "" ] + then + $tput setaf 2 + fi +} + +BlueText() +{ + tput=`GetCommandFullPath "tput"` + if [ "$tput" != "" ] + then + $tput setaf 6 + fi +} +YellowText() +{ + tput=`GetCommandFullPath "tput"` + if [ "$tput" != "" ] + then + $tput setaf 3 + fi +} + +ResetText() +{ + tput=`GetCommandFullPath "tput"` + if [ "$tput" != "" ] + then + $tput sgr0 + fi +} + +# $1 == Status message +WriteStatus() +{ + LogAppend $* + BlueText + echo $1 + ResetText +} + +# $1 == Message. +WriteWarning() +{ + LogAppend $* + YellowText + echo $1 + ResetText +} + +# $1 == Message. +FatalError() +{ + RedText + echo "ERROR: $1" + ResetText + PrintUsage + exit 1 +} + +EnsureRoot() +{ + # Warn non-root users. + if [ `whoami` != "root" ] + then + RedText + echo "This script must be run as root." + ResetText + exit 1; + fi +} + +###################################### +# Command Discovery +###################################### +DiscoverCommands() +{ + perfcmd=`GetCommandFullPath "perf"` + if [ "$(IsDebian)" == "1" ] + then + # Test perf to see if it successfully runs or fails because it doesn't match the kernel version. + $perfcmd --version > /dev/null 2>&1 + if [ "$?" == "1" ] + then + perf49Cmd=`GetCommandFullPath "perf_4.9"` + $perf49Cmd --version > /dev/null 2>&1 + if [ "$?" == "0" ] + then + perfcmd=$perf49Cmd + else + perf419Cmd=`GetCommandFullPath "perf_4.19"` + $perf419Cmd --version > /dev/null 2>&1 + if [ "$?" == "0" ] + then + perfcmd=$perf419Cmd + else + perf316Cmd=`GetCommandFullPath "perf_3.16"` + $perf316Cmd --version > /dev/null 2>&1 + if [ "$?" == "0" ] + then + perfcmd=$perf316Cmd + fi + fi + fi + fi + fi + + # Check to see if perf is installed, but doesn't exactly match the current kernel version. + # This happens when the kernel gets upgraded, or when running inside of a container where the + # host and container OS versions don't match. + perfoutput=$($perfcmd 2>&1) + if [ $? -eq 2 ] + then + # Check the beginning of the output to see if it matches the kernel warning. + warningText="WARNING: perf not found for kernel" + if [[ "$perfoutput" == "$warningText"* ]] + then + foundWorkingPerf=0 + WriteWarning "Perf is installed, but does not exactly match the version of the running kernel." + WriteWarning "This is often OK, and we'll try to workaround this." + WriteStatus "Attempting to find a working copy of perf." + # Attempt to find an existing version of perf to use. + # Order the search by newest directory first. + baseDir="/usr/lib/linux-tools" + searchPath="$baseDir/*/" + for dirName in $(ls -d --sort=time $searchPath) + do + candidatePerfPath="$dirName""perf" + $($candidatePerfPath > /dev/null 2>&1) + if [ $? -eq 1 ] + then + # If $? == 1, then use this copy of perf. + perfcmd=$candidatePerfPath + foundWorkingPerf=1 + break; + fi + done + if [ $foundWorkingPerf -eq 0 ] + then + FatalError "Unable to find a working copy of perf. Try re-installing via ./perfcollect install." + fi + WriteStatus "...FINISHED" + fi + fi + + lttngcmd=`GetCommandFullPath "lttng"` + zipcmd=`GetCommandFullPath "zip"` + unzipcmd=`GetCommandFullPath "unzip"` + objdumpcmd=`GetCommandFullPath "objdump"` +} + +GetCommandFullPath() +{ + echo `command -v $1` +} + +###################################### +# Prerequisite Installation +###################################### +IsAlpine() +{ + local alpine=0 + local apk=`GetCommandFullPath "apk"` + if [ "$apk" != "" ] + then + alpine=1 + fi + + echo $alpine +} + +InstallPerf_Alpine() +{ + # Disallow non-root users. + EnsureRoot + + # Install perf + apk add perf --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community + + # Install zip and unzip + apk add zip unzip +} + +IsRHEL() +{ + local rhel=0 + if [ -f /etc/redhat-release ] + then + rhel=1 + fi + + echo $rhel +} + +InstallPerf_RHEL() +{ + # Disallow non-root users. + EnsureRoot + + # Install perf + yum install perf zip unzip +} + +IsDebian() +{ + local debian=0 + local uname=`uname -a` + if [[ $uname =~ .*Debian.* ]] + then + debian=1 + elif [ -f /etc/debian_version ] + then + debian=1 + fi + + echo $debian +} + +InstallPerf_Debian() +{ + # Disallow non-root users. + EnsureRoot + + # Check for the existence of the linux-tools package. + pkgName='linux-tools' + pkgCount=`apt-cache search $pkgName | grep -c $pkgName` + if [ "$pkgCount" == "0" ] + then + pkgName='linux-perf' + pkgCount=`apt-cache search $pkgName | grep -c $pkgName` + if [ "$pkgCount" == "0" ] + then + FatalError "Unable to find a perf package to install." + fi + fi + + # Install zip and perf. + apt-get install -y zip binutils $pkgName +} + +IsSUSE() +{ + local suse=0 + if [ -f /usr/bin/zypper ] + then + suse=1 + fi + + echo $suse +} + +InstallPerf_SUSE() +{ + # Disallow non-root users. + EnsureRoot + + # Install perf. + zypper install perf zip unzip +} + +IsUbuntu() +{ + local ubuntu=0 + if [ -f /etc/lsb-release ] + then + local flavor=`cat /etc/lsb-release | grep DISTRIB_ID` + if [ "$flavor" == "DISTRIB_ID=Ubuntu" ] + then + ubuntu=1 + fi + fi + + echo $ubuntu +} + +InstallPerf_Ubuntu() +{ + # Disallow non-root users. + EnsureRoot + + # Install packages. + BlueText + echo "Installing perf_event packages." + ResetText + + # Handle Azure instances. + release=`uname -r` + if [[ "$release" == *"-azure" ]] + then + apt-get install -y linux-tools-azure zip software-properties-common + else + apt-get install -y linux-tools-common linux-tools-`uname -r` linux-cloud-tools-`uname -r` zip software-properties-common + fi +} + +InstallPerf() +{ + if [ "$(IsUbuntu)" == "1" ] + then + InstallPerf_Ubuntu + elif [ "$(IsSUSE)" == "1" ] + then + InstallPerf_SUSE + elif [ "$(IsDebian)" == "1" ] + then + InstallPerf_Debian + elif [ "$(IsRHEL)" == "1" ] + then + InstallPerf_RHEL + elif [ "$(IsAlpine)" == "1" ] + then + InstallPerf_Alpine + else + FatalError "Auto install unsupported for this distribution. Install perf manually to continue." + fi +} + +InstallLTTng_RHEL() +{ + # Disallow non-root users. + EnsureRoot + + local isRHEL7=0 + local isRHEL8=0 + if [ -e /etc/redhat-release ]; then + local redhatRelease=$(.dump files. + # Convert to jit dump file name. + local pid=`echo $mapFile | awk -F"-" '{print $NF}' | awk -F"." '{print $1}'` + local path=`echo $mapFile | awk -F"/" '{OFS="/";NF--;print $0;}'` + local jitDumpFile="$path/jit-$pid.dump" + + if [ -f $jitDumpFile ] + then + LogAppend "Saving $jitDumpFile" + RunSilent "cp $jitDumpFile ." + writeCrossgenWarning=0 + fi + done + + WriteStatus "Generating native image symbol files" + + # Get the list of loaded images and use the path to libcoreclr.so to find crossgen. + # crossgen is expected to sit next to libcoreclr.so. + local buildidList=`$perfcmd buildid-list | grep libcoreclr.so | cut -d ' ' -f 2` + local crossgenCmd='' + local crossgenDir='' + for file in $buildidList + do + crossgenDir=`dirname "${file}"` + if [ -f ${crossgenDir}/crossgen ] + then + crossgenCmd=${crossgenDir}/crossgen + LogAppend "Found crossgen at ${crossgenCmd}" + break + fi + done + + OLDIFS=$IFS + + imagePaths="" + + if [ "$crossgenCmd" != "" ] + then + local perfinfos=`ls . | grep perfinfo | cut -d ' ' -f 2` + for perfinfo in $perfinfos + do + if [ -f $perfinfo ] + then + IFS=";" + while read command dll guid; do + if [ $command ]; then + if [ $command = "ImageLoad" ]; then + if [ -f $dll ]; then + imagePaths="${dll}:${imagePaths}" + fi + fi + fi + done < $perfinfo + IFS=$OLDIFS + fi + done + + IFS=":" + LogAppend "Generating PerfMaps for native images" + for path in $imagePaths + do + if [ `echo ${path} | grep ^.*\.dll$` ] + then + IFS="" + LogAppend "Generating PerfMap for ${path}" + LogAppend "Running ${crossgenCmd} /r $imagePaths /CreatePerfMap . ${path}" + ${crossgenCmd} /r $imagePaths /CreatePerfMap . ${path} >> $logFile 2>&1 + IFS=":" + else + LogAppend "Skipping ${path}" + fi + done + else + if [ "$buildidList" != "" ] && [ $writeCrossgenWarning -eq 1 ] + then + LogAppend "crossgen not found, skipping native image map generation." + WriteStatus "...SKIPPED" + WriteWarning "Crossgen not found. Framework symbols will be unavailable." + WriteWarning "See https://github.com/dotnet/coreclr/blob/master/Documentation/project-docs/linux-performance-tracing.md#resolving-framework-symbols for details." + fi + fi + + IFS=$OLDIFS + + WriteStatus "...FINISHED" + + if [ "$objdumpcmd" != "" ] + then + # Create debuginfo files (separate symbols) for all modules in the trace. + WriteStatus "Saving native symbols" + + # Get the list of DSOs with hits in the trace file (those that are actually used). + # Filter out /tmp/perf-$pid.map files and files that end in .dll. + local dsosWithHits=`$perfcmd buildid-list --with-hits | grep -v /tmp/perf- | grep -v .dll$` + for dso in $dsosWithHits + do + # Build up tuples of buildid and binary path. + local processEntry=0 + if [ -f $dso ] + then + local pathToBinary=$dso + processEntry=1 + else + local buildid=$dso + pathToBinary='' + fi + + # Once we have a tuple for a binary path that exists, process it. + if [ "$processEntry" == "1" ] + then + # Get the binary name without path. + local binaryName=`basename $pathToBinary` + + # Build the debuginfo file name. + local destFileName=$binaryName.debuginfo + + # Build the destination directory for the debuginfo file. + local currentDir=`pwd` + local destDir=$currentDir/debuginfo/$buildid + + # Build the full path to the debuginfo file. + local destPath=$destDir/$destFileName + + # Check to see if the DSO contains symbols, and if so, build the debuginfo file. + local noSymbols=`$objdumpcmd -t $pathToBinary | grep "no symbols" -c` + if [ "$noSymbols" == "0" ] + then + LogAppend "Generating debuginfo for $binaryName with buildid=$buildid" + RunSilent "mkdir -p $destDir" + RunSilent "objcopy --only-keep-debug $pathToBinary $destPath" + else + LogAppend "Skipping $binaryName with buildid=$buildid. No symbol information." + fi + fi + done + + WriteStatus "...FINISHED" + fi + + WriteStatus "Resolving JIT and R2R symbols" + + originalFile="perf.data" + inputFile="perf-jit.data" + RunSilent $perfcmd inject --input $originalFile --jit --output $inputFile + + WriteStatus "...FINISHED" + + WriteStatus "Exporting perf.data file" + + outputDumpFile="perf.data.txt" + + # I've not found a good way to get the behavior that we want here - running the command and redirecting the output + # when passing the command line to a function. Thus, this case is hardcoded. + + # There is a breaking change where the capitalization of the -f parameter changed. + LogAppend "Running $perfcmd script -i $inputFile -F comm,pid,tid,cpu,time,period,event,ip,sym,dso,trace > $outputDumpFile" + $perfcmd script -i $inputFile -F comm,pid,tid,cpu,time,period,event,ip,sym,dso,trace > $outputDumpFile 2>>$logFile + LogAppend + + if [ $? -ne 0 ] + then + LogAppend "Running $perfcmd script -i $inputFile -f comm,pid,tid,cpu,time,period,event,ip,sym,dso,trace > $outputDumpFile" + $perfcmd script -i $inputFile -f comm,pid,tid,cpu,time,period,event,ip,sym,dso,trace > $outputDumpFile 2>>$logFile + LogAppend + fi + + # If the dump file is zero length, try to collect without the period field, which was added recently. + if [ ! -s $outputDumpFile ] + then + LogAppend "Running $perfcmd script -i $inputFile -f comm,pid,tid,cpu,time,event,ip,sym,dso,trace > $outputDumpFile" + $perfcmd script -i $inputFile -f comm,pid,tid,cpu,time,event,ip,sym,dso,trace > $outputDumpFile 2>>$logFile + LogAppend + fi + + WriteStatus "...FINISHED" + fi + + WriteStatus "Compressing trace files" + + # Move all collected files to the new directory. + RunSilent "mv -f * $directoryName" + + # Close the log - this stops all writing to the log, so that we can move it into the archive. + CloseLog + + # Move the log file to the new directory and rename it to the standard log name. + RunSilent "mv $logFile $directoryName/perfcollect.log" + + # Compress the data. + local archiveSuffix=".zip" + local archiveName=$directoryName$archiveSuffix + RunSilent "$zipcmd -r $archiveName $directoryName" + + # Move back to the original directory. + popd > /dev/null + + # Move the archive. + RunSilent "mv $tempDir/$archiveName ." + + WriteStatus "...FINISHED" + + WriteStatus "Cleaning up artifacts" + + # Delete the temp directory. + RunSilent "rm -rf $tempDir $collectInfoFile" + + WriteStatus "...FINISHED" + + # Tell the user where the trace is. + WriteStatus + WriteStatus "Trace saved to $archiveName" +} + +## +# Handle the CTRL+C signal. +## +CTRLC_Handler() +{ + # Mark the handler invoked. + handlerInvoked=1 +} + +EndCollect() +{ + # The user must either use "collect stop" or CTRL+C to stop collection. + if [ "$action" == "stop" ] + then + # Recover trace info in the previous program. + source $collectInfoFile # TODO: exit and dispose upon missing file + # New program started, so go back to the temp directory. + pushd $tempDir > /dev/null + fi + + if [ "$useLTTng" == "1" ] + then + StopLTTngCollection + fi + + # Update the user. + WriteStatus + WriteStatus "...STOPPED." + WriteStatus + WriteStatus "Starting post-processing. This may take some time." + WriteStatus + + # The user used CTRL+C to stop collection. + # When this happens, we catch the signal and finish our work. + ProcessCollectedData +} + +## +# Print usage information. +## +PrintUsage() +{ + echo "This script uses perf_event and LTTng to collect and view performance traces for .NET applications." + echo "For detailed collection and viewing steps, view this script in a text editor or viewer." + echo "" + echo "./perfcollect " + echo "Valid Actions: collect start/stop view livetrace install" + echo "" + echo "collect options:" + echo "By default, collection includes CPU samples collected every ms." + echo " -pid : Only collect data from the specified process id." + echo " -threadtime : Collect events for thread time analysis (on and off cpu)." + echo " -offcpu : Collect events for off-cpu analysis." + echo " -hwevents : Collect (some) hardware counters." + echo "" + echo "start:" + echo " Start collection, but with Lttng trace ONLY. It needs to be used with 'stop' action." + echo "" + echo "stop:" + echo " Stop collection if 'start' action is used." + echo "" + echo "view options:" + echo " -processfilter : Filter data by the specified process name." + echo " -graphtype : Specify the type of graph. Valid values are 'caller' and 'callee'. Default is 'callee'." + echo " -viewer : Specify the data viewer. Valid values are 'perf' and 'lttng'. Default is 'perf'." + echo "" + echo "livetrace:" + echo " Print EventSource events directly to the console. Root privileges not required." + echo "" + echo "install options:" + echo " Useful for first-time setup. Installs/upgrades perf_event and LTTng." + echo " -force : Force installation of required packages without prompt" + echo "" +} + +## +# Validate and set arguments. +## + +BuildPerfRecordArgs() +{ + # Start with default collection arguments that record at realtime priority, all CPUs (-a), and collect call stacks (-g) + collectionArgs="record -k 1 -g" + + # Filter to a single process if desired + if [ "$collectionPid" != "" ] + then + collectionArgs="$collectionArgs --pid=$collectionPid" + else + collectionArgs="$collectionArgs -a" + fi + + # Enable CPU Collection + if [ $collect_cpu -eq 1 ] + then + collectionArgs="$collectionArgs" + eventsToCollect=( "${eventsToCollect[@]}" "cpu-clock" ) + + # If only collecting CPU events, set the sampling rate to 1000. + # Otherwise, use the default sampling rate to avoid sampling sched events. + if [ $collect_threadTime -eq 0 ] && [ $collect_offcpu -eq 0 ] + then + collectionArgs="$collectionArgs -F 1000" + fi + fi + + # Enable HW counters event collection + if [ $collect_HWevents -eq 1 ] + then + collectionArgs="$collectionArgs -e cycles,instructions,branches,cache-misses" + fi + + # Enable context switches. + if [ $collect_threadTime -eq 1 ] + then + eventsToCollect=( "${eventsToCollect[@]}" "sched:sched_stat_sleep" "sched:sched_switch" "sched:sched_process_exit" ) + fi + + # Enable offcpu collection + if [ $collect_offcpu -eq 1 ] + then + eventsToCollect=( "${eventsToCollect[@]}" "sched:sched_stat_sleep" "sched:sched_switch" "sched:sched_process_exit" ) + fi + + # Build up the set of events. + local eventString="" + local comma="," + for (( i=0; i<${#eventsToCollect[@]}; i++ )) + do + # Get the arg. + eventName=${eventsToCollect[$i]} + + # Build up the comma separated list. + if [ "$eventString" == "" ] + then + eventString=$eventName + else + eventString="$eventString$comma$eventName" + fi + + done + + if [ ! -z ${duration} ] + then + durationString="sleep ${duration}" + fi + + # Add the events onto the collection command line args. + collectionArgs="$collectionArgs -e $eventString $durationString" +} + +DoCollect() +{ + # Ensure the script is run as root. + EnsureRoot + + # Build collection args. + # Places the resulting args in $collectionArgs + BuildPerfRecordArgs + + # Trap CTRL+C + trap CTRLC_Handler SIGINT + + # Create a temp directory to use for collection. + local tempDir=`mktemp -d` + LogAppend "Created temp directory $tempDir" + + # Switch to the directory. + pushd $tempDir > /dev/null + + # Start LTTng collection. + if [ "$useLTTng" == "1" ] + then + StartLTTngCollection + fi + + # Tell the user that collection has started and how to exit. + if [ "$duration" != "" ] + then + WriteStatus "Collection started. Collection will automatically stop in $duration second(s). Press CTRL+C to stop early." + elif [ "$action" == "start" ] + then + WriteStatus "Collection started." + else + WriteStatus "Collection started. Press CTRL+C to stop." + fi + + # Start perf record. + if [ "$action" == "start" ] + then + # Skip perf, which can only be stopped by CTRL+C. + # Pass trace directory and session name to the stop process. + popd > /dev/null + usePerf=0 + declare -p | grep -e lttngTraceDir -e lttngSessionName -e tempDir -e usePerf -e useLttng -e logFile > $collectInfoFile + else + if [ "$usePerf" == "1" ] + then + if [ ! -z ${duration} ] + then + RunSilent $perfcmd $collectionArgs + else + RunSilentBackground $perfcmd $collectionArgs + fi + else + # Wait here until CTRL+C handler gets called when user types CTRL+C. + LogAppend "Waiting for CTRL+C handler to get called." + + waitTime=0 + for (( ; ; )) + do + if [ "$handlerInvoked" == "1" ] + then + break; + fi + + # Wait and then check to see if the handler has been invoked or we've crossed the duration threshold. + sleep 1 + waitTime=$waitTime+1 + if (( duration > 0 && duration <= waitTime )) + then + break; + fi + done + fi + # End collection if action is 'collect'. + EndCollect + fi +} + +DoLiveTrace() +{ + # Start the session + StartLTTngCollection + + # View the event stream (until the user hits CTRL+C) + WriteStatus "Listening for events from LTTng. Hit CTRL+C to stop." + $lttngcmd view + + # Stop the LTTng sessoin + StopLTTngCollection +} + +# $1 == Path to directory containing trace files +PropSymbolsAndMapFilesForView() +{ + # Get the current directory + local currentDir=`pwd` + + # Copy map files to /tmp since they aren't supported by perf buildid-cache. + local mapFiles=`find -name *.map` + for mapFile in $mapFiles + do + echo "Copying $mapFile to /tmp." + cp $mapFile /tmp + done + + # Cache all debuginfo files saved with the trace in the buildid cache. + local debugInfoFiles=`find $currentDir -name *.debuginfo` + for debugInfoFile in $debugInfoFiles + do + echo "Caching $debugInfoFile in buildid cache using perf buildid-cache." + $perfcmd buildid-cache --add=$debugInfoFile + done +} + +DoView() +{ + # Generate a temp directory to extract the trace files into. + local tempDir=`mktemp -d` + + # Extract the trace files. + $unzipcmd $inputTraceName -d $tempDir + + # Move the to temp directory. + pushd $tempDir + cd `ls` + + # Select the viewer. + if [ "$viewer" == "perf" ] + then + # Prop symbols and map files. + PropSymbolsAndMapFilesForView `pwd` + + # Choose the view + if [ "$graphType" == "" ] + then + graphType="callee" + elif [ "$graphType" != "callee" ] && [ "$graphType" != "caller"] + then + FatalError "Invalid graph type specified. Valid values are 'callee' and 'caller'." + fi + + # Filter to specific process names if desired. + if [ "$processFilter" != "" ] + then + processFilter="--comms=$processFilter" + fi + + # Execute the viewer. + $perfcmd report -n -g graph,0.5,$graphType $processFilter $perfOpt + elif [ "$viewer" == "lttng" ] + then + babeltrace lttngTrace/ | more + fi + + # Switch back to the original directory. + popd + + # Delete the temp directory. + rm -rf $tempDir +} + +##################################### +## Main Script Start +##################################### + +# No arguments +if [ "$#" == "0" ] +then + PrintUsage + exit 0 +fi + +# Install perf if requested. Do this before all other validation. +if [ "$1" == "install" ] +then + if [ "$2" == "-force" ] + then + forceInstall=1 + fi + InstallPerf + InstallLTTng + exit 0 +fi + +# Discover external commands that will be called by this script. +DiscoverCommands + +# Initialize the log. +if [ "$1" != "stop" ] +then + InitializeLog +fi + +# Process arguments. +ProcessArguments $@ + +# Ensure prerequisites are installed. +EnsurePrereqsInstalled + +# Take the appropriate action. +if [ "$action" == "collect" ] || [ "$action" == "start" ] +then + DoCollect +elif [ "$action" == "stop" ] +then + EndCollect +elif [ "$action" == "view" ] +then + DoView +elif [ "$action" == "livetrace" ] +then + DoLiveTrace +fi \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs index 4592063359..bdcafc06e2 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs @@ -44,13 +44,16 @@ public DotNetCliCommand(string cliPath, string arguments, GenerateResult generat BuildPartition = buildPartition; EnvironmentVariables = environmentVariables; Timeout = timeout; - LogOutput = logOutput || buildPartition.LogBuildOutput; + LogOutput = logOutput || (buildPartition is not null && buildPartition.LogBuildOutput); RetryFailedBuildWithNoDeps = retryFailedBuildWithNoDeps; } public DotNetCliCommand WithArguments(string arguments) => new (CliPath, arguments, GenerateResult, Logger, BuildPartition, EnvironmentVariables, Timeout, logOutput: LogOutput); + public DotNetCliCommand WithCliPath(string cliPath) + => new (cliPath, Arguments, GenerateResult, Logger, BuildPartition, EnvironmentVariables, Timeout, logOutput: LogOutput); + [PublicAPI] public BuildResult RestoreThenBuild() { diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs index 9d4cefea0c..a6583ad734 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs @@ -22,11 +22,11 @@ public static class DotNetCliCommandExecutor [PublicAPI] public static DotNetCliCommandResult Execute(DotNetCliCommand parameters) { - using (var process = new Process { StartInfo = BuildStartInfo(parameters.CliPath, parameters.GenerateResult.ArtifactsPaths.BuildArtifactsDirectoryPath, parameters.Arguments, parameters.EnvironmentVariables) }) + using (var process = new Process { StartInfo = BuildStartInfo(parameters.CliPath, parameters.GenerateResult?.ArtifactsPaths.BuildArtifactsDirectoryPath, parameters.Arguments, parameters.EnvironmentVariables) }) using (var outputReader = new AsyncProcessOutputReader(process, parameters.LogOutput, parameters.Logger)) using (new ConsoleExitHandler(process, parameters.Logger)) { - parameters.Logger.WriteLineInfo($"// start {parameters.CliPath ?? "dotnet"} {parameters.Arguments} in {parameters.GenerateResult.ArtifactsPaths.BuildArtifactsDirectoryPath}"); + parameters.Logger.WriteLineInfo($"// start {process.StartInfo.FileName} {process.StartInfo.Arguments} in {process.StartInfo.WorkingDirectory}"); var stopwatch = Stopwatch.StartNew(); @@ -156,5 +156,27 @@ private static string GetDefaultDotNetCliPath() [DllImport("libc")] private static extern int getppid(); + + internal static string GetSdkPath(string cliPath) + { + DotNetCliCommand cliCommand = new ( + cliPath: cliPath, + arguments: "--info", + generateResult: null, + logger: NullLogger.Instance, + buildPartition: null, + environmentVariables: Array.Empty(), + timeout: TimeSpan.FromMinutes(1), + logOutput: false); + + string sdkPath = Execute(cliCommand) + .StandardOutput.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries) + .Where(line => line.EndsWith("/sdk]")) // sth like " 3.1.423 [/usr/share/dotnet/sdk] + .Select(line => line.Split('[')[1]) + .Distinct() + .Single(); // I assume there will be only one such folder + + return sdkPath.Substring(0, sdkPath.Length - 1); // remove trailing `]` + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchain.cs b/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchain.cs index a2eb6183f0..fb30e94a5a 100644 --- a/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchain.cs @@ -36,8 +36,11 @@ internal NativeAotToolchain(string displayName, new DotNetCliPublisher(customDotNetCliPath, GetExtraArguments(runtimeIdentifier)), new Executor()) { + CustomDotNetCliPath = customDotNetCliPath; } + internal string CustomDotNetCliPath { get; } + public static NativeAotToolchainBuilder CreateBuilder() => NativeAotToolchainBuilder.Create(); public static string GetExtraArguments(string runtimeIdentifier) => $"-r {runtimeIdentifier}"; From de5cf4f56cf10fefe9345d84401ac833c30962de Mon Sep 17 00:00:00 2001 From: Andrey Akinshin Date: Thu, 29 Sep 2022 18:09:56 +0300 Subject: [PATCH 06/58] Bump docfx 2.59.3->2.59.4 --- build/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Program.cs b/build/Program.cs index 6182dfbb8b..03968e14f1 100644 --- a/build/Program.cs +++ b/build/Program.cs @@ -203,7 +203,7 @@ public void RunDocfx(FilePath docfxJson, string args = "") public static class DocumentationHelper { - public const string DocFxVersion = "2.59.3"; + public const string DocFxVersion = "2.59.4"; public static readonly string[] BdnAllVersions = { From 7e87e825e8bfa2be4d5cbda0301b969716ea3f98 Mon Sep 17 00:00:00 2001 From: Andrey Akinshin Date: Thu, 29 Sep 2022 18:10:35 +0300 Subject: [PATCH 07/58] Update workflows/docs-stable.yaml --- .github/workflows/docs-stable.yaml | 44 +++++++++++++++++------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/.github/workflows/docs-stable.yaml b/.github/workflows/docs-stable.yaml index 8129ff2dd3..dc77deb75e 100644 --- a/.github/workflows/docs-stable.yaml +++ b/.github/workflows/docs-stable.yaml @@ -19,11 +19,12 @@ jobs: - name: Build BenchmarkDotNet run: ./build.bat --target Build - - name: Download changelog - run: ./build.bat --target DocFX_Changelog_Download --LatestVersions true --StableVersions true - env: - GITHUB_PRODUCT: ChangelogBuilder - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Temporary disabled because of the API limit + # - name: Download changelog + # run: ./build.bat --target DocFX_Changelog_Download --LatestVersions true --StableVersions true + # env: + # GITHUB_PRODUCT: ChangelogBuilder + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Build documentation run: ./build.bat --target DocFX_Build @@ -40,17 +41,22 @@ jobs: runs-on: ubuntu-latest steps: - - name: Download Artifacts - uses: actions/download-artifact@v1 - with: - name: site - - - name: Deploy documentation - uses: JamesIves/github-pages-deploy-action@3.7.1 - with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BRANCH: gh-pages - FOLDER: site - GIT_CONFIG_NAME: Andrey Akinshin - GIT_CONFIG_EMAIL: andrey.akinshin@gmail.com - CLEAN: true \ No newline at end of file + - name: Download Artifacts + uses: actions/download-artifact@v1 + with: + name: site + + - name: Checkout + uses: actions/checkout@v3 + with: + ref: docs-stable + + - name: Deploy documentation + uses: JamesIves/github-pages-deploy-action@3.7.1 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BRANCH: gh-pages + FOLDER: site + GIT_CONFIG_NAME: Andrey Akinshin + GIT_CONFIG_EMAIL: andrey.akinshin@gmail.com + CLEAN: true \ No newline at end of file From ff443adb5a03acbc42cbd9b99052c772f2bc698c Mon Sep 17 00:00:00 2001 From: Andrey Akinshin Date: Thu, 29 Sep 2022 18:18:14 +0300 Subject: [PATCH 08/58] Update workflows/docs-stable.yaml --- .github/workflows/docs-stable.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docs-stable.yaml b/.github/workflows/docs-stable.yaml index dc77deb75e..0826c801f8 100644 --- a/.github/workflows/docs-stable.yaml +++ b/.github/workflows/docs-stable.yaml @@ -41,16 +41,16 @@ jobs: runs-on: ubuntu-latest steps: - - name: Download Artifacts - uses: actions/download-artifact@v1 - with: - name: site - - name: Checkout uses: actions/checkout@v3 with: ref: docs-stable + - name: Download Artifacts + uses: actions/download-artifact@v1 + with: + name: site + - name: Deploy documentation uses: JamesIves/github-pages-deploy-action@3.7.1 with: From a78c2e6a6e3db79069fb5bbbd6da6e5cbea8c029 Mon Sep 17 00:00:00 2001 From: Andrey Akinshin Date: Sun, 2 Oct 2022 09:48:06 +0300 Subject: [PATCH 09/58] Add write-all permissions to workflows/docs-stable.yaml --- .github/workflows/docs-stable.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docs-stable.yaml b/.github/workflows/docs-stable.yaml index 0826c801f8..f5abc60532 100644 --- a/.github/workflows/docs-stable.yaml +++ b/.github/workflows/docs-stable.yaml @@ -6,6 +6,8 @@ on: - docs-stable workflow_dispatch: +permissions: write-all + jobs: build: runs-on: windows-latest From f4d99ab7b81cccf9322fe6ae084572d8eeb9cade Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Wed, 5 Oct 2022 13:56:27 +0200 Subject: [PATCH 10/58] arm64 disassembler (#2127) * make ClrMdV2Disassembler abstract, introduce Arm and Intel disassemblers * Capstone PoC * Implement TryGetReferencedAddress for relative branches (#2107) * Resolve indirect addresses in disassembly (#2118) Co-authored-by: Jan Vorlicek * Initial version of the Arm64 instruction formatter (#2119) * Added other arm64 constant form extraction plus other changes (#2123) Co-authored-by: Jan Vorlicek Co-authored-by: Jan Vorlicek --- .../BenchmarkDotNet.Samples.csproj | 2 +- .../IntroDisassembly.cs | 3 +- .../BenchmarkDotNet.Disassembler.x64.csproj | 1 + .../ClrMdV1Disassembler.cs | 12 +- .../DataContracts.cs | 49 ++- .../BenchmarkDotNet.Disassembler.x86.csproj | 1 + .../DisassemblyDiagnoserAttribute.cs | 3 + src/BenchmarkDotNet/BenchmarkDotNet.csproj | 3 +- .../Disassemblers/Arm64Disassembler.cs | 282 ++++++++++++++++++ .../Arm64InstructionFormatter.cs | 48 +++ .../Disassemblers/ClrMdV2Disassembler.cs | 222 ++++++-------- .../Disassemblers/DisassemblyDiagnoser.cs | 6 +- .../DisassemblyDiagnoserConfig.cs | 62 ++-- .../Disassemblers/DisassemblySyntax.cs | 18 ++ .../Exporters/CombinedDisassemblyExporter.cs | 2 +- .../Exporters/DisassemblyPrettifier.cs | 20 +- .../GithubMarkdownDisassemblyExporter.cs | 2 +- .../Disassemblers/InstructionFormatter.cs | 49 +-- .../Disassemblers/IntelDisassembler.cs | 133 +++++++++ .../IntelInstructionFormatter.cs | 31 ++ .../SameArchitectureDisassembler.cs | 25 +- .../Disassemblers/WindowsDisassembler.cs | 5 +- .../Exporters/InstructionPointerExporter.cs | 6 +- .../DisassemblyDiagnoserTests.cs | 2 +- .../TargetFrameworkVersionParsingTestscs.cs | 23 ++ 25 files changed, 780 insertions(+), 230 deletions(-) create mode 100644 src/BenchmarkDotNet/Disassemblers/Arm64Disassembler.cs create mode 100644 src/BenchmarkDotNet/Disassemblers/Arm64InstructionFormatter.cs create mode 100644 src/BenchmarkDotNet/Disassemblers/DisassemblySyntax.cs create mode 100644 src/BenchmarkDotNet/Disassemblers/IntelDisassembler.cs create mode 100644 src/BenchmarkDotNet/Disassemblers/IntelInstructionFormatter.cs create mode 100644 tests/BenchmarkDotNet.Tests/TargetFrameworkVersionParsingTestscs.cs diff --git a/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj b/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj index 36f094bbbe..9386dcf6bb 100644 --- a/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj +++ b/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj @@ -24,4 +24,4 @@ - \ No newline at end of file + diff --git a/samples/BenchmarkDotNet.Samples/IntroDisassembly.cs b/samples/BenchmarkDotNet.Samples/IntroDisassembly.cs index 87fd0a3152..238821beb3 100644 --- a/samples/BenchmarkDotNet.Samples/IntroDisassembly.cs +++ b/samples/BenchmarkDotNet.Samples/IntroDisassembly.cs @@ -1,9 +1,10 @@ using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Diagnosers; using System.Linq; namespace BenchmarkDotNet.Samples { - [DisassemblyDiagnoser] + [DisassemblyDiagnoser(printInstructionAddresses: true, syntax: DisassemblySyntax.Masm)] public class IntroDisassembly { private int[] field = Enumerable.Range(0, 100).ToArray(); diff --git a/src/BenchmarkDotNet.Disassembler.x64/BenchmarkDotNet.Disassembler.x64.csproj b/src/BenchmarkDotNet.Disassembler.x64/BenchmarkDotNet.Disassembler.x64.csproj index 9e45545a3f..3162c74e5d 100644 --- a/src/BenchmarkDotNet.Disassembler.x64/BenchmarkDotNet.Disassembler.x64.csproj +++ b/src/BenchmarkDotNet.Disassembler.x64/BenchmarkDotNet.Disassembler.x64.csproj @@ -8,6 +8,7 @@ win7-x64 x64 True + $(DefineConstants);CLRMDV1 ..\BenchmarkDotNet\Disassemblers diff --git a/src/BenchmarkDotNet.Disassembler.x64/ClrMdV1Disassembler.cs b/src/BenchmarkDotNet.Disassembler.x64/ClrMdV1Disassembler.cs index ec48377104..2eff0fe25b 100644 --- a/src/BenchmarkDotNet.Disassembler.x64/ClrMdV1Disassembler.cs +++ b/src/BenchmarkDotNet.Disassembler.x64/ClrMdV1Disassembler.cs @@ -25,7 +25,7 @@ internal static DisassemblyResult AttachAndDisassemble(Settings settings) ConfigureSymbols(dataTarget); - var state = new State(runtime); + var state = new State(runtime, settings.TargetFrameworkMoniker); if (settings.Filters.Length > 0) { @@ -169,21 +169,23 @@ private static IEnumerable Decode(ulong startAddress, uint size, State stat { decoder.Decode(out var instruction); - TryTranslateAddressToName(instruction, state, depth, currentMethod); + TryTranslateAddressToName(instruction, state, depth, currentMethod, out ulong referencedAddress); yield return new Asm { InstructionPointer = instruction.IP, - Instruction = instruction + InstructionLength = instruction.Length, + IntelInstruction = instruction, + ReferencedAddress = (referencedAddress > ushort.MaxValue) ? referencedAddress : null, }; } } - private static void TryTranslateAddressToName(Instruction instruction, State state, int depth, ClrMethod currentMethod) + private static void TryTranslateAddressToName(Instruction instruction, State state, int depth, ClrMethod currentMethod, out ulong address) { var runtime = state.Runtime; - if (!TryGetReferencedAddress(instruction, (uint)runtime.PointerSize, out ulong address)) + if (!TryGetReferencedAddress(instruction, (uint)runtime.PointerSize, out address)) return; if (state.AddressToNameMapping.ContainsKey(address)) diff --git a/src/BenchmarkDotNet.Disassembler.x64/DataContracts.cs b/src/BenchmarkDotNet.Disassembler.x64/DataContracts.cs index a1bcdbbca0..3585c53fd4 100644 --- a/src/BenchmarkDotNet.Disassembler.x64/DataContracts.cs +++ b/src/BenchmarkDotNet.Disassembler.x64/DataContracts.cs @@ -24,7 +24,14 @@ public class Sharp : SourceCode public class Asm : SourceCode { - public Instruction Instruction { get; set; } + public int InstructionLength { get; set; } + public ulong? ReferencedAddress { get; set; } + public bool IsReferencedAddressIndirect { get; set; } + + public Instruction? IntelInstruction { get; set; } +#if !CLRMDV1 + public Gee.External.Capstone.Arm64.Arm64Instruction Arm64Instruction { get; set; } +#endif } public class MonoCode : SourceCode @@ -101,7 +108,7 @@ public static class DisassemblerConstants internal class Settings { - internal Settings(int processId, string typeName, string methodName, bool printSource, int maxDepth, string resultsPath, string[] filters) + internal Settings(int processId, string typeName, string methodName, bool printSource, int maxDepth, string resultsPath, string syntax, string tfm, string[] filters) { ProcessId = processId; TypeName = typeName; @@ -109,6 +116,8 @@ internal Settings(int processId, string typeName, string methodName, bool printS PrintSource = printSource; MaxDepth = methodName == DisassemblerConstants.DisassemblerEntryMethodName && maxDepth != int.MaxValue ? maxDepth + 1 : maxDepth; ResultsPath = resultsPath; + Syntax = syntax; + TargetFrameworkMoniker = tfm; Filters = filters; } @@ -118,6 +127,8 @@ internal Settings(int processId, string typeName, string methodName, bool printS internal bool PrintSource { get; } internal int MaxDepth { get; } internal string[] Filters; + internal string Syntax { get; } + internal string TargetFrameworkMoniker { get; } internal string ResultsPath { get; } internal static Settings FromArgs(string[] args) @@ -128,24 +139,54 @@ internal static Settings FromArgs(string[] args) printSource: bool.Parse(args[3]), maxDepth: int.Parse(args[4]), resultsPath: args[5], - filters: args.Skip(6).ToArray() + syntax: args[6], + tfm: args[7], + filters: args.Skip(8).ToArray() ); } internal class State { - internal State(ClrRuntime runtime) + internal State(ClrRuntime runtime, string targetFrameworkMoniker) { Runtime = runtime; Todo = new Queue(); HandledMethods = new HashSet(new ClrMethodComparer()); AddressToNameMapping = new Dictionary(); + RuntimeVersion = ParseVersion(targetFrameworkMoniker); } internal ClrRuntime Runtime { get; } + internal string TargetFrameworkMoniker { get; } internal Queue Todo { get; } internal HashSet HandledMethods { get; } internal Dictionary AddressToNameMapping { get; } + internal Version RuntimeVersion { get; } + + internal static Version ParseVersion(string targetFrameworkMoniker) + { + int firstDigit = -1, lastDigit = -1; + for (int i = 0; i < targetFrameworkMoniker.Length; i++) + { + if (char.IsDigit(targetFrameworkMoniker[i])) + { + if (firstDigit == -1) + firstDigit = i; + + lastDigit = i; + } + else if (targetFrameworkMoniker[i] == '-') + { + break; // it can be platform specific like net7.0-windows8 + } + } + + string versionToParse = targetFrameworkMoniker.Substring(firstDigit, lastDigit - firstDigit + 1); + if (!versionToParse.Contains(".")) // Full .NET Framework (net48 etc) + versionToParse = string.Join(".", versionToParse.ToCharArray()); + + return Version.Parse(versionToParse); + } private sealed class ClrMethodComparer : IEqualityComparer { diff --git a/src/BenchmarkDotNet.Disassembler.x86/BenchmarkDotNet.Disassembler.x86.csproj b/src/BenchmarkDotNet.Disassembler.x86/BenchmarkDotNet.Disassembler.x86.csproj index 983c78346f..21051d6104 100644 --- a/src/BenchmarkDotNet.Disassembler.x86/BenchmarkDotNet.Disassembler.x86.csproj +++ b/src/BenchmarkDotNet.Disassembler.x86/BenchmarkDotNet.Disassembler.x86.csproj @@ -8,6 +8,7 @@ win7-x86 x86 True + $(DefineConstants);CLRMDV1 ..\BenchmarkDotNet\Disassemblers diff --git a/src/BenchmarkDotNet/Attributes/DisassemblyDiagnoserAttribute.cs b/src/BenchmarkDotNet/Attributes/DisassemblyDiagnoserAttribute.cs index c8fdb60671..9985332d76 100644 --- a/src/BenchmarkDotNet/Attributes/DisassemblyDiagnoserAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/DisassemblyDiagnoserAttribute.cs @@ -8,6 +8,7 @@ namespace BenchmarkDotNet.Attributes public class DisassemblyDiagnoserAttribute : Attribute, IConfigSource { /// Includes called methods to given level. 1 by default, indexed from 1. To print just the benchmark set it to 0. + /// The disassembly syntax. MASM is the default. /// C#|F#|VB source code will be printed. False by default. /// Print instruction addresses. False by default /// Exports to GitHub markdown. True by default. @@ -17,6 +18,7 @@ public class DisassemblyDiagnoserAttribute : Attribute, IConfigSource /// Glob patterns applied to full method signatures by the the disassembler. public DisassemblyDiagnoserAttribute( int maxDepth = 1, + DisassemblySyntax syntax = DisassemblySyntax.Masm, bool printSource = false, bool printInstructionAddresses = false, bool exportGithubMarkdown = true, @@ -29,6 +31,7 @@ public DisassemblyDiagnoserAttribute( new DisassemblyDiagnoser( new DisassemblyDiagnoserConfig( maxDepth: maxDepth, + syntax: syntax, filters: filters, printSource: printSource, printInstructionAddresses: printInstructionAddresses, diff --git a/src/BenchmarkDotNet/BenchmarkDotNet.csproj b/src/BenchmarkDotNet/BenchmarkDotNet.csproj index ca2a18450d..6e700e69bb 100644 --- a/src/BenchmarkDotNet/BenchmarkDotNet.csproj +++ b/src/BenchmarkDotNet/BenchmarkDotNet.csproj @@ -4,7 +4,7 @@ BenchmarkDotNet netstandard2.0;net6.0 true - $(NoWarn);1701;1702;1705;1591;3005;NU1702;CS3001;CS3003 + $(NoWarn);1701;1702;1705;1591;3005;NU1702;CS3001;CS3003;CS8002 BenchmarkDotNet BenchmarkDotNet BenchmarkDotNet @@ -17,6 +17,7 @@ + diff --git a/src/BenchmarkDotNet/Disassemblers/Arm64Disassembler.cs b/src/BenchmarkDotNet/Disassemblers/Arm64Disassembler.cs new file mode 100644 index 0000000000..7c949f576f --- /dev/null +++ b/src/BenchmarkDotNet/Disassemblers/Arm64Disassembler.cs @@ -0,0 +1,282 @@ +using BenchmarkDotNet.Diagnosers; +using Gee.External.Capstone; +using Gee.External.Capstone.Arm64; +using Microsoft.Diagnostics.Runtime; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace BenchmarkDotNet.Disassemblers +{ + internal struct RegisterValueAccumulator + { + private enum State + { + LookingForPattern, + ExpectingMovk, + ExpectingAdd, + LookingForPossibleLdr + } + + private State _state; + private long _value; + private int _expectedMovkShift; + private Arm64RegisterId _registerId; + private ClrRuntime _runtime; + + public void Init(ClrRuntime runtime) + { + _state = State.LookingForPattern; + _expectedMovkShift = 0; + _value = 0; + _registerId = Arm64RegisterId.Invalid; + _runtime = runtime; + } + + public void Feed(Arm64Instruction instruction) + { + Arm64InstructionDetail details = instruction.Details; + + switch (_state) + { + case State.LookingForPattern: + if (instruction.Id == Arm64InstructionId.ARM64_INS_MOVZ) + { + _registerId = details.Operands[0].Register.Id; + _value = details.Operands[1].Immediate; + _state = State.ExpectingMovk; + _expectedMovkShift = 16; + } + else if (instruction.Id == Arm64InstructionId.ARM64_INS_ADRP) + { + _registerId = details.Operands[0].Register.Id; + _value = details.Operands[1].Immediate; + _state = State.ExpectingAdd; + } + break; + case State.ExpectingMovk: + if (instruction.Id == Arm64InstructionId.ARM64_INS_MOVK && + details.Operands[0].Register.Id == _registerId && + details.Operands[1].ShiftOperation == Arm64ShiftOperation.ARM64_SFT_LSL && + details.Operands[1].ShiftValue == _expectedMovkShift) + { + _value = _value | (instruction.Details.Operands[1].Immediate << details.Operands[1].ShiftValue); + _expectedMovkShift += 16; + break; + } + _state = State.LookingForPossibleLdr; + goto case State.LookingForPossibleLdr; + case State.ExpectingAdd: + if (instruction.Id == Arm64InstructionId.ARM64_INS_ADD && + details.Operands[0].Register.Id == _registerId && + details.Operands[1].Register.Id == _registerId && + details.Operands[2].Type == Arm64OperandType.Immediate) + { + _value = _value | instruction.Details.Operands[2].Immediate; + _state = State.LookingForPossibleLdr; + } + break; + case State.LookingForPossibleLdr: + if (instruction.Id == Arm64InstructionId.ARM64_INS_LDR && + details.Operands[1].Type == Arm64OperandType.Memory && + details.Operands[1].Memory.Base.Id == _registerId && // The source address is in the register we are tracking + details.Operands[1].Memory.Displacement == 0 && // There is no displacement + details.Operands[1].Memory.Index == null) // And there is no extra index register + { + // Simulate the LDR instruction. + long newValue = (long)_runtime.DataTarget.DataReader.ReadPointer((ulong)_value); + _value = newValue; + if (_value == 0) + { + _state = State.LookingForPattern; + } + else + { + // The LDR might have loaded the result in another register + _registerId = details.Operands[0].Register.Id; + } + } + else if (instruction.Id == Arm64InstructionId.ARM64_INS_CBZ || + instruction.Id == Arm64InstructionId.ARM64_INS_CBNZ || + instruction.Id == Arm64InstructionId.ARM64_INS_B && details.ConditionCode != Arm64ConditionCode.Invalid) + { + // ignore conditional branches + } + else if (details.BelongsToGroup(Arm64InstructionGroupId.ARM64_GRP_BRANCH_RELATIVE) || + details.BelongsToGroup(Arm64InstructionGroupId.ARM64_GRP_CALL) || + details.BelongsToGroup(Arm64InstructionGroupId.ARM64_GRP_JUMP)) + { + // We've encountered an unconditional jump or call, the accumulated registers value is not valid anymore + _state = State.LookingForPattern; + } + else if (instruction.Id == Arm64InstructionId.ARM64_INS_MOVZ) + { + // Another constant loading is starting + _state = State.LookingForPattern; + goto case State.LookingForPattern; + } + else + { + // Finally check if the current instruction modified the register that was accumulating the constant + // and reset the state machine in case it did. + foreach (Arm64Register reg in details.AllWrittenRegisters) + { + // Some unexpected instruction overwriting the accumulated register + if (reg.Id == _registerId) + { + _state = State.LookingForPattern; + } + } + } + break; + } + } + + public bool HasValue => _state == State.ExpectingMovk || _state == State.LookingForPossibleLdr; + + public long Value { get { return _value; } } + + public Arm64RegisterId RegisterId { get { return _registerId; } } + } + + internal class Arm64Disassembler : ClrMdV2Disassembler + { + // See dotnet/runtime src/coreclr/vm/arm64/thunktemplates.asm/.S for the stub code + // ldr x9, DATA_SLOT(CallCountingStub, RemainingCallCountCell) + // ldrh w10, [x9] + // subs w10, w10, #0x1 + private static byte[] callCountingStubTemplate = new byte[12] { 0x09, 0x00, 0x00, 0x58, 0x2a, 0x01, 0x40, 0x79, 0x4a, 0x05, 0x00, 0x71 }; + // ldr x10, DATA_SLOT(StubPrecode, Target) + // ldr x12, DATA_SLOT(StubPrecode, MethodDesc) + // br x10 + private static byte[] stubPrecodeTemplate = new byte[12] { 0x4a, 0x00, 0x00, 0x58, 0xec, 0x00, 0x00, 0x58, 0x40, 0x01, 0x1f, 0xd6 }; + // ldr x11, DATA_SLOT(FixupPrecode, Target) + // br x11 + // ldr x12, DATA_SLOT(FixupPrecode, MethodDesc) + private static byte[] fixupPrecodeTemplate = new byte[12] { 0x0b, 0x00, 0x00, 0x58, 0x60, 0x01, 0x1f, 0xd6, 0x0c, 0x00, 0x00, 0x58 }; + + static Arm64Disassembler() + { + // The stubs code depends on the current OS memory page size, so we need to update the templates to reflect that + int pageSizeShifted = Environment.SystemPageSize / 32; + // Calculate the ldr x9, #offset instruction with offset based on the page size + callCountingStubTemplate[1] = (byte)(pageSizeShifted & 0xff); + callCountingStubTemplate[2] = (byte)(pageSizeShifted >> 8); + + // Calculate the ldr x10, #offset instruction with offset based on the page size + stubPrecodeTemplate[1] = (byte)(pageSizeShifted & 0xff); + stubPrecodeTemplate[2] = (byte)(pageSizeShifted >> 8); + // Calculate the ldr x12, #offset instruction with offset based on the page size + stubPrecodeTemplate[5] = (byte)((pageSizeShifted - 1) & 0xff); + stubPrecodeTemplate[6] = (byte)((pageSizeShifted - 1) >> 8); + + // Calculate the ldr x11, #offset instruction with offset based on the page size + fixupPrecodeTemplate[1] = (byte)(pageSizeShifted & 0xff); + fixupPrecodeTemplate[2] = (byte)(pageSizeShifted >> 8); + // Calculate the ldr x12, #offset instruction with offset based on the page size + fixupPrecodeTemplate[9] = (byte)(pageSizeShifted & 0xff); + fixupPrecodeTemplate[10] = (byte)(pageSizeShifted >> 8); + } + + protected override IEnumerable Decode(byte[] code, ulong startAddress, State state, int depth, ClrMethod currentMethod, DisassemblySyntax syntax) + { + const Arm64DisassembleMode disassembleMode = Arm64DisassembleMode.Arm; + using (CapstoneArm64Disassembler disassembler = CapstoneDisassembler.CreateArm64Disassembler(disassembleMode)) + { + // Enables disassemble details, which are disabled by default, to provide more detailed information on + // disassembled binary code. + disassembler.EnableInstructionDetails = true; + disassembler.DisassembleSyntax = Map(syntax); + RegisterValueAccumulator accumulator = new RegisterValueAccumulator(); + accumulator.Init(state.Runtime); + + Arm64Instruction[] instructions = disassembler.Disassemble(code, (long)startAddress); + foreach (Arm64Instruction instruction in instructions) + { + bool isIndirect = false; + bool isPrestubMD = false; + + ulong address = 0; + if (TryGetReferencedAddress(instruction, accumulator, (uint)state.Runtime.DataTarget.DataReader.PointerSize, out address, out isIndirect)) + { + if (isIndirect && state.RuntimeVersion.Major >= 7) + { + // Check if the target is a known stub + // The stubs are allocated in interleaved code / data pages in memory. The data part of the stub + // is at an address one memory page higher than the code. + byte[] buffer = new byte[12]; + + if (state.Runtime.DataTarget.DataReader.Read(address, buffer) == buffer.Length) + { + if (buffer.SequenceEqual(callCountingStubTemplate)) + { + const ulong TargetMethodAddressSlotOffset = 8; + address = state.Runtime.DataTarget.DataReader.ReadPointer(address + (ulong)Environment.SystemPageSize + TargetMethodAddressSlotOffset); + } + else if (buffer.SequenceEqual(stubPrecodeTemplate)) + { + const ulong MethodDescSlotOffset = 0; + address = state.Runtime.DataTarget.DataReader.ReadPointer(address + (ulong)Environment.SystemPageSize + MethodDescSlotOffset); + isPrestubMD = true; + } + else if (buffer.SequenceEqual(fixupPrecodeTemplate)) + { + const ulong MethodDescSlotOffset = 8; + address = state.Runtime.DataTarget.DataReader.ReadPointer(address + (ulong)Environment.SystemPageSize + MethodDescSlotOffset); + isPrestubMD = true; + } + } + } + TryTranslateAddressToName(address, isPrestubMD, state, isIndirect, depth, currentMethod); + } + + accumulator.Feed(instruction); + + yield return new Asm() + { + InstructionPointer = (ulong)instruction.Address, + InstructionLength = instruction.Bytes.Length, + Arm64Instruction = instruction, + ReferencedAddress = (address > ushort.MaxValue) ? address : null, + IsReferencedAddressIndirect = isIndirect + }; + } + } + } + + private static bool TryGetReferencedAddress(Arm64Instruction instruction, RegisterValueAccumulator accumulator, uint pointerSize, out ulong referencedAddress, out bool isReferencedAddressIndirect) + { + if ((instruction.Id == Arm64InstructionId.ARM64_INS_BR || instruction.Id == Arm64InstructionId.ARM64_INS_BLR) && instruction.Details.Operands[0].Register.Id == accumulator.RegisterId && accumulator.HasValue) + { + // Branch via register where we have extracted the value of the register by parsing the disassembly + referencedAddress = (ulong)accumulator.Value; + isReferencedAddressIndirect = true; + return true; + } + else if (instruction.Details.BelongsToGroup(Arm64InstructionGroupId.ARM64_GRP_BRANCH_RELATIVE)) + { + // One of the operands is the address + for (int i = 0; i < instruction.Details.Operands.Length; i++) + { + if (instruction.Details.Operands[i].Type == Arm64OperandType.Immediate) + { + referencedAddress = (ulong)instruction.Details.Operands[i].Immediate; + isReferencedAddressIndirect = false; + return true; + } + } + } + referencedAddress = 0; + isReferencedAddressIndirect = false; + return false; + } + + private static DisassembleSyntax Map(DisassemblySyntax syntax) + => syntax switch + { + DisassemblySyntax.Att => DisassembleSyntax.Att, + DisassemblySyntax.Intel => DisassembleSyntax.Intel, + _ => DisassembleSyntax.Masm + }; + } +} diff --git a/src/BenchmarkDotNet/Disassemblers/Arm64InstructionFormatter.cs b/src/BenchmarkDotNet/Disassemblers/Arm64InstructionFormatter.cs new file mode 100644 index 0000000000..b3f6e69325 --- /dev/null +++ b/src/BenchmarkDotNet/Disassemblers/Arm64InstructionFormatter.cs @@ -0,0 +1,48 @@ +using Gee.External.Capstone.Arm64; +using Iced.Intel; +using System.Collections.Generic; +using System.Text; + +namespace BenchmarkDotNet.Disassemblers +{ + internal static class Arm64InstructionFormatter + { + // FormatterOptions is an Intel-specific concept that comes from the Iced library, but since our users can pass custom + // Iced Formatter to DisassemblyDiagnoserConfig and it provides all the settings we need, we just reuse it here. + internal static string Format(Asm asm, FormatterOptions formatterOptions, + bool printInstructionAddresses, uint pointerSize, IReadOnlyDictionary symbols) + { + StringBuilder output = new (); + Arm64Instruction instruction = asm.Arm64Instruction; + + if (printInstructionAddresses) + { + FormatInstructionPointer(instruction, formatterOptions, pointerSize, output); + } + + output.Append(instruction.Mnemonic.ToString().PadRight(formatterOptions.FirstOperandCharIndex)); + + if (asm.ReferencedAddress.HasValue && !asm.IsReferencedAddressIndirect && symbols.TryGetValue(asm.ReferencedAddress.Value, out string name)) + { + string partToReplace = $"#0x{asm.ReferencedAddress.Value:x}"; + output.Append(instruction.Operand.Replace(partToReplace, name)); + } + else + { + output.Append(instruction.Operand); + } + + return output.ToString(); + } + + private static void FormatInstructionPointer(Arm64Instruction instruction, FormatterOptions formatterOptions, uint pointerSize, StringBuilder output) + { + string ipFormat = formatterOptions.LeadingZeroes + ? pointerSize == 4 ? "X8" : "X16" + : "X"; + + output.Append(instruction.Address.ToString(ipFormat)); + output.Append(' '); + } + } +} diff --git a/src/BenchmarkDotNet/Disassemblers/ClrMdV2Disassembler.cs b/src/BenchmarkDotNet/Disassemblers/ClrMdV2Disassembler.cs index ab05710ad7..c2226c4082 100644 --- a/src/BenchmarkDotNet/Disassemblers/ClrMdV2Disassembler.cs +++ b/src/BenchmarkDotNet/Disassemblers/ClrMdV2Disassembler.cs @@ -1,5 +1,5 @@ -using BenchmarkDotNet.Filters; -using Iced.Intel; +using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Filters; using Microsoft.Diagnostics.Runtime; using Microsoft.Diagnostics.Runtime.Utilities; using System; @@ -11,9 +11,9 @@ namespace BenchmarkDotNet.Disassemblers { // This Disassembler uses ClrMd v2x. Please keep it in sync with ClrMdV1Disassembler (if possible). - internal static class ClrMdV2Disassembler + internal abstract class ClrMdV2Disassembler { - internal static DisassemblyResult AttachAndDisassemble(Settings settings) + internal DisassemblyResult AttachAndDisassemble(Settings settings) { using (var dataTarget = DataTarget.AttachToProcess( settings.ProcessId, @@ -23,7 +23,7 @@ internal static DisassemblyResult AttachAndDisassemble(Settings settings) ConfigureSymbols(dataTarget); - var state = new State(runtime); + var state = new State(runtime, settings.TargetFrameworkMoniker); if (settings.Filters.Length > 0) { @@ -93,9 +93,10 @@ private static void FilterAndEnqueue(State state, Settings settings) } } - private static DisassembledMethod[] Disassemble(Settings settings, State state) + private DisassembledMethod[] Disassemble(Settings settings, State state) { var result = new List(); + DisassemblySyntax syntax = (DisassemblySyntax)Enum.Parse(typeof(DisassemblySyntax), settings.Syntax); while (state.Todo.Count != 0) { @@ -105,7 +106,7 @@ private static DisassembledMethod[] Disassemble(Settings settings, State state) continue; // already handled if (settings.MaxDepth >= methodInfo.Depth) - result.Add(DisassembleMethod(methodInfo, state, settings)); + result.Add(DisassembleMethod(methodInfo, state, settings, syntax)); } return result.ToArray(); @@ -113,7 +114,7 @@ private static DisassembledMethod[] Disassemble(Settings settings, State state) private static bool CanBeDisassembled(ClrMethod method) => method.ILOffsetMap.Length > 0 && method.NativeCode > 0; - private static DisassembledMethod DisassembleMethod(MethodInfo methodInfo, State state, Settings settings) + private DisassembledMethod DisassembleMethod(MethodInfo methodInfo, State state, Settings settings, DisassemblySyntax syntax) { var method = methodInfo.Method; @@ -144,7 +145,7 @@ private static DisassembledMethod DisassembleMethod(MethodInfo methodInfo, State foreach (var map in GetCompleteNativeMap(method, state.Runtime)) { - codes.AddRange(Decode(map, state, methodInfo.Depth, method)); + codes.AddRange(Decode(map, state, methodInfo.Depth, method, syntax)); } Map[] maps = settings.PrintSource @@ -159,7 +160,7 @@ private static DisassembledMethod DisassembleMethod(MethodInfo methodInfo, State }; } - private static IEnumerable Decode(ILToNativeMap map, State state, int depth, ClrMethod currentMethod) + private IEnumerable Decode(ILToNativeMap map, State state, int depth, ClrMethod currentMethod, DisassemblySyntax syntax) { ulong startAddress = map.StartAddress; uint size = (uint)(map.EndAddress - map.StartAddress); @@ -177,30 +178,70 @@ private static IEnumerable Decode(ILToNativeMap map, State state, int depth totalBytesRead += bytesRead; } while (totalBytesRead != size); - var reader = new ByteArrayCodeReader(code, 0, (int)size); - var decoder = Decoder.Create(state.Runtime.DataTarget.DataReader.PointerSize * 8, reader); - decoder.IP = startAddress; + return Decode(code, startAddress, state, depth, currentMethod, syntax); + } - while (reader.CanReadByte) - { - decoder.Decode(out var instruction); + protected abstract IEnumerable Decode(byte[] code, ulong startAddress, State state, int depth, ClrMethod currentMethod, DisassemblySyntax syntax); - TryTranslateAddressToName(instruction, state, depth, currentMethod); + private static ILToNativeMap[] GetCompleteNativeMap(ClrMethod method, ClrRuntime runtime) + { + if (!TryReadNativeCodeAddresses(runtime, method, out ulong startAddress, out ulong endAddress)) + { + startAddress = method.NativeCode; + endAddress = ulong.MaxValue; + } - yield return new Asm + ILToNativeMap[] sortedMaps = method.ILOffsetMap // CanBeDisassembled ensures that there is at least one map in ILOffsetMap + .Where(map => map.StartAddress >= startAddress && map.StartAddress < endAddress) // can be false for Tier 0 maps, EndAddress is not checked on purpose here + .Where(map => map.StartAddress < map.EndAddress) // some maps have 0 length (they don't have corresponding assembly code?) + .OrderBy(map => map.StartAddress) // we need to print in the machine code order, not IL! #536 + .Select(map => new ILToNativeMap() { - InstructionPointer = instruction.IP, - Instruction = instruction - }; + StartAddress = map.StartAddress, + // some maps have EndAddress > codeHeaderData.MethodStart + codeHeaderData.MethodSize and contain garbage (#2074). They need to be fixed! + EndAddress = Math.Min(map.EndAddress, endAddress), + ILOffset = map.ILOffset + }) + .ToArray(); + + if (sortedMaps.Length == 0) + { + // In such situation ILOffsetMap most likely describes Tier 0, while CodeHeaderData Tier 1. + // Since we care about Tier 1 (if it's present), we "fake" a Tier 1 map. + return new[] { new ILToNativeMap() { StartAddress = startAddress, EndAddress = endAddress } }; } + else if (sortedMaps[0].StartAddress != startAddress || (sortedMaps[sortedMaps.Length - 1].EndAddress != endAddress && endAddress != ulong.MaxValue)) + { + // In such situation ILOffsetMap most likely is missing few bytes. We just "extend" it to avoid producing "bad" instructions. + return new[] { new ILToNativeMap() { StartAddress = startAddress, EndAddress = endAddress } }; + } + + return sortedMaps; } - private static void TryTranslateAddressToName(Instruction instruction, State state, int depth, ClrMethod currentMethod) + private static DisassembledMethod CreateEmpty(ClrMethod method, string reason) + => DisassembledMethod.Empty(method.Signature, method.NativeCode, reason); + + protected static bool TryReadNativeCodeAddresses(ClrRuntime runtime, ClrMethod method, out ulong startAddress, out ulong endAddress) { - var runtime = state.Runtime; + if (method is not null + && runtime.DacLibrary.SOSDacInterface.GetCodeHeaderData(method.NativeCode, out var codeHeaderData) == HResult.S_OK + && codeHeaderData.MethodSize > 0) // false for extern methods! + { + // HotSize can be missing or be invalid (https://github.com/microsoft/clrmd/issues/1036). + // So we fetch the method size on our own. + startAddress = codeHeaderData.MethodStart; + endAddress = codeHeaderData.MethodStart + codeHeaderData.MethodSize; + return true; + } - if (!TryGetReferencedAddress(instruction, (uint)runtime.DataTarget.DataReader.PointerSize, out ulong address)) - return; + startAddress = endAddress = 0; + return false; + } + + protected void TryTranslateAddressToName(ulong address, bool isAddressPrecodeMD, State state, bool isIndirectCallOrJump, int depth, ClrMethod currentMethod) + { + var runtime = state.Runtime; if (state.AddressToNameMapping.ContainsKey(address)) return; @@ -212,20 +253,6 @@ private static void TryTranslateAddressToName(Instruction instruction, State sta return; } - var methodTableName = runtime.DacLibrary.SOSDacInterface.GetMethodTableName(address); - if (!string.IsNullOrEmpty(methodTableName)) - { - state.AddressToNameMapping.Add(address, $"MT_{methodTableName}"); - return; - } - - var methodDescriptor = runtime.GetMethodByHandle(address); - if (!(methodDescriptor is null)) - { - state.AddressToNameMapping.Add(address, $"MD_{methodDescriptor.Signature}"); - return; - } - var method = runtime.GetMethodByInstructionPointer(address); if (method is null && (address & ((uint)runtime.DataTarget.DataReader.PointerSize - 1)) == 0) { @@ -242,7 +269,35 @@ private static void TryTranslateAddressToName(Instruction instruction, State sta } if (method is null) + { + if (isAddressPrecodeMD || this is not Arm64Disassembler) + { + var methodDescriptor = runtime.GetMethodByHandle(address); + if (!(methodDescriptor is null)) + { + if (isAddressPrecodeMD) + { + state.AddressToNameMapping.Add(address, $"Precode of {methodDescriptor.Signature}"); + } + else + { + state.AddressToNameMapping.Add(address, $"MD_{methodDescriptor.Signature}"); + } + return; + } + } + + if (this is not Arm64Disassembler) + { + var methodTableName = runtime.DacLibrary.SOSDacInterface.GetMethodTableName(address); + if (!string.IsNullOrEmpty(methodTableName)) + { + state.AddressToNameMapping.Add(address, $"MT_{methodTableName}"); + return; + } + } return; + } if (method.NativeCode == currentMethod.NativeCode && method.Signature == currentMethod.Signature) return; // in case of a call which is just a jump within the method or a recursive call @@ -263,95 +318,6 @@ private static ClrMethod WorkaroundGetMethodByInstructionPointerBug(ClrRuntime r ? null : method; - internal static bool TryGetReferencedAddress(Instruction instruction, uint pointerSize, out ulong referencedAddress) - { - for (int i = 0; i < instruction.OpCount; i++) - { - switch (instruction.GetOpKind(i)) - { - case OpKind.NearBranch16: - case OpKind.NearBranch32: - case OpKind.NearBranch64: - referencedAddress = instruction.NearBranchTarget; - return referencedAddress > ushort.MaxValue; - case OpKind.Immediate16: - case OpKind.Immediate8to16: - case OpKind.Immediate8to32: - case OpKind.Immediate8to64: - case OpKind.Immediate32to64: - case OpKind.Immediate32 when pointerSize == 4: - case OpKind.Immediate64: - referencedAddress = instruction.GetImmediate(i); - return referencedAddress > ushort.MaxValue; - case OpKind.Memory when instruction.IsIPRelativeMemoryOperand: - referencedAddress = instruction.IPRelativeMemoryAddress; - return referencedAddress > ushort.MaxValue; - case OpKind.Memory: - referencedAddress = instruction.MemoryDisplacement64; - return referencedAddress > ushort.MaxValue; - } - } - - referencedAddress = default; - return false; - } - - private static ILToNativeMap[] GetCompleteNativeMap(ClrMethod method, ClrRuntime runtime) - { - if (!TryReadNativeCodeAddresses(runtime, method, out ulong startAddress, out ulong endAddress)) - { - startAddress = method.NativeCode; - endAddress = ulong.MaxValue; - } - - ILToNativeMap[] sortedMaps = method.ILOffsetMap // CanBeDisassembled ensures that there is at least one map in ILOffsetMap - .Where(map => map.StartAddress >= startAddress && map.StartAddress < endAddress) // can be false for Tier 0 maps, EndAddress is not checked on purpose here - .Where(map => map.StartAddress < map.EndAddress) // some maps have 0 length (they don't have corresponding assembly code?) - .OrderBy(map => map.StartAddress) // we need to print in the machine code order, not IL! #536 - .Select(map => new ILToNativeMap() - { - StartAddress = map.StartAddress, - // some maps have EndAddress > codeHeaderData.MethodStart + codeHeaderData.MethodSize and contain garbage (#2074). They need to be fixed! - EndAddress = Math.Min(map.EndAddress, endAddress), - ILOffset = map.ILOffset - }) - .ToArray(); - - if (sortedMaps.Length == 0) - { - // In such situation ILOffsetMap most likely describes Tier 0, while CodeHeaderData Tier 1. - // Since we care about Tier 1 (if it's present), we "fake" a Tier 1 map. - return new[] { new ILToNativeMap() { StartAddress = startAddress, EndAddress = endAddress }}; - } - else if (sortedMaps[0].StartAddress != startAddress || (sortedMaps[sortedMaps.Length - 1].EndAddress != endAddress && endAddress != ulong.MaxValue)) - { - // In such situation ILOffsetMap most likely is missing few bytes. We just "extend" it to avoid producing "bad" instructions. - return new[] { new ILToNativeMap() { StartAddress = startAddress, EndAddress = endAddress } }; - } - - return sortedMaps; - } - - private static bool TryReadNativeCodeAddresses(ClrRuntime runtime, ClrMethod method, out ulong startAddress, out ulong endAddress) - { - if (method is not null - && runtime.DacLibrary.SOSDacInterface.GetCodeHeaderData(method.NativeCode, out var codeHeaderData) == HResult.S_OK - && codeHeaderData.MethodSize > 0) // false for extern methods! - { - // HotSize can be missing or be invalid (https://github.com/microsoft/clrmd/issues/1036). - // So we fetch the method size on our own. - startAddress = codeHeaderData.MethodStart; - endAddress = codeHeaderData.MethodStart + codeHeaderData.MethodSize; - return true; - } - - startAddress = endAddress = 0; - return false; - } - - private static DisassembledMethod CreateEmpty(ClrMethod method, string reason) - => DisassembledMethod.Empty(method.Signature, method.NativeCode, reason); - private class SharpComparer : IEqualityComparer { public bool Equals(Sharp x, Sharp y) diff --git a/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs b/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs index 376788ffae..86c7c5bdbd 100644 --- a/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs +++ b/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs @@ -92,9 +92,9 @@ public void DisplayResults(ILogger logger) public IEnumerable Validate(ValidationParameters validationParameters) { var currentPlatform = RuntimeInformation.GetCurrentPlatform(); - if (currentPlatform != Platform.X64 && currentPlatform != Platform.X86) + if (!(currentPlatform is Platform.X64 or Platform.X86 or Platform.Arm64)) { - yield return new ValidationError(true, $"{currentPlatform} is not supported (Iced library limitation)"); + yield return new ValidationError(true, $"{currentPlatform} is not supported"); yield break; } @@ -176,7 +176,7 @@ private static IEnumerable GetExporters(Dictionary disassembly.Methods.Sum(method => method.Maps.Sum(map => map.SourceCodes.OfType().Sum(asm => asm.Instruction.Length))); + => disassembly.Methods.Sum(method => method.Maps.Sum(map => map.SourceCodes.OfType().Sum(asm => asm.InstructionLength))); private class NativeCodeSizeMetricDescriptor : IMetricDescriptor { diff --git a/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoserConfig.cs b/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoserConfig.cs index f49cefa0e3..7c917c80b0 100644 --- a/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoserConfig.cs +++ b/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoserConfig.cs @@ -10,7 +10,8 @@ public class DisassemblyDiagnoserConfig { /// Includes called methods to given level. 1 by default, indexed from 1. To print just the benchmark set it to 0. /// Glob patterns applied to full method signatures by the the disassembler. - /// Assembly code formatter. If not provided, MasmFormatter with the recommended settings will be used. + /// The disassembly syntax. MASM is the default. + /// Code formatter options. If not provided, the recommended settings will be used. /// C#|F#|VB source code will be printed. False by default. /// Print instruction addresses. False by default /// Exports to GitHub markdown. True by default. @@ -20,8 +21,9 @@ public class DisassemblyDiagnoserConfig [PublicAPI] public DisassemblyDiagnoserConfig( int maxDepth = 1, + DisassemblySyntax syntax = DisassemblySyntax.Masm, string[] filters = null, - Formatter formatter = null, + FormatterOptions formatterOptions = null, bool printSource = false, bool printInstructionAddresses = false, bool exportGithubMarkdown = true, @@ -29,9 +31,15 @@ public DisassemblyDiagnoserConfig( bool exportCombinedDisassemblyReport = false, bool exportDiff = false) { + if (!(syntax is DisassemblySyntax.Masm or DisassemblySyntax.Intel or DisassemblySyntax.Att)) + { + throw new ArgumentOutOfRangeException(nameof(syntax), syntax, "Invalid syntax"); + } + MaxDepth = maxDepth; Filters = filters ?? Array.Empty(); - Formatter = formatter ?? CreateDefaultFormatter(); + Syntax = syntax; + Formatting = formatterOptions ?? GetDefaults(syntax); PrintSource = printSource; PrintInstructionAddresses = printInstructionAddresses; ExportGithubMarkdown = exportGithubMarkdown; @@ -44,44 +52,36 @@ public DisassemblyDiagnoserConfig( public bool PrintInstructionAddresses { get; } public int MaxDepth { get; } public string[] Filters { get; } + public DisassemblySyntax Syntax { get; } + public FormatterOptions Formatting { get; } public bool ExportGithubMarkdown { get; } public bool ExportHtml { get; } public bool ExportCombinedDisassemblyReport { get; } public bool ExportDiff { get; } - // it's private to make sure that GetFormatterWithSymbolSolver is always used - private Formatter Formatter { get; } - - private static Formatter CreateDefaultFormatter() - { - var formatter = new MasmFormatter(); - formatter.Options.FirstOperandCharIndex = 10; // pad right the mnemonic - formatter.Options.HexSuffix = default; // don't print "h" at the end of every hex address - formatter.Options.TabSize = 0; // use spaces - return formatter; - } - // user can specify a formatter without symbol solver // so we need to clone the formatter with settings and provide our symbols solver internal Formatter GetFormatterWithSymbolSolver(IReadOnlyDictionary addressToNameMapping) - { - var symbolSolver = new SymbolResolver(addressToNameMapping); + => Syntax switch + { + DisassemblySyntax.Att => new GasFormatter(Formatting, new SymbolResolver(addressToNameMapping)), + DisassemblySyntax.Intel => new IntelFormatter(Formatting, new SymbolResolver(addressToNameMapping)), + _ => new MasmFormatter(Formatting, new SymbolResolver(addressToNameMapping)), + }; - switch (Formatter) + private static FormatterOptions GetDefaults(DisassemblySyntax syntax) + { + FormatterOptions options = syntax switch { - case MasmFormatter masmFormatter: - return new MasmFormatter(masmFormatter.Options, symbolSolver); - case NasmFormatter nasmFormatter: - return new NasmFormatter(nasmFormatter.Options, symbolSolver); - case GasFormatter gasFormatter: - return new GasFormatter(gasFormatter.Options, symbolSolver); - case IntelFormatter intelFormatter: - return new IntelFormatter(intelFormatter.Options, symbolSolver); - default: - // we don't know how to translate it so we just return the original one - // it's better not to solve symbols rather than throw exception ;) - return Formatter; - } + DisassemblySyntax.Att => FormatterOptions.CreateGas(), + DisassemblySyntax.Intel => FormatterOptions.CreateIntel(), + _ => FormatterOptions.CreateMasm(), + }; + + options.FirstOperandCharIndex = 10; // pad right the mnemonic + options.HexSuffix = default; // don't print "h" at the end of every hex address + options.TabSize = 0; // use spaces + return options; } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Disassemblers/DisassemblySyntax.cs b/src/BenchmarkDotNet/Disassemblers/DisassemblySyntax.cs new file mode 100644 index 0000000000..594cf18c31 --- /dev/null +++ b/src/BenchmarkDotNet/Disassemblers/DisassemblySyntax.cs @@ -0,0 +1,18 @@ +namespace BenchmarkDotNet.Diagnosers +{ + public enum DisassemblySyntax + { + /// + /// Indicates a disassembler should use MASM syntax for generated assembly code + /// + Masm, + /// + /// Indicates a disassembler should use Intel syntax for generated assembly code. + /// + Intel, + /// + /// Indicates a disassembler should use AT&T syntax for generated assembly code. + /// + Att + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Disassemblers/Exporters/CombinedDisassemblyExporter.cs b/src/BenchmarkDotNet/Disassemblers/Exporters/CombinedDisassemblyExporter.cs index 4cad71c1b2..51057cd443 100644 --- a/src/BenchmarkDotNet/Disassemblers/Exporters/CombinedDisassemblyExporter.cs +++ b/src/BenchmarkDotNet/Disassemblers/Exporters/CombinedDisassemblyExporter.cs @@ -99,7 +99,7 @@ private void PrintTable(BenchmarkCase[] benchmarksCase, ILogger logger, string t foreach (var map in method.Maps) foreach (var sourceCode in map.SourceCodes) - logger.WriteLine(CodeFormatter.Format(sourceCode, formatter, config.PrintInstructionAddresses, disassemblyResult.PointerSize)); + logger.WriteLine(CodeFormatter.Format(sourceCode, formatter, config.PrintInstructionAddresses, disassemblyResult.PointerSize, disassemblyResult.AddressToNameMapping)); logger.WriteLine(); } diff --git a/src/BenchmarkDotNet/Disassemblers/Exporters/DisassemblyPrettifier.cs b/src/BenchmarkDotNet/Disassemblers/Exporters/DisassemblyPrettifier.cs index f9aaf47ef9..57b8a04026 100644 --- a/src/BenchmarkDotNet/Disassemblers/Exporters/DisassemblyPrettifier.cs +++ b/src/BenchmarkDotNet/Disassemblers/Exporters/DisassemblyPrettifier.cs @@ -39,8 +39,10 @@ internal static IReadOnlyList Prettify(DisassembledMethod method, Disas // first of all, we search of referenced addresses (jump|calls) var referencedAddresses = new HashSet(); foreach (var asm in asmInstructions) - if (ClrMdV2Disassembler.TryGetReferencedAddress(asm.Instruction, disassemblyResult.PointerSize, out ulong referencedAddress)) - referencedAddresses.Add(referencedAddress); + if (asm.ReferencedAddress != null) + { + referencedAddresses.Add(asm.ReferencedAddress.Value); + } // for every IP that is referenced, we emit a uinque label var addressesToLabels = new Dictionary(); @@ -72,24 +74,30 @@ internal static IReadOnlyList Prettify(DisassembledMethod method, Disas prettified.Add(new Label(label)); } - if (ClrMdV2Disassembler.TryGetReferencedAddress(asm.Instruction, disassemblyResult.PointerSize, out ulong referencedAddress)) + if (asm.ReferencedAddress != null) { + ulong referencedAddress = asm.ReferencedAddress.Value; // jump or a call within same method if (addressesToLabels.TryGetValue(referencedAddress, out string translated)) { - prettified.Add(new Reference(InstructionFormatter.Format(asm.Instruction, formatterWithLabelsSymbols, config.PrintInstructionAddresses, disassemblyResult.PointerSize), translated, asm)); + prettified.Add(new Reference(CodeFormatter.Format(asm, formatterWithLabelsSymbols, config.PrintInstructionAddresses, disassemblyResult.PointerSize, addressesToLabels), translated, asm)); continue; } // call to a known method if (disassemblyResult.AddressToNameMapping.ContainsKey(referencedAddress)) { - prettified.Add(new Element(InstructionFormatter.Format(asm.Instruction, formatterWithGlobalSymbols, config.PrintInstructionAddresses, disassemblyResult.PointerSize), asm)); + string comment = string.Empty; + if (asm.IsReferencedAddressIndirect) + { + comment = "; " + disassemblyResult.AddressToNameMapping[referencedAddress]; + } + prettified.Add(new Element(CodeFormatter.Format(asm, formatterWithGlobalSymbols, config.PrintInstructionAddresses, disassemblyResult.PointerSize, disassemblyResult.AddressToNameMapping) + comment, asm)); continue; } } - prettified.Add(new Element(InstructionFormatter.Format(asm.Instruction, formatterWithGlobalSymbols, config.PrintInstructionAddresses, disassemblyResult.PointerSize), asm)); + prettified.Add(new Element(CodeFormatter.Format(asm, formatterWithGlobalSymbols, config.PrintInstructionAddresses, disassemblyResult.PointerSize, disassemblyResult.AddressToNameMapping), asm)); } } diff --git a/src/BenchmarkDotNet/Disassemblers/Exporters/GithubMarkdownDisassemblyExporter.cs b/src/BenchmarkDotNet/Disassemblers/Exporters/GithubMarkdownDisassemblyExporter.cs index 398b82951e..4ebdc708a6 100644 --- a/src/BenchmarkDotNet/Disassemblers/Exporters/GithubMarkdownDisassemblyExporter.cs +++ b/src/BenchmarkDotNet/Disassemblers/Exporters/GithubMarkdownDisassemblyExporter.cs @@ -61,7 +61,7 @@ internal static void Export(ILogger logger, DisassemblyResult disassemblyResult, { checked { - totalSizeInBytes += (uint)asm.Instruction.Length; + totalSizeInBytes += (uint)asm.InstructionLength; } logger.WriteLine($" {element.TextRepresentation}"); diff --git a/src/BenchmarkDotNet/Disassemblers/InstructionFormatter.cs b/src/BenchmarkDotNet/Disassemblers/InstructionFormatter.cs index 19d497d10c..0b9a3fd306 100644 --- a/src/BenchmarkDotNet/Disassemblers/InstructionFormatter.cs +++ b/src/BenchmarkDotNet/Disassemblers/InstructionFormatter.cs @@ -1,50 +1,19 @@ using Iced.Intel; using System; +using System.Collections.Generic; namespace BenchmarkDotNet.Disassemblers { internal static class CodeFormatter { - internal static string Format(SourceCode sourceCode, Formatter formatter, bool printInstructionAddresses, uint pointerSize) - { - switch (sourceCode) + internal static string Format(SourceCode sourceCode, Formatter formatter, bool printInstructionAddresses, uint pointerSize, IReadOnlyDictionary symbols) + => sourceCode switch { - case Asm asm: - return InstructionFormatter.Format(asm.Instruction, formatter, printInstructionAddresses, pointerSize); - case Sharp sharp: - return sharp.Text; - case MonoCode mono: - return mono.Text; - default: - throw new NotSupportedException(); - } - } - } - - internal static class InstructionFormatter - { - internal static string Format(Instruction instruction, Formatter formatter, bool printInstructionAddresses, uint pointerSize) - { - var output = new StringOutput(); - - if (printInstructionAddresses) - { - FormatInstructionPointer(instruction, formatter, pointerSize, output); - } - - formatter.Format(instruction, output); - - return output.ToString(); - } - - private static void FormatInstructionPointer(Instruction instruction, Formatter formatter, uint pointerSize, StringOutput output) - { - string ipFormat = formatter.Options.LeadingZeroes - ? pointerSize == 4 ? "X8" : "X16" - : "X"; - - output.Write(instruction.IP.ToString(ipFormat), FormatterTextKind.Text); - output.Write(" ", FormatterTextKind.Text); - } + Asm asm when asm.IntelInstruction.HasValue => IntelInstructionFormatter.Format(asm.IntelInstruction.Value, formatter, printInstructionAddresses, pointerSize), + Asm asm when asm.Arm64Instruction is not null => Arm64InstructionFormatter.Format(asm, formatter.Options, printInstructionAddresses, pointerSize, symbols), + Sharp sharp => sharp.Text, + MonoCode mono => mono.Text, + _ => throw new NotSupportedException(), + }; } } diff --git a/src/BenchmarkDotNet/Disassemblers/IntelDisassembler.cs b/src/BenchmarkDotNet/Disassemblers/IntelDisassembler.cs new file mode 100644 index 0000000000..37ab4652d5 --- /dev/null +++ b/src/BenchmarkDotNet/Disassemblers/IntelDisassembler.cs @@ -0,0 +1,133 @@ +using BenchmarkDotNet.Diagnosers; +using Iced.Intel; +using Microsoft.Diagnostics.Runtime; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace BenchmarkDotNet.Disassemblers +{ + internal class IntelDisassembler : ClrMdV2Disassembler + { + // See dotnet/runtime src/coreclr/vm/amd64/thunktemplates.asm/.S for the stub code + // mov rax,QWORD PTR [rip + DATA_SLOT(CallCountingStub, RemainingCallCountCell)] + // dec WORD PTR [rax] + // je LOCAL_LABEL(CountReachedZero) + // jmp QWORD PTR [rip + DATA_SLOT(CallCountingStub, TargetForMethod)] + // LOCAL_LABEL(CountReachedZero): + // jmp QWORD PTR [rip + DATA_SLOT(CallCountingStub, TargetForThresholdReached)] + private static byte[] callCountingStubTemplate = new byte[10] { 0x48, 0x8b, 0x05, 0xf9, 0x0f, 0x00, 0x00, 0x66, 0xff, 0x08 }; + // mov r10, [rip + DATA_SLOT(StubPrecode, MethodDesc)] + // jmp [rip + DATA_SLOT(StubPrecode, Target)] + private static byte[] stubPrecodeTemplate = new byte[13] { 0x4c, 0x8b, 0x15, 0xf9, 0x0f, 0x00, 0x00, 0xff, 0x25, 0xfb, 0x0f, 0x00, 0x00 }; + // jmp [rip + DATA_SLOT(FixupPrecode, Target)] + // mov r10, [rip + DATA_SLOT(FixupPrecode, MethodDesc)] + // jmp [rip + DATA_SLOT(FixupPrecode, PrecodeFixupThunk)] + private static byte[] fixupPrecodeTemplate = new byte[19] { 0xff, 0x25, 0xfa, 0x0f, 0x00, 0x00, 0x4c, 0x8b, 0x15, 0xfb, 0x0f, 0x00, 0x00, 0xff, 0x25, 0xfd, 0x0f, 0x00, 0x00 }; + + protected override IEnumerable Decode(byte[] code, ulong startAddress, State state, int depth, ClrMethod currentMethod, DisassemblySyntax syntax) + { + var reader = new ByteArrayCodeReader(code); + var decoder = Decoder.Create(state.Runtime.DataTarget.DataReader.PointerSize * 8, reader); + decoder.IP = startAddress; + + while (reader.CanReadByte) + { + decoder.Decode(out var instruction); + + bool isIndirect = instruction.IsCallFarIndirect || instruction.IsCallNearIndirect || instruction.IsJmpFarIndirect || instruction.IsJmpNearIndirect; + bool isPrestubMD = false; + + ulong address = 0; + if (TryGetReferencedAddress(instruction, (uint)state.Runtime.DataTarget.DataReader.PointerSize, out address)) + { + if (isIndirect) + { + address = state.Runtime.DataTarget.DataReader.ReadPointer(address); + if (state.RuntimeVersion.Major >= 7) + { + // Check if the target is a known stub + // The stubs are allocated in interleaved code / data pages in memory. The data part of the stub + // is at an address one memory page higher than the code. + byte[] buffer = new byte[10]; + + if (state.Runtime.DataTarget.DataReader.Read(address, buffer) == buffer.Length && buffer.SequenceEqual(callCountingStubTemplate)) + { + const ulong TargetMethodAddressSlotOffset = 8; + address = state.Runtime.DataTarget.DataReader.ReadPointer(address + (ulong)Environment.SystemPageSize + TargetMethodAddressSlotOffset); + } + else + { + buffer = new byte[13]; + if (state.Runtime.DataTarget.DataReader.Read(address, buffer) == buffer.Length && buffer.SequenceEqual(stubPrecodeTemplate)) + { + const ulong MethodDescSlotOffset = 0; + address = state.Runtime.DataTarget.DataReader.ReadPointer(address + (ulong)Environment.SystemPageSize + MethodDescSlotOffset); + isPrestubMD = true; + } + else + { + buffer = new byte[19]; + if (state.Runtime.DataTarget.DataReader.Read(address, buffer) == buffer.Length && buffer.SequenceEqual(fixupPrecodeTemplate)) + { + const ulong MethodDescSlotOffset = 8; + address = state.Runtime.DataTarget.DataReader.ReadPointer(address + (ulong)Environment.SystemPageSize + MethodDescSlotOffset); + isPrestubMD = true; + } + + } + } + } + } + + if (address > ushort.MaxValue) + { + TryTranslateAddressToName(address, isPrestubMD, state, isIndirect, depth, currentMethod); + } + } + + yield return new Asm + { + InstructionPointer = instruction.IP, + InstructionLength = instruction.Length, + IntelInstruction = instruction, + ReferencedAddress = (address > ushort.MaxValue) ? address : null, + IsReferencedAddressIndirect = isIndirect, + }; + } + } + + private static bool TryGetReferencedAddress(Instruction instruction, uint pointerSize, out ulong referencedAddress) + { + for (int i = 0; i < instruction.OpCount; i++) + { + switch (instruction.GetOpKind(i)) + { + case OpKind.NearBranch16: + case OpKind.NearBranch32: + case OpKind.NearBranch64: + referencedAddress = instruction.NearBranchTarget; + return referencedAddress > ushort.MaxValue; + case OpKind.Immediate16: + case OpKind.Immediate8to16: + case OpKind.Immediate8to32: + case OpKind.Immediate8to64: + case OpKind.Immediate32to64: + case OpKind.Immediate32 when pointerSize == 4: + case OpKind.Immediate64: + referencedAddress = instruction.GetImmediate(i); + return referencedAddress > ushort.MaxValue; + case OpKind.Memory when instruction.IsIPRelativeMemoryOperand: + referencedAddress = instruction.IPRelativeMemoryAddress; + return referencedAddress > ushort.MaxValue; + case OpKind.Memory: + referencedAddress = instruction.MemoryDisplacement64; + return referencedAddress > ushort.MaxValue; + } + } + + referencedAddress = default; + return false; + } + } +} diff --git a/src/BenchmarkDotNet/Disassemblers/IntelInstructionFormatter.cs b/src/BenchmarkDotNet/Disassemblers/IntelInstructionFormatter.cs new file mode 100644 index 0000000000..b2b0269080 --- /dev/null +++ b/src/BenchmarkDotNet/Disassemblers/IntelInstructionFormatter.cs @@ -0,0 +1,31 @@ +using Iced.Intel; + +namespace BenchmarkDotNet.Disassemblers +{ + internal static class IntelInstructionFormatter + { + internal static string Format(Instruction instruction, Formatter formatter, bool printInstructionAddresses, uint pointerSize) + { + var output = new StringOutput(); + + if (printInstructionAddresses) + { + FormatInstructionPointer(instruction, formatter, pointerSize, output); + } + + formatter.Format(instruction, output); + + return output.ToString(); + } + + private static void FormatInstructionPointer(Instruction instruction, Formatter formatter, uint pointerSize, StringOutput output) + { + string ipFormat = formatter.Options.LeadingZeroes + ? pointerSize == 4 ? "X8" : "X16" + : "X"; + + output.Write(instruction.IP.ToString(ipFormat), FormatterTextKind.Text); + output.Write(" ", FormatterTextKind.Text); + } + } +} diff --git a/src/BenchmarkDotNet/Disassemblers/SameArchitectureDisassembler.cs b/src/BenchmarkDotNet/Disassemblers/SameArchitectureDisassembler.cs index 494632bdc4..623b9e3e67 100644 --- a/src/BenchmarkDotNet/Disassemblers/SameArchitectureDisassembler.cs +++ b/src/BenchmarkDotNet/Disassemblers/SameArchitectureDisassembler.cs @@ -1,24 +1,43 @@ using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Portability; +using BenchmarkDotNet.Toolchains; +using System; namespace BenchmarkDotNet.Disassemblers { internal class SameArchitectureDisassembler { private readonly DisassemblyDiagnoserConfig config; + private readonly ClrMdV2Disassembler clrMdV2Disassembler; - internal SameArchitectureDisassembler(DisassemblyDiagnoserConfig config) => this.config = config; + internal SameArchitectureDisassembler(DisassemblyDiagnoserConfig config) + { + this.config = config; + clrMdV2Disassembler = CreateDisassemblerForCurrentArchitecture(); + } internal DisassemblyResult Disassemble(DiagnoserActionParameters parameters) - => ClrMdV2Disassembler.AttachAndDisassemble(BuildDisassemblerSettings(parameters)); + => clrMdV2Disassembler.AttachAndDisassemble(BuildDisassemblerSettings(parameters)); + + private static ClrMdV2Disassembler CreateDisassemblerForCurrentArchitecture() + => RuntimeInformation.GetCurrentPlatform() switch + { + Platform.X86 or Platform.X64 => new IntelDisassembler(), + Platform.Arm64 => new Arm64Disassembler(), + _ => throw new NotSupportedException($"{RuntimeInformation.GetCurrentPlatform()} is not supported") + }; private Settings BuildDisassemblerSettings(DiagnoserActionParameters parameters) - => new Settings( + => new ( processId: parameters.Process.Id, typeName: $"BenchmarkDotNet.Autogenerated.Runnable_{parameters.BenchmarkId.Value}", methodName: DisassemblerConstants.DisassemblerEntryMethodName, printSource: config.PrintSource, maxDepth: config.MaxDepth, filters: config.Filters, + syntax: config.Syntax.ToString(), + tfm: parameters.BenchmarkCase.Job.Environment.GetRuntime().MsBuildMoniker, resultsPath: default ); } diff --git a/src/BenchmarkDotNet/Disassemblers/WindowsDisassembler.cs b/src/BenchmarkDotNet/Disassemblers/WindowsDisassembler.cs index c1d6f91882..5ed8c3b534 100644 --- a/src/BenchmarkDotNet/Disassemblers/WindowsDisassembler.cs +++ b/src/BenchmarkDotNet/Disassemblers/WindowsDisassembler.cs @@ -64,7 +64,8 @@ public DisassemblyResult Disassemble(DiagnoserActionParameters parameters) internal static Platform GetDisassemblerArchitecture(Process process, Platform platform) => platform switch { - Platform.AnyCpu => NativeMethods.Is64Bit(process) ? Platform.X64 : Platform.X86, // currently ARM is not supported + Platform.AnyCpu when System.Runtime.InteropServices.RuntimeInformation.OSArchitecture is Architecture.Arm or Architecture.Arm64 => RuntimeInformation.GetCurrentPlatform(), + Platform.AnyCpu => NativeMethods.Is64Bit(process) ? Platform.X64 : Platform.X86, _ => platform }; @@ -143,6 +144,8 @@ private string BuildArguments(DiagnoserActionParameters parameters, string resul .Append(config.MaxDepth).Append(' ') .Append(Escape(resultsPath)) .Append(' ') + .Append(config.Syntax.ToString()) + .Append(' ') .Append(string.Join(" ", config.Filters.Select(Escape))) .ToString(); diff --git a/src/BenchmarkDotNet/Exporters/InstructionPointerExporter.cs b/src/BenchmarkDotNet/Exporters/InstructionPointerExporter.cs index ec20babee5..9bd783b88f 100644 --- a/src/BenchmarkDotNet/Exporters/InstructionPointerExporter.cs +++ b/src/BenchmarkDotNet/Exporters/InstructionPointerExporter.cs @@ -80,7 +80,7 @@ private string Export(Summary summary, BenchmarkCase benchmarkCase, DisassemblyR IEnumerable Range(Asm asm) { // most probably asm.StartAddress would be enough, but I don't want to miss any edge case - for (ulong instructionPointer = asm.InstructionPointer; instructionPointer < asm.InstructionPointer + (ulong)asm.Instruction.Length; instructionPointer++) + for (ulong instructionPointer = asm.InstructionPointer; instructionPointer < asm.InstructionPointer + (ulong)asm.InstructionLength; instructionPointer++) yield return instructionPointer; } @@ -130,7 +130,7 @@ private static IReadOnlyList SumHardwareCountersPerMethod(Di foreach (var hardwareCounter in pmcStats.Counters) { // most probably asm.StartAddress would be enough, but I don't want to miss any edge case - for (ulong instructionPointer = asm.InstructionPointer; instructionPointer < asm.InstructionPointer + (ulong)asm.Instruction.Length; instructionPointer++) + for (ulong instructionPointer = asm.InstructionPointer; instructionPointer < asm.InstructionPointer + (ulong)asm.InstructionLength; instructionPointer++) if (hardwareCounter.Value.PerInstructionPointer.TryGetValue(instructionPointer, out ulong value)) totalsPerCounter[hardwareCounter.Key] = totalsPerCounter[hardwareCounter.Key] + value; } @@ -225,7 +225,7 @@ private void Export(ILogger logger, BenchmarkCase benchmarkCase, Dictionary"); - string formatted = CodeFormatter.Format(instruction.Code, formatterWithSymbols, config.PrintInstructionAddresses, disassemblyResult.PointerSize); + string formatted = CodeFormatter.Format(instruction.Code, formatterWithSymbols, config.PrintInstructionAddresses, disassemblyResult.PointerSize, disassemblyResult.AddressToNameMapping); logger.WriteLine($"
{formatted}
"); } diff --git a/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs index 620c1e75d4..475c462764 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs @@ -143,7 +143,7 @@ public void CanDisassembleInlinableBenchmarks(Jit jit, Platform platform, Runtim var disassemblyResult = disassemblyDiagnoser.Results.Values.Single(result => result.Methods.Count(method => method.Name.Contains(nameof(WithInlineable.JustReturn))) == 1); - Assert.Contains(disassemblyResult.Methods, method => method.Maps.Any(map => map.SourceCodes.OfType().All(asm => asm.Instruction.ToString().Contains("ret")))); + Assert.Contains(disassemblyResult.Methods, method => method.Maps.Any(map => map.SourceCodes.OfType().All(asm => asm.IntelInstruction.ToString().Contains("ret")))); } private IConfig CreateConfig(Jit jit, Platform platform, Runtime runtime, IDiagnoser disassemblyDiagnoser, RunStrategy runStrategy) diff --git a/tests/BenchmarkDotNet.Tests/TargetFrameworkVersionParsingTestscs.cs b/tests/BenchmarkDotNet.Tests/TargetFrameworkVersionParsingTestscs.cs new file mode 100644 index 0000000000..88f0fc8447 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/TargetFrameworkVersionParsingTestscs.cs @@ -0,0 +1,23 @@ +using BenchmarkDotNet.Disassemblers; +using System; +using Xunit; + +namespace BenchmarkDotNet.Tests +{ + public class TargetFrameworkVersionParsingTestscs + { + [Theory] + [InlineData("net461", 4, 6, 1)] + [InlineData("net48", 4, 8, -1)] + [InlineData("net7.0", 7, 0, -1)] + [InlineData("net7.0-windows8", 7, 0, -1)] + public void RuntimeVersionCanBeParsedFromTfm(string tfm, int major, int minor, int build) + { + Version version = State.ParseVersion(tfm); + + Assert.Equal(major, version.Major); + Assert.Equal(minor, version.Minor); + Assert.Equal(build, version.Build); + } + } +} From 97c2d14f548823206cb26db5d8316fe475ae4e52 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Wed, 5 Oct 2022 19:03:31 +0200 Subject: [PATCH 11/58] ensure access to logger is synchronized for async output reader (#2133) fixes #2125 --- .../Loggers/AsyncProcessOutputReader.cs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/BenchmarkDotNet/Loggers/AsyncProcessOutputReader.cs b/src/BenchmarkDotNet/Loggers/AsyncProcessOutputReader.cs index 480b7513cc..958d11138b 100644 --- a/src/BenchmarkDotNet/Loggers/AsyncProcessOutputReader.cs +++ b/src/BenchmarkDotNet/Loggers/AsyncProcessOutputReader.cs @@ -106,8 +106,14 @@ private void ProcessOnOutputDataReceived(object sender, DataReceivedEventArgs e) if (!string.IsNullOrEmpty(e.Data)) { output.Enqueue(e.Data); + if (logOutput) - logger.WriteLine(e.Data); + { + lock (this) // #2125 + { + logger.WriteLine(e.Data); + } + } } } @@ -116,8 +122,14 @@ private void ProcessOnErrorDataReceived(object sender, DataReceivedEventArgs e) if (!string.IsNullOrEmpty(e.Data)) { error.Enqueue(e.Data); + if (logOutput) - logger.WriteLineError(e.Data); + { + lock (this) // #2125 + { + logger.WriteLineError(e.Data); + } + } } } From 9938c3c649fe2a9709eeafad5858e4a0eb2a5423 Mon Sep 17 00:00:00 2001 From: blouflashdb <45914736+blouflashdb@users.noreply.github.com> Date: Wed, 5 Oct 2022 19:05:10 +0200 Subject: [PATCH 12/58] include argument and param names in --filter (#2132) --- .../ConsoleArguments/CorrectionsSuggester.cs | 3 +- src/BenchmarkDotNet/Filters/GlobFilter.cs | 7 ++-- .../BenchmarkDotNet.Tests/GlobFilterTests.cs | 35 +++++++++++++++++-- 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/src/BenchmarkDotNet/ConsoleArguments/CorrectionsSuggester.cs b/src/BenchmarkDotNet/ConsoleArguments/CorrectionsSuggester.cs index d93728f97d..e77d373f8d 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/CorrectionsSuggester.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/CorrectionsSuggester.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Linq; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Running; using JetBrains.Annotations; @@ -21,7 +22,7 @@ public CorrectionsSuggester(IReadOnlyList types) { foreach (var benchmarkCase in benchmarkRunInfo.BenchmarksCases) { - string fullBenchmarkName = benchmarkCase.Descriptor.GetFilterName(); + string fullBenchmarkName = FullNameProvider.GetBenchmarkName(benchmarkCase); actualFullBenchmarkNames.Add(fullBenchmarkName); diff --git a/src/BenchmarkDotNet/Filters/GlobFilter.cs b/src/BenchmarkDotNet/Filters/GlobFilter.cs index 784555645a..e2f25b181c 100644 --- a/src/BenchmarkDotNet/Filters/GlobFilter.cs +++ b/src/BenchmarkDotNet/Filters/GlobFilter.cs @@ -1,5 +1,7 @@ +using System; using System.Linq; using System.Text.RegularExpressions; +using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Running; namespace BenchmarkDotNet.Filters @@ -16,9 +18,10 @@ public class GlobFilter : IFilter public bool Predicate(BenchmarkCase benchmarkCase) { var benchmark = benchmarkCase.Descriptor.WorkloadMethod; - string fullBenchmarkName = benchmarkCase.Descriptor.GetFilterName(); + string nameWithoutArgs = benchmarkCase.Descriptor.GetFilterName(); + string fullBenchmarkName = FullNameProvider.GetBenchmarkName(benchmarkCase); - return patterns.Any(pattern => pattern.IsMatch(fullBenchmarkName)); + return patterns.Any(pattern => pattern.IsMatch(fullBenchmarkName) || pattern.IsMatch(nameWithoutArgs)); } internal static Regex[] ToRegex(string[] patterns) diff --git a/tests/BenchmarkDotNet.Tests/GlobFilterTests.cs b/tests/BenchmarkDotNet.Tests/GlobFilterTests.cs index fecfb455e9..82347de4cc 100644 --- a/tests/BenchmarkDotNet.Tests/GlobFilterTests.cs +++ b/tests/BenchmarkDotNet.Tests/GlobFilterTests.cs @@ -1,4 +1,6 @@ using System.Linq; +using System.Threading; +using ApprovalUtilities.Utilities; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Filters; using BenchmarkDotNet.Running; @@ -27,9 +29,38 @@ public void TheFilterIsCaseInsensitive(string pattern, bool expected) Assert.Equal(expected, filter.Predicate(benchmarkCase)); } - public class TypeWithBenchmarks + [Theory] + [InlineData(nameof(TypeWithBenchmarksAndParams), 0)] // type name + [InlineData("typewithbenchmarksandparams", 0)] // type name lowercase + [InlineData("TYPEWITHBENCHMARKSANDPARAMS", 0)] // type name uppercase + [InlineData("*TypeWithBenchmarksAndParams*", 2)] // regular expression + [InlineData("*typewithbenchmarksandparams*", 2)] // regular expression lowercase + [InlineData("*TYPEWITHBENCHMARKSANDPARAMS*", 2)] // regular expression uppercase + [InlineData("*", 2)] + [InlineData("WRONG", 0)] + [InlineData("*stillWRONG*", 0)] + [InlineData("BenchmarkDotNet.Tests.TypeWithBenchmarksAndParams.TheBenchmark", 2)] + [InlineData("BenchmarkDotNet.Tests.TypeWithBenchmarksAndParams.TheBenchmark(A: 100)", 1)] + public void TheFilterWorksWithParams(string pattern, int expectedBenchmarks) { - [Benchmark] public void TheBenchmark() { } + var benchmarkCases = BenchmarkConverter.TypeToBenchmarks(typeof(TypeWithBenchmarksAndParams)).BenchmarksCases; + + var filter = new GlobFilter(new[] { pattern }); + + Assert.Equal(expectedBenchmarks, benchmarkCases.Where(benchmarkCase => filter.Predicate(benchmarkCase)).Count()); } } + + public class TypeWithBenchmarks + { + [Benchmark] public void TheBenchmark() { } + } + + public class TypeWithBenchmarksAndParams + { + [Params(100, 200)] + public int A { get; set; } + + [Benchmark] public void TheBenchmark() { } + } } \ No newline at end of file From 7f9590afbbcf8ed7e983c39634e5915c60913d78 Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Thu, 6 Oct 2022 01:13:02 -0700 Subject: [PATCH 13/58] Update doc links and reduce redirects (#2137) * Update links * Fix formatting * http -> https, docs.microsoft -> learn.microsoft * learn.microsoft.com/en-us/ -> learn.microsoft.com/ * remove brackets because the site adds whitespaces around them (markdown doesn't do it) * revert whitespaces and put it inside markdown block --- docs/_changelog/footer/v0.10.7.md | 2 +- docs/_changelog/header/v0.10.10.md | 4 ++-- docs/_changelog/header/v0.10.12.md | 2 +- docs/_changelog/header/v0.10.13.md | 2 +- docs/_changelog/header/v0.10.14.md | 4 ++-- docs/_changelog/header/v0.11.2.md | 4 ++-- docs/_changelog/header/v0.11.3.md | 4 ++-- docs/_changelog/header/v0.12.0.md | 8 ++++---- docs/_changelog/header/v0.13.2.md | 2 +- docs/articles/configs/diagnosers.md | 8 ++++---- docs/articles/configs/powerplans.md | 4 ++-- docs/articles/configs/toolchains.md | 4 ++-- docs/articles/contributing/building.md | 12 ++++++------ docs/articles/contributing/running-tests.md | 2 +- docs/articles/faq.md | 2 +- docs/articles/features/etwprofiler.md | 2 +- docs/articles/features/event-pipe-profiler.md | 2 +- docs/articles/guides/dotnet-new-templates.md | 4 ++-- docs/articles/guides/good-practices.md | 2 +- docs/articles/samples/IntroDeferredExecution.md | 2 +- docs/articles/samples/IntroDisassemblyAllJits.md | 2 +- docs/articles/samples/IntroEnvVars.md | 6 ++++-- .../samples/IntroEventPipeProfilerAdvanced.md | 2 +- docs/articles/samples/IntroStaThread.md | 2 +- docs/articles/samples/IntroStatisticalTesting.md | 2 +- docs/articles/samples/IntroXamarin.md | 4 ++-- docs/articles/team.md | 2 +- docs/index.md | 2 +- 28 files changed, 50 insertions(+), 48 deletions(-) diff --git a/docs/_changelog/footer/v0.10.7.md b/docs/_changelog/footer/v0.10.7.md index 4b17f328ef..5ee21fee21 100644 --- a/docs/_changelog/footer/v0.10.7.md +++ b/docs/_changelog/footer/v0.10.7.md @@ -2,7 +2,7 @@ _Date: June 05, 2017_ _Milestone: [v0.10.7](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.7)_ -_Overview post: http://aakinshin.net/blog/post/bdn-v0_10_7/_ +_Overview post: https://aakinshin.net/posts/bdn-v0_10_7/_ _NuGet Packages:_ * https://www.nuget.org/packages/BenchmarkDotNet/0.10.7 diff --git a/docs/_changelog/header/v0.10.10.md b/docs/_changelog/header/v0.10.10.md index db3b76bdf0..ef34841958 100644 --- a/docs/_changelog/header/v0.10.10.md +++ b/docs/_changelog/header/v0.10.10.md @@ -1,10 +1,10 @@ Highlights: -* Disassembly Diagnoser (read more here: [Disassembling .NET Code with BenchmarkDotNet](http://adamsitnik.com/Disassembly-Diagnoser/)) +* Disassembly Diagnoser (read more here: [Disassembling .NET Code with BenchmarkDotNet](https://adamsitnik.com/Disassembly-Diagnoser/)) * ParamsSources * .NET Core x86 support * Environment variables and Mono args support * Better environment description * More: additional sections in the documentation, bug fixes, build script improvements, internal refactoring. -Overview post: [BenchmarkDotNet v0.10.10](http://aakinshin.net/blog/post/bdn-v0_10_10/) \ No newline at end of file +Overview post: [BenchmarkDotNet v0.10.10](https://aakinshin.net/posts/bdn-v0_10_10/) \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.12.md b/docs/_changelog/header/v0.10.12.md index f4d63273c8..43cff33827 100644 --- a/docs/_changelog/header/v0.10.12.md +++ b/docs/_changelog/header/v0.10.12.md @@ -1,4 +1,4 @@ -Overview post: [BenchmarkDotNet v0.10.12](http://aakinshin.net/blog/post/bdn-v0_10_12/) +Overview post: [BenchmarkDotNet v0.10.12](https://aakinshin.net/posts/bdn-v0_10_12/) ### Highlights diff --git a/docs/_changelog/header/v0.10.13.md b/docs/_changelog/header/v0.10.13.md index 2da7b40619..73f5aa1e53 100644 --- a/docs/_changelog/header/v0.10.13.md +++ b/docs/_changelog/header/v0.10.13.md @@ -1 +1 @@ -Overview post: [BenchmarkDotNet v0.10.13](http://aakinshin.net/blog/post/bdn-v0_10_13/) \ No newline at end of file +Overview post: [BenchmarkDotNet v0.10.13](https://aakinshin.net/posts/bdn-v0_10_13/) \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.14.md b/docs/_changelog/header/v0.10.14.md index a2a71f9427..aefb183459 100644 --- a/docs/_changelog/header/v0.10.14.md +++ b/docs/_changelog/header/v0.10.14.md @@ -1,4 +1,4 @@ -* Per-method parameterization ([Read more](http://benchmarkdotnet.org/Advanced/Arguments.htm)) +* Per-method parameterization ([Read more](https://benchmarkdotnet.org/articles/features/parameterization.html)) * Console histograms and multimodal disribution detection -* Many improvements for Mono disassembly support on Windows ([Read more](https://aakinshin.net/blog/post/dotnet-crossruntime-disasm/)) +* Many improvements for Mono disassembly support on Windows ([Read more](https://aakinshin.net/posts/dotnet-crossruntime-disasm/)) * Many bugfixes diff --git a/docs/_changelog/header/v0.11.2.md b/docs/_changelog/header/v0.11.2.md index 4aa6272808..2e6c435a92 100644 --- a/docs/_changelog/header/v0.11.2.md +++ b/docs/_changelog/header/v0.11.2.md @@ -45,7 +45,7 @@ In this release, we have many improvements in different areas: ### EtwProfiler -`EtwProfiler` allows to profile the benchmarked .NET code on Windows and exports the data to a trace file which can be opened with [PerfView](https://github.com/Microsoft/perfview) or [Windows Performance Analyzer](https://docs.microsoft.com/en-us/windows-hardware/test/wpt/windows-performance-analyzer). +`EtwProfiler` allows to profile the benchmarked .NET code on Windows and exports the data to a trace file which can be opened with [PerfView](https://github.com/Microsoft/perfview) or [Windows Performance Analyzer](https://learn.microsoft.com/windows-hardware/test/wpt/windows-performance-analyzer). `EtwProfiler` uses `TraceEvent` library which internally uses Event Tracing for Windows (ETW) to capture stack traces and important .NET Runtime events. Before the process with benchmarked code is started, EtwProfiler starts User and Kernel ETW sessions. Every session writes data to it's own file and captures different data. User session listens for the .NET Runtime events (GC, JIT etc) while the Kernel session gets CPU stacks and Hardware Counter events. After this, the process with benchmarked code is started. During the benchmark execution all the data is captured and written to a trace file. Moreover, BenchmarkDotNet Engine emits it's own events to be able to differentiate jitting, warmup, pilot and actual workload when analyzing the trace file. When the benchmarking is over, both sessions are closed and the two trace files are merged into one. @@ -116,7 +116,7 @@ Now it's possible to run benchmarks on preview versions of .NET Core 3.0. ### Deferred Execution Validator -In LINQ, execution of a query is usually [deferred](https://blogs.msdn.microsoft.com/charlie/2007/12/10/linq-and-deferred-execution/) until the moment when you actually request the data. If your benchmark just returns `IEnumerable` or `IQueryable` it's not measuring the execution of the query, just the creation. +In LINQ, execution of a query is usually [deferred](https://learn.microsoft.com/dotnet/standard/linq/deferred-execution-example) until the moment when you actually request the data. If your benchmark just returns `IEnumerable` or `IQueryable` it's not measuring the execution of the query, just the creation. This is why we decided to warn you about this issue whenever it happens: diff --git a/docs/_changelog/header/v0.11.3.md b/docs/_changelog/header/v0.11.3.md index e944bb8821..f3be5d81af 100644 --- a/docs/_changelog/header/v0.11.3.md +++ b/docs/_changelog/header/v0.11.3.md @@ -23,9 +23,9 @@ This release is focused mainly on bug fixes that were affecting user experience. ### ConcurrencyVisualizerProfiler -`ConcurrencyVisualizerProfiler` allows to profile the benchmarked .NET code on Windows and exports the data to a CVTrace file which can be opened with [Concurrency Visualizer](https://docs.microsoft.com/en-us/visualstudio/profiling/concurrency-visualizer). +`ConcurrencyVisualizerProfiler` allows to profile the benchmarked .NET code on Windows and exports the data to a CVTrace file which can be opened with [Concurrency Visualizer](https://learn.microsoft.com/visualstudio/profiling/concurrency-visualizer). -`ConcurrencyVisualizerProfiler` uses `EtwProfiler` to get a `.etl` file which still can be opened with [PerfView](https://github.com/Microsoft/perfview) or [Windows Performance Analyzer](https://docs.microsoft.com/en-us/windows-hardware/test/wpt/windows-performance-analyzer). The difference is that it also enables all Task and Thread related ETW Providers and exports a simple `xml` which can be opened with Visual Studio if you install [Concurrency Visualizer plugin](https://marketplace.visualstudio.com/items?itemName=Diagnostics.ConcurrencyVisualizer2017) +`ConcurrencyVisualizerProfiler` uses `EtwProfiler` to get a `.etl` file which still can be opened with [PerfView](https://github.com/Microsoft/perfview) or [Windows Performance Analyzer](https://learn.microsoft.com/windows-hardware/test/wpt/windows-performance-analyzer). The difference is that it also enables all Task and Thread related ETW Providers and exports a simple `xml` which can be opened with Visual Studio if you install [Concurrency Visualizer plugin](https://marketplace.visualstudio.com/items?itemName=Diagnostics.ConcurrencyVisualizer2017) ![open trace](https://user-images.githubusercontent.com/6011991/48638184-2b13fe00-e9d0-11e8-8a94-0e951e4606ae.png) diff --git a/docs/_changelog/header/v0.12.0.md b/docs/_changelog/header/v0.12.0.md index ccd5691009..0e6a00582f 100644 --- a/docs/_changelog/header/v0.12.0.md +++ b/docs/_changelog/header/v0.12.0.md @@ -216,8 +216,8 @@ Full .NET Framework always runs every .NET executable using the latest .NET Fram If you try to run the benchmarks for a few .NET TFMs, they are all going to be executed using the latest .NET Framework from your machine. The only difference is that they are all going to have different features enabled depending on the target version they were compiled for. You can read more about this - [here](https://docs.microsoft.com/en-us/dotnet/framework/migration-guide/version-compatibility) and - [here](https://docs.microsoft.com/en-us/dotnet/framework/migration-guide/application-compatibility). + [here](https://learn.microsoft.com/dotnet/framework/migration-guide/version-compatibility) and + [here](https://learn.microsoft.com/dotnet/framework/migration-guide/application-compatibility). This is **.NET Framework behavior which can not be controlled by BenchmarkDotNet or any other tool**. **Note:** Console arguments support works only if you pass the `args` to `BenchmarkSwitcher`: @@ -282,7 +282,7 @@ Moreover, if you share the source code of the benchmark, other people can run it ## Official templates for BenchmarkDotNet-based projects Since v0.12.0, BenchmarkDotNet provides project templates to setup your benchmarks easily. -The template exists for each major .NET language ([C#](https://docs.microsoft.com/en-us/dotnet/csharp/), [F#](https://docs.microsoft.com/en-us/dotnet/fsharp/) and [VB](https://docs.microsoft.com/en-us/dotnet/visual-basic/)) with equivalent features and structure. +The template exists for each major .NET language ([C#](https://learn.microsoft.com/dotnet/csharp/), [F#](https://learn.microsoft.com/dotnet/fsharp/) and [VB](https://learn.microsoft.com/dotnet/visual-basic/)) with equivalent features and structure. The templates require the [.NET Core SDK](https://www.microsoft.com/net/download). Once installed, run the following command to install the templates: ```log @@ -352,7 +352,7 @@ dotnet new benchmark --help The version of the template NuGet package is synced with the [BenchmarkDotNet](https://www.nuget.org/packages/BenchmarkDotNet/) package. For instance, the template version `0.12.0` is referencing [BenchmarkDotnet 0.12.0](https://www.nuget.org/packages/BenchmarkDotNet/0.12.0) - there is no floating version behavior. -For more info about the `dotnet new` CLI, please read [the documentation](https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet). +For more info about the `dotnet new` CLI, please read [the documentation](https://learn.microsoft.com/dotnet/core/tools/dotnet). ## New NativeMemoryProfiler diff --git a/docs/_changelog/header/v0.13.2.md b/docs/_changelog/header/v0.13.2.md index b6a2780f50..7410958448 100644 --- a/docs/_changelog/header/v0.13.2.md +++ b/docs/_changelog/header/v0.13.2.md @@ -56,7 +56,7 @@ We are really excited to see the experimental CoreRT project grow and become off As every AOT solution, NativeAOT has some [limitations](https://github.com/dotnet/runtime/blob/main/src/coreclr/nativeaot/docs/limitations.md) like limited reflection support or lack of dynamic assembly loading. Because of that, the host process (what you run from command line) is never an AOT process, but just a regular .NET process. This process (called Host process) uses reflection to read benchmarks metadata (find all `[Benchmark]` methods etc.), generates a new project that references the benchmarks and compiles it using ILCompiler. The boilerplate code is not using reflection, so the project is built with `TrimmerDefaultAction=link` (we have greatly reduced build time thanks to that). Such compilation produces a native executable, which is later started by the Host process. This process (called Benchmark or Child process) performs the actual benchmarking and reports the results back to the Host process. By default BenchmarkDotNet uses the latest version of `Microsoft.DotNet.ILCompiler` to build the NativeAOT benchmark according to [this instructions](https://github.com/dotnet/runtime/blob/main/src/coreclr/nativeaot/docs/compiling.md). Moreover, BenchmarkDotNet by default uses current machines CPU features (change: [#1994](https://github.com/dotnet/BenchmarkDotNet/pull/1994), discussion: [#2061](https://github.com/dotnet/BenchmarkDotNet/issues/2061)) and if you don't like this behavior, you can [disable it](https://github.com/dotnet/BenchmarkDotNet/issues/2061#issuecomment-1203602177). This is why you need to: -- install [pre-requisites](https://docs.microsoft.com/en-us/dotnet/core/deploying/native-aot/#prerequisites) required by NativeAOT compiler +- install [pre-requisites](https://learn.microsoft.com/dotnet/core/deploying/native-aot/#prerequisites) required by NativeAOT compiler - target .NET to be able to run NativeAOT benchmarks (example: `net7.0` in the .csproj file) - run the app as a .NET process (example: `dotnet run -c Release -f net7.0`). - specify the NativeAOT runtime in an explicit way, either by using console line arguments `--runtimes nativeaot7.0` (the recommended approach), or by using`[SimpleJob]` attribute or by using the fluent Job config API `Job.ShortRun.With(NativeAotRuntime.Net70)`: diff --git a/docs/articles/configs/diagnosers.md b/docs/articles/configs/diagnosers.md index 0532e375af..d942ab44f4 100644 --- a/docs/articles/configs/diagnosers.md +++ b/docs/articles/configs/diagnosers.md @@ -10,7 +10,7 @@ A **diagnoser** can attach to your benchmark and get some useful info. The current Diagnosers are: - GC and Memory Allocation (`MemoryDiagnoser`) which is cross platform, built-in and **is not enabled by default anymore**. - Please see Adam Sitnik's [blog post](http://adamsitnik.com/the-new-Memory-Diagnoser/) for all the details. + Please see Adam Sitnik's [blog post](https://adamsitnik.com/the-new-Memory-Diagnoser/) for all the details. - JIT Inlining Events (`InliningDiagnoser`). You can find this diagnoser in a separate package with diagnosers for Windows (`BenchmarkDotNet.Diagnostics.Windows`): [![NuGet](https://img.shields.io/nuget/v/BenchmarkDotNet.svg)](https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/) @@ -20,10 +20,10 @@ The current Diagnosers are: - Hardware Counter Diagnoser. You can find this diagnoser in a separate package with diagnosers for Windows (`BenchmarkDotNet.Diagnostics.Windows`): [![NuGet](https://img.shields.io/nuget/v/BenchmarkDotNet.svg)](https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/). - Please see Adam Sitnik's [blog post](http://adamsitnik.com/Hardware-Counters-Diagnoser/) for all the details. + Please see Adam Sitnik's [blog post](https://adamsitnik.com/Hardware-Counters-Diagnoser/) for all the details. - Disassembly Diagnoser. It allows you to disassemble the benchmarked code to asm, IL and C#/F#. - Please see Adam Sitnik's [blog post](http://adamsitnik.com/Disassembly-Diagnoser/) for all the details. + Please see Adam Sitnik's [blog post](https://adamsitnik.com/Disassembly-Diagnoser/) for all the details. - ETW Profiler (`EtwProfiler`). It allows you to not only benchmark, but also profile the code. It's using TraceEvent, which internally uses ETW and exports all the information to a trace file. The trace file contains all of the stack traces captured by the profiler, PDBs to resolve symbols for both native and managed code and captured GC, JIT and CLR events. Please use one of the free tools: PerfView or Windows Performance Analyzer to analyze and visualize the data from trace file. You can find this diagnoser in a separate package with diagnosers for Windows (`BenchmarkDotNet.Diagnostics.Windows`): [![NuGet](https://img.shields.io/nuget/v/BenchmarkDotNet.svg)](https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/) Please see Adam Sitnik's [blog post](https://adamsitnik.com/ETW-Profiler/) for all the details. @@ -80,7 +80,7 @@ In BenchmarkDotNet, 1kB = 1024B, 1MB = 1024kB, and so on. The column Gen X means * In order to not affect main results we perform a separate run if any diagnoser is used. That's why it might take more time to execute benchmarks. * MemoryDiagnoser: - * Mono currently [does not](http://stackoverflow.com/questions/40234948/how-to-get-the-number-of-allocated-bytes-in-mono) expose any api to get the number of allocated bytes. That's why our Mono users will get `?` in Allocated column. + * Mono currently [does not](https://stackoverflow.com/questions/40234948/how-to-get-the-number-of-allocated-bytes-in-mono) expose any api to get the number of allocated bytes. That's why our Mono users will get `?` in Allocated column. * In order to get the number of allocated bytes in cross platform way we are using `GC.GetAllocatedBytesForCurrentThread` which recently got [exposed](https://github.com/dotnet/corefx/pull/12489) for netcoreapp1.1. That's why BenchmarkDotNet does not support netcoreapp1.0 from version 0.10.1. * MemoryDiagnoser is `99.5%` accurate about allocated memory when using default settings or Job.ShortRun (or any longer job than it). * Threading Diagnoser: diff --git a/docs/articles/configs/powerplans.md b/docs/articles/configs/powerplans.md index 18296cc90a..988746a198 100644 --- a/docs/articles/configs/powerplans.md +++ b/docs/articles/configs/powerplans.md @@ -11,8 +11,8 @@ Please note. During an execution, BenchmarkDotNet saves the current power plan a ### Links -* Power policy settings: https://docs.microsoft.com/en-us/windows/desktop/power/power-policy-settings -* Powercfg command: https://docs.microsoft.com/en-us/windows-hardware/design/device-experiences/powercfg-command-line-options +* Power policy settings: https://learn.microsoft.com/windows/win32/power/power-policy-settings +* Powercfg command: https://learn.microsoft.com/windows-hardware/design/device-experiences/powercfg-command-line-options * @BenchmarkDotNet.Samples.IntroPowerPlan --- \ No newline at end of file diff --git a/docs/articles/configs/toolchains.md b/docs/articles/configs/toolchains.md index 4e21752405..8f73bc3784 100644 --- a/docs/articles/configs/toolchains.md +++ b/docs/articles/configs/toolchains.md @@ -355,7 +355,7 @@ BenchmarkDotNet is going to follow [these instructrions](https://github.com/dotn BenchmarkDotNet supports Web Assembly on Unix! However, currently you need to build the **dotnet runtime** yourself to be able to run the benchmarks. -For up-to-date docs, you should visit [dotnet/runtime repository](https://github.com/dotnet/runtime/blob/master/docs/workflow/testing/libraries/testing-wasm.md). +For up-to-date docs, you should visit [dotnet/runtime repository](https://github.com/dotnet/runtime/blob/main/docs/workflow/testing/libraries/testing-wasm.md). The docs below are specific to Ubuntu 18.04 at the moment of writing this document (16/07/2020). @@ -407,7 +407,7 @@ git clone https://github.com/dotnet/runtime cd runtime ``` -Install [all Mono prerequisites](https://github.com/dotnet/runtime/blob/master/docs/workflow/testing/libraries/testing-wasm.md): +Install [all Mono prerequisites](https://github.com/dotnet/runtime/blob/main/docs/workflow/testing/libraries/testing-wasm.md): ```cmd sudo apt-get install cmake llvm-9 clang-9 autoconf automake libtool build-essential python curl git lldb-6.0 liblldb-6.0-dev libunwind8 libunwind8-dev gettext libicu-dev liblttng-ust-dev libssl-dev libnuma-dev libkrb5-dev zlib1g-dev diff --git a/docs/articles/contributing/building.md b/docs/articles/contributing/building.md index 400bf74174..d613961e30 100644 --- a/docs/articles/contributing/building.md +++ b/docs/articles/contributing/building.md @@ -12,7 +12,7 @@ Once all the necessary tools are in place, building is trivial. Simply open solu ## Cake (C# Make) -[Cake (C# Make)](http://cakebuild.net/) is a cross platform build automation system with a C# DSL to do things like compiling code, copy files/folders, running unit tests, compress files and build NuGet packages. +[Cake (C# Make)](https://cakebuild.net/) is a cross platform build automation system with a C# DSL to do things like compiling code, copy files/folders, running unit tests, compress files and build NuGet packages. The build currently depends on the following prerequisites: @@ -22,8 +22,8 @@ The build currently depends on the following prerequisites: - .NET Framework 4.6 or higher - Linux: - - Install [Mono version 5 or higher](http://www.mono-project.com/download/#download-lin) - - Install [fsharp package](http://fsharp.org/use/linux/) + - Install [Mono version 5 or higher](https://www.mono-project.com/download/stable/#download-lin) + - Install [fsharp package](https://fsharp.org/use/linux/) - Install packages required to .NET Core SDK - gettext - libcurl4-openssl-dev @@ -32,9 +32,9 @@ The build currently depends on the following prerequisites: - libunwind8 - macOS - - Install [Mono version 5 or higher](http://www.mono-project.com/download/#download-mac) - - Install [fsharp package](http://fsharp.org/use/mac/) - - Install the latest version of [OpenSSL](https://www.microsoft.com/net/core#macos). + - Install [Mono version 5 or higher](https://www.mono-project.com/download/stable/#download-mac) + - Install [fsharp package](https://fsharp.org/use/mac/) + - Install the latest version of [OpenSSL](https://www.openssl.org/source/). After you have installed these pre-requisites, you can build the BenchmarkDotNet by invoking the build script (`build.ps1` on Windows, or `build.sh` on Linux and macOS) at the base of the BenchmarkDotNet repository. By default the build process also run all the tests. There are quite a few tests, taking a significant amount of time that is not necessary if you just want to experiment with changes. You can skip the tests phase by adding the `skiptests` argument to the build script, e.g. `.\build.ps1 --SkipTests=True` or `./build.sh --skiptests=true`. diff --git a/docs/articles/contributing/running-tests.md b/docs/articles/contributing/running-tests.md index 26cde491c1..431acd9671 100644 --- a/docs/articles/contributing/running-tests.md +++ b/docs/articles/contributing/running-tests.md @@ -16,6 +16,6 @@ You should be able to run all of tests from your IDE as well. ## Approval Tests -For some unit tests (e.g. for exporter tests) BenchmarkDotNet uses [approval tests'](http://approvaltests.sourceforge.net/) implementation for .NET: [ApprovalTests.Net](https://github.com/approvals/ApprovalTests.Net). +For some unit tests (e.g. for exporter tests) BenchmarkDotNet uses [approval tests'](https://approvaltests.com/) implementation for .NET: [ApprovalTests.Net](https://github.com/approvals/ApprovalTests.Net). * The expected value for each test is stored in a `*.approved.txt` file located near the test source file in the repository. ApprovalTests.NET generates approved file's names automatically according test name and its parameters. This files must be added under the source control. * It also creates a `*.received` file for each failed test. You can use different reporters for convenient file comparison. By default we use XUnit2Reporter, so you can find test run results on the test runner console as usual. You can add [UseReporter(typeof(KDiffReporter))] on test class and then ApprovalTests will open KDiff for each failed test. This way you can easily understand what's the difference between approved and received values and choose the correct one. diff --git a/docs/articles/faq.md b/docs/articles/faq.md index b0ccf827fe..e966e3795e 100644 --- a/docs/articles/faq.md +++ b/docs/articles/faq.md @@ -8,7 +8,7 @@ name: FAQ * **Q** Why can't I install BenchmarkDotNet in Visual Studio 2010/2012/2013? **A** BenchmarkDotNet requires NuGet 3.x+ and can't be installed in old versions of Visual Studio which use NuGet 2.x. -Consider to use Visual Studio 2015/2017 or [Rider](http://jetbrains.com/rider/). +Consider to use Visual Studio 2015/2017 or [Rider](https://www.jetbrains.com/rider/). See also: [BenchmarkDotNet#237](https://github.com/dotnet/BenchmarkDotNet/issues/237), [roslyn#12780](https://github.com/dotnet/roslyn/issues/12780). * **Q** Why can't I install BenchmarkDotNet in a new .NET Core Console App in Visual Studio 2017? diff --git a/docs/articles/features/etwprofiler.md b/docs/articles/features/etwprofiler.md index c19192e202..3b8118a759 100644 --- a/docs/articles/features/etwprofiler.md +++ b/docs/articles/features/etwprofiler.md @@ -5,7 +5,7 @@ name: EtwProfiler # EtwProfiler -`EtwProfiler` allows to profile the benchmarked .NET code on Windows and exports the data to a trace file which can be opened with [PerfView](https://github.com/Microsoft/perfview) or [Windows Performance Analyzer](https://docs.microsoft.com/en-us/windows-hardware/test/wpt/windows-performance-analyzer). +`EtwProfiler` allows to profile the benchmarked .NET code on Windows and exports the data to a trace file which can be opened with [PerfView](https://github.com/Microsoft/perfview) or [Windows Performance Analyzer](https://learn.microsoft.com/windows-hardware/test/wpt/windows-performance-analyzer). ![](https://adamsitnik.com/images/etwprofiler/flamegraph.png) diff --git a/docs/articles/features/event-pipe-profiler.md b/docs/articles/features/event-pipe-profiler.md index 5fa9057835..3b0ed69bc6 100644 --- a/docs/articles/features/event-pipe-profiler.md +++ b/docs/articles/features/event-pipe-profiler.md @@ -5,7 +5,7 @@ name: EventPipeProfiler # EventPipeProfiler -`EventPipeProfiler` is a cross-platform profiler that allows profile .NET code on every platform - Windows, Linux, macOS. Collected data are exported to trace files (`.speedscope.json` and `.nettrace`) which can be analyzed using [SpeedScope](https://www.speedscope.app/), [PerfView](https://github.com/Microsoft/perfview), and [Visual Studio Profiler](https://docs.microsoft.com/en-us/visualstudio/profiling/profiling-feature-tour). This new profiler is available from the 0.12.1 version. +`EventPipeProfiler` is a cross-platform profiler that allows profile .NET code on every platform - Windows, Linux, macOS. Collected data are exported to trace files (`.speedscope.json` and `.nettrace`) which can be analyzed using [SpeedScope](https://www.speedscope.app/), [PerfView](https://github.com/Microsoft/perfview), and [Visual Studio Profiler](https://learn.microsoft.com/visualstudio/profiling/profiling-feature-tour). This new profiler is available from the 0.12.1 version. ![](https://wojciechnagorski.com/images/EventPipeProfiler/SpeedScopeAdvance.png) diff --git a/docs/articles/guides/dotnet-new-templates.md b/docs/articles/guides/dotnet-new-templates.md index 2f75360c74..a11064a1f2 100644 --- a/docs/articles/guides/dotnet-new-templates.md +++ b/docs/articles/guides/dotnet-new-templates.md @@ -6,7 +6,7 @@ name: BenchmarkDotNet templates # BenchmarkDotNet templates BenchmarkDotNet provides project templates to setup your benchmarks easily -The template exists for each major .NET language ([C#](https://docs.microsoft.com/en-us/dotnet/csharp/), [F#](https://docs.microsoft.com/en-us/dotnet/fsharp/) and [VB](https://docs.microsoft.com/en-us/dotnet/visual-basic/)) with equivalent features and structure. +The template exists for each major .NET language ([C#](https://learn.microsoft.com/dotnet/csharp/), [F#](https://learn.microsoft.com/dotnet/fsharp/) and [VB](https://learn.microsoft.com/dotnet/visual-basic/)) with equivalent features and structure. ## How to install the templates @@ -97,5 +97,5 @@ For instance, the template version `0.11.5` is referencing [BenchmarkDotnet 0.11 ## References -For more info about the `dotnet new` CLI, please read [the documentation](https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet). +For more info about the `dotnet new` CLI, please read [the documentation](https://learn.microsoft.com/dotnet/core/tools/dotnet). diff --git a/docs/articles/guides/good-practices.md b/docs/articles/guides/good-practices.md index c4e8c33040..854ead5679 100644 --- a/docs/articles/guides/good-practices.md +++ b/docs/articles/guides/good-practices.md @@ -4,7 +4,7 @@ Never use the Debug build for benchmarking. *Never*. The debug version of the target method can run 10–100 times slower. The release mode means that you should have `true` in your csproj file -or use [/optimize](https://msdn.microsoft.com/en-us/library/t0hfscdc.aspx) for `csc`. Also your never +or use [/optimize](https://learn.microsoft.com/dotnet/csharp/language-reference/compiler-options/) for `csc`. Also your never should use an attached debugger (e.g. Visual Studio or WinDbg) during the benchmarking. The best way is build our benchmark in the Release mode and run it from the command line. diff --git a/docs/articles/samples/IntroDeferredExecution.md b/docs/articles/samples/IntroDeferredExecution.md index 8f49553aac..8ecbd3512d 100644 --- a/docs/articles/samples/IntroDeferredExecution.md +++ b/docs/articles/samples/IntroDeferredExecution.md @@ -4,7 +4,7 @@ uid: BenchmarkDotNet.Samples.IntroDeferredExecution ## Sample: IntroDeferredExecution -In LINQ, execution of a query is usually [deferred](https://blogs.msdn.microsoft.com/charlie/2007/12/10/linq-and-deferred-execution/) until the moment when you actually request the data. If your benchmark just returns `IEnumerable` or `IQueryable` it's not measuring the execution of the query, just the creation. +In LINQ, execution of a query is usually [deferred](https://learn.microsoft.com/dotnet/standard/linq/deferred-execution-example) until the moment when you actually request the data. If your benchmark just returns `IEnumerable` or `IQueryable` it's not measuring the execution of the query, just the creation. This is why we decided to warn you about this issue whenever it happens: diff --git a/docs/articles/samples/IntroDisassemblyAllJits.md b/docs/articles/samples/IntroDisassemblyAllJits.md index 7a742a9961..1e9868a279 100644 --- a/docs/articles/samples/IntroDisassemblyAllJits.md +++ b/docs/articles/samples/IntroDisassemblyAllJits.md @@ -20,7 +20,7 @@ But to allow benchmarking any target platform architecture the project which def ### Output -The disassembly result can be obtained [here](http://adamsitnik.com/files/disasm/Jit_Devirtualization-disassembly-report.html). +The disassembly result can be obtained [here](https://adamsitnik.com/files/disasm/Jit_Devirtualization-disassembly-report.html). The file was too big to embed it in this doc page. ### Links diff --git a/docs/articles/samples/IntroEnvVars.md b/docs/articles/samples/IntroEnvVars.md index 9c9f598642..286a449682 100644 --- a/docs/articles/samples/IntroEnvVars.md +++ b/docs/articles/samples/IntroEnvVars.md @@ -6,8 +6,10 @@ uid: BenchmarkDotNet.Samples.IntroEnvVars You can configure custom environment variables for the process that is running your benchmarks. One reason for doing this might be checking out how different - [runtime knobs](https://github.com/dotnet/coreclr/blob/master/Documentation/project-docs/clr-configuration-knobs.md) - affect the performance of .NET Core. + [compilation](https://learn.microsoft.com/dotnet/core/runtime-config/compilation), + [threading](https://learn.microsoft.com/dotnet/core/runtime-config/threading), + [garbage collector](https://learn.microsoft.com/dotnet/core/runtime-config/garbage-collector) + settings affect the performance of .NET Core. ### Source code diff --git a/docs/articles/samples/IntroEventPipeProfilerAdvanced.md b/docs/articles/samples/IntroEventPipeProfilerAdvanced.md index 49b0e171f2..1ae651472b 100644 --- a/docs/articles/samples/IntroEventPipeProfilerAdvanced.md +++ b/docs/articles/samples/IntroEventPipeProfilerAdvanced.md @@ -5,7 +5,7 @@ uid: BenchmarkDotNet.Samples.IntroEventPipeProfilerAdvanced ## Sample: EventPipeProfilerAdvanced The most advanced and powerful way to use `EventPipeProfiler` is a custom configuration. As you can see the below configuration adds `EventPipeProfiler` that constructor can take the profile and/or a list of providers. -Both `EventPipeProfiler` and `dotnet trace` use the `Microsoft.Diagnostics.NETCore.Client` package internally. So before you start using the custom configuration of this profiler, it is worth reading the documentation [here](https://github.com/dotnet/diagnostics/blob/master/documentation/dotnet-trace-instructions.md) and [here](https://docs.microsoft.com/en-us/dotnet/core/diagnostics/dotnet-trace#dotnet-trace-collect) where you can find more information about how to configure provider list. +Both `EventPipeProfiler` and `dotnet trace` use the `Microsoft.Diagnostics.NETCore.Client` package internally. So before you start using the custom configuration of this profiler, it is worth reading the documentation [here](https://github.com/dotnet/diagnostics/blob/main/documentation/dotnet-trace-instructions.md) and [here](https://learn.microsoft.com/dotnet/core/diagnostics/dotnet-trace#dotnet-trace-collect) where you can find more information about how to configure provider list. ### Source code diff --git a/docs/articles/samples/IntroStaThread.md b/docs/articles/samples/IntroStaThread.md index cc3a8e21b4..d115ebc618 100644 --- a/docs/articles/samples/IntroStaThread.md +++ b/docs/articles/samples/IntroStaThread.md @@ -8,7 +8,7 @@ If the code you want to benchmark requires `[System.STAThread]` then you need to apply this attribute to the benchmarked method. BenchmarkDotNet will generate executable with `[STAThread]` applied to it's `Main` method. -Currently it does not work for .NET Core 2.0 due to [this](https://github.com/dotnet/coreclr/issues/13688) bug. +Currently it does not work for .NET Core 2.0 due to [this](https://github.com/dotnet/runtime/issues/8834) bug. ### Source code diff --git a/docs/articles/samples/IntroStatisticalTesting.md b/docs/articles/samples/IntroStatisticalTesting.md index 71992802f5..40b647f640 100644 --- a/docs/articles/samples/IntroStatisticalTesting.md +++ b/docs/articles/samples/IntroStatisticalTesting.md @@ -9,7 +9,7 @@ uid: BenchmarkDotNet.Samples.IntroStatisticalTesting [!code-csharp[IntroStatisticalTesting.cs](../../../samples/BenchmarkDotNet.Samples/IntroStatisticalTesting.cs)] ### Output - +```markdown | Method | Mean | Error | StdDev | Ratio | Welch(1us)/p-values | Welch(3%)/p-values | MannWhitney(1us)/p-values | MannWhitney(3%)/p-values | |--------- |----------:|----------:|----------:|------:|---------------------- |---------------------- |-------------------------- |------------------------- | | Sleep50 | 53.13 ms | 0.5901 ms | 0.1532 ms | 0.51 | Faster: 1.0000/0.0000 | Faster: 1.0000/0.0000 | Faster: 1.0000/0.0040 | Faster: 1.0000/0.0040 | diff --git a/docs/articles/samples/IntroXamarin.md b/docs/articles/samples/IntroXamarin.md index 4484063e97..79a26c1543 100644 --- a/docs/articles/samples/IntroXamarin.md +++ b/docs/articles/samples/IntroXamarin.md @@ -22,8 +22,8 @@ Other notes: ### Links -* [Xamarin.Android linker settings](https://docs.microsoft.com/xamarin/android/deploy-test/linker#linker-behavior) -* [Xamarin.iOS linker settings](https://docs.microsoft.com/xamarin/ios/deploy-test/linker#dont-link) +* [Xamarin.Android linker settings](https://learn.microsoft.com/xamarin/android/deploy-test/linker#linker-behavior) +* [Xamarin.iOS linker settings](https://learn.microsoft.com/xamarin/ios/deploy-test/linker#dont-link) * The permanent link to this sample: @BenchmarkDotNet.Samples.Xamarin --- \ No newline at end of file diff --git a/docs/articles/team.md b/docs/articles/team.md index eadacc90b3..663c35958c 100644 --- a/docs/articles/team.md +++ b/docs/articles/team.md @@ -73,4 +73,4 @@ Contributors: * [@factormystic](https://github.com/factormystic) * [@nietras](https://github.com/nietras) -[All contributors on GitHub](https://github.com/PerfDotNet/BenchmarkDotNet/graphs/contributors) \ No newline at end of file +[All contributors on GitHub](https://github.com/dotnet/BenchmarkDotNet/graphs/contributors) \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index e948bc68ca..73a353f28c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -314,6 +314,6 @@ Let's build the best tool for benchmarking together! ## Code of Conduct -This project has adopted the code of conduct defined by the [Contributor Covenant](http://contributor-covenant.org/) +This project has adopted the code of conduct defined by the [Contributor Covenant](https://www.contributor-covenant.org/) to clarify expected behavior in our community. For more information, see the [.NET Foundation Code of Conduct](https://dotnetfoundation.org/code-of-conduct). From 95bb2aa2392a8541a6c815c6a9f95bafb3f56e5d Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Thu, 6 Oct 2022 01:28:22 -0700 Subject: [PATCH 14/58] Escape Param data for the exporters (#2135) * Rename: Escape -> EscapeCommandLine * Escape special characters for the all exporters --- .../Extensions/StringAndTextExtensions.cs | 12 +++++++++++- .../Parameters/ParameterInstance.cs | 6 +++--- src/BenchmarkDotNet/Running/BenchmarkId.cs | 2 +- .../Toolchains/DotNetCli/DotNetCliExecutor.cs | 2 +- .../Toolchains/Roslyn/Generator.cs | 4 ++-- ...est.Escape_ParamsAndArguments.approved.txt | 19 +++++++++++++++++++ .../MarkdownExporterApprovalTests.cs | 11 +++++++++++ 7 files changed, 48 insertions(+), 8 deletions(-) create mode 100644 tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.Escape_ParamsAndArguments.approved.txt diff --git a/src/BenchmarkDotNet/Extensions/StringAndTextExtensions.cs b/src/BenchmarkDotNet/Extensions/StringAndTextExtensions.cs index 4408cab658..962d3b659a 100644 --- a/src/BenchmarkDotNet/Extensions/StringAndTextExtensions.cs +++ b/src/BenchmarkDotNet/Extensions/StringAndTextExtensions.cs @@ -16,7 +16,7 @@ private static readonly Lazy> InvalidFileNameCharacte internal static string ToLowerCase(this bool value) => value ? "true" : "false"; // to avoid .ToString().ToLower() allocation // source: https://stackoverflow.com/a/12364234/5852046 - internal static string Escape(this string cliArg) + internal static string EscapeCommandLine(this string cliArg) { if (string.IsNullOrEmpty(cliArg)) return cliArg; @@ -27,6 +27,16 @@ internal static string Escape(this string cliArg) return value; } + /// + /// Escapes UNICODE control characters + /// + /// string to escape + /// True to put (double) quotes around the string literal + internal static string EscapeSpecialCharacters(this string str, bool quote) + { + return Microsoft.CodeAnalysis.CSharp.SymbolDisplay.FormatLiteral(str, quote); + } + /// /// replaces all invalid file name chars with their number representation /// diff --git a/src/BenchmarkDotNet/Parameters/ParameterInstance.cs b/src/BenchmarkDotNet/Parameters/ParameterInstance.cs index ab1bb7d8bd..276ca6f669 100644 --- a/src/BenchmarkDotNet/Parameters/ParameterInstance.cs +++ b/src/BenchmarkDotNet/Parameters/ParameterInstance.cs @@ -44,14 +44,14 @@ private string ToDisplayText(CultureInfo cultureInfo, int maxParameterColumnWidt case null: return NullParameterTextRepresentation; case IParam parameter: - return Trim(parameter.DisplayText, maxParameterColumnWidth); + return Trim(parameter.DisplayText, maxParameterColumnWidth).EscapeSpecialCharacters(false); case IFormattable formattable: - return Trim(formattable.ToString(null, cultureInfo), maxParameterColumnWidth); + return Trim(formattable.ToString(null, cultureInfo), maxParameterColumnWidth).EscapeSpecialCharacters(false); // no trimming for types! case Type type: return type.IsNullable() ? $"{Nullable.GetUnderlyingType(type).GetDisplayName()}?" : type.GetDisplayName(); default: - return Trim(value.ToString(), maxParameterColumnWidth); + return Trim(value.ToString(), maxParameterColumnWidth).EscapeSpecialCharacters(false); } } diff --git a/src/BenchmarkDotNet/Running/BenchmarkId.cs b/src/BenchmarkDotNet/Running/BenchmarkId.cs index acfc89ce85..52085b905a 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkId.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkId.cs @@ -30,7 +30,7 @@ public BenchmarkId(int value, BenchmarkCase benchmarkCase) public override int GetHashCode() => Value; - public string ToArguments() => $"--benchmarkName {FullBenchmarkName.Escape()} --job {JobId.Escape()} --benchmarkId {Value}"; + public string ToArguments() => $"--benchmarkName {FullBenchmarkName.EscapeCommandLine()} --job {JobId.EscapeCommandLine()} --benchmarkId {Value}"; public string ToArguments(string fromBenchmark, string toBenchmark) => $"{AnonymousPipesHost.AnonymousPipesDescriptors} {fromBenchmark} {toBenchmark} {ToArguments()}"; diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliExecutor.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliExecutor.cs index 403f38bf02..1ce1d88944 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliExecutor.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliExecutor.cs @@ -69,7 +69,7 @@ private ExecuteResult Execute(BenchmarkCase benchmarkCase, var startInfo = DotNetCliCommandExecutor.BuildStartInfo( CustomDotNetCliPath, artifactsPaths.BinariesDirectoryPath, - $"{executableName.Escape()} {benchmarkId.ToArguments(inputFromBenchmark.GetClientHandleAsString(), acknowledgments.GetClientHandleAsString())}", + $"{executableName.EscapeCommandLine()} {benchmarkId.ToArguments(inputFromBenchmark.GetClientHandleAsString(), acknowledgments.GetClientHandleAsString())}", redirectStandardOutput: true, redirectStandardInput: false, redirectStandardError: false); // #1629 diff --git a/src/BenchmarkDotNet/Toolchains/Roslyn/Generator.cs b/src/BenchmarkDotNet/Toolchains/Roslyn/Generator.cs index ef11c17d91..0718d6b8cb 100644 --- a/src/BenchmarkDotNet/Toolchains/Roslyn/Generator.cs +++ b/src/BenchmarkDotNet/Toolchains/Roslyn/Generator.cs @@ -38,8 +38,8 @@ protected override void GenerateBuildScript(BuildPartition buildPartition, Artif list.Add("/unsafe"); list.Add("/deterministic"); list.Add("/platform:" + buildPartition.Platform.ToConfig()); - list.Add("/appconfig:" + artifactsPaths.AppConfigPath.Escape()); - var references = GetAllReferences(buildPartition.RepresentativeBenchmarkCase).Select(assembly => assembly.Location.Escape()); + list.Add("/appconfig:" + artifactsPaths.AppConfigPath.EscapeCommandLine()); + var references = GetAllReferences(buildPartition.RepresentativeBenchmarkCase).Select(assembly => assembly.Location.EscapeCommandLine()); list.Add("/reference:" + string.Join(",", references)); list.Add(Path.GetFileName(artifactsPaths.ProgramCodePath)); diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.Escape_ParamsAndArguments.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.Escape_ParamsAndArguments.approved.txt new file mode 100644 index 0000000000..5bc80750d2 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.Escape_ParamsAndArguments.approved.txt @@ -0,0 +1,19 @@ +=== Escape_ParamsAndArguments === + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + DefaultJob : extra output line + + + Method | StringParam | charArg | Mean | Error | StdDev | +------- |------------ |-------- |---------:|--------:|--------:| + Foo | \t | \t | 102.0 ns | 6.09 ns | 1.58 ns | ^ + Foo | \t | \n | 202.0 ns | 6.09 ns | 1.58 ns | ^ + Bar | \t | ? | 302.0 ns | 6.09 ns | 1.58 ns | ^ + Foo | \n | \t | 402.0 ns | 6.09 ns | 1.58 ns | ^ + Foo | \n | \n | 502.0 ns | 6.09 ns | 1.58 ns | ^ + Bar | \n | ? | 602.0 ns | 6.09 ns | 1.58 ns | ^ + +Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/MarkdownExporterApprovalTests.cs b/tests/BenchmarkDotNet.Tests/Exporters/MarkdownExporterApprovalTests.cs index 2d673d7742..9e502b192a 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/MarkdownExporterApprovalTests.cs +++ b/tests/BenchmarkDotNet.Tests/Exporters/MarkdownExporterApprovalTests.cs @@ -255,6 +255,17 @@ public class Invalid_TwoJobBaselines [Benchmark] public void Foo() {} [Benchmark] public void Bar() {} } + + /* Escape Params */ + + public class Escape_ParamsAndArguments + { + [Params("\t", "\n"), UsedImplicitly] public string StringParam; + + [Arguments('\t')] [Arguments('\n')] + [Benchmark] public void Foo(char charArg) {} + [Benchmark] public void Bar() {} + } } } } From ecb25bf413923ac9976ced1b55223fd89f6fb664 Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Thu, 6 Oct 2022 10:19:29 -0700 Subject: [PATCH 15/58] Pass non escaped strings to generated project (#2136) --- .../Exporters/FullNameProvider.cs | 8 +-- .../Extensions/StringAndTextExtensions.cs | 10 ++++ .../Helpers/SourceCodeHelper.cs | 4 +- .../ParamsTests.cs | 51 +++++++++++++++++++ .../SourceCodeHelperTests.cs | 39 ++++++++++++-- 5 files changed, 99 insertions(+), 13 deletions(-) diff --git a/src/BenchmarkDotNet/Exporters/FullNameProvider.cs b/src/BenchmarkDotNet/Exporters/FullNameProvider.cs index 3986aaa202..fc6ab3d8b8 100644 --- a/src/BenchmarkDotNet/Exporters/FullNameProvider.cs +++ b/src/BenchmarkDotNet/Exporters/FullNameProvider.cs @@ -141,9 +141,9 @@ private static string GetArgument(object argumentValue, Type argumentType) case object[] array when array.Length == 1: return GetArgument(array[0], argumentType); case string text: - return $"\"{EscapeWhitespaces(text)}\""; + return text.EscapeSpecialCharacters(true); case char character: - return $"'{character}'"; + return character.EscapeSpecialCharacter(true); case DateTime time: return time.ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss.fffffffK"); case Type type: @@ -183,10 +183,6 @@ private static string GetArray(IEnumerable collection) return buffer.ToString(); } - private static string EscapeWhitespaces(string text) - => text.Replace("\t", "\\t") - .Replace("\r\n", "\\r\\n"); - private static string GetTypeArgumentName(Type type) { if (Aliases.TryGetValue(type, out string alias)) diff --git a/src/BenchmarkDotNet/Extensions/StringAndTextExtensions.cs b/src/BenchmarkDotNet/Extensions/StringAndTextExtensions.cs index 962d3b659a..6244d4832e 100644 --- a/src/BenchmarkDotNet/Extensions/StringAndTextExtensions.cs +++ b/src/BenchmarkDotNet/Extensions/StringAndTextExtensions.cs @@ -37,6 +37,16 @@ internal static string EscapeSpecialCharacters(this string str, bool quote) return Microsoft.CodeAnalysis.CSharp.SymbolDisplay.FormatLiteral(str, quote); } + /// + /// Escapes UNICODE control character + /// + /// char to escape + /// True to put (single) quotes around the character literal. + internal static string EscapeSpecialCharacter(this char c, bool quote) + { + return Microsoft.CodeAnalysis.CSharp.SymbolDisplay.FormatLiteral(c, quote); + } + /// /// replaces all invalid file name chars with their number representation /// diff --git a/src/BenchmarkDotNet/Helpers/SourceCodeHelper.cs b/src/BenchmarkDotNet/Helpers/SourceCodeHelper.cs index 3553ece5f6..3883de7546 100644 --- a/src/BenchmarkDotNet/Helpers/SourceCodeHelper.cs +++ b/src/BenchmarkDotNet/Helpers/SourceCodeHelper.cs @@ -18,9 +18,9 @@ public static string ToSourceCode(object value) case bool b: return b.ToLowerCase(); case string text: - return $"$@\"{text.Replace("\"", "\"\"").Replace("{", "{{").Replace("}", "}}")}\""; + return text.EscapeSpecialCharacters(true); case char c: - return c == '\\' ? "'\\\\'" : $"'{value}'"; + return c.EscapeSpecialCharacter(true); case float f: return ToSourceCode(f); case double d: diff --git a/tests/BenchmarkDotNet.IntegrationTests/ParamsTests.cs b/tests/BenchmarkDotNet.IntegrationTests/ParamsTests.cs index d610793bf1..e4a23dc121 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ParamsTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ParamsTests.cs @@ -139,6 +139,57 @@ public class InvalidFileNamesInParams public void Benchmark() => Console.WriteLine("// " + Field); } + [Fact] + public void SpecialCharactersInStringAreSupported() => CanExecute(); + + public class CompileSpecialCharactersInString + { + [Params("\0")] public string Null; + [Params("\t")] public string Tab; + [Params("\n")] public string NewLine; + [Params("\\")] public string Slash; + [Params("\"")] public string Quote; + [Params("\u0061")] public string Unicode; + [Params("{")] public string Bracket; + + [Params("\n \0 \n")] public string Combo; + + [Params("C:\\file1.txt")] public string Path1; + [Params(@"C:\file2.txt")] public string Path2; + + [Benchmark] + public void Benchmark() + { + var isPassedAsSingleCharacter = + Null.Length == 1 && + Tab.Length == 1 && + NewLine.Length == 1 && + Slash.Length == 1 && + Quote.Length == 1 && + Unicode.Length == 1 && + Bracket.Length == 1; + + if (!isPassedAsSingleCharacter) + throw new InvalidOperationException("Some Param has an invalid escaped string"); + } + } + + [Fact] + public void SpecialCharactersInCharAreSupported() => CanExecute(); + + public class CompileSpecialCharactersInChar + { + [Params('\0')] public char Null; + [Params('\t')] public char Tab; + [Params('\n')] public char NewLine; + [Params('\\')] public char Slash; + [Params('\"')] public char Quote; + [Params('\u0061')] public char Unicode; + + [Benchmark] + public void Benchmark() { } + } + [Fact] public void ParamsMustBeEscapedProperly() => CanExecute(); diff --git a/tests/BenchmarkDotNet.Tests/SourceCodeHelperTests.cs b/tests/BenchmarkDotNet.Tests/SourceCodeHelperTests.cs index a1403cacc1..846d59faa9 100644 --- a/tests/BenchmarkDotNet.Tests/SourceCodeHelperTests.cs +++ b/tests/BenchmarkDotNet.Tests/SourceCodeHelperTests.cs @@ -9,7 +9,7 @@ namespace BenchmarkDotNet.Tests // TODO: add decimal, typeof, CreateInstance, TimeValue, IntPtr, IFormattable public class SourceCodeHelperTests { - private ITestOutputHelper output; + private readonly ITestOutputHelper output; public SourceCodeHelperTests(ITestOutputHelper output) => this.output = output; @@ -17,8 +17,8 @@ public class SourceCodeHelperTests [InlineData(null, "null")] [InlineData(false, "false")] [InlineData(true, "true")] - [InlineData("string", "$@\"string\"")] - [InlineData("string/\\", @"$@""string/\""")] + [InlineData("string", "\"string\"")] + [InlineData("string/\\", @"""string/\\""")] [InlineData('a', "'a'")] [InlineData('\\', "'\\\\'")] [InlineData(0.123f, "0.123f")] @@ -43,7 +43,7 @@ public void SupportsGuid() [Fact] public void CanEscapeJson() { - const string expected = "$@\"{{ \"\"message\"\": \"\"Hello, World!\"\" }}\""; + const string expected = "\"{ \\\"message\\\": \\\"Hello, World!\\\" }\""; var actual = SourceCodeHelper.ToSourceCode("{ \"message\": \"Hello, World!\" }"); @@ -53,11 +53,40 @@ public void CanEscapeJson() [Fact] public void CanEscapePath() { - const string expected = @"$@""C:\Projects\BenchmarkDotNet\samples\BenchmarkDotNet.Samples"""; + const string expected = @"""C:\\Projects\\BenchmarkDotNet\\samples\\BenchmarkDotNet.Samples"""; var actual = SourceCodeHelper.ToSourceCode(@"C:\Projects\BenchmarkDotNet\samples\BenchmarkDotNet.Samples"); Assert.Equal(expected, actual); } + + [Fact] + public void CanEscapeControlCharacters() + { + const string expected = @""" \0 \b \f \n \t \v \"" a a a a { } """; + + var actual = SourceCodeHelper.ToSourceCode(" \0 \b \f \n \t \v \" \u0061 \x0061 \x61 \U00000061 { } "); + + Assert.Equal(expected, actual); + } + + [Theory] + [InlineData('\0', @"'\0'")] + [InlineData('\b', @"'\b'")] + [InlineData('\f', @"'\f'")] + [InlineData('\n', @"'\n'")] + [InlineData('\t', @"'\t'")] + [InlineData('\v', @"'\v'")] + [InlineData('\'', @"'\''")] + [InlineData('\u0061', "'a'")] + [InlineData('"', "'\"'")] + [InlineData('{', "'{'")] + [InlineData('}', "'}'")] + public void CanEscapeControlCharactersInChar(char original, string excepted) + { + var actual = SourceCodeHelper.ToSourceCode(original); + + Assert.Equal(excepted, actual); + } } } \ No newline at end of file From 5ed46d575ead7f3a85de5605e234f5d534f536e3 Mon Sep 17 00:00:00 2001 From: Fan Yang <52458914+fanyang-mono@users.noreply.github.com> Date: Thu, 6 Oct 2022 13:31:13 -0400 Subject: [PATCH 16/58] [Mono] Disable LLVM JIT (#2134) --- src/BenchmarkDotNet/Extensions/ProcessExtensions.cs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/BenchmarkDotNet/Extensions/ProcessExtensions.cs b/src/BenchmarkDotNet/Extensions/ProcessExtensions.cs index 9dffbf5eff..5c0ae5e82f 100644 --- a/src/BenchmarkDotNet/Extensions/ProcessExtensions.cs +++ b/src/BenchmarkDotNet/Extensions/ProcessExtensions.cs @@ -149,15 +149,6 @@ internal static void SetEnvironmentVariables(this ProcessStartInfo start, Benchm // disable ReSharper's Dynamic Program Analysis (see https://github.com/dotnet/BenchmarkDotNet/issues/1871 for details) start.EnvironmentVariables["JETBRAINS_DPA_AGENT_ENABLE"] = "0"; - if (benchmarkCase.Job.Infrastructure.Toolchain is MonoAotLLVMToolChain) - { - MonoAotLLVMRuntime aotruntime = (MonoAotLLVMRuntime)benchmarkCase.GetRuntime(); - - if (aotruntime.AOTCompilerMode == MonoAotCompilerMode.llvm) - { - start.EnvironmentVariables["MONO_ENV_OPTIONS"] = "--llvm"; - } - } if (!benchmarkCase.Job.HasValue(EnvironmentMode.EnvironmentVariablesCharacteristic)) return; From 1f5637c784ad9887a2d5abdba54d1337655281b1 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Fri, 7 Oct 2022 12:27:31 +0200 Subject: [PATCH 17/58] throw an exception when multiple benchmark projects with the same name are found, fixes #2126 (#2143) --- .../Toolchains/CsProj/CsProjGenerator.cs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjGenerator.cs b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjGenerator.cs index 86a08ac3c6..996355a6df 100644 --- a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjGenerator.cs @@ -154,19 +154,24 @@ protected virtual FileInfo GetProjectFilePath(Type benchmarkTarget, ILogger logg // important assumption! project's file name === output dll name string projectName = benchmarkTarget.GetTypeInfo().Assembly.GetName().Name; - // I was afraid of using .GetFiles with some smart search pattern due to the fact that the method was designed for Windows - // and now .NET is cross platform so who knows if the pattern would be supported for other OSes var possibleNames = new HashSet { $"{projectName}.csproj", $"{projectName}.fsproj", $"{projectName}.vbproj" }; - var projectFile = rootDirectory - .EnumerateFiles("*.*", SearchOption.AllDirectories) - .FirstOrDefault(file => possibleNames.Contains(file.Name)); + var projectFiles = rootDirectory + .EnumerateFiles("*proj", SearchOption.AllDirectories) + .Where(file => possibleNames.Contains(file.Name)) + .ToArray(); - if (projectFile == default(FileInfo)) + if (projectFiles.Length == 0) { throw new NotSupportedException( $"Unable to find {projectName} in {rootDirectory.FullName} and its subfolders. Most probably the name of output exe is different than the name of the .(c/f)sproj"); } - return projectFile; + else if (projectFiles.Length > 1) + { + throw new NotSupportedException( + $"Found more than one matching project file for {projectName} in {rootDirectory.FullName} and its subfolders: {string.Join(",", projectFiles.Select(pf => $"'{pf.FullName}'"))}. Benchmark project names needs to be unique."); + } + + return projectFiles[0]; } public override bool Equals(object obj) => obj is CsProjGenerator other && Equals(other); From 095975f40252f265254acb517a114690674d5f5a Mon Sep 17 00:00:00 2001 From: leonvandermeer <64834803+leonvandermeer@users.noreply.github.com> Date: Wed, 12 Oct 2022 20:23:56 +0200 Subject: [PATCH 18/58] Handle addresses larger than 2GB for 32-bit benchmarks #1469 (#2145) * Handle addresses larger than 2GB for 32-bit benchmarks #1469 * Apply suggestions from code review Co-authored-by: Adam Sitnik --- docs/articles/configs/jobs.md | 3 + .../samples/IntroLargeAddressAware.md | 16 +++++ docs/articles/samples/toc.yml | 2 + .../IntroLargeAddressAware.cs | 38 ++++++++++ src/BenchmarkDotNet/Jobs/EnvironmentMode.cs | 20 ++++++ src/BenchmarkDotNet/Jobs/JobExtensions.cs | 5 ++ .../Running/BenchmarkPartitioner.cs | 3 + .../Toolchains/DotNetCli/DotNetCliBuilder.cs | 10 ++- .../Toolchains/LargeAddressAware.cs | 60 ++++++++++++++++ .../Toolchains/Roslyn/Builder.cs | 39 +++++----- .../LargeAddressAwareTest.cs | 71 +++++++++++++++++++ 11 files changed, 250 insertions(+), 17 deletions(-) create mode 100644 docs/articles/samples/IntroLargeAddressAware.md create mode 100644 samples/BenchmarkDotNet.Samples/IntroLargeAddressAware.cs create mode 100644 src/BenchmarkDotNet/Toolchains/LargeAddressAware.cs create mode 100644 tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs diff --git a/docs/articles/configs/jobs.md b/docs/articles/configs/jobs.md index f0203ce58d..cddba24d8e 100644 --- a/docs/articles/configs/jobs.md +++ b/docs/articles/configs/jobs.md @@ -33,6 +33,9 @@ It's a single string characteristic. It allows to name your job. This name will * `CpuGroups`: Specifies whether garbage collection supports multiple CPU groups * `Force`: Specifies whether the BenchmarkDotNet's benchmark runner forces full garbage collection after each benchmark invocation * `AllowVeryLargeObjects`: On 64-bit platforms, enables arrays that are greater than 2 gigabytes (GB) in total size +* `LargeAddressAware`: Specifies that benchmark can handle addresses larger than 2 gigabytes. See also: @BenchmarkDotNet.Samples.IntroLargeAddressAware and [LARGEADDRESSAWARE](https://learn.microsoft.com/cpp/build/reference/largeaddressaware-handle-large-addresses) + * `false`: Benchmark uses the defaults (64-bit: enabled; 32-bit: disabled). + * `true`: Explicitly specify that benchmark can handle addresses larger than 2 gigabytes. * `EnvironmentVariables`: customized environment variables for target benchmark. See also: @BenchmarkDotNet.Samples.IntroEnvVars BenchmarkDotNet will use host process environment characteristics for non specified values. diff --git a/docs/articles/samples/IntroLargeAddressAware.md b/docs/articles/samples/IntroLargeAddressAware.md new file mode 100644 index 0000000000..1824e0281f --- /dev/null +++ b/docs/articles/samples/IntroLargeAddressAware.md @@ -0,0 +1,16 @@ +--- +uid: BenchmarkDotNet.Samples.IntroLargeAddressAware +title: "Sample: IntroLargeAddressAware" +--- + +## Sample: IntroLargeAddressAware + +### Source code + +[!code-csharp[IntroLargeAddressAware.cs](../../../samples/BenchmarkDotNet.Samples/IntroLargeAddressAware.cs)] + +### Links + +* The permanent link to this sample: @BenchmarkDotNet.Samples.IntroLargeAddressAware + +--- \ No newline at end of file diff --git a/docs/articles/samples/toc.yml b/docs/articles/samples/toc.yml index 45423c20c8..3ec492a07b 100644 --- a/docs/articles/samples/toc.yml +++ b/docs/articles/samples/toc.yml @@ -64,6 +64,8 @@ href: IntroJobBaseline.md - name: IntroJoin href: IntroJoin.md +- name: IntroLargeAddressAware + href: IntroLargeAddressAware.md - name: IntroMonitoring href: IntroMonitoring.md - name: IntroMultimodal diff --git a/samples/BenchmarkDotNet.Samples/IntroLargeAddressAware.cs b/samples/BenchmarkDotNet.Samples/IntroLargeAddressAware.cs new file mode 100644 index 0000000000..964901c3ad --- /dev/null +++ b/samples/BenchmarkDotNet.Samples/IntroLargeAddressAware.cs @@ -0,0 +1,38 @@ +using System; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Jobs; + +namespace BenchmarkDotNet.Samples +{ + [MemoryDiagnoser] + [Config(typeof(Config))] + public class IntroLargeAddressAware + { + private class Config : ManualConfig + { + public Config() + { + AddJob(Job.Default + .WithRuntime(ClrRuntime.Net462) + .WithPlatform(Platform.X86) + .WithLargeAddressAware() + .WithId("Framework")); + } + } + + [Benchmark] + public void AllocateMoreThan2GB() + { + const int oneGB = 1024 * 1024 * 1024; + const int halfGB = oneGB / 2; + byte[] bytes1 = new byte[oneGB]; + byte[] bytes2 = new byte[oneGB]; + byte[] bytes3 = new byte[halfGB]; + GC.KeepAlive(bytes1); + GC.KeepAlive(bytes2); + GC.KeepAlive(bytes3); + } + } +} diff --git a/src/BenchmarkDotNet/Jobs/EnvironmentMode.cs b/src/BenchmarkDotNet/Jobs/EnvironmentMode.cs index 28de7ada1d..21a4ff3ead 100644 --- a/src/BenchmarkDotNet/Jobs/EnvironmentMode.cs +++ b/src/BenchmarkDotNet/Jobs/EnvironmentMode.cs @@ -16,6 +16,7 @@ public sealed class EnvironmentMode : JobMode public static readonly Characteristic GcCharacteristic = CreateCharacteristic(nameof(Gc)); public static readonly Characteristic> EnvironmentVariablesCharacteristic = CreateCharacteristic>(nameof(EnvironmentVariables)); public static readonly Characteristic PowerPlanModeCharacteristic = CreateCharacteristic(nameof(PowerPlanMode)); + public static readonly Characteristic LargeAddressAwareCharacteristic = CreateCharacteristic(nameof(LargeAddressAware)); public static readonly EnvironmentMode LegacyJitX86 = new EnvironmentMode(nameof(LegacyJitX86), Jit.LegacyJit, Platform.X86).Freeze(); public static readonly EnvironmentMode LegacyJitX64 = new EnvironmentMode(nameof(LegacyJitX64), Jit.LegacyJit, Platform.X64).Freeze(); @@ -96,6 +97,25 @@ public Guid? PowerPlanMode set => PowerPlanModeCharacteristic[this] = value; } + /// + /// Specifies that benchmark can handle addresses larger than 2 gigabytes. + /// false: Benchmark uses the default (64-bit: enabled; 32-bit:disabled). This is the default. + /// true: Explicitly specify that benchmark can handle addresses larger than 2 gigabytes. + /// + public bool LargeAddressAware + { + get => LargeAddressAwareCharacteristic[this]; + set + { + if (!RuntimeInformation.IsWindows()) + { + throw new NotSupportedException("LargeAddressAware is a Windows-specific concept."); + } + + LargeAddressAwareCharacteristic[this] = value; + } + } + /// /// Adds the specified to . /// If already contains a variable with the same key, diff --git a/src/BenchmarkDotNet/Jobs/JobExtensions.cs b/src/BenchmarkDotNet/Jobs/JobExtensions.cs index 34605e273d..ca731ad723 100644 --- a/src/BenchmarkDotNet/Jobs/JobExtensions.cs +++ b/src/BenchmarkDotNet/Jobs/JobExtensions.cs @@ -76,6 +76,11 @@ public static class JobExtensions /// public static Job WithGcAllowVeryLargeObjects(this Job job, bool value) => job.WithCore(j => j.Environment.Gc.AllowVeryLargeObjects = value); + /// + /// Specifies that benchmark can handle addresses larger than 2 gigabytes. + /// + public static Job WithLargeAddressAware(this Job job, bool value = true) => job.WithCore(j => j.Environment.LargeAddressAware = value); + /// /// Put segments that should be deleted on a standby list for future use instead of releasing them back to the OS /// The default is false diff --git a/src/BenchmarkDotNet/Running/BenchmarkPartitioner.cs b/src/BenchmarkDotNet/Running/BenchmarkPartitioner.cs index 373d956b77..861038e114 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkPartitioner.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkPartitioner.cs @@ -41,6 +41,8 @@ public bool Equals(BenchmarkCase x, BenchmarkCase y) return false; if (jobX.Environment.Platform != jobY.Environment.Platform) // platform is set in .csproj return false; + if (jobX.Environment.LargeAddressAware != jobY.Environment.LargeAddressAware) + return false; if (AreDifferent(jobX.Infrastructure.BuildConfiguration, jobY.Infrastructure.BuildConfiguration)) // Debug vs Release return false; if (AreDifferent(jobX.Infrastructure.Arguments, jobY.Infrastructure.Arguments)) // arguments can be anything (Mono runtime settings or MsBuild parameters) @@ -74,6 +76,7 @@ public int GetHashCode(BenchmarkCase obj) hashCode ^= obj.Descriptor.Type.Assembly.Location.GetHashCode(); hashCode ^= (int)job.Environment.Jit; hashCode ^= (int)job.Environment.Platform; + hashCode ^= job.Environment.LargeAddressAware.GetHashCode(); hashCode ^= job.Environment.Gc.GetHashCode(); if (job.Infrastructure.BuildConfiguration != null) diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs index 834a254df0..bad11c91be 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs @@ -26,7 +26,8 @@ public DotNetCliBuilder(string targetFrameworkMoniker, string customDotNetCliPat } public BuildResult Build(GenerateResult generateResult, BuildPartition buildPartition, ILogger logger) - => new DotNetCliCommand( + { + BuildResult buildResult = new DotNetCliCommand( CustomDotNetCliPath, string.Empty, generateResult, @@ -37,5 +38,12 @@ public BuildResult Build(GenerateResult generateResult, BuildPartition buildPart logOutput: LogOutput, retryFailedBuildWithNoDeps: RetryFailedBuildWithNoDeps) .RestoreThenBuild(); + if (buildResult.IsBuildSuccess && + buildPartition.RepresentativeBenchmarkCase.Job.Environment.LargeAddressAware) + { + LargeAddressAware.SetLargeAddressAware(generateResult.ArtifactsPaths.ExecutablePath); + } + return buildResult; + } } } diff --git a/src/BenchmarkDotNet/Toolchains/LargeAddressAware.cs b/src/BenchmarkDotNet/Toolchains/LargeAddressAware.cs new file mode 100644 index 0000000000..7727c2afb1 --- /dev/null +++ b/src/BenchmarkDotNet/Toolchains/LargeAddressAware.cs @@ -0,0 +1,60 @@ +using System; +using System.IO; + +namespace BenchmarkDotNet.Toolchains +{ + // From https://github.com/KirillOsenkov/LargeAddressAware/blob/95e5fc6024438f94325df4bac6ef73c77bf90e71/SetLargeAddressAware/LargeAddressAware.cs + internal class LargeAddressAware + { + + public static void SetLargeAddressAware(string filePath) + { + PrepareStream(filePath, (stream, binaryReader) => + { + var value = binaryReader.ReadInt16(); + if ((value & 0x20) == 0) + { + value = (short)(value | 0x20); + stream.Position -= 2; + var binaryWriter = new BinaryWriter(stream); + binaryWriter.Write(value); + binaryWriter.Flush(); + } + }); + } + + private static void PrepareStream(string filePath, Action action) + { + using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.ReadWrite, FileShare.Read)) + { + if (stream.Length < 0x3C) + { + return; + } + + var binaryReader = new BinaryReader(stream); + + // MZ header + if (binaryReader.ReadInt16() != 0x5A4D) + { + return; + } + + stream.Position = 0x3C; + var peHeaderLocation = binaryReader.ReadInt32(); + + stream.Position = peHeaderLocation; + + // PE header + if (binaryReader.ReadInt32() != 0x4550) + { + return; + } + + stream.Position += 0x12; + + action(stream, binaryReader); + } + } + } +} diff --git a/src/BenchmarkDotNet/Toolchains/Roslyn/Builder.cs b/src/BenchmarkDotNet/Toolchains/Roslyn/Builder.cs index fdaf1498f7..dc21423de8 100644 --- a/src/BenchmarkDotNet/Toolchains/Roslyn/Builder.cs +++ b/src/BenchmarkDotNet/Toolchains/Roslyn/Builder.cs @@ -12,6 +12,7 @@ using JetBrains.Annotations; using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.Emit; using OurPlatform = BenchmarkDotNet.Environments.Platform; namespace BenchmarkDotNet.Toolchains.Roslyn @@ -70,17 +71,17 @@ private BuildResult Build(GenerateResult generateResult, BuildPartition buildPar .Select(uniqueMetadata => uniqueMetadata.GetReference()) .ToList(); - var (result, missingReferences) = Build(generateResult, syntaxTree, compilationOptions, references, cancellationToken); + var (result, missingReferences) = Build(generateResult, buildPartition, syntaxTree, compilationOptions, references, cancellationToken); if (result.IsBuildSuccess || !missingReferences.Any()) return result; var withMissingReferences = references.Union(missingReferences.Select(assemblyMetadata => assemblyMetadata.GetReference())); - return Build(generateResult, syntaxTree, compilationOptions, withMissingReferences, cancellationToken).result; + return Build(generateResult, buildPartition, syntaxTree, compilationOptions, withMissingReferences, cancellationToken).result; } - private static (BuildResult result, AssemblyMetadata[] missingReference) Build(GenerateResult generateResult, SyntaxTree syntaxTree, + private static (BuildResult result, AssemblyMetadata[] missingReference) Build(GenerateResult generateResult, BuildPartition buildPartition, SyntaxTree syntaxTree, CSharpCompilationOptions compilationOptions, IEnumerable references, CancellationToken cancellationToken) { var compilation = CSharpCompilation @@ -89,25 +90,31 @@ private static (BuildResult result, AssemblyMetadata[] missingReference) Build(G .WithOptions(compilationOptions) .AddReferences(references); + EmitResult emitResult; using (var executable = File.Create(generateResult.ArtifactsPaths.ExecutablePath)) { - var emitResult = compilation.Emit(executable, cancellationToken: cancellationToken); - - if (emitResult.Success) - return (BuildResult.Success(generateResult), default); + emitResult = compilation.Emit(executable, cancellationToken: cancellationToken); + } + if (emitResult.Success) + { + if (buildPartition.RepresentativeBenchmarkCase.Job.Environment.LargeAddressAware) + { + LargeAddressAware.SetLargeAddressAware(generateResult.ArtifactsPaths.ExecutablePath); + } + return (BuildResult.Success(generateResult), default); + } - var compilationErrors = emitResult.Diagnostics - .Where(diagnostic => diagnostic.IsWarningAsError || diagnostic.Severity == DiagnosticSeverity.Error) - .ToArray(); + var compilationErrors = emitResult.Diagnostics + .Where(diagnostic => diagnostic.IsWarningAsError || diagnostic.Severity == DiagnosticSeverity.Error) + .ToArray(); - var errors = new StringBuilder("The build has failed!").AppendLine(); - foreach (var diagnostic in compilationErrors) - errors.AppendLine($"{diagnostic.Id}: {diagnostic.GetMessage(CultureInfo.InvariantCulture)}"); + var errors = new StringBuilder("The build has failed!").AppendLine(); + foreach (var diagnostic in compilationErrors) + errors.AppendLine($"{diagnostic.Id}: {diagnostic.GetMessage(CultureInfo.InvariantCulture)}"); - var missingReferences = GetMissingReferences(compilationErrors); + var missingReferences = GetMissingReferences(compilationErrors); - return (BuildResult.Failure(generateResult, errors.ToString()), missingReferences); - } + return (BuildResult.Failure(generateResult, errors.ToString()), missingReferences); } private Platform GetPlatform(OurPlatform platform) diff --git a/tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs b/tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs new file mode 100644 index 0000000000..0fd81bf793 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs @@ -0,0 +1,71 @@ +using System; +using System.Linq; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Tests.Loggers; +using BenchmarkDotNet.Tests.XUnit; +using Xunit; +using Xunit.Abstractions; + +namespace BenchmarkDotNet.IntegrationTests +{ + public class LargeAddressAwareTest + { + private readonly ITestOutputHelper output; + + public LargeAddressAwareTest(ITestOutputHelper outputHelper) => output = outputHelper; + + [FactWindowsOnly("CLR is a valid job only on Windows")] + public void BenchmarkCanAllocateMoreThan2Gb() + { + var summary = BenchmarkRunner + .Run( + ManualConfig.CreateEmpty() + .AddJob(Job.Dry.WithRuntime(CoreRuntime.Core60).WithPlatform(Platform.X64).WithId("Core")) + .AddJob(Job.Dry.WithRuntime(ClrRuntime.Net462).WithPlatform(Platform.X86).WithLargeAddressAware().WithId("Framework")) + .AddColumnProvider(DefaultColumnProviders.Instance) + .AddLogger(new OutputLogger(output))); + + Assert.True(summary.Reports + .All(report => report.ExecuteResults + .All(executeResult => executeResult.FoundExecutable))); + + Assert.True(summary.Reports.All(report => report.AllMeasurements.Any())); + + Assert.True(summary.Reports + .Single(report => report.BenchmarkCase.Job.Environment.Runtime is ClrRuntime) + .ExecuteResults + .Any()); + + Assert.True(summary.Reports + .Single(report => report.BenchmarkCase.Job.Environment.Runtime is CoreRuntime) + .ExecuteResults + .Any()); + + Assert.Contains(".NET Framework", summary.AllRuntimes); + Assert.Contains(".NET 6.0", summary.AllRuntimes); + } + } + + public class NeedsMoreThan2GB + { + [Benchmark] + public void AllocateMoreThan2GB() + { + Console.WriteLine($"Is64BitProcess = {Environment.Is64BitProcess}"); + + const int oneGB = 1024 * 1024 * 1024; + const int halfGB = oneGB / 2; + byte[] bytes1 = new byte[oneGB]; + byte[] bytes2 = new byte[oneGB]; + byte[] bytes3 = new byte[halfGB]; + GC.KeepAlive(bytes1); + GC.KeepAlive(bytes2); + GC.KeepAlive(bytes3); + } + } +} \ No newline at end of file From 8d2379626ecfa547a2e7e6eaab808785baf484cf Mon Sep 17 00:00:00 2001 From: franciscomoloureiro Date: Wed, 12 Oct 2022 18:26:19 +0000 Subject: [PATCH 19/58] Update console title with benchmark information (#2140) --- .../Running/BenchmarkRunnerClean.cs | 34 +++++++++++++++++-- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs index 7e5b56c118..c283d40d76 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs @@ -16,6 +16,7 @@ using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Mathematics; +using BenchmarkDotNet.Portability; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Toolchains; using BenchmarkDotNet.Toolchains.Parameters; @@ -144,12 +145,15 @@ private static Summary Run(BenchmarkRunInfo benchmarkRunInfo, var cultureInfo = config.CultureInfo ?? DefaultCultureInfo.Instance; var reports = new List(); string title = GetTitle(new[] { benchmarkRunInfo }); + var consoleTitle = RuntimeInformation.IsWindows() ? Console.Title : string.Empty; logger.WriteLineInfo($"// Found {benchmarks.Length} benchmarks:"); foreach (var benchmark in benchmarks) logger.WriteLineInfo($"// {benchmark.DisplayInfo}"); logger.WriteLine(); + UpdateTitle(totalBenchmarkCount, benchmarksToRunCount); + using (var powerManagementApplier = new PowerManagementApplier(logger)) { bool stop = false; @@ -218,6 +222,11 @@ private static Summary Run(BenchmarkRunInfo benchmarkRunInfo, } } + if (RuntimeInformation.IsWindows()) + { + Console.Title = consoleTitle; + } + var runEnd = runsChronometer.GetElapsed(); return new Summary(title, @@ -227,7 +236,7 @@ private static Summary Run(BenchmarkRunInfo benchmarkRunInfo, logFilePath, runEnd.GetTimeSpan() - runStart.GetTimeSpan(), cultureInfo, - Validate(new[] {benchmarkRunInfo }, NullLogger.Instance), // validate them once again, but don't print the output + Validate(new[] { benchmarkRunInfo }, NullLogger.Instance), // validate them once again, but don't print the output config.GetColumnHidingRules().ToImmutableArray()); } @@ -614,15 +623,34 @@ private static void Cleanup(HashSet artifactsToCleanup) } } + private static void UpdateTitle(int totalBenchmarkCount, int benchmarksToRunCount) + { + if (!Console.IsOutputRedirected && (RuntimeInformation.IsWindows() || RuntimeInformation.IsLinux() || RuntimeInformation.IsMacOSX())) + { + Console.Title = $"{benchmarksToRunCount}/{totalBenchmarkCount} Remaining"; + } + } + private static void LogProgress(ILogger logger, in StartedClock runsChronometer, int totalBenchmarkCount, int benchmarksToRunCount) { int executedBenchmarkCount = totalBenchmarkCount - benchmarksToRunCount; - double avgSecondsPerBenchmark = runsChronometer.GetElapsed().GetTimeSpan().TotalSeconds / executedBenchmarkCount; - TimeSpan fromNow = TimeSpan.FromSeconds(avgSecondsPerBenchmark * benchmarksToRunCount); + TimeSpan fromNow = GetEstimatedFinishTime(runsChronometer, benchmarksToRunCount, executedBenchmarkCount); DateTime estimatedEnd = DateTime.Now.Add(fromNow); string message = $"// ** Remained {benchmarksToRunCount} ({(double)benchmarksToRunCount / totalBenchmarkCount:P1}) benchmark(s) to run." + $" Estimated finish {estimatedEnd:yyyy-MM-dd H:mm} ({(int)fromNow.TotalHours}h {fromNow.Minutes}m from now) **"; logger.WriteLineHeader(message); + + if (!Console.IsOutputRedirected && (RuntimeInformation.IsWindows() || RuntimeInformation.IsLinux() || RuntimeInformation.IsMacOSX())) + { + Console.Title = $"{benchmarksToRunCount}/{totalBenchmarkCount} Remaining - {(int)fromNow.TotalHours}h {fromNow.Minutes}m to finish"; + } + } + + private static TimeSpan GetEstimatedFinishTime(in StartedClock runsChronometer, int benchmarksToRunCount, int executedBenchmarkCount) + { + double avgSecondsPerBenchmark = executedBenchmarkCount > 0 ? runsChronometer.GetElapsed().GetTimeSpan().TotalSeconds / executedBenchmarkCount : 0; + TimeSpan fromNow = TimeSpan.FromSeconds(avgSecondsPerBenchmark * benchmarksToRunCount); + return fromNow; } } } From f8e0a5c23883fa6ea4a975cd08a0071c4648472c Mon Sep 17 00:00:00 2001 From: Sean Killeen Date: Thu, 13 Oct 2022 03:55:17 -0400 Subject: [PATCH 20/58] Automated spellcheck for docs via GitHub Actions (and address all raised issues) (#2144) * Add GH Action * add cSpell Config * Ignore team page Mostly names * capitalize GUID * add some terms * add known terms * fix: "disassemblers" * add known words * ignore changelog many names and many link to issues that will be copy/paste and could contain spelling errors * more known terms * Known terms. * Fix: "priorities" * fix: priorities * known term * add back-ticks * fix: characteristic / minor grammar * fix: variance * add word * back-ticks * fix: "specify" * back-ticks * add terms * front-matter ignore of configoptions * back-ticks * back-ticks * add terms * frontmatter ignore Since term is only in front-matte * fix: "log file" * separate words from ignoreWords * Fix branch for GH Action * ignores of runstrategy in frontmatter --- .github/workflows/spellcheck.yml | 28 +++++ cSpell.json | 108 ++++++++++++++++++ docs/articles/configs/configoptions.md | 1 + docs/articles/configs/filters.md | 22 ++-- docs/articles/configs/jobs.md | 19 ++- docs/articles/configs/validators.md | 6 +- docs/articles/contributing/building.md | 30 ++--- docs/articles/contributing/disassembler.md | 3 +- docs/articles/features/etwprofiler.md | 2 +- docs/articles/guides/choosing-run-strategy.md | 3 +- docs/articles/guides/console-args.md | 20 ++-- docs/articles/samples/IntroColdStart.md | 3 +- docs/articles/samples/IntroMonitoring.md | 3 +- docs/articles/samples/IntroParamsPriority.md | 2 +- docs/articles/samples/IntroPowerPlan.md | 7 +- 15 files changed, 208 insertions(+), 49 deletions(-) create mode 100644 .github/workflows/spellcheck.yml create mode 100644 cSpell.json diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml new file mode 100644 index 0000000000..ec5ad83f7b --- /dev/null +++ b/.github/workflows/spellcheck.yml @@ -0,0 +1,28 @@ +name: Documentation Checks + +on: + push: + branches: + - master + paths: + - "docs/**/*" + pull_request: + branches: + - master + paths: + - "docs/**/*" +jobs: + spellcheck: + name: "Docs: Spellcheck" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + name: Check out the code + - uses: actions/setup-node@v1 + name: Setup node + with: + node-version: "16" + - run: npm install -g cspell + name: Install cSpell + - run: cspell --config ./cSpell.json "docs/**/*.md" --no-progress + name: Run cSpell diff --git a/cSpell.json b/cSpell.json new file mode 100644 index 0000000000..a224a62a9c --- /dev/null +++ b/cSpell.json @@ -0,0 +1,108 @@ +{ + "version": "0.2", + "language": "en", + "words": [ + "Alloc", + "analyse", + "analyser", + "Analysers", + "Autofac", + "bitness", + "corlib", + "Cygwin", + "Diagnoser", + "diagnosers", + "disassemblers", + "disassm", + "Jits", + "Jitting", + "LINQ", + "microbenchmarking", + "microbenchmarks", + "Mispredict", + "Mispredictions", + "msbuild", + "Multimodal", + "multimodality", + "netcoreapp", + "powerplans", + "Pseudocode", + "runtimes", + "Serilog", + "Tailcall", + "toolchains", + "unmanaged" + ], + "ignoreWords": [ + "Akinshin", + "Andrey", + "Expecto", + "Jint", + "LoongArch64", + "macrobenchmark", + "MediatR", + "Nagórski's", + "Newtonsoft", + "NodaTime", + "Npgsql", + "Sitnik's", + "Wojciech", + "Avalonia", + "Gitter" + ], + "patterns": [ + { + "name": "Markdown links", + "pattern": "\\((.*)\\)", + "description": "" + }, + { + "name": "Markdown code blocks", + "pattern": "/^(\\s*`{3,}).*[\\s\\S]*?^\\1/gmx", + "description": "Taken from the cSpell example at https://cspell.org/configuration/patterns/#verbose-regular-expressions" + }, + { + "name": "Inline code blocks", + "pattern": "\\`([^\\`\\r\\n]+?)\\`", + "description": "https://stackoverflow.com/questions/41274241/how-to-capture-inline-markdown-code-but-not-a-markdown-code-fence-with-regex" + }, + { + "name": "Link contents", + "pattern": "\\", + "description": "" + }, + { + "name": "Snippet references", + "pattern": "-- snippet:(.*)", + "description": "" + }, + { + "name": "Snippet references 2", + "pattern": "\\<\\[sample:(.*)", + "description": "another kind of snippet reference" + }, + { + "name": "Multi-line code blocks", + "pattern": "/^\\s*```[\\s\\S]*?^\\s*```/gm" + }, + { + "name": "HTML Tags", + "pattern": "<[^>]*>", + "description": "Reference: https://stackoverflow.com/questions/11229831/regular-expression-to-remove-html-tags-from-a-string" + } + ], + "ignoreRegExpList": [ + "Markdown links", + "Markdown code blocks", + "Inline code blocks", + "Link contents", + "Snippet references", + "Snippet references 2", + "Multi-line code blocks", + "HTML Tags" + ], + "ignorePaths": [ + "docs/_changelog/**/*.md", + "docs/articles/team.md" + ] +} diff --git a/docs/articles/configs/configoptions.md b/docs/articles/configs/configoptions.md index a06e4dcd1c..fe46284bf2 100644 --- a/docs/articles/configs/configoptions.md +++ b/docs/articles/configs/configoptions.md @@ -1,4 +1,5 @@ --- +#cspell:ignore configoptions uid: docs.configoptions name: Configoptions --- diff --git a/docs/articles/configs/filters.md b/docs/articles/configs/filters.md index acecda4fd2..eb89d3138c 100644 --- a/docs/articles/configs/filters.md +++ b/docs/articles/configs/filters.md @@ -10,16 +10,16 @@ In this case, you can *filter* some of them with the help of *filters*. Predefined filters: -| Filter Type | Filters benchmarks by | Console argument | Console example | -|---------------------|-----------------------------|------------------|----------------------------------| -| GlobFilter | Provided glob pattern | filter | --filter *Serializer*.ToStream | -| AttributesFilter | Provided attribute names | attribute | --attribute STAThread | -| AllCategoriesFilter | All Provided category names | categories | --allCategories Priority1 CoreFX | -| AnyCategoriesFilter | Any provided category names | anycategories | --anyCategories Json Xml | -| SimpleFilter | Provided lambda predicate | - | | -| NameFilter | Provided lambda predicate | - | | -| UnionFilter | Logical AND | - | | -| DisjunctionFilter | Logical OR | - | | +| Filter Type | Filters benchmarks by | Console argument | Console example | +|---------------------|-----------------------------|--------------------|----------------------------------| +| GlobFilter | Provided glob pattern | `filter` | --filter *Serializer*.ToStream | +| AttributesFilter | Provided attribute names | `attribute` | --attribute STAThread | +| AllCategoriesFilter | All Provided category names | `categories` | --allCategories Priority1 CoreFX | +| AnyCategoriesFilter | Any provided category names | `anycategories` | --anyCategories Json Xml | +| SimpleFilter | Provided lambda predicate | - | | +| NameFilter | Provided lambda predicate | - | | +| UnionFilter | Logical AND | - | | +| DisjunctionFilter | Logical OR | - | | --- @@ -27,4 +27,4 @@ Predefined filters: [!include[IntroCategories](../samples/IntroCategories.md)] -[!include[IntroJoin](../samples/IntroJoin.md)] \ No newline at end of file +[!include[IntroJoin](../samples/IntroJoin.md)] diff --git a/docs/articles/configs/jobs.md b/docs/articles/configs/jobs.md index cddba24d8e..edc6f708e8 100644 --- a/docs/articles/configs/jobs.md +++ b/docs/articles/configs/jobs.md @@ -12,9 +12,11 @@ Basically, a *job* describes how to run your benchmark. Practically, it's a set There are several categories of characteristics which you can specify. Let's consider each category in detail. ### Id + It's a single string characteristic. It allows to name your job. This name will be used in logs and a part of a folder name with generated files for this job. `Id` doesn't affect benchmark results, but it can be useful for diagnostics. If you don't specify `Id`, random value will be chosen based on other characteristics ### Environment + `Environment` specifies an environment of the job. You can specify the following characteristics: * `Platform`: `x86` or `x64` @@ -41,7 +43,8 @@ It's a single string characteristic. It allows to name your job. This name will BenchmarkDotNet will use host process environment characteristics for non specified values. ### Run -In this category, you can specifiy how to benchmark each method. + +In this category, you can specify how to benchmark each method. * `RunStrategy`: * `Throughput`: default strategy which allows to get good precision level @@ -61,16 +64,18 @@ In this category, you can specifiy how to benchmark each method. Usually, you shouldn't specify such characteristics like `LaunchCount`, `WarmupCount`, `TargetCount`, or `IterationTime` because BenchmarkDotNet has a smart algorithm to choose these values automatically based on received measurements. You can specify it for testing purposes or when you are damn sure that you know the right characteristics for your benchmark (when you set `TargetCount` = `20` you should understand why `20` is a good value for your case). ### Accuracy + If you want to change the accuracy level, you should use the following characteristics instead of manually adjusting values of `WarmupCount`, `TargetCount`, and so on. * `MaxRelativeError`, `MaxAbsoluteError`: Maximum acceptable error for a benchmark (by default, BenchmarkDotNet continue iterations until the actual error is less than the specified error). *In these two characteristics*, the error means half of 99.9% confidence interval. `MaxAbsoluteError` is an absolute `TimeInterval`; doesn't have a default value. `MaxRelativeError` defines max acceptable (`() / Mean`). * `MinIterationTime`: Minimum time of a single iteration. Unlike `Run.IterationTime`, this characteristic specifies only the lower limit. In case of need, BenchmarkDotNet can increase this value. * `MinInvokeCount`: Minimum about of target method invocation. Default value if `4` but you can decrease this value for cases when single invocations takes a lot of time. -* `EvaluateOverhead`: if you benchmark method takes nanoseconds, BenchmarkDotNet overhead can significantly affect measurements. If this characterics is enable, the overhead will be evaluated and subtracted from the result measurements. Default value is `true`. +* `EvaluateOverhead`: if your benchmark method takes nanoseconds, BenchmarkDotNet overhead can significantly affect measurements. If this characteristic is enabled, the overhead will be evaluated and subtracted from the result measurements. Default value is `true`. * `WithOutlierMode`: sometimes you could have outliers in your measurements. Usually these are unexpected outliers which arose because of other processes activities. By default (`OutlierMode.RemoveUpper`), all upper outliers (which is larger than Q3) will be removed from the result measurements. However, some of benchmarks have *expected* outliers. In these situation, you expect that some of invocation can produce outliers measurements (e.g. in case of network activities, cache operations, and so on). If you want to see result statistics with these outliers, you should use `OutlierMode.DontRemove`. If you can also choose `OutlierMode.RemoveLower` (outliers which are smaller than Q1 will be removed) or `OutlierMode.RemoveAll` (all outliers will be removed). See also: @BenchmarkDotNet.Mathematics.OutlierMode -* `AnalyzeLaunchVariance`: this characteristic makes sense only if `Run.LaunchCount` is default. If this mode is enabled and, BenchmarkDotNet will try to perform several launches and detect if there is a veriance between launches. If this mode is disable, only one launch will be performed. +* `AnalyzeLaunchVariance`: this characteristic makes sense only if `Run.LaunchCount` is default. If this mode is enabled and, BenchmarkDotNet will try to perform several launches and detect if there is a variance between launches. If this mode is disable, only one launch will be performed. ### Infrastructure + Usually, you shouldn't specify any characteristics from this section, it can be used for advanced cases only. * `Toolchain`: a toolchain which generates source code for target benchmark methods, builds it, and executes it. BenchmarkDotNet has own toolchains for .NET, .NET Core, Mono and CoreRT projects. If you want, you can define own toolchain. @@ -121,18 +126,24 @@ Basically, it's a good idea to start with predefined values (e.g. `EnvMode.RyuJi Note that the job cannot be modified after it's added into config. Trying to set a value on property of the frozen job will throw an `InvalidOperationException`. Use the `Job.Frozen` property to determine if the code properties can be altered. -If you do want to create a new job based on frozen one (all predefined job values are frozen) you can use the `.With()` extension method +If you do want to create a new job based on frozen one (all predefined job values are frozen) you can use the `.With()` extension method + ```cs var newJob = Job.Dry.With(Platform.X64); ``` + or pass the frozen value as a constructor argument + ```c# var newJob = new Job(Job.Dry) { Env = { Platform = Platform.X64 } }; ``` + or use the `.Apply()` method on unfrozen job + ```c# var newJob = new Job() { Env = { Platform = Platform.X64 } }.Apply(Job.Dry); ``` + in any case the Id property will not be transfered and you must pass it explicitly (using the .ctor id argument or the `.WithId()` extension method). ### Attribute style diff --git a/docs/articles/configs/validators.md b/docs/articles/configs/validators.md index abbf13fc5e..17c0588d29 100644 --- a/docs/articles/configs/validators.md +++ b/docs/articles/configs/validators.md @@ -5,12 +5,12 @@ name: Validators # Validators -A **validator** can validate your benchmarks before they are executed and produce validation errors. -If any of the validation errors is critical, then none of the benchmarks will get executed. +A **validator** can validate your benchmarks before they are executed and produce validation errors. +If any of the validation errors is critical, then none of the benchmarks will get executed. Available validators are: * `BaselineValidator.FailOnError` - it checks if more than 1 Benchmark per class has `Baseline = true` applied. This validator is mandatory. -* `JitOptimizationsValidator.(Dont)FailOnError` - it checks whether any of the referenced assemblies is non-optimized. DontFailOnError version is enabled by default. +* `JitOptimizationsValidator.(Dont)FailOnError` - it checks whether any of the referenced assemblies is non-optimized. `DontFailOnError` version is enabled by default. * `ExecutionValidator.(Dont)FailOnError` - it checks if it is possible to run your benchmarks by executing each of them once. Optional. * `ReturnValueValidator.(Dont)FailOnError` - it checks if non-void benchmarks return equal values. Optional. diff --git a/docs/articles/contributing/building.md b/docs/articles/contributing/building.md index d613961e30..da016f2fc5 100644 --- a/docs/articles/contributing/building.md +++ b/docs/articles/contributing/building.md @@ -25,11 +25,11 @@ The build currently depends on the following prerequisites: - Install [Mono version 5 or higher](https://www.mono-project.com/download/stable/#download-lin) - Install [fsharp package](https://fsharp.org/use/linux/) - Install packages required to .NET Core SDK - - gettext - - libcurl4-openssl-dev - - libicu-dev - - libssl-dev - - libunwind8 + - `gettext` + - `libcurl4-openssl-dev` + - `libicu-dev` + - `libssl-dev` + - `libunwind8` - macOS - Install [Mono version 5 or higher](https://www.mono-project.com/download/stable/#download-mac) @@ -40,15 +40,15 @@ After you have installed these pre-requisites, you can build the BenchmarkDotNet Build has a number of options that you use. Some of the more important options are -- **skiptests** - do not run the tests. This can shorten build times quite a bit. On Windows: `.\build.ps1 --SkipTests=True` or `./build.sh --skiptests=true` on Linux/macOS. +- **`skiptests`** - do not run the tests. This can shorten build times quite a bit. On Windows: `.\build.ps1 --SkipTests=True` or `./build.sh --skiptests=true` on Linux/macOS. -- **configuration** - build the 'Release' or 'Debug' build type. Default value is 'Release'. On Windows: `.\build.ps1 -Configuration Debug` or `./build.sh --configuration debug` on Linux/macOS. +- **`configuration`** - build the 'Release' or 'Debug' build type. Default value is 'Release'. On Windows: `.\build.ps1 -Configuration Debug` or `./build.sh --configuration debug` on Linux/macOS. -- **target** - with this parameter you can run a specific target from build pipeline. Default value is 'Default' target. On Windows: `.\build.ps1 -Target Default` or `./build.sh --target default` on Linux/macOS. Available targets: - - **Default** - run all actions one by one. - - **Clean** - clean all `obj`, `bin` and `artifacts` directories. - - **Restore** - automatically execute `Clean` action and after that restore all NuGet dependencies. - - **Build** - automatically execute `Restore` action, then run MSBuild for the solution file. - - **FastTests** - automatically execute `Build` action, then run all tests from the BenchmarkDotNet.Tests project. - - **SlowTests** - automatically execute `Build` action, then run all tests from the BenchmarkDotNet.IntegrationTests project. - - **Pack** - automatically execute `Build` action and after that creates local NuGet packages. \ No newline at end of file +- **`target`** - with this parameter you can run a specific target from build pipeline. Default value is 'Default' target. On Windows: `.\build.ps1 -Target Default` or `./build.sh --target default` on Linux/macOS. Available targets: + - **`Default`** - run all actions one by one. + - **`Clean`** - clean all `obj`, `bin` and `artifacts` directories. + - **`Restore`** - automatically execute `Clean` action and after that restore all NuGet dependencies. + - **`Build`** - automatically execute `Restore` action, then run MSBuild for the solution file. + - **`FastTests`** - automatically execute `Build` action, then run all tests from the BenchmarkDotNet.Tests project. + - **`SlowTests`** - automatically execute `Build` action, then run all tests from the BenchmarkDotNet.IntegrationTests project. + - **`Pack`** - automatically execute `Build` action and after that creates local NuGet packages. diff --git a/docs/articles/contributing/disassembler.md b/docs/articles/contributing/disassembler.md index ca7e691a08..8d01e42994 100644 --- a/docs/articles/contributing/disassembler.md +++ b/docs/articles/contributing/disassembler.md @@ -13,11 +13,10 @@ We have 3 disassemblers: The MonoDisassembler is very simple: it spawns Mono with the right arguments to get the asm, Mono prints the output to the console and we just parse it. Single class does the job: `MonoDisassembler`. When it comes to Windows disassemblers it's not so easy. To obtain the disassm we are using ClrMD. ClrMD can attach only to the process of same bitness (architecture). -This is why we have two dissasemblers: x64 and x86. The code is the same (single class, linked in two projects) but compiled for two different architectures. We keep both disassemblers in the resources of the BenchmarkDotNet.dll. When we need the disassembler, we search for it in the resources, copy it to the disk and run (it's an exe). +This is why we have two disassemblers: x64 and x86. The code is the same (single class, linked in two projects) but compiled for two different architectures. We keep both disassemblers in the resources of the BenchmarkDotNet.dll. When we need the disassembler, we search for it in the resources, copy it to the disk and run (it's an exe). On Linux it's simpler (only x64 is supported) and we don't spawn a new process (everything is done in-proc). - ### How to debug the disassembler You need to create a new console app project which executes the code that you would like to disassemble. In this app, you need to run the desired code (to get it jitted) and just don't exit before attaching the disassembler and getting the disassembly. diff --git a/docs/articles/features/etwprofiler.md b/docs/articles/features/etwprofiler.md index 3b8118a759..c5e56e2cbb 100644 --- a/docs/articles/features/etwprofiler.md +++ b/docs/articles/features/etwprofiler.md @@ -1,4 +1,5 @@ --- +#cspell:ignore etwprofiler uid: docs.etwprofiler name: EtwProfiler --- @@ -60,7 +61,6 @@ class Program * Passing `-p ETW` or `--profiler ETW` command line argument to `BenchmarkSwitcher` - ## Configuration To configure the new diagnoser you need to create an instance of `EtwProfilerConfig` class and pass it to the `EtwProfiler` constructor. The parameters that `EtwProfilerConfig` ctor takes are: diff --git a/docs/articles/guides/choosing-run-strategy.md b/docs/articles/guides/choosing-run-strategy.md index e216069f85..d399fe4a81 100644 --- a/docs/articles/guides/choosing-run-strategy.md +++ b/docs/articles/guides/choosing-run-strategy.md @@ -1,4 +1,5 @@ --- +#cspell:ignore runstrategy uid: docs.runstrategy name: Choosing RunStrategy --- @@ -28,4 +29,4 @@ public class MyBenchmarkClass [!include[IntroColdStart](../samples/IntroColdStart.md)] -[!include[IntroMonitoring](../samples/IntroMonitoring.md)] \ No newline at end of file +[!include[IntroMonitoring](../samples/IntroMonitoring.md)] diff --git a/docs/articles/guides/console-args.md b/docs/articles/guides/console-args.md index eb179bb636..6540ebab6a 100644 --- a/docs/articles/guides/console-args.md +++ b/docs/articles/guides/console-args.md @@ -31,9 +31,10 @@ Examples: ## List of benchmarks -The `--list` allows you to print all of the available benchmark names. Available options are: +The `--list` allows you to print all of the available benchmark names. Available options are: * `flat` - prints list of the available benchmarks: `--list flat` + ```ini BenchmarkDotNet.Samples.Algo_Md5VsSha256.Md5 BenchmarkDotNet.Samples.Algo_Md5VsSha256.Sha256 @@ -45,7 +46,9 @@ BenchmarkDotNet.Samples.IntroArrayParam.ManualIndexOf BenchmarkDotNet.Samples.IntroBasic.Sleep [...] ``` + * `tree` - prints tree of the available benchmarks: `--list tree` + ```ini BenchmarkDotNet └─Samples @@ -68,6 +71,7 @@ BenchmarkDotNet The `--list` option works with the `--filter` option. Examples: * `--list flat --filter *IntroSetupCleanup*` prints: + ```ini BenchmarkDotNet.Samples.IntroSetupCleanupGlobal.Logic BenchmarkDotNet.Samples.IntroSetupCleanupIteration.Benchmark @@ -76,7 +80,9 @@ BenchmarkDotNet.Samples.IntroSetupCleanupTarget.BenchmarkB BenchmarkDotNet.Samples.IntroSetupCleanupTarget.BenchmarkC BenchmarkDotNet.Samples.IntroSetupCleanupTarget.BenchmarkD ``` + * `--list tree --filter *IntroSetupCleanup*` prints: + ```ini BenchmarkDotNet └─Samples @@ -103,12 +109,12 @@ You can also filter the benchmarks by categories: * `-m`, `--memory` - enables MemoryDiagnoser and prints memory statistics * `-t`, `--threading` - enables `ThreadingDiagnoser` and prints threading statistics * `-d`, `--disasm`- enables DisassemblyDiagnoser and exports diassembly of benchmarked code. When you enable this option, you can use: - - `--disasmDepth` - Sets the recursive depth for the disassembler. - - `--disasmDiff` - Generates diff reports for the disassembler. + * `--disasmDepth` - Sets the recursive depth for the disassembler. + * `--disasmDiff` - Generates diff reports for the disassembler. ## Runtimes -The `--runtimes` or just `-r` allows you to run the benchmarks for selected Runtimes. Available options are: +The `--runtimes` or just `-r` allows you to run the benchmarks for selected Runtimes. Available options are: * Clr - BDN will either use Roslyn (if you run it as .NET app) or latest installed .NET SDK to build the benchmarks (if you run it as .NET Core app). * Core - if you run it as .NET Core app, BDN will use the same target framework moniker, if you run it as .NET app it's going to use netcoreapp2.1. @@ -174,7 +180,7 @@ static IConfig GetGlobalConfig() .AsDefault()); // the KEY to get it working ``` -Now, the default settings are: `WarmupCount=1` but you might still overwrite it from console args like in the example below: +Now, the default settings are: `WarmupCount=1` but you might still overwrite it from console args like in the example below: ```log dotnet run -c Release -- --warmupCount 2 @@ -198,7 +204,7 @@ dotnet run -c Release -- --filter * --runtimes netcoreapp2.0 netcoreapp2.1 --sta * `-e`, `--exporters` GitHub/StackOverflow/RPlot/CSV/JSON/HTML/XML. * `-i`, `--inProcess` (default: false) run benchmarks in the same process, without spawning child process per benchmark. * `-a`, `--artifacts` valid path to an accessible directory where output artifacts will be stored. -* `--outliers` (default: RemoveUpper) DontRemove/RemoveUpper/RemoveLower/RemoveAll. +* `--outliers` (default: RemoveUpper) `DontRemove`/`RemoveUpper`/`RemoveLower`/`RemoveAll`. * `--affinity` affinity mask to set for the benchmark process. * `--allStats` (default: false) Displays all statistics (min, max & more). * `--allCategories` categories to run. If few are provided, only the benchmarks which belong to all of them are going to be executed. @@ -214,7 +220,7 @@ dotnet run -c Release -- --filter * --runtimes netcoreapp2.0 netcoreapp2.1 --sta * `--version` display version information. * `--keepFiles` (default: false) determines if all auto-generated files should be kept or removed after running the benchmarks. * `--noOverwrite` (default: false) determines if the exported result files should not be overwritten. -* `--disableLogFile` disables the logfile. +* `--disableLogFile` disables the log file. * `--maxWidth` max parameter column width, the default is 20. * `--envVars` colon separated environment variables (key:value). * `--strategy` the RunStrategy that should be used. Throughput/ColdStart/Monitoring. diff --git a/docs/articles/samples/IntroColdStart.md b/docs/articles/samples/IntroColdStart.md index d886bcf5c6..7360a66f68 100644 --- a/docs/articles/samples/IntroColdStart.md +++ b/docs/articles/samples/IntroColdStart.md @@ -1,4 +1,5 @@ --- +#cspell:ignore runstrategy uid: BenchmarkDotNet.Samples.IntroColdStart --- @@ -38,4 +39,4 @@ Result 5: 1 op, 10449400.00 ns, 10.4494 ms/op * @docs.runstrategy * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroColdStart ---- \ No newline at end of file +--- diff --git a/docs/articles/samples/IntroMonitoring.md b/docs/articles/samples/IntroMonitoring.md index f1a0a91478..5f2e256417 100644 --- a/docs/articles/samples/IntroMonitoring.md +++ b/docs/articles/samples/IntroMonitoring.md @@ -1,4 +1,5 @@ --- +#cspell:ignore runstrategy uid: BenchmarkDotNet.Samples.IntroMonitoring --- @@ -47,4 +48,4 @@ Result 10: 1 op, 70496300.00 ns, 70.4963 ms/op * @docs.runstrategy * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroMonitoring ---- \ No newline at end of file +--- diff --git a/docs/articles/samples/IntroParamsPriority.md b/docs/articles/samples/IntroParamsPriority.md index 44e3e0c6d2..3de534a0df 100644 --- a/docs/articles/samples/IntroParamsPriority.md +++ b/docs/articles/samples/IntroParamsPriority.md @@ -4,7 +4,7 @@ uid: BenchmarkDotNet.Samples.IntroParamsPriority ## Sample: IntroParamsPriority -In order to sort columns of parameters in the results table you can use the Property `Priority` inside the params attribute. The priority range is `[Int32.MinValue;Int32.MaxValue]`, lower priorites will appear earlier in the column order. The default priority is set to `0`. +In order to sort columns of parameters in the results table you can use the Property `Priority` inside the params attribute. The priority range is `[Int32.MinValue;Int32.MaxValue]`, lower priorities will appear earlier in the column order. The default priority is set to `0`. ### Source code diff --git a/docs/articles/samples/IntroPowerPlan.md b/docs/articles/samples/IntroPowerPlan.md index 2fee0b78b8..a4657f867a 100644 --- a/docs/articles/samples/IntroPowerPlan.md +++ b/docs/articles/samples/IntroPowerPlan.md @@ -5,15 +5,18 @@ uid: BenchmarkDotNet.Samples.IntroPowerPlan ## Sample: IntroPowerPlan This sample shows how we can manipulate with power plans. In BenchmarkDotNet we could change power plan in two ways. The first one is to set one from the list: + * PowerSaver, guid: a1841308-3541-4fab-bc81-f71556f20b4a * Balanced, guid: 381b4222-f694-41f0-9685-ff5bb260df2e * High-Performance, guid: 8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c (the default one) * Ultimate Performance, guid: e9a42b02-d5df-448d-aa00-03f14749eb61 * UserPowerPlan (a current power plan set in computer) -The second one rely on guid string. We could easily found currently set guids with cmd command +The second one rely on guid string. We could easily found currently set GUIDs with cmd command + ```cmd powercfg /list ``` + If we set power plans in two ways at the same time, the second one will be used. ### Source code @@ -25,4 +28,4 @@ If we set power plans in two ways at the same time, the second one will be used. * @docs.powerplans * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroPowerPlan ---- \ No newline at end of file +--- From db8f8d85db44f35af97e763b90f2ffd256eb99e0 Mon Sep 17 00:00:00 2001 From: Andrey Akinshin Date: Thu, 13 Oct 2022 10:59:53 +0300 Subject: [PATCH 21/58] Fix spelling warning in docs/articles/configs.jobs.md --- docs/articles/configs/jobs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/articles/configs/jobs.md b/docs/articles/configs/jobs.md index edc6f708e8..dd52110a63 100644 --- a/docs/articles/configs/jobs.md +++ b/docs/articles/configs/jobs.md @@ -35,7 +35,7 @@ It's a single string characteristic. It allows to name your job. This name will * `CpuGroups`: Specifies whether garbage collection supports multiple CPU groups * `Force`: Specifies whether the BenchmarkDotNet's benchmark runner forces full garbage collection after each benchmark invocation * `AllowVeryLargeObjects`: On 64-bit platforms, enables arrays that are greater than 2 gigabytes (GB) in total size -* `LargeAddressAware`: Specifies that benchmark can handle addresses larger than 2 gigabytes. See also: @BenchmarkDotNet.Samples.IntroLargeAddressAware and [LARGEADDRESSAWARE](https://learn.microsoft.com/cpp/build/reference/largeaddressaware-handle-large-addresses) +* `LargeAddressAware`: Specifies that benchmark can handle addresses larger than 2 gigabytes. See also: @BenchmarkDotNet.Samples.IntroLargeAddressAware and [`LARGEADDRESSAWARE`](https://learn.microsoft.com/cpp/build/reference/largeaddressaware-handle-large-addresses) * `false`: Benchmark uses the defaults (64-bit: enabled; 32-bit: disabled). * `true`: Explicitly specify that benchmark can handle addresses larger than 2 gigabytes. * `EnvironmentVariables`: customized environment variables for target benchmark. See also: @BenchmarkDotNet.Samples.IntroEnvVars From adf9d603471afd0ca14bdc9ae71c89fe8369aa00 Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Mon, 17 Oct 2022 00:54:20 -0700 Subject: [PATCH 22/58] Fix BenchmarkSwitcher user input matching (#2151) --- src/BenchmarkDotNet/Running/UserInteraction.cs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/BenchmarkDotNet/Running/UserInteraction.cs b/src/BenchmarkDotNet/Running/UserInteraction.cs index 8d591e85bd..b7a10426ae 100644 --- a/src/BenchmarkDotNet/Running/UserInteraction.cs +++ b/src/BenchmarkDotNet/Running/UserInteraction.cs @@ -90,18 +90,23 @@ private static IEnumerable GetMatching(IReadOnlyList allTypes, strin if (userInput.IsEmpty()) yield break; + var integerInput = userInput.Where(arg => IsInteger(arg)).ToArray(); + var stringInput = userInput.Where(arg => !IsInteger(arg)).ToArray(); + for (int i = 0; i < allTypes.Count; i++) { var type = allTypes[i]; - if (userInput.Any(arg => type.GetDisplayName().ContainsWithIgnoreCase(arg)) - || userInput.Contains($"#{i}") - || userInput.Contains(i.ToString()) - || userInput.Contains("*")) + if (stringInput.Any(arg => type.GetDisplayName().ContainsWithIgnoreCase(arg)) + || stringInput.Contains($"#{i}") + || integerInput.Contains($"{i}") + || stringInput.Contains("*")) { yield return type; } } + + static bool IsInteger(string str) => int.TryParse(str, out _); } private static void PrintAvailable([NotNull] IReadOnlyList allTypes, ILogger logger) From b758d20b44e6779f1d0cbbce1e000dfe91c448b2 Mon Sep 17 00:00:00 2001 From: norepro <30921834+norepro@users.noreply.github.com> Date: Mon, 17 Oct 2022 04:35:37 -0700 Subject: [PATCH 23/58] Update StaThread intro documentation (#2150) Call out that support for `[STAThread]` was fixed in .NET Core 2.1. Fix a couple minor grammatical errors. --- docs/articles/samples/IntroStaThread.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/articles/samples/IntroStaThread.md b/docs/articles/samples/IntroStaThread.md index d115ebc618..95823b7516 100644 --- a/docs/articles/samples/IntroStaThread.md +++ b/docs/articles/samples/IntroStaThread.md @@ -6,9 +6,9 @@ uid: BenchmarkDotNet.Samples.IntroStaThread If the code you want to benchmark requires `[System.STAThread]` then you need to apply this attribute to the benchmarked method. -BenchmarkDotNet will generate executable with `[STAThread]` applied to it's `Main` method. +BenchmarkDotNet will generate an executable with `[STAThread]` applied to its `Main` method. -Currently it does not work for .NET Core 2.0 due to [this](https://github.com/dotnet/runtime/issues/8834) bug. +Using this feature on .NET Core requires .NET Core 2.1 or newer. Older versions will not work due to [this](https://github.com/dotnet/runtime/issues/8834) bug. ### Source code From eda1a412418ba59eda676cd909accbed6756eeee Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Mon, 17 Oct 2022 15:31:04 +0200 Subject: [PATCH 24/58] remove dependency to Mono.Posix.NETStandard to unblock WASM benchmarks runs (#2154) * remove dependency to Mono.Posix.NETStandard to unblock WASM benchmark runs * Unix samples fix * increase the default timeout to 5 minutes --- .../IntroLargeAddressAware.cs | 3 +- .../PerfCollectProfilerAttribute.cs | 4 +-- src/BenchmarkDotNet/BenchmarkDotNet.csproj | 1 - .../Diagnosers/PerfCollectProfiler.cs | 14 +++++---- .../Diagnosers/PerfCollectProfilerConfig.cs | 4 +-- src/BenchmarkDotNet/Jobs/EnvironmentMode.cs | 2 +- src/BenchmarkDotNet/Portability/Libc.cs | 29 +++++++++++++++++++ .../DotNetCli/DotNetCliCommandExecutor.cs | 7 ++--- 8 files changed, 46 insertions(+), 18 deletions(-) create mode 100644 src/BenchmarkDotNet/Portability/Libc.cs diff --git a/samples/BenchmarkDotNet.Samples/IntroLargeAddressAware.cs b/samples/BenchmarkDotNet.Samples/IntroLargeAddressAware.cs index 964901c3ad..9751c1837b 100644 --- a/samples/BenchmarkDotNet.Samples/IntroLargeAddressAware.cs +++ b/samples/BenchmarkDotNet.Samples/IntroLargeAddressAware.cs @@ -1,4 +1,5 @@ using System; +using System.Runtime.InteropServices; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Environments; @@ -17,7 +18,7 @@ public Config() AddJob(Job.Default .WithRuntime(ClrRuntime.Net462) .WithPlatform(Platform.X86) - .WithLargeAddressAware() + .WithLargeAddressAware(value: RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) .WithId("Framework")); } } diff --git a/src/BenchmarkDotNet/Attributes/PerfCollectProfilerAttribute.cs b/src/BenchmarkDotNet/Attributes/PerfCollectProfilerAttribute.cs index 8b3e0bb1d2..ec72e927c4 100644 --- a/src/BenchmarkDotNet/Attributes/PerfCollectProfilerAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/PerfCollectProfilerAttribute.cs @@ -8,8 +8,8 @@ namespace BenchmarkDotNet.Attributes public class PerfCollectProfilerAttribute : Attribute, IConfigSource { /// When set to true, benchmarks will be executed one more time with the profiler attached. If set to false, there will be no extra run but the results will contain overhead. False by default. - /// How long should we wait for the perfcollect script to finish processing the trace. 120s by default. - public PerfCollectProfilerAttribute(bool performExtraBenchmarksRun = false, int timeoutInSeconds = 120) + /// How long should we wait for the perfcollect script to finish processing the trace. 300s by default. + public PerfCollectProfilerAttribute(bool performExtraBenchmarksRun = false, int timeoutInSeconds = 300) { Config = ManualConfig.CreateEmpty().AddDiagnoser(new PerfCollectProfiler(new PerfCollectProfilerConfig(performExtraBenchmarksRun, timeoutInSeconds))); } diff --git a/src/BenchmarkDotNet/BenchmarkDotNet.csproj b/src/BenchmarkDotNet/BenchmarkDotNet.csproj index 6e700e69bb..4e28175a44 100644 --- a/src/BenchmarkDotNet/BenchmarkDotNet.csproj +++ b/src/BenchmarkDotNet/BenchmarkDotNet.csproj @@ -21,7 +21,6 @@ - diff --git a/src/BenchmarkDotNet/Diagnosers/PerfCollectProfiler.cs b/src/BenchmarkDotNet/Diagnosers/PerfCollectProfiler.cs index a050beb5dd..c31896dcaf 100644 --- a/src/BenchmarkDotNet/Diagnosers/PerfCollectProfiler.cs +++ b/src/BenchmarkDotNet/Diagnosers/PerfCollectProfiler.cs @@ -3,6 +3,7 @@ using System.Diagnostics; using System.IO; using System.Linq; +using System.Runtime.InteropServices; using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Exporters; @@ -20,7 +21,7 @@ using BenchmarkDotNet.Toolchains.NativeAot; using BenchmarkDotNet.Validators; using JetBrains.Annotations; -using Mono.Unix.Native; +using RuntimeInformation = BenchmarkDotNet.Portability.RuntimeInformation; namespace BenchmarkDotNet.Diagnosers { @@ -58,7 +59,7 @@ public IEnumerable Validate(ValidationParameters validationPara yield break; } - if (Syscall.getuid() != 0) + if (libc.getuid() != 0) { yield return new ValidationError(true, "You must run as root to use PerfCollectProfiler."); yield break; @@ -102,9 +103,10 @@ private bool TryInstallPerfCollect(ValidationParameters validationParameters) string script = ResourceHelper.LoadTemplate(perfCollectFile.Name); File.WriteAllText(perfCollectFile.FullName, script); - if (Syscall.chmod(perfCollectFile.FullName, FilePermissions.S_IXUSR) != 0) + if (libc.chmod(perfCollectFile.FullName, libc.FilePermissions.S_IXUSR) != 0) { - logger.WriteError($"Unable to make perfcollect script an executable, the last error was: {Syscall.GetLastError()}"); + int lastError = Marshal.GetLastWin32Error(); + logger.WriteError($"Unable to make perfcollect script an executable, the last error was: {lastError}"); } else { @@ -158,9 +160,9 @@ private void StopCollection(DiagnoserActionParameters parameters) { if (!perfCollectProcess.HasExited) { - if (Syscall.kill(perfCollectProcess.Id, Signum.SIGINT) != 0) + if (libc.kill(perfCollectProcess.Id, libc.Signals.SIGINT) != 0) { - var lastError = Stdlib.GetLastError(); + int lastError = Marshal.GetLastWin32Error(); logger.WriteLineError($"kill(perfcollect, SIGINT) failed with {lastError}"); } diff --git a/src/BenchmarkDotNet/Diagnosers/PerfCollectProfilerConfig.cs b/src/BenchmarkDotNet/Diagnosers/PerfCollectProfilerConfig.cs index 8d025c3366..6b1ad2d7d1 100644 --- a/src/BenchmarkDotNet/Diagnosers/PerfCollectProfilerConfig.cs +++ b/src/BenchmarkDotNet/Diagnosers/PerfCollectProfilerConfig.cs @@ -5,8 +5,8 @@ namespace BenchmarkDotNet.Diagnosers public class PerfCollectProfilerConfig { /// When set to true, benchmarks will be executed one more time with the profiler attached. If set to false, there will be no extra run but the results will contain overhead. False by default. - /// How long should we wait for the perfcollect script to finish processing the trace. 120s by default. - public PerfCollectProfilerConfig(bool performExtraBenchmarksRun = false, int timeoutInSeconds = 120) + /// How long should we wait for the perfcollect script to finish processing the trace. 300s by default. + public PerfCollectProfilerConfig(bool performExtraBenchmarksRun = false, int timeoutInSeconds = 300) { RunMode = performExtraBenchmarksRun ? RunMode.ExtraRun : RunMode.NoOverhead; Timeout = TimeSpan.FromSeconds(timeoutInSeconds); diff --git a/src/BenchmarkDotNet/Jobs/EnvironmentMode.cs b/src/BenchmarkDotNet/Jobs/EnvironmentMode.cs index 21a4ff3ead..8fe04e5764 100644 --- a/src/BenchmarkDotNet/Jobs/EnvironmentMode.cs +++ b/src/BenchmarkDotNet/Jobs/EnvironmentMode.cs @@ -107,7 +107,7 @@ public bool LargeAddressAware get => LargeAddressAwareCharacteristic[this]; set { - if (!RuntimeInformation.IsWindows()) + if (value && !RuntimeInformation.IsWindows()) { throw new NotSupportedException("LargeAddressAware is a Windows-specific concept."); } diff --git a/src/BenchmarkDotNet/Portability/Libc.cs b/src/BenchmarkDotNet/Portability/Libc.cs new file mode 100644 index 0000000000..c9a3cf1322 --- /dev/null +++ b/src/BenchmarkDotNet/Portability/Libc.cs @@ -0,0 +1,29 @@ +using System.Runtime.InteropServices; + +namespace BenchmarkDotNet.Portability +{ + internal static class libc + { + [DllImport(nameof(libc))] + internal static extern int getppid(); + + [DllImport(nameof(libc))] + internal static extern uint getuid(); + + [DllImport(nameof(libc), SetLastError = true)] + internal static extern int kill(int pid, int sig); + + [DllImport(nameof(libc), SetLastError = true)] + internal static extern int chmod(string path, uint mode); + + internal static class Signals + { + internal const int SIGINT = 2; + } + + internal static class FilePermissions + { + internal const uint S_IXUSR = 0x40u; + } + } +} diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs index a6583ad734..9b41dedcb0 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs @@ -3,13 +3,13 @@ using System.ComponentModel; using System.Diagnostics; using System.Linq; -using System.Runtime.InteropServices; using System.Text; using System.Text.RegularExpressions; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Portability; using JetBrains.Annotations; namespace BenchmarkDotNet.Toolchains.DotNetCli @@ -140,7 +140,7 @@ private static string GetDefaultDotNetCliPath() if (!Portability.RuntimeInformation.IsLinux()) return "dotnet"; - using (var parentProcess = Process.GetProcessById(getppid())) + using (var parentProcess = Process.GetProcessById(libc.getppid())) { string parentPath = parentProcess.MainModule?.FileName ?? string.Empty; // sth like /snap/dotnet-sdk/112/dotnet and we should use the exact path instead of just "dotnet" @@ -154,9 +154,6 @@ private static string GetDefaultDotNetCliPath() } } - [DllImport("libc")] - private static extern int getppid(); - internal static string GetSdkPath(string cliPath) { DotNetCliCommand cliCommand = new ( From 58d4bae8996b713f21d6472702d95b21a77f79c9 Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Mon, 17 Oct 2022 12:53:26 -0700 Subject: [PATCH 25/58] Make ParamsAllValues validator mandatory (#2152) --- src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs index 5aa1746121..677574bf86 100644 --- a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs +++ b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs @@ -26,7 +26,8 @@ public static class ImmutableConfigBuilder ConfigValidator.DontFailOnError, ShadowCopyValidator.DontFailOnError, JitOptimizationsValidator.DontFailOnError, - DeferredExecutionValidator.DontFailOnError + DeferredExecutionValidator.DontFailOnError, + ParamsAllValuesValidator.FailOnError }; /// From 28bf214dee0b1f6937851e6f1d09f1f5ae133df1 Mon Sep 17 00:00:00 2001 From: Emanuel Ramos <40024962+emanuel-v-r@users.noreply.github.com> Date: Mon, 17 Oct 2022 21:15:55 +0100 Subject: [PATCH 26/58] adding validation errors when the benchmarks are unsupported (#2148) * Refactoring tool chains in order to return validation errors * Replace IsSupported method by Validate which returns the errors instead of only a bool * Extract printing logic from tool chains * Remove logger dependency from IToolChain Co-authored-by: emanuelra-tr Co-authored-by: Adam Sitnik Co-authored-by: Yegor Stepanov --- samples/BenchmarkDotNet.Samples/IntroWasm.cs | 2 +- .../ConsoleArguments/ConfigParser.cs | 25 +++++--- .../Loggers/LoggerExtensions.cs | 2 +- src/BenchmarkDotNet/Reports/Summary.cs | 9 +-- .../Running/BenchmarkRunnerClean.cs | 60 +++++++++++++------ .../Running/BenchmarkRunnerDirty.cs | 4 +- src/BenchmarkDotNet/Running/BuildPartition.cs | 7 +-- .../Toolchains/CoreRun/CoreRunToolchain.cs | 19 +++--- .../CsProj/CsProjClassicNetToolchain.cs | 27 +++++---- .../Toolchains/CsProj/CsProjCoreToolchain.cs | 45 ++++++++------ src/BenchmarkDotNet/Toolchains/IToolchain.cs | 7 ++- .../InProcessNoEmitToolchain.cs | 14 ++--- .../InProcess/InProcessToolchain.cs | 13 ++-- .../InProcess/InProcessValidator.cs | 29 +++------ .../Toolchains/Mono/MonoAotToolchain.cs | 39 ++++++------ .../{WasmToolChain.cs => WasmToolchain.cs} | 40 +++++++------ .../Toolchains/Roslyn/RoslynToolchain.cs | 33 +++++----- src/BenchmarkDotNet/Toolchains/Toolchain.cs | 38 +++++++----- .../Toolchains/ToolchainExtensions.cs | 25 ++++++-- .../Validators/ValidationError.cs | 4 +- .../InProcessTest.cs | 23 ++++++- .../NugetReferenceTests.cs | 31 +++++----- 22 files changed, 289 insertions(+), 207 deletions(-) rename src/BenchmarkDotNet/Toolchains/MonoWasm/{WasmToolChain.cs => WasmToolchain.cs} (59%) diff --git a/samples/BenchmarkDotNet.Samples/IntroWasm.cs b/samples/BenchmarkDotNet.Samples/IntroWasm.cs index d2e55aca68..aaf513d79a 100644 --- a/samples/BenchmarkDotNet.Samples/IntroWasm.cs +++ b/samples/BenchmarkDotNet.Samples/IntroWasm.cs @@ -39,7 +39,7 @@ public static void Run() NetCoreAppSettings netCoreAppSettings = new NetCoreAppSettings( targetFrameworkMoniker: "net5.0", runtimeFrameworkVersion: null, name: "Wasm", customDotNetCliPath: cliPath); - IToolchain toolChain = WasmToolChain.From(netCoreAppSettings); + IToolchain toolChain = WasmToolchain.From(netCoreAppSettings); BenchmarkRunner.Run(DefaultConfig.Instance .AddJob(Job.ShortRun.WithRuntime(runtime).WithToolchain(toolChain))); diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index 5f5203cc30..a208cbd2c6 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -17,13 +17,13 @@ using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Reports; -using BenchmarkDotNet.Toolchains.NativeAot; using BenchmarkDotNet.Toolchains.CoreRun; using BenchmarkDotNet.Toolchains.CsProj; using BenchmarkDotNet.Toolchains.DotNetCli; using BenchmarkDotNet.Toolchains.InProcess.Emit; -using BenchmarkDotNet.Toolchains.MonoWasm; using BenchmarkDotNet.Toolchains.MonoAotLLVM; +using BenchmarkDotNet.Toolchains.MonoWasm; +using BenchmarkDotNet.Toolchains.NativeAot; using CommandLine; using Perfolizer.Horology; using Perfolizer.Mathematics.OutlierDetection; @@ -113,7 +113,7 @@ private static bool Validate(CommandLineOptions options, ILogger logger) } else if (runtimeMoniker == RuntimeMoniker.MonoAOTLLVM && (options.AOTCompilerPath == null || options.AOTCompilerPath.IsNotNullButDoesNotExist())) { - logger.WriteLineError($"The provided {nameof(options.AOTCompilerPath)} \"{ options.AOTCompilerPath }\" does NOT exist. It MUST be provided."); + logger.WriteLineError($"The provided {nameof(options.AOTCompilerPath)} \"{options.AOTCompilerPath}\" does NOT exist. It MUST be provided."); } } @@ -265,7 +265,7 @@ private static Job GetBaseJob(CommandLineOptions options, IConfig globalConfig) baseJob = baseJob.WithOutlierMode(options.Outliers); if (options.Affinity.HasValue) - baseJob = baseJob.WithAffinity((IntPtr) options.Affinity.Value); + baseJob = baseJob.WithAffinity((IntPtr)options.Affinity.Value); if (options.LaunchCount.HasValue) baseJob = baseJob.WithLaunchCount(options.LaunchCount.Value); @@ -376,6 +376,7 @@ private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, Comma return baseJob .WithRuntime(runtimeMoniker.GetRuntime()) .WithToolchain(CsProjClassicNetToolchain.From(runtimeId, options.RestorePath?.FullName)); + case RuntimeMoniker.NetCoreApp20: case RuntimeMoniker.NetCoreApp21: case RuntimeMoniker.NetCoreApp22: @@ -390,26 +391,37 @@ private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, Comma return baseJob .WithRuntime(runtimeMoniker.GetRuntime()) .WithToolchain(CsProjCoreToolchain.From(new NetCoreAppSettings(runtimeId, null, runtimeId, options.CliPath?.FullName, options.RestorePath?.FullName))); + case RuntimeMoniker.Mono: return baseJob.WithRuntime(new MonoRuntime("Mono", options.MonoPath?.FullName)); + case RuntimeMoniker.NativeAot60: return CreateAotJob(baseJob, options, runtimeMoniker, "6.0.0-*", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json"); + case RuntimeMoniker.NativeAot70: return CreateAotJob(baseJob, options, runtimeMoniker, "", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet7/nuget/v3/index.json"); + case RuntimeMoniker.Wasm: return MakeWasmJob(baseJob, options, RuntimeInformation.IsNetCore ? CoreRuntime.GetCurrentVersion().MsBuildMoniker : "net5.0", runtimeMoniker); + case RuntimeMoniker.WasmNet50: return MakeWasmJob(baseJob, options, "net5.0", runtimeMoniker); + case RuntimeMoniker.WasmNet60: return MakeWasmJob(baseJob, options, "net6.0", runtimeMoniker); + case RuntimeMoniker.WasmNet70: return MakeWasmJob(baseJob, options, "net7.0", runtimeMoniker); + case RuntimeMoniker.MonoAOTLLVM: return MakeMonoAOTLLVMJob(baseJob, options, RuntimeInformation.IsNetCore ? CoreRuntime.GetCurrentVersion().MsBuildMoniker : "net6.0"); + case RuntimeMoniker.MonoAOTLLVMNet60: return MakeMonoAOTLLVMJob(baseJob, options, "net6.0"); + case RuntimeMoniker.MonoAOTLLVMNet70: return MakeMonoAOTLLVMJob(baseJob, options, "net7.0"); + default: throw new NotSupportedException($"Runtime {runtimeId} is not supported"); } @@ -467,7 +479,7 @@ private static Job MakeWasmJob(Job baseJob, CommandLineOptions options, string m wasmDataDir: options.WasmDataDirectory?.FullName, moniker: moniker); - var toolChain = WasmToolChain.From(new NetCoreAppSettings( + var toolChain = WasmToolchain.From(new NetCoreAppSettings( targetFrameworkMoniker: wasmRuntime.MsBuildMoniker, runtimeFrameworkVersion: null, name: wasmRuntime.Name, @@ -555,7 +567,6 @@ private static string GetCoreRunToolchainDisplayName(IReadOnlyList pat } } - if (commonLongestPrefixIndex <= 1) return coreRunPath.FullName; @@ -573,4 +584,4 @@ private static bool TryParse(string runtime, out RuntimeMoniker runtimeMoniker) : Enum.TryParse(runtime.Substring(0, index).Replace(".", string.Empty), ignoreCase: true, out runtimeMoniker); } } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Loggers/LoggerExtensions.cs b/src/BenchmarkDotNet/Loggers/LoggerExtensions.cs index 89355e2f2b..866620c3fb 100644 --- a/src/BenchmarkDotNet/Loggers/LoggerExtensions.cs +++ b/src/BenchmarkDotNet/Loggers/LoggerExtensions.cs @@ -20,7 +20,7 @@ public static class LoggerExtensions public static void WriteLineHint(this ILogger logger, string text) => logger.WriteLine(LogKind.Hint, text); - public static void Write(this ILogger logger, string text) => logger.Write(LogKind.Default, text); + public static void Write(this ILogger logger, string text) => logger.Write(LogKind.Default, text); [PublicAPI] public static void WriteHelp(this ILogger logger, string text) => logger.Write(LogKind.Help, text); diff --git a/src/BenchmarkDotNet/Reports/Summary.cs b/src/BenchmarkDotNet/Reports/Summary.cs index f57d4868fe..08a1efecc9 100644 --- a/src/BenchmarkDotNet/Reports/Summary.cs +++ b/src/BenchmarkDotNet/Reports/Summary.cs @@ -83,11 +83,8 @@ public Summary( public bool IsMultipleRuntimes => isMultipleRuntimes ??= BenchmarksCases.Length > 1 ? BenchmarksCases.Select(benchmark => benchmark.GetRuntime()).Distinct().Count() > 1 : false; - internal static Summary NothingToRun(string title, string resultsDirectoryPath, string logFilePath) - => new Summary(title, ImmutableArray.Empty, HostEnvironmentInfo.GetCurrent(), resultsDirectoryPath, logFilePath, TimeSpan.Zero, DefaultCultureInfo.Instance, ImmutableArray.Empty, ImmutableArray.Empty); - - internal static Summary ValidationFailed(string title, string resultsDirectoryPath, string logFilePath, ImmutableArray validationErrors) - => new Summary(title, ImmutableArray.Empty, HostEnvironmentInfo.GetCurrent(), resultsDirectoryPath, logFilePath, TimeSpan.Zero, DefaultCultureInfo.Instance, validationErrors, ImmutableArray.Empty); + internal static Summary ValidationFailed(string title, string resultsDirectoryPath, string logFilePath, ImmutableArray? validationErrors = null) + => new Summary(title, ImmutableArray.Empty, HostEnvironmentInfo.GetCurrent(), resultsDirectoryPath, logFilePath, TimeSpan.Zero, DefaultCultureInfo.Instance, validationErrors ?? ImmutableArray.Empty, ImmutableArray.Empty); internal static Summary Join(List summaries, ClockSpan clockSpan) => new Summary( @@ -169,4 +166,4 @@ private static SummaryStyle GetConfiguredSummaryStyleOrDefaultOne(ImmutableArray .SingleOrDefault() ?? SummaryStyle.Default; } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs index c283d40d76..0de7f8c7a9 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs @@ -47,13 +47,19 @@ internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos) { var compositeLogger = CreateCompositeLogger(benchmarkRunInfos, streamLogger); - var supportedBenchmarks = GetSupportedBenchmarks(benchmarkRunInfos, compositeLogger, resolver); - if (!supportedBenchmarks.Any(benchmarks => benchmarks.BenchmarksCases.Any())) - return new[] { Summary.NothingToRun(title, resultsFolderPath, logFilePath) }; + compositeLogger.WriteLineInfo("// Validating benchmarks:"); + + var (supportedBenchmarks, validationErrors) = GetSupportedBenchmarks(benchmarkRunInfos, compositeLogger, resolver); + + validationErrors.AddRange(Validate(supportedBenchmarks)); + + PrintValidationErrors(compositeLogger, validationErrors); - var validationErrors = Validate(supportedBenchmarks, compositeLogger); if (validationErrors.Any(validationError => validationError.IsCritical)) - return new[] { Summary.ValidationFailed(title, resultsFolderPath, logFilePath, validationErrors) }; + return new[] { Summary.ValidationFailed(title, resultsFolderPath, logFilePath, validationErrors.ToImmutableArray()) }; + + if (!supportedBenchmarks.Any(benchmarks => benchmarks.BenchmarksCases.Any())) + return new[] { Summary.ValidationFailed(title, resultsFolderPath, logFilePath) }; int totalBenchmarkCount = supportedBenchmarks.Sum(benchmarkInfo => benchmarkInfo.BenchmarksCases.Length); int benchmarksToRunCount = totalBenchmarkCount; @@ -145,7 +151,7 @@ private static Summary Run(BenchmarkRunInfo benchmarkRunInfo, var cultureInfo = config.CultureInfo ?? DefaultCultureInfo.Instance; var reports = new List(); string title = GetTitle(new[] { benchmarkRunInfo }); - var consoleTitle = RuntimeInformation.IsWindows() ? Console.Title : string.Empty; + var consoleTitle = RuntimeInformation.IsWindows() ? Console.Title : string.Empty; logger.WriteLineInfo($"// Found {benchmarks.Length} benchmarks:"); foreach (var benchmark in benchmarks) @@ -236,7 +242,7 @@ private static Summary Run(BenchmarkRunInfo benchmarkRunInfo, logFilePath, runEnd.GetTimeSpan() - runStart.GetTimeSpan(), cultureInfo, - Validate(new[] { benchmarkRunInfo }, NullLogger.Instance), // validate them once again, but don't print the output + Validate(benchmarkRunInfo), // validate them once again, but don't print the output config.GetColumnHidingRules().ToImmutableArray()); } @@ -306,10 +312,8 @@ private static void PrintSummary(ILogger logger, ImmutableConfig config, Summary logger.WriteLineHeader("// ***** BenchmarkRunner: End *****"); } - private static ImmutableArray Validate(BenchmarkRunInfo[] benchmarks, ILogger logger) + private static ImmutableArray Validate(params BenchmarkRunInfo[] benchmarks) { - logger.WriteLineInfo("// Validating benchmarks:"); - var validationErrors = new List(); if (benchmarks.Any(b => b.Config.Options.IsSet(ConfigOptions.JoinSummary))) @@ -326,9 +330,6 @@ private static ImmutableArray Validate(BenchmarkRunInfo[] bench foreach (var benchmarkRunInfo in benchmarks) validationErrors.AddRange(benchmarkRunInfo.Config.GetCompositeValidator().Validate(new ValidationParameters(benchmarkRunInfo.BenchmarksCases, benchmarkRunInfo.Config))); - foreach (var validationError in validationErrors.Distinct()) - logger.WriteLineError(validationError.Message); - return validationErrors.ToImmutableArray(); } @@ -531,14 +532,25 @@ private static ExecuteResult RunExecute(ILogger logger, BenchmarkCase benchmarkC private static void LogTotalTime(ILogger logger, TimeSpan time, int executedBenchmarksCount, string message = "Total time") => logger.WriteLineStatistic($"{message}: {time.ToFormattedTotalTime(DefaultCultureInfo.Instance)}, executed benchmarks: {executedBenchmarksCount}"); - private static BenchmarkRunInfo[] GetSupportedBenchmarks(BenchmarkRunInfo[] benchmarkRunInfos, ILogger logger, IResolver resolver) - => benchmarkRunInfos.Select(info => new BenchmarkRunInfo( - info.BenchmarksCases.Where(benchmark => benchmark.GetToolchain().IsSupported(benchmark, logger, resolver)).ToArray(), + private static (BenchmarkRunInfo[], List) GetSupportedBenchmarks(BenchmarkRunInfo[] benchmarkRunInfos, ILogger logger, IResolver resolver) + { + List validationErrors = new (); + + var benchmarksRunInfo = benchmarkRunInfos.Select(info => new BenchmarkRunInfo( + info.BenchmarksCases.Where(benchmark => + { + var errors = benchmark.GetToolchain().Validate(benchmark, resolver).ToArray(); + validationErrors.AddRange(errors); + return !errors.Any(); + }).ToArray(), info.Type, info.Config)) .Where(infos => infos.BenchmarksCases.Any()) .ToArray(); + return (benchmarkRunInfos, validationErrors); + } + private static string GetRootArtifactsFolderPath(BenchmarkRunInfo[] benchmarkRunInfos) { var defaultPath = DefaultConfig.Instance.ArtifactsPath; @@ -652,5 +664,19 @@ private static TimeSpan GetEstimatedFinishTime(in StartedClock runsChronometer, TimeSpan fromNow = TimeSpan.FromSeconds(avgSecondsPerBenchmark * benchmarksToRunCount); return fromNow; } + + private static void PrintValidationErrors(ILogger logger, IEnumerable validationErrors) + { + foreach (var validationError in validationErrors.Distinct()) + { + if (validationError.BenchmarkCase != null) + { + logger.WriteLineInfo($"// Benchmark {validationError.BenchmarkCase.DisplayInfo}"); + } + + logger.WriteLineError($"// * {validationError.Message}"); + logger.WriteLine(); + } + } } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Running/BenchmarkRunnerDirty.cs b/src/BenchmarkDotNet/Running/BenchmarkRunnerDirty.cs index 0d51ae08a7..f353261111 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkRunnerDirty.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkRunnerDirty.cs @@ -136,7 +136,7 @@ private static Summary RunWithExceptionHandling(Func run) catch (InvalidBenchmarkDeclarationException e) { ConsoleLogger.Default.WriteLineError(e.Message); - return Summary.NothingToRun(e.Message, string.Empty, string.Empty); + return Summary.ValidationFailed(e.Message, string.Empty, string.Empty); } } @@ -149,7 +149,7 @@ private static Summary[] RunWithExceptionHandling(Func run) catch (InvalidBenchmarkDeclarationException e) { ConsoleLogger.Default.WriteLineError(e.Message); - return new[] { Summary.NothingToRun(e.Message, string.Empty, string.Empty) }; + return new[] { Summary.ValidationFailed(e.Message, string.Empty, string.Empty) }; } } } diff --git a/src/BenchmarkDotNet/Running/BuildPartition.cs b/src/BenchmarkDotNet/Running/BuildPartition.cs index 5275acd61b..f89863215c 100644 --- a/src/BenchmarkDotNet/Running/BuildPartition.cs +++ b/src/BenchmarkDotNet/Running/BuildPartition.cs @@ -1,13 +1,10 @@ using System; using System.IO; -using System.Linq; using System.Reflection; using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Portability; -using BenchmarkDotNet.Toolchains.NativeAot; using BenchmarkDotNet.Toolchains.CsProj; using BenchmarkDotNet.Toolchains.MonoWasm; using BenchmarkDotNet.Toolchains.Roslyn; @@ -51,7 +48,7 @@ public BuildPartition(BenchmarkBuildInfo[] benchmarks, IResolver resolver) public bool IsNativeAot => RepresentativeBenchmarkCase.Job.IsNativeAOT(); public bool IsWasm => Runtime is WasmRuntime // given job can have Wasm toolchain set, but Runtime == default ;) - || (RepresentativeBenchmarkCase.Job.Infrastructure.TryGetToolchain(out var toolchain) && toolchain is WasmToolChain); + || (RepresentativeBenchmarkCase.Job.Infrastructure.TryGetToolchain(out var toolchain) && toolchain is WasmToolchain); public bool IsNetFramework => Runtime is ClrRuntime || (RepresentativeBenchmarkCase.Job.Infrastructure.TryGetToolchain(out var toolchain) && (toolchain is RoslynToolchain || toolchain is CsProjClassicNetToolchain)); @@ -75,4 +72,4 @@ private static string GetResolvedAssemblyLocation(Assembly assembly) => // manually construct the path. assembly.Location.Length == 0 ? Path.Combine(AppContext.BaseDirectory, assembly.GetName().Name) : assembly.Location; } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunToolchain.cs b/src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunToolchain.cs index d741988424..03d677a5aa 100644 --- a/src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunToolchain.cs @@ -1,9 +1,10 @@ using System; +using System.Collections.Generic; using System.IO; using BenchmarkDotNet.Characteristics; -using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.DotNetCli; +using BenchmarkDotNet.Validators; namespace BenchmarkDotNet.Toolchains.CoreRun { @@ -57,18 +58,18 @@ public CoreRunToolchain(FileInfo coreRun, bool createCopy = true, public override string ToString() => Name; - public bool IsSupported(BenchmarkCase benchmark, ILogger logger, IResolver resolver) + public IEnumerable Validate(BenchmarkCase benchmark, IResolver resolver) { if (!SourceCoreRun.Exists) { - logger.WriteLineError($"Provided CoreRun path does not exist, benchmark '{benchmark.DisplayInfo}' will not be executed. Please remember that BDN expects path to CoreRun.exe (corerun on Unix), not to Core_Root folder."); - return false; + yield return new ValidationError(true, + $"Provided CoreRun path does not exist, benchmark '{benchmark.DisplayInfo}' will not be executed. Please remember that BDN expects path to CoreRun.exe (corerun on Unix), not to Core_Root folder.", + benchmark); + } + else if (Toolchain.IsCliPathInvalid(CustomDotNetCliPath?.FullName, benchmark, out var invalidCliError)) + { + yield return invalidCliError; } - - if (Toolchain.InvalidCliPath(CustomDotNetCliPath?.FullName, benchmark, logger)) - return false; - - return true; } private static FileInfo GetShadowCopyPath(FileInfo coreRunPath) diff --git a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjClassicNetToolchain.cs b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjClassicNetToolchain.cs index c670870820..be6fae3f4b 100644 --- a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjClassicNetToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjClassicNetToolchain.cs @@ -1,8 +1,9 @@ -using BenchmarkDotNet.Characteristics; -using BenchmarkDotNet.Loggers; +using System.Collections.Generic; +using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.DotNetCli; +using BenchmarkDotNet.Validators; using JetBrains.Annotations; namespace BenchmarkDotNet.Toolchains.CsProj @@ -33,21 +34,23 @@ private CsProjClassicNetToolchain(string targetFrameworkMoniker, string name, st public static IToolchain From(string targetFrameworkMoniker, string packagesPath = null) => new CsProjClassicNetToolchain(targetFrameworkMoniker, targetFrameworkMoniker, packagesPath); - public override bool IsSupported(BenchmarkCase benchmarkCase, ILogger logger, IResolver resolver) + public override IEnumerable Validate(BenchmarkCase benchmarkCase, IResolver resolver) { - if (!base.IsSupported(benchmarkCase, logger, resolver)) - return false; + foreach (var validationError in base.Validate(benchmarkCase, resolver)) + { + yield return validationError; + } if (!RuntimeInformation.IsWindows()) { - logger.WriteLineError($"Classic .NET toolchain is supported only for Windows, benchmark '{benchmarkCase.DisplayInfo}' will not be executed"); - return false; + yield return new ValidationError(true, + $"Classic .NET toolchain is supported only for Windows, benchmark '{benchmarkCase.DisplayInfo}' will not be executed", + benchmarkCase); + } + else if (IsCliPathInvalid(customDotNetCliPath: null, benchmarkCase, out var invalidCliError)) + { + yield return invalidCliError; } - - if (InvalidCliPath(customDotNetCliPath: null, benchmarkCase, logger)) - return false; - - return true; } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjCoreToolchain.cs b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjCoreToolchain.cs index 624824715c..4b7c43feab 100644 --- a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjCoreToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjCoreToolchain.cs @@ -1,13 +1,14 @@ -using BenchmarkDotNet.Characteristics; +using System; +using System.Collections.Generic; +using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.DotNetCli; using BenchmarkDotNet.Toolchains.InProcess.Emit; +using BenchmarkDotNet.Validators; using JetBrains.Annotations; -using System; namespace BenchmarkDotNet.Toolchains.CsProj { @@ -39,38 +40,44 @@ public static IToolchain From(NetCoreAppSettings settings) new DotNetCliExecutor(settings.CustomDotNetCliPath), settings.CustomDotNetCliPath); - public override bool IsSupported(BenchmarkCase benchmarkCase, ILogger logger, IResolver resolver) + public override IEnumerable Validate(BenchmarkCase benchmarkCase, IResolver resolver) { - if (!base.IsSupported(benchmarkCase, logger, resolver)) - return false; + foreach (var validationError in base.Validate(benchmarkCase, resolver)) + { + yield return validationError; + } - if (InvalidCliPath(CustomDotNetCliPath, benchmarkCase, logger)) - return false; + if (IsCliPathInvalid(CustomDotNetCliPath, benchmarkCase, out var invalidCliError)) + { + yield return invalidCliError; + } if (benchmarkCase.Job.HasValue(EnvironmentMode.JitCharacteristic) && benchmarkCase.Job.ResolveValue(EnvironmentMode.JitCharacteristic, resolver) == Jit.LegacyJit) { - logger.WriteLineError($"Currently dotnet cli toolchain supports only RyuJit, benchmark '{benchmarkCase.DisplayInfo}' will not be executed"); - return false; + yield return new ValidationError(true, + $"Currently dotnet cli toolchain supports only RyuJit, benchmark '{benchmarkCase.DisplayInfo}' will not be executed", + benchmarkCase); } if (benchmarkCase.Job.ResolveValue(GcMode.CpuGroupsCharacteristic, resolver)) { - logger.WriteLineError($"Currently project.json does not support CpuGroups (app.config does), benchmark '{benchmarkCase.DisplayInfo}' will not be executed"); - return false; + yield return new ValidationError(true, + $"Currently project.json does not support CpuGroups (app.config does), benchmark '{benchmarkCase.DisplayInfo}' will not be executed", + benchmarkCase); } if (benchmarkCase.Job.ResolveValue(GcMode.AllowVeryLargeObjectsCharacteristic, resolver)) { - logger.WriteLineError($"Currently project.json does not support gcAllowVeryLargeObjects (app.config does), benchmark '{benchmarkCase.DisplayInfo}' will not be executed"); - return false; + yield return new ValidationError(true, + $"Currently project.json does not support gcAllowVeryLargeObjects (app.config does), benchmark '{benchmarkCase.DisplayInfo}' will not be executed", + benchmarkCase); } var benchmarkAssembly = benchmarkCase.Descriptor.Type.Assembly; if (benchmarkAssembly.IsLinqPad()) { - logger.WriteLineError($"Currently CsProjCoreToolchain does not support LINQPad 6+. Please use {nameof(InProcessEmitToolchain)} instead. Benchmark '{benchmarkCase.DisplayInfo}' will not be executed"); - return false; + yield return new ValidationError(true, + $"Currently CsProjCoreToolchain does not support LINQPad 6+. Please use {nameof(InProcessEmitToolchain)} instead.", + benchmarkCase); } - - return true; } public override bool Equals(object obj) => obj is CsProjCoreToolchain typed && Equals(typed); @@ -79,4 +86,4 @@ public override bool IsSupported(BenchmarkCase benchmarkCase, ILogger logger, IR public override int GetHashCode() => Generator.GetHashCode(); } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/IToolchain.cs b/src/BenchmarkDotNet/Toolchains/IToolchain.cs index ce87e07eec..7085b820fe 100644 --- a/src/BenchmarkDotNet/Toolchains/IToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/IToolchain.cs @@ -1,6 +1,7 @@ -using BenchmarkDotNet.Characteristics; -using BenchmarkDotNet.Loggers; +using System.Collections.Generic; +using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Running; +using BenchmarkDotNet.Validators; using JetBrains.Annotations; namespace BenchmarkDotNet.Toolchains @@ -13,6 +14,6 @@ public interface IToolchain IExecutor Executor { get; } bool IsInProcess { get; } - bool IsSupported(BenchmarkCase benchmarkCase, ILogger logger, IResolver resolver); + IEnumerable Validate(BenchmarkCase benchmarkCase, IResolver resolver); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/InProcess.NoEmit/InProcessNoEmitToolchain.cs b/src/BenchmarkDotNet/Toolchains/InProcess.NoEmit/InProcessNoEmitToolchain.cs index 247ea3906a..269661aeb5 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess.NoEmit/InProcessNoEmitToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess.NoEmit/InProcessNoEmitToolchain.cs @@ -1,9 +1,8 @@ using System; - +using System.Collections.Generic; using BenchmarkDotNet.Characteristics; -using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Running; - +using BenchmarkDotNet.Validators; using JetBrains.Annotations; namespace BenchmarkDotNet.Toolchains.InProcess.NoEmit @@ -39,13 +38,8 @@ public InProcessNoEmitToolchain(TimeSpan timeout, bool logOutput) Executor = new InProcessNoEmitExecutor(timeout, logOutput); } - /// Determines whether the specified benchmark is supported. - /// The benchmark. - /// The logger. - /// The resolver. - /// true if the benchmark can be run with the toolchain. - public bool IsSupported(BenchmarkCase benchmarkCase, ILogger logger, IResolver resolver) => - InProcessValidator.IsSupported(benchmarkCase, logger); + public IEnumerable Validate(BenchmarkCase benchmarkCase, IResolver resolver) => + InProcessValidator.Validate(benchmarkCase); /// Name of the toolchain. /// The name of the toolchain. diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/InProcessToolchain.cs b/src/BenchmarkDotNet/Toolchains/InProcess/InProcessToolchain.cs index e3fd301ab6..9dd55eed99 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/InProcessToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/InProcessToolchain.cs @@ -1,8 +1,8 @@ using System; - +using System.Collections.Generic; using BenchmarkDotNet.Characteristics; -using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Running; +using BenchmarkDotNet.Validators; namespace BenchmarkDotNet.Toolchains.InProcess { @@ -39,13 +39,12 @@ public InProcessToolchain(TimeSpan timeout, BenchmarkActionCodegen codegenMode, Executor = new InProcessExecutor(timeout, codegenMode, logOutput); } - /// Determines whether the specified benchmark is supported. + /// Validates the specified benchmark. /// The benchmark. - /// The logger. /// The resolver. - /// true if the benchmark can be run with the toolchain. - public bool IsSupported(BenchmarkCase benchmarkCase, ILogger logger, IResolver resolver) => - InProcessValidator.IsSupported(benchmarkCase, logger); + /// Collection of validation errors, when not empty means that toolchain does not support provided benchmark. + public IEnumerable Validate(BenchmarkCase benchmarkCase, IResolver resolver) => + InProcessValidator.Validate(benchmarkCase); /// Name of the toolchain. /// The name of the toolchain. diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/InProcessValidator.cs b/src/BenchmarkDotNet/Toolchains/InProcess/InProcessValidator.cs index f04287f41d..77cdcfe135 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/InProcessValidator.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/InProcessValidator.cs @@ -1,11 +1,9 @@ using System; using System.Collections.Generic; using System.Linq; - using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.InProcess.Emit; using BenchmarkDotNet.Toolchains.InProcess.NoEmit; @@ -95,26 +93,18 @@ job.Infrastructure.Toolchain is InProcessEmitToolchain /// The instance of validator that DOES fail on error. public static readonly IValidator FailOnError = new InProcessValidator(true); - public static bool IsSupported(BenchmarkCase benchmarkCase, ILogger logger) + public static IEnumerable Validate(BenchmarkCase benchmarkCase) { - var result = new List(); - result.AddRange(ValidateJob(benchmarkCase.Job, true)); - if (benchmarkCase.HasArguments) - result.Add(new ValidationError(true, "Arguments are not supported by the InProcessToolchain yet, see #687 for more details")); - - if (result.Any()) + foreach (var validationError in ValidateJob(benchmarkCase.Job, true)) { - logger.WriteLineInfo($"// Benchmark {benchmarkCase.DisplayInfo}"); - logger.WriteLineInfo("// cannot be run in-process. Validation errors:"); - foreach (var validationError in result) - { - logger.WriteLineInfo($"// * {validationError.Message}"); - } - logger.WriteLine(); - - return false; + yield return new ValidationError( + validationError.IsCritical, + validationError.Message, + benchmarkCase); } - return true; + + if (benchmarkCase.HasArguments) + yield return new ValidationError(true, "Arguments are not supported by the InProcessToolchain yet, see #687 for more details", benchmarkCase); } private static IEnumerable ValidateJob(Job job, bool isCritical) @@ -140,7 +130,6 @@ private static IEnumerable ValidateJob(Job job, bool isCritical } } - private InProcessValidator(bool failOnErrors) { TreatsWarningsAsErrors = failOnErrors; diff --git a/src/BenchmarkDotNet/Toolchains/Mono/MonoAotToolchain.cs b/src/BenchmarkDotNet/Toolchains/Mono/MonoAotToolchain.cs index ec93ace883..acbeeffe38 100644 --- a/src/BenchmarkDotNet/Toolchains/Mono/MonoAotToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/Mono/MonoAotToolchain.cs @@ -1,11 +1,12 @@ -using BenchmarkDotNet.Characteristics; +using System.Collections.Generic; +using System.IO; +using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.Roslyn; +using BenchmarkDotNet.Validators; using JetBrains.Annotations; -using System.IO; namespace BenchmarkDotNet.Toolchains.Mono { @@ -18,39 +19,41 @@ public class MonoAotToolchain : Toolchain { } - public override bool IsSupported(BenchmarkCase benchmarkCase, ILogger logger, IResolver resolver) + public override IEnumerable Validate(BenchmarkCase benchmarkCase, IResolver resolver) { - if (!base.IsSupported(benchmarkCase, logger, resolver)) + foreach (var validationError in base.Validate(benchmarkCase, resolver)) { - return false; + yield return validationError; } - if (!benchmarkCase.Job.Environment.HasValue(EnvironmentMode.RuntimeCharacteristic) || !(benchmarkCase.Job.Environment.Runtime is MonoRuntime)) + if (!benchmarkCase.Job.Environment.HasValue(EnvironmentMode.RuntimeCharacteristic) || benchmarkCase.Job.Environment.Runtime is not MonoRuntime) { - logger.WriteLineError("The MonoAOT toolchain requires the Runtime property to be configured explicitly to an instance of MonoRuntime class"); - return false; + yield return new ValidationError(true, + "The MonoAOT toolchain requires the Runtime property to be configured explicitly to an instance of MonoRuntime class", + benchmarkCase); } if ((benchmarkCase.Job.Environment.Runtime is MonoRuntime monoRuntime) && !string.IsNullOrEmpty(monoRuntime.MonoBclPath) && !Directory.Exists(monoRuntime.MonoBclPath)) { - logger.WriteLineError($"The MonoBclPath provided for MonoAOT toolchain: {monoRuntime.MonoBclPath} does NOT exist."); - return false; + yield return new ValidationError(true, + $"The MonoBclPath provided for MonoAOT toolchain: {monoRuntime.MonoBclPath} does NOT exist.", + benchmarkCase); } if (benchmarkCase.Job.HasValue(InfrastructureMode.BuildConfigurationCharacteristic) && benchmarkCase.Job.ResolveValue(InfrastructureMode.BuildConfigurationCharacteristic, resolver) != InfrastructureMode.ReleaseConfigurationName) { - logger.WriteLineError("The MonoAOT toolchain does not allow to rebuild source project, so defining custom build configuration makes no sense"); - return false; + yield return new ValidationError(true, + "The MonoAOT toolchain does not allow to rebuild source project, so defining custom build configuration makes no sense", + benchmarkCase); } if (benchmarkCase.Job.HasValue(InfrastructureMode.NuGetReferencesCharacteristic)) { - logger.WriteLineError("The MonoAOT toolchain does not allow specifying NuGet package dependencies"); - return false; + yield return new ValidationError(true, + "The MonoAOT toolchain does not allow specifying NuGet package dependencies", + benchmarkCase); } - - return true; } } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmToolChain.cs b/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmToolchain.cs similarity index 59% rename from src/BenchmarkDotNet/Toolchains/MonoWasm/WasmToolChain.cs rename to src/BenchmarkDotNet/Toolchains/MonoWasm/WasmToolchain.cs index 326957d74a..37f816bde9 100644 --- a/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmToolChain.cs +++ b/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmToolchain.cs @@ -1,40 +1,46 @@ -using BenchmarkDotNet.Toolchains.DotNetCli; -using JetBrains.Annotations; +using System.Collections.Generic; using BenchmarkDotNet.Characteristics; -using BenchmarkDotNet.Running; -using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Portability; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains.DotNetCli; +using BenchmarkDotNet.Validators; +using JetBrains.Annotations; namespace BenchmarkDotNet.Toolchains.MonoWasm { [PublicAPI] - public class WasmToolChain : Toolchain + public class WasmToolchain : Toolchain { private string CustomDotNetCliPath { get; } - private WasmToolChain(string name, IGenerator generator, IBuilder builder, IExecutor executor, string customDotNetCliPath) + private WasmToolchain(string name, IGenerator generator, IBuilder builder, IExecutor executor, string customDotNetCliPath) : base(name, generator, builder, executor) { CustomDotNetCliPath = customDotNetCliPath; } - public override bool IsSupported(BenchmarkCase benchmarkCase, ILogger logger, IResolver resolver) + public override IEnumerable Validate(BenchmarkCase benchmarkCase, IResolver resolver) { - if (!base.IsSupported(benchmarkCase, logger, resolver)) - return false; - - if (InvalidCliPath(CustomDotNetCliPath, benchmarkCase, logger)) - return false; + foreach (var validationError in base.Validate(benchmarkCase, resolver)) + { + yield return validationError; + } if (RuntimeInformation.IsWindows()) - logger.WriteLineInfo($"{nameof(WasmToolChain)} is supported only on Unix, benchmark '{benchmarkCase.DisplayInfo}' might not work correctly"); - - return true; + { + yield return new ValidationError(true, + $"{nameof(WasmToolchain)} is supported only on Unix, benchmark '{benchmarkCase.DisplayInfo}' might not work correctly", + benchmarkCase); + } + else if (IsCliPathInvalid(CustomDotNetCliPath, benchmarkCase, out var invalidCliError)) + { + yield return invalidCliError; + } } [PublicAPI] public static IToolchain From(NetCoreAppSettings netCoreAppSettings) - => new WasmToolChain(netCoreAppSettings.Name, + => new WasmToolchain(netCoreAppSettings.Name, new WasmGenerator(netCoreAppSettings.TargetFrameworkMoniker, netCoreAppSettings.CustomDotNetCliPath, netCoreAppSettings.PackagesPath, @@ -48,4 +54,4 @@ public static IToolchain From(NetCoreAppSettings netCoreAppSettings) new WasmExecutor(), netCoreAppSettings.CustomDotNetCliPath); } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/Roslyn/RoslynToolchain.cs b/src/BenchmarkDotNet/Toolchains/Roslyn/RoslynToolchain.cs index df40c09139..31c4d68c29 100644 --- a/src/BenchmarkDotNet/Toolchains/Roslyn/RoslynToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/Roslyn/RoslynToolchain.cs @@ -1,8 +1,9 @@ -using BenchmarkDotNet.Characteristics; +using System.Collections.Generic; +using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Running; +using BenchmarkDotNet.Validators; using JetBrains.Annotations; namespace BenchmarkDotNet.Toolchains.Roslyn @@ -21,39 +22,41 @@ public class RoslynToolchain : Toolchain } [PublicAPI] - public override bool IsSupported(BenchmarkCase benchmarkCase, ILogger logger, IResolver resolver) + public override IEnumerable Validate(BenchmarkCase benchmarkCase, IResolver resolver) { - if (!base.IsSupported(benchmarkCase, logger, resolver)) + foreach (var validationError in base.Validate(benchmarkCase, resolver)) { - return false; + yield return validationError; } if (!RuntimeInformation.IsFullFramework) { - logger.WriteLineError("The Roslyn toolchain is only supported on .NET Framework"); - return false; + yield return new ValidationError(true, + "The Roslyn toolchain is only supported on .NET Framework", + benchmarkCase); } if (benchmarkCase.Job.ResolveValue(GcMode.RetainVmCharacteristic, resolver)) { - logger.WriteLineError($"Currently App.config does not support RetainVM option, benchmark '{benchmarkCase.DisplayInfo}' will not be executed"); - return false; + yield return new ValidationError(true, + $"Currently App.config does not support RetainVM option, benchmark '{benchmarkCase.DisplayInfo}' will not be executed", + benchmarkCase); } if (benchmarkCase.Job.HasValue(InfrastructureMode.BuildConfigurationCharacteristic) && benchmarkCase.Job.ResolveValue(InfrastructureMode.BuildConfigurationCharacteristic, resolver) != InfrastructureMode.ReleaseConfigurationName) { - logger.WriteLineError("The Roslyn toolchain does not allow to rebuild source project, so defining custom build configuration makes no sense"); - return false; + yield return new ValidationError(true, + "The Roslyn toolchain does not allow to rebuild source project, so defining custom build configuration makes no sense", + benchmarkCase); } if (benchmarkCase.Job.HasValue(InfrastructureMode.NuGetReferencesCharacteristic)) { - logger.WriteLineError("The Roslyn toolchain does not allow specifying NuGet package dependencies"); - return false; + yield return new ValidationError(true, + "The Roslyn toolchain does not allow specifying NuGet package dependencies", + benchmarkCase); } - - return true; } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/Toolchain.cs b/src/BenchmarkDotNet/Toolchains/Toolchain.cs index 0e9da89bb8..4424f20291 100644 --- a/src/BenchmarkDotNet/Toolchains/Toolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/Toolchain.cs @@ -1,9 +1,10 @@ -using System.IO; +using System.Collections.Generic; +using System.IO; using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Running; +using BenchmarkDotNet.Validators; namespace BenchmarkDotNet.Toolchains { @@ -27,45 +28,54 @@ public Toolchain(string name, IGenerator generator, IBuilder builder, IExecutor Executor = executor; } - public virtual bool IsSupported(BenchmarkCase benchmarkCase, ILogger logger, IResolver resolver) + public virtual IEnumerable Validate(BenchmarkCase benchmarkCase, IResolver resolver) { var runtime = benchmarkCase.Job.ResolveValue(EnvironmentMode.RuntimeCharacteristic, resolver); var jit = benchmarkCase.Job.ResolveValue(EnvironmentMode.JitCharacteristic, resolver); if (!(runtime is MonoRuntime) && jit == Jit.Llvm) { - logger.WriteLineError($"Llvm is supported only for Mono, benchmark '{benchmarkCase.DisplayInfo}' will not be executed"); - return false; + yield return new ValidationError(true, + $"Llvm is supported only for Mono, benchmark '{benchmarkCase.DisplayInfo}' will not be executed", + benchmarkCase); } if (runtime is MonoRuntime mono && !benchmarkCase.GetToolchain().IsInProcess) { if (string.IsNullOrEmpty(mono.CustomPath) && !HostEnvironmentInfo.GetCurrent().IsMonoInstalled.Value) { - logger.WriteLineError($"Mono is not installed or added to PATH, benchmark '{benchmarkCase.DisplayInfo}' will not be executed"); - return false; + yield return new ValidationError(true, + $"Mono is not installed or added to PATH, benchmark '{benchmarkCase.DisplayInfo}' will not be executed", + benchmarkCase); } if (!string.IsNullOrEmpty(mono.CustomPath) && !File.Exists(mono.CustomPath)) { - logger.WriteLineError($"We could not find Mono in provided path ({mono.CustomPath}), benchmark '{benchmarkCase.DisplayInfo}' will not be executed"); - return false; + yield return new ValidationError(true, + $"We could not find Mono in provided path ({mono.CustomPath}), benchmark '{benchmarkCase.DisplayInfo}' will not be executed", + benchmarkCase); } } - - return true; } - internal static bool InvalidCliPath(string customDotNetCliPath, BenchmarkCase benchmarkCase, ILogger logger) + internal static bool IsCliPathInvalid(string customDotNetCliPath, BenchmarkCase benchmarkCase, out ValidationError validationError) { + validationError = null; + if (string.IsNullOrEmpty(customDotNetCliPath) && !HostEnvironmentInfo.GetCurrent().IsDotNetCliInstalled()) { - logger.WriteLineError($"BenchmarkDotNet requires dotnet cli to be installed or path to local dotnet cli provided in explicit way using `--cli` argument, benchmark '{benchmarkCase.DisplayInfo}' will not be executed"); + validationError = new ValidationError(true, + $"BenchmarkDotNet requires dotnet cli to be installed or path to local dotnet cli provided in explicit way using `--cli` argument, benchmark '{benchmarkCase.DisplayInfo}' will not be executed", + benchmarkCase); + return true; } if (!string.IsNullOrEmpty(customDotNetCliPath) && !File.Exists(customDotNetCliPath)) { - logger.WriteLineError($"Provided custom dotnet cli path does not exist, benchmark '{benchmarkCase.DisplayInfo}' will not be executed"); + validationError = new ValidationError(true, + $"Provided custom dotnet cli path does not exist, benchmark '{benchmarkCase.DisplayInfo}' will not be executed", + benchmarkCase); + return true; } diff --git a/src/BenchmarkDotNet/Toolchains/ToolchainExtensions.cs b/src/BenchmarkDotNet/Toolchains/ToolchainExtensions.cs index fd0fb5b63d..51bb7c88bd 100644 --- a/src/BenchmarkDotNet/Toolchains/ToolchainExtensions.cs +++ b/src/BenchmarkDotNet/Toolchains/ToolchainExtensions.cs @@ -4,15 +4,14 @@ using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Running; -using BenchmarkDotNet.Toolchains.NativeAot; using BenchmarkDotNet.Toolchains.CsProj; using BenchmarkDotNet.Toolchains.DotNetCli; -using BenchmarkDotNet.Toolchains.InProcess; using BenchmarkDotNet.Toolchains.InProcess.Emit; using BenchmarkDotNet.Toolchains.InProcess.NoEmit; using BenchmarkDotNet.Toolchains.Mono; -using BenchmarkDotNet.Toolchains.Roslyn; using BenchmarkDotNet.Toolchains.MonoWasm; +using BenchmarkDotNet.Toolchains.NativeAot; +using BenchmarkDotNet.Toolchains.Roslyn; namespace BenchmarkDotNet.Toolchains { @@ -66,7 +65,7 @@ internal static IToolchain GetToolchain(this Runtime runtime, Descriptor descrip : NativeAotToolchain.CreateBuilder().UseNuGet().TargetFrameworkMoniker(nativeAotRuntime.MsBuildMoniker).ToToolchain(); case WasmRuntime wasmRuntime: - return WasmToolChain.From(new NetCoreAppSettings(targetFrameworkMoniker: wasmRuntime.MsBuildMoniker, name: wasmRuntime.Name, runtimeFrameworkVersion: null)); + return WasmToolchain.From(new NetCoreAppSettings(targetFrameworkMoniker: wasmRuntime.MsBuildMoniker, name: wasmRuntime.Name, runtimeFrameworkVersion: null)); default: throw new ArgumentOutOfRangeException(nameof(runtime), runtime, "Runtime not supported"); @@ -79,26 +78,37 @@ private static IToolchain GetToolchain(RuntimeMoniker runtimeMoniker) { case RuntimeMoniker.Net461: return CsProjClassicNetToolchain.Net461; + case RuntimeMoniker.Net462: return CsProjClassicNetToolchain.Net462; + case RuntimeMoniker.Net47: return CsProjClassicNetToolchain.Net47; + case RuntimeMoniker.Net471: return CsProjClassicNetToolchain.Net471; + case RuntimeMoniker.Net472: return CsProjClassicNetToolchain.Net472; + case RuntimeMoniker.Net48: return CsProjClassicNetToolchain.Net48; + case RuntimeMoniker.Net481: return CsProjClassicNetToolchain.Net481; + case RuntimeMoniker.NetCoreApp20: return CsProjCoreToolchain.NetCoreApp20; + case RuntimeMoniker.NetCoreApp21: return CsProjCoreToolchain.NetCoreApp21; + case RuntimeMoniker.NetCoreApp22: return CsProjCoreToolchain.NetCoreApp22; + case RuntimeMoniker.NetCoreApp30: return CsProjCoreToolchain.NetCoreApp30; + case RuntimeMoniker.NetCoreApp31: return CsProjCoreToolchain.NetCoreApp31; #pragma warning disable CS0618 // Type or member is obsolete @@ -106,17 +116,22 @@ private static IToolchain GetToolchain(RuntimeMoniker runtimeMoniker) #pragma warning restore CS0618 // Type or member is obsolete case RuntimeMoniker.Net50: return CsProjCoreToolchain.NetCoreApp50; + case RuntimeMoniker.Net60: return CsProjCoreToolchain.NetCoreApp60; + case RuntimeMoniker.Net70: return CsProjCoreToolchain.NetCoreApp70; + case RuntimeMoniker.NativeAot60: return NativeAotToolchain.Net60; + case RuntimeMoniker.NativeAot70: return NativeAotToolchain.Net70; + default: throw new ArgumentOutOfRangeException(nameof(runtimeMoniker), runtimeMoniker, "RuntimeMoniker not supported"); } } } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Validators/ValidationError.cs b/src/BenchmarkDotNet/Validators/ValidationError.cs index 4d8526b87b..5cb7b63863 100644 --- a/src/BenchmarkDotNet/Validators/ValidationError.cs +++ b/src/BenchmarkDotNet/Validators/ValidationError.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using BenchmarkDotNet.Running; using JetBrains.Annotations; @@ -36,7 +37,7 @@ public override bool Equals(object obj) return true; if (obj.GetType() != this.GetType()) return false; - return Equals((ValidationError) obj); + return Equals((ValidationError)obj); } public override int GetHashCode() @@ -50,6 +51,7 @@ public override int GetHashCode() } } + public static bool operator ==(ValidationError left, ValidationError right) => Equals(left, right); public static bool operator !=(ValidationError left, ValidationError right) => !Equals(left, right); diff --git a/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs b/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs index 89120e84c4..6a2a0e2dd7 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs @@ -8,6 +8,7 @@ using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Running; @@ -56,10 +57,28 @@ public InProcessTest(ITestOutputHelper output) : base(output) [Fact] public void BenchmarkActionValueTaskOfTSupported() => TestInvoke(x => x.InvokeOnceValueTaskOfT(), UnrollFactor, DecimalResult); + [Fact] + public void BenchmarkDifferentPlatformReturnsValidationError() + { + var otherPlatform = IntPtr.Size == 8 + ? Platform.X86 + : Platform.X64; + + var otherPlatformConfig = new ManualConfig() + .With(Job.Dry.With(InProcessToolchain.Instance).With(otherPlatform)) + .With(new OutputLogger(Output)) + .With(DefaultColumnProviders.Instance); + + var runInfo = BenchmarkConverter.TypeToBenchmarks(typeof(BenchmarkAllCases), otherPlatformConfig); + var summary = BenchmarkRunner.Run(runInfo); + + Assert.NotEmpty(summary.ValidationErrors); + } + [AssertionMethod] private void TestInvoke(Expression> methodCall, int unrollFactor) { - var targetMethod = ((MethodCallExpression) methodCall.Body).Method; + var targetMethod = ((MethodCallExpression)methodCall.Body).Method; var descriptor = new Descriptor(typeof(BenchmarkAllCases), targetMethod, targetMethod, targetMethod); // Run mode @@ -97,7 +116,7 @@ private void TestInvoke(Expression> methodCall, int un [AssertionMethod] private void TestInvoke(Expression> methodCall, int unrollFactor, object expectedResult) { - var targetMethod = ((MethodCallExpression) methodCall.Body).Method; + var targetMethod = ((MethodCallExpression)methodCall.Body).Method; var descriptor = new Descriptor(typeof(BenchmarkAllCases), targetMethod); // Run mode diff --git a/tests/BenchmarkDotNet.IntegrationTests/NugetReferenceTests.cs b/tests/BenchmarkDotNet.IntegrationTests/NugetReferenceTests.cs index bff8b60a74..1a122f5343 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/NugetReferenceTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/NugetReferenceTests.cs @@ -1,22 +1,22 @@ -using BenchmarkDotNet.Jobs; -using Xunit; -using Xunit.Abstractions; -using BenchmarkDotNet.Portability; -using BenchmarkDotNet.Toolchains; +using System; using BenchmarkDotNet.Attributes; -using Newtonsoft.Json; -using System; -using BenchmarkDotNet.Toolchains.Roslyn; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Portability; using BenchmarkDotNet.Running; -using BenchmarkDotNet.Loggers; -using System.Collections.Immutable; using BenchmarkDotNet.Tests.XUnit; +using BenchmarkDotNet.Toolchains; +using BenchmarkDotNet.Toolchains.Roslyn; +using Newtonsoft.Json; +using Xunit; +using Xunit.Abstractions; namespace BenchmarkDotNet.IntegrationTests { public class NuGetReferenceTests : BenchmarkTestExecutor { - public NuGetReferenceTests(ITestOutputHelper output) : base(output) { } + public NuGetReferenceTests(ITestOutputHelper output) : base(output) + { + } [FactNotLinux("For some reason this test is unstable on Ubuntu for both AzureDevOps and Travis CI")] public void UserCanSpecifyCustomNuGetPackageDependency() @@ -37,19 +37,18 @@ public void RoslynToolchainDoesNotSupportNuGetPackageDependency() var unsupportedJob = Job.Dry.WithToolchain(toolchain).WithNuGet("Newtonsoft.Json", "11.0.2"); var unsupportedJobConfig = CreateSimpleConfig(job: unsupportedJob); var unsupportedJobBenchmark = BenchmarkConverter.TypeToBenchmarks(typeof(WithCallToNewtonsoft), unsupportedJobConfig); - var unsupportedJobLogger = new CompositeLogger(unsupportedJobConfig.GetLoggers().ToImmutableHashSet()); + foreach (var benchmarkCase in unsupportedJobBenchmark.BenchmarksCases) { - Assert.False(toolchain.IsSupported(benchmarkCase, unsupportedJobLogger, BenchmarkRunnerClean.DefaultResolver)); + Assert.NotEmpty(toolchain.Validate(benchmarkCase, BenchmarkRunnerClean.DefaultResolver)); } var supportedJob = Job.Dry.WithToolchain(toolchain); var supportedConfig = CreateSimpleConfig(job: supportedJob); var supportedBenchmark = BenchmarkConverter.TypeToBenchmarks(typeof(WithCallToNewtonsoft), supportedConfig); - var supportedLogger = new CompositeLogger(supportedConfig.GetLoggers().ToImmutableHashSet()); foreach (var benchmarkCase in supportedBenchmark.BenchmarksCases) { - Assert.True(toolchain.IsSupported(benchmarkCase, supportedLogger, BenchmarkRunnerClean.DefaultResolver)); + Assert.Empty(toolchain.Validate(benchmarkCase, BenchmarkRunnerClean.DefaultResolver)); } } @@ -58,4 +57,4 @@ public class WithCallToNewtonsoft [Benchmark] public void SerializeAnonymousObject() => JsonConvert.SerializeObject(new { hello = "world", price = 1.99, now = DateTime.UtcNow }); } } -} +} \ No newline at end of file From 1fb1015556039664451dd101b4e2ef9501e03805 Mon Sep 17 00:00:00 2001 From: farQtech <31050786+farQtech@users.noreply.github.com> Date: Tue, 18 Oct 2022 17:00:11 +0530 Subject: [PATCH 27/58] Corrected logic to restore foreground color in ConsoleLogger.cs (#2160) Co-authored-by: Ikramuddin Farooqui --- src/BenchmarkDotNet/Loggers/ConsoleLogger.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/BenchmarkDotNet/Loggers/ConsoleLogger.cs b/src/BenchmarkDotNet/Loggers/ConsoleLogger.cs index 8d9a96e939..6f3cd34ae3 100644 --- a/src/BenchmarkDotNet/Loggers/ConsoleLogger.cs +++ b/src/BenchmarkDotNet/Loggers/ConsoleLogger.cs @@ -53,8 +53,7 @@ private void Write(LogKind logKind, Action write, string text) } finally { - if (colorBefore != Console.ForegroundColor && colorBefore != Console.BackgroundColor) - Console.ForegroundColor = colorBefore; + Console.ForegroundColor = colorBefore; } } From c02c3d82bbefa6236660a7b7405f38e739679daf Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Tue, 25 Oct 2022 01:10:20 -0700 Subject: [PATCH 28/58] Fix bugs and typos (#2168) * Fix bugs * Fix typos in comments --- .../Tracing/NativeMemoryLogParser.cs | 2 +- .../Tracing/TraceLogParser.cs | 2 +- .../Attributes/Filters/OperatingSystemsFilterAttribute.cs | 2 +- .../Disassemblers/Exporters/DisassemblyPrettifier.cs | 2 +- src/BenchmarkDotNet/Environments/HostEnvironmentInfo.cs | 2 +- src/BenchmarkDotNet/Jobs/EnvironmentVariable.cs | 2 +- src/BenchmarkDotNet/Jobs/NugetReference.cs | 2 +- src/BenchmarkDotNet/Reports/BenchmarkReport.cs | 2 +- src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs | 8 ++++---- .../Running/BenchmarkConverterTests.cs | 2 +- .../Running/JobRuntimePropertiesComparerTests.cs | 4 ++-- 11 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/Tracing/NativeMemoryLogParser.cs b/src/BenchmarkDotNet.Diagnostics.Windows/Tracing/NativeMemoryLogParser.cs index 11cce65947..711c23f0c5 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/Tracing/NativeMemoryLogParser.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/Tracing/NativeMemoryLogParser.cs @@ -217,7 +217,7 @@ bool IsCallStackIn(StackSourceCallStackIndex index) return; } - // Heap is dieing, kill all objects in it. + // Heap is dying, kill all objects in it. var allocs = lastHeapAllocs; if (data.HeapHandle != lastHeapHandle) { diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/Tracing/TraceLogParser.cs b/src/BenchmarkDotNet.Diagnostics.Windows/Tracing/TraceLogParser.cs index f2173efc91..f274151ccb 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/Tracing/TraceLogParser.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/Tracing/TraceLogParser.cs @@ -160,7 +160,7 @@ public IEnumerable CalculateMetrics(Dictionary profileSourceId private IterationData[] CreateIterationData(List startStopTimeStamps) { - // collection contains mixted .Start and .Stop intervals, if we sort it we know that n is Start and n + 1 is Stop + // collection contains mixed .Start and .Stop intervals, if we sort it we know that n is Start and n + 1 is Stop startStopTimeStamps.Sort(); var iterations = new IterationData[startStopTimeStamps.Count / 2]; diff --git a/src/BenchmarkDotNet/Attributes/Filters/OperatingSystemsFilterAttribute.cs b/src/BenchmarkDotNet/Attributes/Filters/OperatingSystemsFilterAttribute.cs index 5acd061b47..41d1a2032e 100644 --- a/src/BenchmarkDotNet/Attributes/Filters/OperatingSystemsFilterAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/Filters/OperatingSystemsFilterAttribute.cs @@ -25,7 +25,7 @@ public class OperatingSystemsFilterAttribute : FilterConfigBaseAttribute // CLS-Compliant Code requires a constructor without an array in the argument list public OperatingSystemsFilterAttribute() { } - /// if set to true, the OSes beloning to platforms are enabled, if set to false, disabled + /// if set to true, the OSes belonging to platforms are enabled, if set to false, disabled /// the platform(s) for which the filter should be applied public OperatingSystemsFilterAttribute(bool allowed, params OS[] platforms) : base(new SimpleFilter(_ => diff --git a/src/BenchmarkDotNet/Disassemblers/Exporters/DisassemblyPrettifier.cs b/src/BenchmarkDotNet/Disassemblers/Exporters/DisassemblyPrettifier.cs index 57b8a04026..d566ed6eab 100644 --- a/src/BenchmarkDotNet/Disassemblers/Exporters/DisassemblyPrettifier.cs +++ b/src/BenchmarkDotNet/Disassemblers/Exporters/DisassemblyPrettifier.cs @@ -44,7 +44,7 @@ internal static IReadOnlyList Prettify(DisassembledMethod method, Disas referencedAddresses.Add(asm.ReferencedAddress.Value); } - // for every IP that is referenced, we emit a uinque label + // for every IP that is referenced, we emit a unique label var addressesToLabels = new Dictionary(); int currentLabelIndex = 0; foreach (var instruction in asmInstructions) diff --git a/src/BenchmarkDotNet/Environments/HostEnvironmentInfo.cs b/src/BenchmarkDotNet/Environments/HostEnvironmentInfo.cs index 3eb95632cb..a3c5999700 100644 --- a/src/BenchmarkDotNet/Environments/HostEnvironmentInfo.cs +++ b/src/BenchmarkDotNet/Environments/HostEnvironmentInfo.cs @@ -98,7 +98,7 @@ public override IEnumerable ToFormattedString() if (RuntimeInformation.IsNetCore && IsDotNetCliInstalled()) { - // this wonderfull version number contains words like "preview" and ... 5 segments so it can not be parsed by Version.Parse. Example: "5.0.100-preview.8.20362.3" + // this wonderful version number contains words like "preview" and ... 5 segments so it can not be parsed by Version.Parse. Example: "5.0.100-preview.8.20362.3" if (int.TryParse(new string(DotNetSdkVersion.Value.TrimStart().TakeWhile(char.IsDigit).ToArray()), out int major) && major >= 5) yield return $".NET SDK={DotNetSdkVersion.Value}"; else diff --git a/src/BenchmarkDotNet/Jobs/EnvironmentVariable.cs b/src/BenchmarkDotNet/Jobs/EnvironmentVariable.cs index 164efc0af8..728ecee480 100644 --- a/src/BenchmarkDotNet/Jobs/EnvironmentVariable.cs +++ b/src/BenchmarkDotNet/Jobs/EnvironmentVariable.cs @@ -8,7 +8,7 @@ public class EnvironmentVariable : IEquatable public EnvironmentVariable([NotNull] string key, [NotNull] string value) { Key = key ?? throw new ArgumentNullException(nameof(key)); - Value = value ?? throw new ArgumentNullException(nameof(Value)); + Value = value ?? throw new ArgumentNullException(nameof(value)); } [NotNull] diff --git a/src/BenchmarkDotNet/Jobs/NugetReference.cs b/src/BenchmarkDotNet/Jobs/NugetReference.cs index 3f72f73b99..8f4567c9a8 100644 --- a/src/BenchmarkDotNet/Jobs/NugetReference.cs +++ b/src/BenchmarkDotNet/Jobs/NugetReference.cs @@ -13,7 +13,7 @@ public NuGetReference(string packageName, string packageVersion, Uri source = nu PackageName = packageName; - if (!string.IsNullOrWhiteSpace(PackageVersion) && !IsValidVersion(packageVersion)) + if (!string.IsNullOrWhiteSpace(packageVersion) && !IsValidVersion(packageVersion)) throw new InvalidOperationException($"Invalid version specified: {packageVersion}"); PackageVersion = packageVersion ?? string.Empty; diff --git a/src/BenchmarkDotNet/Reports/BenchmarkReport.cs b/src/BenchmarkDotNet/Reports/BenchmarkReport.cs index 71bb2deb33..4938216fce 100644 --- a/src/BenchmarkDotNet/Reports/BenchmarkReport.cs +++ b/src/BenchmarkDotNet/Reports/BenchmarkReport.cs @@ -44,7 +44,7 @@ public BenchmarkReport( BuildResult = buildResult; ExecuteResults = executeResults ?? Array.Empty(); AllMeasurements = ExecuteResults.SelectMany((results, index) => results.Measurements).ToArray(); - GcStats = ExecuteResults.Count > 0 ? executeResults[executeResults.Count -1].GcStats : default; + GcStats = ExecuteResults.Count > 0 ? ExecuteResults[ExecuteResults.Count - 1].GcStats : default; Metrics = metrics?.ToDictionary(metric => metric.Descriptor.Id) ?? (IReadOnlyDictionary)ImmutableDictionary.Empty; } diff --git a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs index 0de7f8c7a9..d7fb3ae7be 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs @@ -49,7 +49,7 @@ internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos) compositeLogger.WriteLineInfo("// Validating benchmarks:"); - var (supportedBenchmarks, validationErrors) = GetSupportedBenchmarks(benchmarkRunInfos, compositeLogger, resolver); + var (supportedBenchmarks, validationErrors) = GetSupportedBenchmarks(benchmarkRunInfos, resolver); validationErrors.AddRange(Validate(supportedBenchmarks)); @@ -532,11 +532,11 @@ private static ExecuteResult RunExecute(ILogger logger, BenchmarkCase benchmarkC private static void LogTotalTime(ILogger logger, TimeSpan time, int executedBenchmarksCount, string message = "Total time") => logger.WriteLineStatistic($"{message}: {time.ToFormattedTotalTime(DefaultCultureInfo.Instance)}, executed benchmarks: {executedBenchmarksCount}"); - private static (BenchmarkRunInfo[], List) GetSupportedBenchmarks(BenchmarkRunInfo[] benchmarkRunInfos, ILogger logger, IResolver resolver) + private static (BenchmarkRunInfo[], List) GetSupportedBenchmarks(BenchmarkRunInfo[] benchmarkRunInfos, IResolver resolver) { List validationErrors = new (); - var benchmarksRunInfo = benchmarkRunInfos.Select(info => new BenchmarkRunInfo( + var runInfos = benchmarkRunInfos.Select(info => new BenchmarkRunInfo( info.BenchmarksCases.Where(benchmark => { var errors = benchmark.GetToolchain().Validate(benchmark, resolver).ToArray(); @@ -548,7 +548,7 @@ private static (BenchmarkRunInfo[], List) GetSupportedBenchmark .Where(infos => infos.BenchmarksCases.Any()) .ToArray(); - return (benchmarkRunInfos, validationErrors); + return (runInfos, validationErrors); } private static string GetRootArtifactsFolderPath(BenchmarkRunInfo[] benchmarkRunInfos) diff --git a/tests/BenchmarkDotNet.Tests/Running/BenchmarkConverterTests.cs b/tests/BenchmarkDotNet.Tests/Running/BenchmarkConverterTests.cs index 5e9f9072e4..4eb72b5d59 100644 --- a/tests/BenchmarkDotNet.Tests/Running/BenchmarkConverterTests.cs +++ b/tests/BenchmarkDotNet.Tests/Running/BenchmarkConverterTests.cs @@ -215,7 +215,7 @@ public void MethodDeclarationOrderIsPreserved() public class BAC { - // BAC is not sorted in either desceding or ascending way + // BAC is not sorted in either descending or ascending way [Benchmark] public void B() { } [Benchmark] public void A() { } [Benchmark] public void C() { } diff --git a/tests/BenchmarkDotNet.Tests/Running/JobRuntimePropertiesComparerTests.cs b/tests/BenchmarkDotNet.Tests/Running/JobRuntimePropertiesComparerTests.cs index 54697af929..96e0bd2fc9 100644 --- a/tests/BenchmarkDotNet.Tests/Running/JobRuntimePropertiesComparerTests.cs +++ b/tests/BenchmarkDotNet.Tests/Running/JobRuntimePropertiesComparerTests.cs @@ -166,8 +166,8 @@ public void CustomTargetPlatformJobsAreGroupedByTargetFrameworkMoniker() Assert.Equal(2, grouped.Length); - Assert.Single(grouped, group => group.Count() == 3); // Plain1 (3 methods) runing against "net5.0" - Assert.Single(grouped, group => group.Count() == 6); // Plain2 (3 methods) and Plain3 (3 methods) runing against "net5.0-windows" + Assert.Single(grouped, group => group.Count() == 3); // Plain1 (3 methods) running against "net5.0" + Assert.Single(grouped, group => group.Count() == 6); // Plain2 (3 methods) and Plain3 (3 methods) running against "net5.0-windows" } } } \ No newline at end of file From 0f7eb25eb4097e7926b75bf765d21a0a175a86a3 Mon Sep 17 00:00:00 2001 From: Maykon Elias Date: Wed, 26 Oct 2022 13:43:19 +0100 Subject: [PATCH 29/58] Implement --resume support (#2164), fixes #1799 Co-authored-by: Maykon Elias Co-authored-by: Adam Sitnik --- src/BenchmarkDotNet/Configs/ConfigOptions.cs | 6 ++- .../ConsoleArguments/CommandLineOptions.cs | 3 ++ .../ConsoleArguments/ConfigParser.cs | 1 + .../Running/BenchmarkRunnerClean.cs | 45 ++++++++++++++++++- .../BenchmarkSwitcherTest.cs | 15 +++++++ 5 files changed, 68 insertions(+), 2 deletions(-) diff --git a/src/BenchmarkDotNet/Configs/ConfigOptions.cs b/src/BenchmarkDotNet/Configs/ConfigOptions.cs index 29e9e1af6f..c1ad75d642 100644 --- a/src/BenchmarkDotNet/Configs/ConfigOptions.cs +++ b/src/BenchmarkDotNet/Configs/ConfigOptions.cs @@ -44,7 +44,11 @@ public enum ConfigOptions /// /// Performs apples-to-apples comparison for provided benchmarks and jobs. Experimental, will change in the near future! /// - ApplesToApples = 1 << 8 + ApplesToApples = 1 << 8, + /// + /// Continue the execution if the last run was stopped. + /// + Resume = 1 << 9 } internal static class ConfigOptionsExtensions diff --git a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs index 8a52323ad2..d1c8a115af 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs @@ -215,6 +215,9 @@ public bool UseDisassemblyDiagnoser [Option("noForcedGCs", Required = false, HelpText = "Specifying would not forcefully induce any GCs.")] public bool NoForcedGCs { get; set; } + [Option("resume", Required = false, Default = false, HelpText = "Continue the execution if the last run was stopped.")] + public bool Resume { get; set; } + internal bool UserProvidedFilters => Filters.Any() || AttributeNames.Any() || AllCategories.Any() || AnyCategories.Any(); [Usage(ApplicationAlias = "")] diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index a208cbd2c6..97a96abf47 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -245,6 +245,7 @@ private static IConfig CreateConfig(CommandLineOptions options, IConfig globalCo config.WithOption(ConfigOptions.LogBuildOutput, options.LogBuildOutput); config.WithOption(ConfigOptions.GenerateMSBuildBinLog, options.GenerateMSBuildBinLog); config.WithOption(ConfigOptions.ApplesToApples, options.ApplesToApples); + config.WithOption(ConfigOptions.Resume, options.Resume); if (options.MaxParameterColumnWidth.HasValue) config.WithSummaryStyle(SummaryStyle.Default.WithMaxParameterColumnWidth(options.MaxParameterColumnWidth.Value)); diff --git a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs index d7fb3ae7be..b6dc5670d2 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs @@ -3,6 +3,7 @@ using System.Collections.Immutable; using System.IO; using System.Linq; +using System.Text.RegularExpressions; using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Columns; @@ -42,6 +43,7 @@ internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos) var rootArtifactsFolderPath = GetRootArtifactsFolderPath(benchmarkRunInfos); var resultsFolderPath = GetResultsFolderPath(rootArtifactsFolderPath, benchmarkRunInfos); var logFilePath = Path.Combine(rootArtifactsFolderPath, title + ".log"); + var idToResume = GetIdToResume(rootArtifactsFolderPath, title, benchmarkRunInfos); using (var streamLogger = new StreamLogger(GetLogFileStreamWriter(benchmarkRunInfos, logFilePath))) { @@ -62,7 +64,7 @@ internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos) return new[] { Summary.ValidationFailed(title, resultsFolderPath, logFilePath) }; int totalBenchmarkCount = supportedBenchmarks.Sum(benchmarkInfo => benchmarkInfo.BenchmarksCases.Length); - int benchmarksToRunCount = totalBenchmarkCount; + int benchmarksToRunCount = totalBenchmarkCount - (idToResume + 1); // ids are indexed from 0 compositeLogger.WriteLineHeader("// ***** BenchmarkRunner: Start *****"); compositeLogger.WriteLineHeader($"// ***** Found {totalBenchmarkCount} benchmark(s) in total *****"); var globalChronometer = Chronometer.Start(); @@ -84,6 +86,16 @@ internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos) foreach (var benchmarkRunInfo in supportedBenchmarks) // we run them in the old order now using the new build artifacts { + if (idToResume >= 0) + { + var benchmarkWithHighestIdForGivenType = benchmarkRunInfo.BenchmarksCases.Last(); + if (benchmarkToBuildResult[benchmarkWithHighestIdForGivenType].Id.Value <= idToResume) + { + compositeLogger.WriteLineInfo($"Skipping {benchmarkRunInfo.BenchmarksCases.Length} benchmark(s) defined by {benchmarkRunInfo.Type.GetCorrectCSharpTypeName()}."); + continue; + } + } + var summary = Run(benchmarkRunInfo, benchmarkToBuildResult, resolver, compositeLogger, artifactsToCleanup, resultsFolderPath, logFilePath, totalBenchmarkCount, in runsChronometer, ref benchmarksToRunCount); @@ -678,5 +690,36 @@ private static void PrintValidationErrors(ILogger logger, IEnumerable benchmark.Config.Options.IsSet(ConfigOptions.Resume))) + { + var directoryInfo = new DirectoryInfo(rootArtifactsFolderPath); + var logFilesExceptCurrent = directoryInfo + .GetFiles($"{currentLogFileName.Split('-')[0]}*") + .Where(file => Path.GetFileNameWithoutExtension(file.Name) != currentLogFileName) + .ToArray(); + + if (logFilesExceptCurrent.Length > 0) + { + var previousRunLogFile = logFilesExceptCurrent + .OrderByDescending(o => o.LastWriteTime) + .First(); + + var regex = new Regex("--benchmarkId (.*?) in", RegexOptions.Compiled); + foreach (var line in File.ReadLines(previousRunLogFile.FullName).Reverse()) + { + var match = regex.Match(line); + if (match.Success) + { + return int.Parse(match.Groups[1].Value); + } + } + } + } + + return -1; + } } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs index f7dc6e7c0b..0ae17a5380 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs @@ -332,6 +332,21 @@ public void WhenUserCreatesStaticBenchmarkMethodWeDisplayAnError_FromAssembly() Assert.Contains("static", logger.GetLog()); } + [Fact] + public void WhenUserAddTheResumeAttributeAndRunTheBenchmarks() + { + var logger = new OutputLogger(Output); + var config = ManualConfig.CreateEmpty().AddLogger(logger); + + var types = new[] { typeof(WithDryAttributeAndCategory) }; + var switcher = new BenchmarkSwitcher(types); + + // the first run should execute all benchmarks + Assert.Single(switcher.Run(new[] { "--filter", "*WithDryAttributeAndCategory*" }, config)); + // resuming after succesfull run should run nothing + Assert.Empty(switcher.Run(new[] { "--resume", "--filter", "*WithDryAttributeAndCategory*" }, config)); + } + private class UserInteractionMock : IUserInteraction { private readonly IReadOnlyList returnValue; From 7d83758b271d901dd0c67225fcfeea1084731ed6 Mon Sep 17 00:00:00 2001 From: Sergey Aseev Date: Wed, 26 Oct 2022 16:03:16 +0300 Subject: [PATCH 30/58] Issue #1736: Add ExceptionDiagnoser (#2169) Co-authored-by: Adam Sitnik --- docs/articles/configs/diagnosers.md | 5 ++ .../samples/IntroExceptionDiagnoser.md | 26 ++++++++ .../IntroExceptionDiagnoser.cs | 22 +++++++ .../Attributes/ExceptionDiagnoserAttribute.cs | 14 ++++ src/BenchmarkDotNet/Columns/Column.cs | 1 + .../Configs/ImmutableConfig.cs | 4 +- .../ConsoleArguments/CommandLineOptions.cs | 3 + .../ConsoleArguments/ConfigParser.cs | 2 + .../Diagnosers/DiagnoserResults.cs | 3 + .../Diagnosers/ExceptionDiagnoser.cs | 54 +++++++++++++++ src/BenchmarkDotNet/Engines/Engine.cs | 16 +++-- .../Engines/ExceptionsStats.cs | 43 ++++++++++++ src/BenchmarkDotNet/Engines/RunResults.cs | 8 ++- .../Toolchains/Results/ExecuteResult.cs | 15 +++-- .../CustomEngineTests.cs | 3 +- .../ExceptionDiagnoserTests.cs | 66 +++++++++++++++++++ .../Engine/EngineResultStageTests.cs | 2 +- .../Mocks/MockFactory.cs | 2 +- .../Reports/RatioPrecisionTests.cs | 2 +- .../Reports/RatioStyleTests.cs | 2 +- 20 files changed, 276 insertions(+), 17 deletions(-) create mode 100644 docs/articles/samples/IntroExceptionDiagnoser.md create mode 100644 samples/BenchmarkDotNet.Samples/IntroExceptionDiagnoser.cs create mode 100644 src/BenchmarkDotNet/Attributes/ExceptionDiagnoserAttribute.cs create mode 100644 src/BenchmarkDotNet/Diagnosers/ExceptionDiagnoser.cs create mode 100644 src/BenchmarkDotNet/Engines/ExceptionsStats.cs create mode 100644 tests/BenchmarkDotNet.IntegrationTests/ExceptionDiagnoserTests.cs diff --git a/docs/articles/configs/diagnosers.md b/docs/articles/configs/diagnosers.md index d942ab44f4..b1d5b7164f 100644 --- a/docs/articles/configs/diagnosers.md +++ b/docs/articles/configs/diagnosers.md @@ -37,6 +37,7 @@ The current Diagnosers are: It is a cross-platform profiler that allows profile .NET code on every platform - Windows, Linux, macOS. Please see Wojciech Nagórski's [blog post](https://wojciechnagorski.com/2020/04/cross-platform-profiling-.net-code-with-benchmarkdotnet/) for all the details. - Threading Diagnoser (`ThreadingDiagnoser`) - .NET Core 3.0+ diagnoser that reports some Threading statistics. +- Exception Diagnoser (`ExceptionDiagnoser`) - a diagnoser that reports the frequency of exceptions thrown during the operation. ## Usage @@ -59,6 +60,7 @@ private class Config : ManualConfig Add(new InliningDiagnoser()); Add(new EtwProfiler()); Add(ThreadingDiagnoser.Default); + Add(ExceptionDiagnoser.Default); } } ``` @@ -72,6 +74,7 @@ You can also use one of the following attributes (apply it on a class that conta [ConcurrencyVisualizerProfiler] [NativeMemoryProfiler] [ThreadingDiagnoser] +[ExceptionDiagnoser] ``` In BenchmarkDotNet, 1kB = 1024B, 1MB = 1024kB, and so on. The column Gen X means number of GC collections per 1000 operations for that generation. @@ -123,3 +126,5 @@ In BenchmarkDotNet, 1kB = 1024B, 1MB = 1024kB, and so on. The column Gen X means [!include[IntroNativeMemory](../samples/IntroNativeMemory.md)] [!include[IntroThreadingDiagnoser](../samples/IntroThreadingDiagnoser.md)] + +[!include[IntroExceptionDiagnoser](../samples/IntroExceptionDiagnoser.md)] diff --git a/docs/articles/samples/IntroExceptionDiagnoser.md b/docs/articles/samples/IntroExceptionDiagnoser.md new file mode 100644 index 0000000000..1a654d5fb7 --- /dev/null +++ b/docs/articles/samples/IntroExceptionDiagnoser.md @@ -0,0 +1,26 @@ +--- +uid: BenchmarkDotNet.Samples.IntroExceptionDiagnoser +--- + +## Sample: IntroExceptionDiagnoser + +The `ExceptionDiagnoser` uses [AppDomain.FirstChanceException](https://learn.microsoft.com/en-us/dotnet/api/system.appdomain.firstchanceexception) API to report: + +* Exception frequency: The number of exceptions thrown during the operations divided by the number of operations. + +### Source code + +[!code-csharp[IntroExceptionDiagnoser.cs](../../../samples/BenchmarkDotNet.Samples/IntroExceptionDiagnoser.cs)] + +### Output + +| Method | Mean | Error | StdDev | Exception frequency | +|----------------------- |---------:|----------:|----------:|--------------------:| +| ThrowExceptionRandomly | 4.936 us | 0.1542 us | 0.4499 us | 0.1381 | + +### Links + +* @docs.diagnosers +* The permanent link to this sample: @BenchmarkDotNet.Samples.IntroExceptionDiagnoser + +--- \ No newline at end of file diff --git a/samples/BenchmarkDotNet.Samples/IntroExceptionDiagnoser.cs b/samples/BenchmarkDotNet.Samples/IntroExceptionDiagnoser.cs new file mode 100644 index 0000000000..a43844e592 --- /dev/null +++ b/samples/BenchmarkDotNet.Samples/IntroExceptionDiagnoser.cs @@ -0,0 +1,22 @@ +using BenchmarkDotNet.Attributes; +using System; + +namespace BenchmarkDotNet.Samples +{ + [ExceptionDiagnoser] + public class IntroExceptionDiagnoser + { + [Benchmark] + public void ThrowExceptionRandomly() + { + try + { + if (new Random().Next(0, 5) > 1) + { + throw new Exception(); + } + } + catch { } + } + } +} diff --git a/src/BenchmarkDotNet/Attributes/ExceptionDiagnoserAttribute.cs b/src/BenchmarkDotNet/Attributes/ExceptionDiagnoserAttribute.cs new file mode 100644 index 0000000000..bff956e968 --- /dev/null +++ b/src/BenchmarkDotNet/Attributes/ExceptionDiagnoserAttribute.cs @@ -0,0 +1,14 @@ +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Diagnosers; +using System; + +namespace BenchmarkDotNet.Attributes +{ + [AttributeUsage(AttributeTargets.Class)] + public class ExceptionDiagnoserAttribute : Attribute, IConfigSource + { + public IConfig Config { get; } + + public ExceptionDiagnoserAttribute() => Config = ManualConfig.CreateEmpty().AddDiagnoser(ExceptionDiagnoser.Default); + } +} diff --git a/src/BenchmarkDotNet/Columns/Column.cs b/src/BenchmarkDotNet/Columns/Column.cs index c4207b744b..4d7cad48a4 100644 --- a/src/BenchmarkDotNet/Columns/Column.cs +++ b/src/BenchmarkDotNet/Columns/Column.cs @@ -55,6 +55,7 @@ public static class Column public const string CompletedWorkItems = "Completed Work Items"; public const string LockContentions = "Lock Contentions"; public const string CodeSize = "Code Size"; + public const string Exceptions = "Exceptions"; //Characteristics: public const string Id = "Id"; diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfig.cs b/src/BenchmarkDotNet/Configs/ImmutableConfig.cs index 83b7b9374a..185aa03f0e 100644 --- a/src/BenchmarkDotNet/Configs/ImmutableConfig.cs +++ b/src/BenchmarkDotNet/Configs/ImmutableConfig.cs @@ -106,9 +106,11 @@ internal ImmutableConfig( public bool HasThreadingDiagnoser() => diagnosers.Contains(ThreadingDiagnoser.Default); + public bool HasExceptionDiagnoser() => diagnosers.Contains(ExceptionDiagnoser.Default); + internal bool HasPerfCollectProfiler() => diagnosers.OfType().Any(); - public bool HasExtraStatsDiagnoser() => HasMemoryDiagnoser() || HasThreadingDiagnoser(); + public bool HasExtraStatsDiagnoser() => HasMemoryDiagnoser() || HasThreadingDiagnoser() || HasExceptionDiagnoser(); public IDiagnoser GetCompositeDiagnoser(BenchmarkCase benchmarkCase, RunMode runMode) { diff --git a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs index d1c8a115af..9e3e3ca350 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs @@ -37,6 +37,9 @@ public class CommandLineOptions [Option('t', "threading", Required = false, Default = false, HelpText = "Prints threading statistics")] public bool UseThreadingDiagnoser { get; set; } + [Option("exceptions", Required = false, Default = false, HelpText = "Prints exception statistics")] + public bool UseExceptionDiagnoser { get; set; } + [Option('d', "disasm", Required = false, Default = false, HelpText = "Gets disassembly of benchmarked code")] public bool UseDisassemblyDiagnoser { diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index 97a96abf47..691d8d9e75 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -213,6 +213,8 @@ private static IConfig CreateConfig(CommandLineOptions options, IConfig globalCo config.AddDiagnoser(MemoryDiagnoser.Default); if (options.UseThreadingDiagnoser) config.AddDiagnoser(ThreadingDiagnoser.Default); + if (options.UseExceptionDiagnoser) + config.AddDiagnoser(ExceptionDiagnoser.Default); if (options.UseDisassemblyDiagnoser) config.AddDiagnoser(new DisassemblyDiagnoser(new DisassemblyDiagnoserConfig( maxDepth: options.DisassemblerRecursiveDepth, diff --git a/src/BenchmarkDotNet/Diagnosers/DiagnoserResults.cs b/src/BenchmarkDotNet/Diagnosers/DiagnoserResults.cs index 6eee446ba0..68591ad8c8 100644 --- a/src/BenchmarkDotNet/Diagnosers/DiagnoserResults.cs +++ b/src/BenchmarkDotNet/Diagnosers/DiagnoserResults.cs @@ -15,6 +15,7 @@ public DiagnoserResults(BenchmarkCase benchmarkCase, ExecuteResult executeResult GcStats = executeResult.GcStats; ThreadingStats = executeResult.ThreadingStats; BuildResult = buildResult; + ExceptionFrequency = executeResult.ExceptionFrequency; } public BenchmarkCase BenchmarkCase { get; } @@ -25,6 +26,8 @@ public DiagnoserResults(BenchmarkCase benchmarkCase, ExecuteResult executeResult public ThreadingStats ThreadingStats { get; } + public double ExceptionFrequency { get; } + public BuildResult BuildResult { get; } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Diagnosers/ExceptionDiagnoser.cs b/src/BenchmarkDotNet/Diagnosers/ExceptionDiagnoser.cs new file mode 100644 index 0000000000..d30495d02b --- /dev/null +++ b/src/BenchmarkDotNet/Diagnosers/ExceptionDiagnoser.cs @@ -0,0 +1,54 @@ +using BenchmarkDotNet.Analysers; +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Validators; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace BenchmarkDotNet.Diagnosers +{ + public class ExceptionDiagnoser : IDiagnoser + { + public static readonly ExceptionDiagnoser Default = new ExceptionDiagnoser(); + + private ExceptionDiagnoser() { } + + public IEnumerable Ids => new[] { nameof(ExceptionDiagnoser) }; + + public IEnumerable Exporters => Array.Empty(); + + public IEnumerable Analysers => Array.Empty(); + + public void DisplayResults(ILogger logger) { } + + public RunMode GetRunMode(BenchmarkCase benchmarkCase) => RunMode.NoOverhead; + + public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { } + + public IEnumerable ProcessResults(DiagnoserResults results) + { + yield return new Metric(ExceptionsFrequencyMetricDescriptor.Instance, results.ExceptionFrequency); + } + + public IEnumerable Validate(ValidationParameters validationParameters) => Enumerable.Empty(); + + private class ExceptionsFrequencyMetricDescriptor : IMetricDescriptor + { + internal static readonly IMetricDescriptor Instance = new ExceptionsFrequencyMetricDescriptor(); + + public string Id => "ExceptionFrequency"; + public string DisplayName => Column.Exceptions; + public string Legend => "Exceptions thrown per single operation"; + public string NumberFormat => "#0.0000"; + public UnitType UnitType => UnitType.Dimensionless; + public string Unit => "Count"; + public bool TheGreaterTheBetter => false; + public int PriorityInCategory => 0; + } + } +} diff --git a/src/BenchmarkDotNet/Engines/Engine.cs b/src/BenchmarkDotNet/Engines/Engine.cs index f626eddac4..708a126b92 100644 --- a/src/BenchmarkDotNet/Engines/Engine.cs +++ b/src/BenchmarkDotNet/Engines/Engine.cs @@ -132,16 +132,16 @@ public RunResults Run() Host.AfterMainRun(); - (GcStats workGcHasDone, ThreadingStats threadingStats) = includeExtraStats + (GcStats workGcHasDone, ThreadingStats threadingStats, double exceptionFrequency) = includeExtraStats ? GetExtraStats(new IterationData(IterationMode.Workload, IterationStage.Actual, 0, invokeCount, UnrollFactor)) - : (GcStats.Empty, ThreadingStats.Empty); + : (GcStats.Empty, ThreadingStats.Empty, 0); if (EngineEventSource.Log.IsEnabled()) EngineEventSource.Log.BenchmarkStop(BenchmarkName); var outlierMode = TargetJob.ResolveValue(AccuracyMode.OutlierModeCharacteristic, Resolver); - return new RunResults(idle, main, outlierMode, workGcHasDone, threadingStats); + return new RunResults(idle, main, outlierMode, workGcHasDone, threadingStats, exceptionFrequency); } public Measurement RunIteration(IterationData data) @@ -189,7 +189,7 @@ public Measurement RunIteration(IterationData data) return measurement; } - private (GcStats, ThreadingStats) GetExtraStats(IterationData data) + private (GcStats, ThreadingStats, double) GetExtraStats(IterationData data) { // we enable monitoring after main target run, for this single iteration which is executed at the end // so even if we enable AppDomain monitoring in separate process @@ -199,19 +199,23 @@ public Measurement RunIteration(IterationData data) IterationSetupAction(); // we run iteration setup first, so even if it allocates, it is not included in the results var initialThreadingStats = ThreadingStats.ReadInitial(); // this method might allocate + var exceptionsStats = new ExceptionsStats(); // allocates + exceptionsStats.StartListening(); // this method might allocate var initialGcStats = GcStats.ReadInitial(); WorkloadAction(data.InvokeCount / data.UnrollFactor); + exceptionsStats.Stop(); var finalGcStats = GcStats.ReadFinal(); var finalThreadingStats = ThreadingStats.ReadFinal(); IterationCleanupAction(); // we run iteration cleanup after collecting GC stats - GcStats gcStats = (finalGcStats - initialGcStats).WithTotalOperations(data.InvokeCount * OperationsPerInvoke); + var totalOperationsCount = data.InvokeCount * OperationsPerInvoke; + GcStats gcStats = (finalGcStats - initialGcStats).WithTotalOperations(totalOperationsCount); ThreadingStats threadingStats = (finalThreadingStats - initialThreadingStats).WithTotalOperations(data.InvokeCount * OperationsPerInvoke); - return (gcStats, threadingStats); + return (gcStats, threadingStats, exceptionsStats.ExceptionsCount / (double)totalOperationsCount); } [MethodImpl(MethodImplOptions.NoInlining)] diff --git a/src/BenchmarkDotNet/Engines/ExceptionsStats.cs b/src/BenchmarkDotNet/Engines/ExceptionsStats.cs new file mode 100644 index 0000000000..35aa387f97 --- /dev/null +++ b/src/BenchmarkDotNet/Engines/ExceptionsStats.cs @@ -0,0 +1,43 @@ +using System; +using System.Runtime.ExceptionServices; + +namespace BenchmarkDotNet.Engines +{ + internal class ExceptionsStats + { + internal const string ResultsLinePrefix = "// Exceptions: "; + + internal ulong ExceptionsCount { get; private set; } + + public void StartListening() + { + AppDomain.CurrentDomain.FirstChanceException += OnFirstChanceException; + } + + public void Stop() + { + AppDomain.CurrentDomain.FirstChanceException -= OnFirstChanceException; + } + + private void OnFirstChanceException(object sender, FirstChanceExceptionEventArgs e) + { + ExceptionsCount++; + } + + public static string ToOutputLine(double exceptionCount) => $"{ResultsLinePrefix} {exceptionCount}"; + + public static double Parse(string line) + { + if (!line.StartsWith(ResultsLinePrefix)) + throw new NotSupportedException($"Line must start with {ResultsLinePrefix}"); + + var measurement = line.Remove(0, ResultsLinePrefix.Length); + if (!double.TryParse(measurement, out var exceptionsNumber)) + { + throw new NotSupportedException("Invalid string"); + } + + return exceptionsNumber; + } + } +} diff --git a/src/BenchmarkDotNet/Engines/RunResults.cs b/src/BenchmarkDotNet/Engines/RunResults.cs index 17f0f52b3e..5a7651f46a 100644 --- a/src/BenchmarkDotNet/Engines/RunResults.cs +++ b/src/BenchmarkDotNet/Engines/RunResults.cs @@ -23,17 +23,21 @@ public struct RunResults public ThreadingStats ThreadingStats { get; } + public double ExceptionFrequency { get; } + public RunResults([CanBeNull] IReadOnlyList overhead, [NotNull] IReadOnlyList workload, OutlierMode outlierMode, GcStats gcStats, - ThreadingStats threadingStats) + ThreadingStats threadingStats, + double exceptionFrequency) { this.outlierMode = outlierMode; Overhead = overhead; Workload = workload; GCStats = gcStats; ThreadingStats = threadingStats; + ExceptionFrequency = exceptionFrequency; } public IEnumerable GetMeasurements() @@ -68,6 +72,8 @@ public void Print(TextWriter outWriter) outWriter.WriteLine(GCStats.ToOutputLine()); if (!ThreadingStats.Equals(ThreadingStats.Empty)) outWriter.WriteLine(ThreadingStats.ToOutputLine()); + if (ExceptionFrequency > 0) + outWriter.WriteLine(ExceptionsStats.ToOutputLine(ExceptionFrequency)); outWriter.WriteLine(); } diff --git a/src/BenchmarkDotNet/Toolchains/Results/ExecuteResult.cs b/src/BenchmarkDotNet/Toolchains/Results/ExecuteResult.cs index 769558ebeb..dcd68000d8 100644 --- a/src/BenchmarkDotNet/Toolchains/Results/ExecuteResult.cs +++ b/src/BenchmarkDotNet/Toolchains/Results/ExecuteResult.cs @@ -33,6 +33,7 @@ public class ExecuteResult internal readonly GcStats GcStats; internal readonly ThreadingStats ThreadingStats; + internal readonly double ExceptionFrequency; private readonly List errors; private readonly List measurements; @@ -48,10 +49,10 @@ public ExecuteResult(bool foundExecutable, int? exitCode, int? processId, IReadO ExitCode = exitCode; PrefixedLines = prefixedLines; StandardOutput = standardOutput; - Parse(results, prefixedLines, launchIndex, out measurements, out errors, out GcStats, out ThreadingStats); + Parse(results, prefixedLines, launchIndex, out measurements, out errors, out GcStats, out ThreadingStats, out ExceptionFrequency); } - internal ExecuteResult(List measurements, GcStats gcStats, ThreadingStats threadingStats) + internal ExecuteResult(List measurements, GcStats gcStats, ThreadingStats threadingStats, double exceptionFrequency) { FoundExecutable = true; ExitCode = 0; @@ -60,12 +61,13 @@ internal ExecuteResult(List measurements, GcStats gcStats, Threadin this.measurements = measurements; GcStats = gcStats; ThreadingStats = threadingStats; + ExceptionFrequency = exceptionFrequency; } internal static ExecuteResult FromRunResults(RunResults runResults, int exitCode) => exitCode != 0 ? CreateFailed(exitCode) - : new ExecuteResult(runResults.GetMeasurements().ToList(), runResults.GCStats, runResults.ThreadingStats); + : new ExecuteResult(runResults.GetMeasurements().ToList(), runResults.GCStats, runResults.ThreadingStats, runResults.ExceptionFrequency); internal static ExecuteResult CreateFailed(int exitCode = -1) => new ExecuteResult(false, exitCode, default, Array.Empty(), Array.Empty(), Array.Empty(), 0); @@ -98,12 +100,13 @@ public void LogIssues(ILogger logger, BuildResult buildResult) } private static void Parse(IReadOnlyList results, IReadOnlyList prefixedLines, int launchIndex, out List measurements, - out List errors, out GcStats gcStats, out ThreadingStats threadingStats) + out List errors, out GcStats gcStats, out ThreadingStats threadingStats, out double exceptionFrequency) { measurements = new List(); errors = new List(); gcStats = default; threadingStats = default; + exceptionFrequency = 0; foreach (string line in results.Where(text => !string.IsNullOrEmpty(text))) { @@ -128,6 +131,10 @@ private static void Parse(IReadOnlyList results, IReadOnlyList p { threadingStats = ThreadingStats.Parse(line); } + else if (line.StartsWith(ExceptionsStats.ResultsLinePrefix)) + { + exceptionFrequency = ExceptionsStats.Parse(line); + } } if (errors.Count > 0) diff --git a/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs b/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs index 29707c917f..5f423c2595 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs @@ -76,7 +76,8 @@ public RunResults Run() new List { new Measurement(1, IterationMode.Workload, IterationStage.Actual, 1, 1, 1) }, OutlierMode.DontRemove, default, - default); + default, + 0); } public void Dispose() => GlobalCleanupAction?.Invoke(); diff --git a/tests/BenchmarkDotNet.IntegrationTests/ExceptionDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/ExceptionDiagnoserTests.cs new file mode 100644 index 0000000000..1fbf86afe6 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/ExceptionDiagnoserTests.cs @@ -0,0 +1,66 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using System; +using System.Collections.Generic; +using System.Linq; +using Xunit; + +namespace BenchmarkDotNet.IntegrationTests +{ + public class ExceptionDiagnoserTests + { + [Fact] + public void ExceptionCountIsAccurate() + { + var config = CreateConfig(); + + var summary = BenchmarkRunner.Run(config); + + AssertStats(summary, new Dictionary + { + { nameof(ExceptionCount.DoNothing), ("ExceptionFrequency", 0.0) }, + { nameof(ExceptionCount.ThrowOneException), ("ExceptionFrequency", 1.0) } + }); + } + + public class ExceptionCount + { + [Benchmark] + public void ThrowOneException() + { + try + { + throw new Exception(); + } + catch { } + } + + [Benchmark] + public void DoNothing() { } + } + + private IConfig CreateConfig() + => ManualConfig.CreateEmpty() + .AddJob(Job.ShortRun + .WithEvaluateOverhead(false) // no need to run idle for this test + .WithWarmupCount(0) // don't run warmup to save some time for our CI runs + .WithIterationCount(1)) // single iteration is enough for us + .AddColumnProvider(DefaultColumnProviders.Instance) + .AddDiagnoser(ExceptionDiagnoser.Default); + + private void AssertStats(Summary summary, Dictionary assertions) + { + foreach (var assertion in assertions) + { + var selectedReport = summary.Reports.Single(report => report.BenchmarkCase.DisplayInfo.Contains(assertion.Key)); + var metric = selectedReport.Metrics.Single(m => m.Key == assertion.Value.metricName); + Assert.Equal(assertion.Value.expectedValue, metric.Value.Value); + } + } + } +} diff --git a/tests/BenchmarkDotNet.Tests/Engine/EngineResultStageTests.cs b/tests/BenchmarkDotNet.Tests/Engine/EngineResultStageTests.cs index 84d24a6b10..468ef87629 100644 --- a/tests/BenchmarkDotNet.Tests/Engine/EngineResultStageTests.cs +++ b/tests/BenchmarkDotNet.Tests/Engine/EngineResultStageTests.cs @@ -31,7 +31,7 @@ public void OutliersTest() [AssertionMethod] private static void CheckResults(int expectedResultCount, List measurements, OutlierMode outlierMode) { - Assert.Equal(expectedResultCount, new RunResults(null, measurements, outlierMode, default, default).GetMeasurements().Count()); + Assert.Equal(expectedResultCount, new RunResults(null, measurements, outlierMode, default, default, 0).GetMeasurements().Count()); } private static void Add(List measurements, int time) diff --git a/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs b/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs index 813d25f4d1..f1393f8b62 100644 --- a/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs +++ b/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs @@ -89,7 +89,7 @@ private static BenchmarkReport CreateReport(BenchmarkCase benchmarkCase, bool hu new Measurement(1, IterationMode.Workload, IterationStage.Result, 5, 1, hugeSd && isBar ? 3 : 1), new Measurement(1, IterationMode.Workload, IterationStage.Result, 6, 1, 1) }; - var executeResult = new ExecuteResult(measurements, default, default); + var executeResult = new ExecuteResult(measurements, default, default, 0); return new BenchmarkReport(true, benchmarkCase, buildResult, buildResult, new List { executeResult }, metrics); } diff --git a/tests/BenchmarkDotNet.Tests/Reports/RatioPrecisionTests.cs b/tests/BenchmarkDotNet.Tests/Reports/RatioPrecisionTests.cs index 27f2db779c..b8e0c6a6b7 100644 --- a/tests/BenchmarkDotNet.Tests/Reports/RatioPrecisionTests.cs +++ b/tests/BenchmarkDotNet.Tests/Reports/RatioPrecisionTests.cs @@ -91,7 +91,7 @@ private static BenchmarkReport CreateReport(BenchmarkCase benchmarkCase, int mea new Measurement(1, IterationMode.Workload, IterationStage.Result, 5, 1, measurementValue), new Measurement(1, IterationMode.Workload, IterationStage.Result, 6, 1, measurementValue), }; - var executeResult = new ExecuteResult(measurements, default, default); + var executeResult = new ExecuteResult(measurements, default, default, 0); return new BenchmarkReport(true, benchmarkCase, buildResult, buildResult, new List { executeResult }, Array.Empty()); } diff --git a/tests/BenchmarkDotNet.Tests/Reports/RatioStyleTests.cs b/tests/BenchmarkDotNet.Tests/Reports/RatioStyleTests.cs index c6e2f72191..2fe3ad2e5a 100644 --- a/tests/BenchmarkDotNet.Tests/Reports/RatioStyleTests.cs +++ b/tests/BenchmarkDotNet.Tests/Reports/RatioStyleTests.cs @@ -111,7 +111,7 @@ private static BenchmarkReport CreateReport(BenchmarkCase benchmarkCase, int mea new Measurement(1, IterationMode.Workload, IterationStage.Result, 4, 1, measurementValue + 2 * noise), new Measurement(1, IterationMode.Workload, IterationStage.Result, 5, 1, measurementValue - 3 * noise) }; - var executeResult = new ExecuteResult(measurements, default, default); + var executeResult = new ExecuteResult(measurements, default, default, 0); return new BenchmarkReport(true, benchmarkCase, buildResult, buildResult, new List { executeResult }, Array.Empty()); } From 64c3a3cb4ad4057b6065a2783a4b333206f8b5e5 Mon Sep 17 00:00:00 2001 From: Sergey Aseev Date: Wed, 26 Oct 2022 16:12:09 +0300 Subject: [PATCH 31/58] Implement MonoVM toolchain for net6.0 and net7.0 monikers (#2142) fixes #2064 Co-authored-by: Adam Sitnik --- .../Jobs/RuntimeMoniker.cs | 12 +++++- .../Attributes/Jobs/MonoJobAttribute.cs | 5 +++ src/BenchmarkDotNet/BenchmarkDotNet.csproj | 2 +- .../ConsoleArguments/ConfigParser.cs | 20 +++++++++ .../Environments/Runtimes/MonoRuntime.cs | 9 ++++ .../Extensions/RuntimeMonikerExtensions.cs | 4 ++ .../Portability/RuntimeInformation.cs | 43 +++++++++++-------- .../Toolchains/CsProj/CsProjCoreToolchain.cs | 2 +- .../Toolchains/Mono/MonoGenerator.cs | 19 ++++++++ .../Toolchains/Mono/MonoPublisher.cs | 38 ++++++++++++++++ .../Toolchains/Mono/MonoToolchain.cs | 35 +++++++++++++++ src/BenchmarkDotNet/Toolchains/Toolchain.cs | 2 +- .../Toolchains/ToolchainExtensions.cs | 8 ++++ .../MonoTests.cs | 31 +++++++++++++ 14 files changed, 209 insertions(+), 21 deletions(-) create mode 100644 src/BenchmarkDotNet/Toolchains/Mono/MonoGenerator.cs create mode 100644 src/BenchmarkDotNet/Toolchains/Mono/MonoPublisher.cs create mode 100644 src/BenchmarkDotNet/Toolchains/Mono/MonoToolchain.cs create mode 100644 tests/BenchmarkDotNet.IntegrationTests/MonoTests.cs diff --git a/src/BenchmarkDotNet.Annotations/Jobs/RuntimeMoniker.cs b/src/BenchmarkDotNet.Annotations/Jobs/RuntimeMoniker.cs index d74ac3cde9..7b6d6830db 100644 --- a/src/BenchmarkDotNet.Annotations/Jobs/RuntimeMoniker.cs +++ b/src/BenchmarkDotNet.Annotations/Jobs/RuntimeMoniker.cs @@ -143,6 +143,16 @@ public enum RuntimeMoniker /// /// Mono with the Ahead of Time LLVM Compiler backend and .net7.0 /// - MonoAOTLLVMNet70 + MonoAOTLLVMNet70, + + /// + /// .NET 6 using MonoVM (not CLR which is the default) + /// + Mono60, + + /// + /// .NET 7 using MonoVM (not CLR which is the default) + /// + Mono70, } } diff --git a/src/BenchmarkDotNet/Attributes/Jobs/MonoJobAttribute.cs b/src/BenchmarkDotNet/Attributes/Jobs/MonoJobAttribute.cs index b102033693..f47e5b77eb 100644 --- a/src/BenchmarkDotNet/Attributes/Jobs/MonoJobAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/Jobs/MonoJobAttribute.cs @@ -1,5 +1,6 @@ using System; using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Jobs; namespace BenchmarkDotNet.Attributes @@ -11,6 +12,10 @@ public MonoJobAttribute(bool baseline = false) : base(Job.Default.WithRuntime(Mo { } + public MonoJobAttribute(RuntimeMoniker runtimeMoniker, bool baseline = false) : base(Job.Default.WithRuntime(runtimeMoniker.GetRuntime()).WithBaseline(baseline)) + { + } + public MonoJobAttribute(string name, string path, bool baseline = false) : base(new Job(name, new EnvironmentMode(new MonoRuntime(name, path)).Freeze()).WithBaseline(baseline).Freeze()) { diff --git a/src/BenchmarkDotNet/BenchmarkDotNet.csproj b/src/BenchmarkDotNet/BenchmarkDotNet.csproj index 4e28175a44..4c401072ee 100644 --- a/src/BenchmarkDotNet/BenchmarkDotNet.csproj +++ b/src/BenchmarkDotNet/BenchmarkDotNet.csproj @@ -33,7 +33,7 @@ - + false diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index 691d8d9e75..93d1cb3c53 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -29,6 +29,7 @@ using Perfolizer.Mathematics.OutlierDetection; using Perfolizer.Mathematics.SignificanceTesting; using Perfolizer.Mathematics.Thresholds; +using BenchmarkDotNet.Toolchains.Mono; namespace BenchmarkDotNet.ConsoleArguments { @@ -425,6 +426,12 @@ private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, Comma case RuntimeMoniker.MonoAOTLLVMNet70: return MakeMonoAOTLLVMJob(baseJob, options, "net7.0"); + case RuntimeMoniker.Mono60: + return MakeMonoJob(baseJob, options, MonoRuntime.Mono60); + + case RuntimeMoniker.Mono70: + return MakeMonoJob(baseJob, options, MonoRuntime.Mono70); + default: throw new NotSupportedException($"Runtime {runtimeId} is not supported"); } @@ -452,6 +459,19 @@ private static Job CreateAotJob(Job baseJob, CommandLineOptions options, Runtime return baseJob.WithRuntime(runtime).WithToolchain(builder.ToToolchain()); } + private static Job MakeMonoJob(Job baseJob, CommandLineOptions options, MonoRuntime runtime) + { + return baseJob + .WithRuntime(runtime) + .WithToolchain(MonoToolchain.From( + new NetCoreAppSettings( + targetFrameworkMoniker: runtime.MsBuildMoniker, + runtimeFrameworkVersion: null, + name: runtime.Name, + customDotNetCliPath: options.CliPath?.FullName, + packagesPath: options.RestorePath?.FullName))); + } + private static Job MakeMonoAOTLLVMJob(Job baseJob, CommandLineOptions options, string msBuildMoniker) { var monoAotLLVMRuntime = new MonoAotLLVMRuntime(aotCompilerPath: options.AOTCompilerPath, aotCompilerMode: options.AOTCompilerMode, msBuildMoniker: msBuildMoniker); diff --git a/src/BenchmarkDotNet/Environments/Runtimes/MonoRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/MonoRuntime.cs index ad045cc7b7..36ef6e4976 100644 --- a/src/BenchmarkDotNet/Environments/Runtimes/MonoRuntime.cs +++ b/src/BenchmarkDotNet/Environments/Runtimes/MonoRuntime.cs @@ -6,6 +6,8 @@ namespace BenchmarkDotNet.Environments public class MonoRuntime : Runtime, IEquatable { public static readonly MonoRuntime Default = new MonoRuntime("Mono"); + public static readonly MonoRuntime Mono60 = new MonoRuntime("Mono with .NET 6.0", RuntimeMoniker.Mono60, "net6.0", isDotNetBuiltIn: true); + public static readonly MonoRuntime Mono70 = new MonoRuntime("Mono with .NET 7.0", RuntimeMoniker.Mono70, "net7.0", isDotNetBuiltIn: true); public string CustomPath { get; } @@ -15,8 +17,15 @@ public class MonoRuntime : Runtime, IEquatable public string MonoBclPath { get; } + internal bool IsDotNetBuiltIn { get; } + private MonoRuntime(string name) : base(RuntimeMoniker.Mono, "mono", name) { } + private MonoRuntime(string name, RuntimeMoniker runtimeMoniker, string msBuildMoniker, bool isDotNetBuiltIn) : base(runtimeMoniker, msBuildMoniker, name) + { + IsDotNetBuiltIn = isDotNetBuiltIn; + } + public MonoRuntime(string name, string customPath) : this(name) => CustomPath = customPath; public MonoRuntime(string name, string customPath, string aotArgs, string monoBclPath) : this(name) diff --git a/src/BenchmarkDotNet/Extensions/RuntimeMonikerExtensions.cs b/src/BenchmarkDotNet/Extensions/RuntimeMonikerExtensions.cs index e88ae890d1..a8ba6c5632 100644 --- a/src/BenchmarkDotNet/Extensions/RuntimeMonikerExtensions.cs +++ b/src/BenchmarkDotNet/Extensions/RuntimeMonikerExtensions.cs @@ -49,6 +49,10 @@ internal static Runtime GetRuntime(this RuntimeMoniker runtimeMoniker) return NativeAotRuntime.Net60; case RuntimeMoniker.NativeAot70: return NativeAotRuntime.Net70; + case RuntimeMoniker.Mono60: + return MonoRuntime.Mono60; + case RuntimeMoniker.Mono70: + return MonoRuntime.Mono70; default: throw new ArgumentOutOfRangeException(nameof(runtimeMoniker), runtimeMoniker, "Runtime Moniker not supported"); } diff --git a/src/BenchmarkDotNet/Portability/RuntimeInformation.cs b/src/BenchmarkDotNet/Portability/RuntimeInformation.cs index 1e0d88c67d..a630e2daf9 100644 --- a/src/BenchmarkDotNet/Portability/RuntimeInformation.cs +++ b/src/BenchmarkDotNet/Portability/RuntimeInformation.cs @@ -25,7 +25,7 @@ internal static class RuntimeInformation internal const string ReleaseConfigurationName = "RELEASE"; internal const string Unknown = "?"; - public static bool IsMono { get; } = Type.GetType("Mono.Runtime") != null; // it allocates a lot of memory, we need to check it once in order to keep Engine non-allocating! + public static bool IsMono { get; } = Type.GetType("Mono.RuntimeStructs") != null; // it allocates a lot of memory, we need to check it once in order to keep Engine non-allocating! public static bool IsFullFramework => #if NET6_0_OR_GREATER @@ -184,6 +184,10 @@ internal static string GetRuntimeVersion() return "Mono " + version; } + else + { + return $"{GetNetCoreVersion()} using MonoVM"; + } } else if (IsFullFramework) { @@ -208,22 +212,7 @@ internal static string GetRuntimeVersion() } else if (IsNetCore) { - var coreclrAssemblyInfo = FileVersionInfo.GetVersionInfo(typeof(object).GetTypeInfo().Assembly.Location); - var corefxAssemblyInfo = FileVersionInfo.GetVersionInfo(typeof(Regex).GetTypeInfo().Assembly.Location); - - if (CoreRuntime.TryGetVersion(out var version) && version >= new Version(5, 0)) - { - // after the merge of dotnet/corefx and dotnet/coreclr into dotnet/runtime the version should always be the same - Debug.Assert(coreclrAssemblyInfo.FileVersion == corefxAssemblyInfo.FileVersion); - - return $".NET {version} ({coreclrAssemblyInfo.FileVersion})"; - } - else - { - string runtimeVersion = version != default ? version.ToString() : "?"; - - return $".NET Core {runtimeVersion} (CoreCLR {coreclrAssemblyInfo.FileVersion}, CoreFX {corefxAssemblyInfo.FileVersion})"; - } + return GetNetCoreVersion(); } else if (IsNativeAOT) { @@ -233,6 +222,26 @@ internal static string GetRuntimeVersion() return Unknown; } + private static string GetNetCoreVersion() + { + var coreclrAssemblyInfo = FileVersionInfo.GetVersionInfo(typeof(object).GetTypeInfo().Assembly.Location); + var corefxAssemblyInfo = FileVersionInfo.GetVersionInfo(typeof(Regex).GetTypeInfo().Assembly.Location); + + if (CoreRuntime.TryGetVersion(out var version) && version >= new Version(5, 0)) + { + // after the merge of dotnet/corefx and dotnet/coreclr into dotnet/runtime the version should always be the same + Debug.Assert(coreclrAssemblyInfo.FileVersion == corefxAssemblyInfo.FileVersion); + + return $".NET {version} ({coreclrAssemblyInfo.FileVersion})"; + } + else + { + string runtimeVersion = version != default ? version.ToString() : "?"; + + return $".NET Core {runtimeVersion} (CoreCLR {coreclrAssemblyInfo.FileVersion}, CoreFX {corefxAssemblyInfo.FileVersion})"; + } + } + internal static Runtime GetCurrentRuntime() { //do not change the order of conditions because it may cause incorrect determination of runtime diff --git a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjCoreToolchain.cs b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjCoreToolchain.cs index 4b7c43feab..9dbaacc68c 100644 --- a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjCoreToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjCoreToolchain.cs @@ -24,7 +24,7 @@ public class CsProjCoreToolchain : Toolchain, IEquatable [PublicAPI] public static readonly IToolchain NetCoreApp60 = From(NetCoreAppSettings.NetCoreApp60); [PublicAPI] public static readonly IToolchain NetCoreApp70 = From(NetCoreAppSettings.NetCoreApp70); - private CsProjCoreToolchain(string name, IGenerator generator, IBuilder builder, IExecutor executor, string customDotNetCliPath) + internal CsProjCoreToolchain(string name, IGenerator generator, IBuilder builder, IExecutor executor, string customDotNetCliPath) : base(name, generator, builder, executor) { CustomDotNetCliPath = customDotNetCliPath; diff --git a/src/BenchmarkDotNet/Toolchains/Mono/MonoGenerator.cs b/src/BenchmarkDotNet/Toolchains/Mono/MonoGenerator.cs new file mode 100644 index 0000000000..f71a0f0733 --- /dev/null +++ b/src/BenchmarkDotNet/Toolchains/Mono/MonoGenerator.cs @@ -0,0 +1,19 @@ +using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Toolchains.CsProj; + +namespace BenchmarkDotNet.Toolchains.Mono +{ + public class MonoGenerator : CsProjGenerator + { + public MonoGenerator(string targetFrameworkMoniker, string cliPath, string packagesPath, string runtimeFrameworkVersion) : base(targetFrameworkMoniker, cliPath, packagesPath, runtimeFrameworkVersion, true) + { + } + + protected override string GetRuntimeSettings(GcMode gcMode, IResolver resolver) + { + // Workaround for 'Found multiple publish output files with the same relative path' error + return base.GetRuntimeSettings(gcMode, resolver) + "false"; + } + } +} diff --git a/src/BenchmarkDotNet/Toolchains/Mono/MonoPublisher.cs b/src/BenchmarkDotNet/Toolchains/Mono/MonoPublisher.cs new file mode 100644 index 0000000000..ff98540cbf --- /dev/null +++ b/src/BenchmarkDotNet/Toolchains/Mono/MonoPublisher.cs @@ -0,0 +1,38 @@ +using System.Collections.Generic; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains.DotNetCli; +using BenchmarkDotNet.Toolchains.Results; + +namespace BenchmarkDotNet.Toolchains.Mono +{ + public class MonoPublisher : IBuilder + { + public MonoPublisher(string customDotNetCliPath) + { + CustomDotNetCliPath = customDotNetCliPath; + var runtimeIdentifier = CustomDotNetCliToolchainBuilder.GetPortableRuntimeIdentifier(); + + // /p:RuntimeIdentifiers is set explicitly here because --self-contained requires it, see https://github.com/dotnet/sdk/issues/10566 + ExtraArguments = $"--self-contained -r {runtimeIdentifier} /p:UseMonoRuntime=true /p:RuntimeIdentifiers={runtimeIdentifier}"; + } + + private string CustomDotNetCliPath { get; } + + private string ExtraArguments { get; } + + private IReadOnlyList EnvironmentVariables { get; } + + public BuildResult Build(GenerateResult generateResult, BuildPartition buildPartition, ILogger logger) + => new DotNetCliCommand( + CustomDotNetCliPath, + ExtraArguments, + generateResult, + logger, + buildPartition, + EnvironmentVariables, + buildPartition.Timeout) + .Publish().ToBuildResult(generateResult); + } +} diff --git a/src/BenchmarkDotNet/Toolchains/Mono/MonoToolchain.cs b/src/BenchmarkDotNet/Toolchains/Mono/MonoToolchain.cs new file mode 100644 index 0000000000..28ea147c97 --- /dev/null +++ b/src/BenchmarkDotNet/Toolchains/Mono/MonoToolchain.cs @@ -0,0 +1,35 @@ +using BenchmarkDotNet.Toolchains.CsProj; +using BenchmarkDotNet.Toolchains.DotNetCli; +using JetBrains.Annotations; +using System; + +namespace BenchmarkDotNet.Toolchains.Mono +{ + [PublicAPI] + public class MonoToolchain : CsProjCoreToolchain, IEquatable + { + [PublicAPI] public static readonly IToolchain Mono60 = From(new NetCoreAppSettings("net6.0", null, "mono60")); + [PublicAPI] public static readonly IToolchain Mono70 = From(new NetCoreAppSettings("net7.0", null, "mono70")); + + private MonoToolchain(string name, IGenerator generator, IBuilder builder, IExecutor executor, string customDotNetCliPath) + : base(name, generator, builder, executor, customDotNetCliPath) + { + } + + [PublicAPI] + public static new IToolchain From(NetCoreAppSettings settings) + { + return new MonoToolchain(settings.Name, + new MonoGenerator(settings.TargetFrameworkMoniker, settings.CustomDotNetCliPath, settings.PackagesPath, settings.RuntimeFrameworkVersion), + new MonoPublisher(settings.CustomDotNetCliPath), + new DotNetCliExecutor(settings.CustomDotNetCliPath), + settings.CustomDotNetCliPath); + } + + public override bool Equals(object obj) => obj is MonoToolchain typed && Equals(typed); + + public bool Equals(MonoToolchain other) => Generator.Equals(other.Generator); + + public override int GetHashCode() => Generator.GetHashCode(); + } +} diff --git a/src/BenchmarkDotNet/Toolchains/Toolchain.cs b/src/BenchmarkDotNet/Toolchains/Toolchain.cs index 4424f20291..8c5d1596a7 100644 --- a/src/BenchmarkDotNet/Toolchains/Toolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/Toolchain.cs @@ -39,7 +39,7 @@ public virtual IEnumerable Validate(BenchmarkCase benchmarkCase benchmarkCase); } - if (runtime is MonoRuntime mono && !benchmarkCase.GetToolchain().IsInProcess) + if (runtime is MonoRuntime mono && !mono.IsDotNetBuiltIn && !benchmarkCase.GetToolchain().IsInProcess) { if (string.IsNullOrEmpty(mono.CustomPath) && !HostEnvironmentInfo.GetCurrent().IsMonoInstalled.Value) { diff --git a/src/BenchmarkDotNet/Toolchains/ToolchainExtensions.cs b/src/BenchmarkDotNet/Toolchains/ToolchainExtensions.cs index 51bb7c88bd..6741f7b86e 100644 --- a/src/BenchmarkDotNet/Toolchains/ToolchainExtensions.cs +++ b/src/BenchmarkDotNet/Toolchains/ToolchainExtensions.cs @@ -48,6 +48,8 @@ internal static IToolchain GetToolchain(this Runtime runtime, Descriptor descrip return InProcessNoEmitToolchain.Instance; if (!string.IsNullOrEmpty(mono.AotArgs)) return MonoAotToolchain.Instance; + if (mono.IsDotNetBuiltIn) + return MonoToolchain.From(new NetCoreAppSettings(targetFrameworkMoniker: mono.MsBuildMoniker, runtimeFrameworkVersion: null, name: mono.Name)); return RoslynToolchain.Instance; @@ -129,6 +131,12 @@ private static IToolchain GetToolchain(RuntimeMoniker runtimeMoniker) case RuntimeMoniker.NativeAot70: return NativeAotToolchain.Net70; + case RuntimeMoniker.Mono60: + return MonoToolchain.Mono60; + + case RuntimeMoniker.Mono70: + return MonoToolchain.Mono70; + default: throw new ArgumentOutOfRangeException(nameof(runtimeMoniker), runtimeMoniker, "RuntimeMoniker not supported"); } diff --git a/tests/BenchmarkDotNet.IntegrationTests/MonoTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MonoTests.cs new file mode 100644 index 0000000000..2dacf85173 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/MonoTests.cs @@ -0,0 +1,31 @@ +using System; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Tests.XUnit; + +namespace BenchmarkDotNet.IntegrationTests +{ + public class MonoTests : BenchmarkTestExecutor + { + [FactDotNetCoreOnly("UseMonoRuntime option is available in .NET Core only starting from .NET 6")] + public void Mono60IsSupported() + { + var config = ManualConfig.CreateEmpty().AddJob(Job.Dry.WithRuntime(MonoRuntime.Mono60)); + CanExecute(config); + } + + public class MonoBenchmark + { + [Benchmark] + public void Check() + { + if (Type.GetType("Mono.RuntimeStructs") == null) + { + throw new Exception("This is not Mono runtime"); + } + } + } + } +} From c6989543d2e42f1bdc316d53d3e2711cc9d3aa82 Mon Sep 17 00:00:00 2001 From: Tim Cassell <35501420+timcassell@users.noreply.github.com> Date: Thu, 27 Oct 2022 08:02:51 -0400 Subject: [PATCH 32/58] Add taskbar progress (#2158) --- .../Helpers/Taskbar/TaskbarProgress.cs | 197 ++++++++++++++++++ .../Running/BenchmarkRunnerClean.cs | 14 +- 2 files changed, 207 insertions(+), 4 deletions(-) create mode 100644 src/BenchmarkDotNet/Helpers/Taskbar/TaskbarProgress.cs diff --git a/src/BenchmarkDotNet/Helpers/Taskbar/TaskbarProgress.cs b/src/BenchmarkDotNet/Helpers/Taskbar/TaskbarProgress.cs new file mode 100644 index 0000000000..d385f933d8 --- /dev/null +++ b/src/BenchmarkDotNet/Helpers/Taskbar/TaskbarProgress.cs @@ -0,0 +1,197 @@ +using System; +using System.Runtime.InteropServices; + +namespace BenchmarkDotNet.Helpers +{ + internal class TaskbarProgress : IDisposable + { + private static readonly bool OsVersionIsSupported = Portability.RuntimeInformation.IsWindows() + // Must be windows 7 or greater + && Environment.OSVersion.Version >= new Version(6, 1); + + private IntPtr consoleWindowHandle = IntPtr.Zero; + private IntPtr consoleHandle = IntPtr.Zero; + + [DllImport("kernel32.dll")] + private static extern IntPtr GetConsoleWindow(); + [DllImport("kernel32.dll", SetLastError = true)] + private static extern IntPtr GetStdHandle(int nStdHandle); + + private const int STD_OUTPUT_HANDLE = -11; + + internal TaskbarProgress() + { + if (OsVersionIsSupported) + { + consoleWindowHandle = GetConsoleWindow(); + consoleHandle = GetStdHandle(STD_OUTPUT_HANDLE); + Console.CancelKeyPress += OnConsoleCancelEvent; + } + } + + internal void SetState(TaskbarProgressState state) + { + if (OsVersionIsSupported) + { + TaskbarProgressCom.SetState(consoleWindowHandle, consoleHandle, state); + } + } + + internal void SetProgress(float progressValue) + { + if (OsVersionIsSupported) + { + TaskbarProgressCom.SetValue(consoleWindowHandle, consoleHandle, progressValue); + } + } + + private void OnConsoleCancelEvent(object sender, ConsoleCancelEventArgs e) + { + Dispose(); + } + + public void Dispose() + { + if (OsVersionIsSupported) + { + TaskbarProgressCom.SetState(consoleWindowHandle, consoleHandle, TaskbarProgressState.NoProgress); + consoleWindowHandle = IntPtr.Zero; + consoleHandle = IntPtr.Zero; + Console.CancelKeyPress -= OnConsoleCancelEvent; + } + } + } + + internal enum TaskbarProgressState + { + NoProgress = 0, + Indeterminate = 0x1, + Normal = 0x2, + Error = 0x4, + Paused = 0x8, + Warning = Paused + } + + internal static class TaskbarProgressCom + { + [DllImport("kernel32.dll", SetLastError = true)] + private static extern bool GetConsoleMode(IntPtr hConsoleHandle, out ConsoleModes lpMode); + [DllImport("kernel32.dll", SetLastError = true)] + private static extern bool SetConsoleMode(IntPtr hConsoleHandle, ConsoleModes dwMode); + + [Flags] + private enum ConsoleModes : uint + { + ENABLE_PROCESSED_INPUT = 0x0001, + ENABLE_LINE_INPUT = 0x0002, + ENABLE_ECHO_INPUT = 0x0004, + ENABLE_WINDOW_INPUT = 0x0008, + ENABLE_MOUSE_INPUT = 0x0010, + ENABLE_INSERT_MODE = 0x0020, + ENABLE_QUICK_EDIT_MODE = 0x0040, + ENABLE_EXTENDED_FLAGS = 0x0080, + ENABLE_AUTO_POSITION = 0x0100, + + ENABLE_PROCESSED_OUTPUT = 0x0001, + ENABLE_WRAP_AT_EOL_OUTPUT = 0x0002, + ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004, + DISABLE_NEWLINE_AUTO_RETURN = 0x0008, + ENABLE_LVB_GRID_WORLDWIDE = 0x0010 + } + + [ComImport] + [Guid("ea1afb91-9e28-4b86-90e9-9e9f8a5eefaf")] + [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + private interface ITaskbarList3 + { + // ITaskbarList + [PreserveSig] + void HrInit(); + [PreserveSig] + void AddTab(IntPtr hwnd); + [PreserveSig] + void DeleteTab(IntPtr hwnd); + [PreserveSig] + void ActivateTab(IntPtr hwnd); + [PreserveSig] + void SetActiveAlt(IntPtr hwnd); + + // ITaskbarList2 + [PreserveSig] + void MarkFullscreenWindow(IntPtr hwnd, [MarshalAs(UnmanagedType.Bool)] bool fFullscreen); + + // ITaskbarList3 + [PreserveSig] + void SetProgressValue(IntPtr hwnd, ulong ullCompleted, ulong ullTotal); + [PreserveSig] + void SetProgressState(IntPtr hwnd, TaskbarProgressState state); + } + + [Guid("56FDF344-FD6D-11d0-958A-006097C9A090")] + [ClassInterface(ClassInterfaceType.None)] + [ComImport] + private class TaskbarInstance + { + } + + private static readonly ITaskbarList3 s_taskbarInstance = (ITaskbarList3) new TaskbarInstance(); + + internal static void SetState(IntPtr consoleWindowHandle, IntPtr consoleHandle, TaskbarProgressState taskbarState) + { + if (consoleWindowHandle != IntPtr.Zero) + { + s_taskbarInstance.SetProgressState(consoleWindowHandle, taskbarState); + } + + if (consoleHandle != IntPtr.Zero) + { + // Write progress state to console for Windows Terminal (https://github.com/microsoft/terminal/issues/6700). + GetConsoleMode(consoleHandle, out ConsoleModes previousConsoleMode); + SetConsoleMode(consoleHandle, ConsoleModes.ENABLE_VIRTUAL_TERMINAL_PROCESSING | ConsoleModes.ENABLE_PROCESSED_OUTPUT); + switch (taskbarState) + { + case TaskbarProgressState.NoProgress: + Console.Write("\x1b]9;4;0;0\x1b\\"); + break; + case TaskbarProgressState.Indeterminate: + Console.Write("\x1b]9;4;3;0\x1b\\"); + break; + case TaskbarProgressState.Normal: + // Do nothing, this is set automatically when SetValue is called (and WT has no documented way to set this). + break; + case TaskbarProgressState.Error: + Console.Write("\x1b]9;4;2;0\x1b\\"); + break; + case TaskbarProgressState.Warning: + Console.Write("\x1b]9;4;4;0\x1b\\"); + break; + }; + SetConsoleMode(consoleHandle, previousConsoleMode); + } + } + + internal static void SetValue(IntPtr consoleWindowHandle, IntPtr consoleHandle, float progressValue) + { + bool isValidRange = progressValue >= 0 & progressValue <= 1; + if (!isValidRange) + { + throw new ArgumentOutOfRangeException(nameof(progressValue), "progressValue must be between 0 and 1 inclusive."); + } + uint value = (uint) (progressValue * 100); + + if (consoleWindowHandle != IntPtr.Zero) + { + s_taskbarInstance.SetProgressValue(consoleWindowHandle, value, 100); + } + + if (consoleHandle != IntPtr.Zero) + { + // Write progress sequence to console for Windows Terminal (https://github.com/microsoft/terminal/discussions/14268). + GetConsoleMode(consoleHandle, out ConsoleModes previousConsoleMode); + SetConsoleMode(consoleHandle, ConsoleModes.ENABLE_VIRTUAL_TERMINAL_PROCESSING | ConsoleModes.ENABLE_PROCESSED_OUTPUT); + Console.Write($"\x1b]9;4;1;{value}\x1b\\"); + SetConsoleMode(consoleHandle, previousConsoleMode); + } + } + } +} diff --git a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs index b6dc5670d2..66a1f3d7da 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs @@ -36,6 +36,9 @@ internal static class BenchmarkRunnerClean internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos) { + using var taskbarProgress = new TaskbarProgress(); + taskbarProgress.SetState(TaskbarProgressState.Indeterminate); + var resolver = DefaultResolver; var artifactsToCleanup = new List(); @@ -97,7 +100,8 @@ internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos) } var summary = Run(benchmarkRunInfo, benchmarkToBuildResult, resolver, compositeLogger, artifactsToCleanup, - resultsFolderPath, logFilePath, totalBenchmarkCount, in runsChronometer, ref benchmarksToRunCount); + resultsFolderPath, logFilePath, totalBenchmarkCount, in runsChronometer, ref benchmarksToRunCount, + taskbarProgress); if (!benchmarkRunInfo.Config.Options.IsSet(ConfigOptions.JoinSummary)) PrintSummary(compositeLogger, benchmarkRunInfo.Config, summary); @@ -153,7 +157,8 @@ private static Summary Run(BenchmarkRunInfo benchmarkRunInfo, string logFilePath, int totalBenchmarkCount, in StartedClock runsChronometer, - ref int benchmarksToRunCount) + ref int benchmarksToRunCount, + TaskbarProgress taskbarProgress) { var runStart = runsChronometer.GetElapsed(); @@ -236,7 +241,7 @@ private static Summary Run(BenchmarkRunInfo benchmarkRunInfo, benchmarksToRunCount -= stop ? benchmarks.Length - i : 1; - LogProgress(logger, in runsChronometer, totalBenchmarkCount, benchmarksToRunCount); + LogProgress(logger, in runsChronometer, totalBenchmarkCount, benchmarksToRunCount, taskbarProgress); } } @@ -655,7 +660,7 @@ private static void UpdateTitle(int totalBenchmarkCount, int benchmarksToRunCoun } } - private static void LogProgress(ILogger logger, in StartedClock runsChronometer, int totalBenchmarkCount, int benchmarksToRunCount) + private static void LogProgress(ILogger logger, in StartedClock runsChronometer, int totalBenchmarkCount, int benchmarksToRunCount, TaskbarProgress taskbarProgress) { int executedBenchmarkCount = totalBenchmarkCount - benchmarksToRunCount; TimeSpan fromNow = GetEstimatedFinishTime(runsChronometer, benchmarksToRunCount, executedBenchmarkCount); @@ -668,6 +673,7 @@ private static void LogProgress(ILogger logger, in StartedClock runsChronometer, { Console.Title = $"{benchmarksToRunCount}/{totalBenchmarkCount} Remaining - {(int)fromNow.TotalHours}h {fromNow.Minutes}m to finish"; } + taskbarProgress.SetProgress((float) executedBenchmarkCount / totalBenchmarkCount); } private static TimeSpan GetEstimatedFinishTime(in StartedClock runsChronometer, int benchmarksToRunCount, int executedBenchmarkCount) From 1638995e405fcfca4b6d094614a12baf20b275d1 Mon Sep 17 00:00:00 2001 From: Youssef Victor Date: Fri, 28 Oct 2022 08:46:34 +0200 Subject: [PATCH 33/58] Cleanup NuGet.config (#2172) --- NuGet.Config | 3 --- 1 file changed, 3 deletions(-) diff --git a/NuGet.Config b/NuGet.Config index f493c9284a..b121982021 100644 --- a/NuGet.Config +++ b/NuGet.Config @@ -8,9 +8,6 @@ - - - From f4c0a718201cebe169d9695bbb11ecb32e100b2d Mon Sep 17 00:00:00 2001 From: "Mukund Raghav Sharma (Moko)" <68247673+mrsharm@users.noreply.github.com> Date: Fri, 28 Oct 2022 00:25:10 -0700 Subject: [PATCH 34/58] Added the ability to not run with Evaluation Overhead from command line (#2174) Co-authored-by: Adam Sitnik --- .../ConsoleArguments/CommandLineOptions.cs | 3 +++ src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs | 2 ++ tests/BenchmarkDotNet.Tests/ConfigParserTests.cs | 12 ++++++++++++ 3 files changed, 17 insertions(+) diff --git a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs index 9e3e3ca350..e3aed1fedd 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs @@ -218,6 +218,9 @@ public bool UseDisassemblyDiagnoser [Option("noForcedGCs", Required = false, HelpText = "Specifying would not forcefully induce any GCs.")] public bool NoForcedGCs { get; set; } + [Option("noOverheadEvaluation", Required = false, HelpText = "Specifying would not run the evaluation overhead iterations.")] + public bool NoEvaluationOverhead { get; set; } + [Option("resume", Required = false, Default = false, HelpText = "Continue the execution if the last run was stopped.")] public bool Resume { get; set; } diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index 93d1cb3c53..004639181c 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -301,6 +301,8 @@ private static Job GetBaseJob(CommandLineOptions options, IConfig globalConfig) baseJob = baseJob.WithMemoryRandomization(); if (options.NoForcedGCs) baseJob = baseJob.WithGcForce(false); + if (options.NoEvaluationOverhead) + baseJob = baseJob.WithEvaluateOverhead(false); if (options.EnvironmentVariables.Any()) { diff --git a/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs b/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs index 88bdc7ea5c..7276ee79e1 100644 --- a/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs +++ b/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs @@ -572,5 +572,17 @@ public void UserCanSpecifyNoForceGCs() Assert.False(job.Environment.Gc.Force); } } + + [Fact] + public void UsersCanSpecifyWithoutOverheadEvalution() + { + var parsedConfiguration = ConfigParser.Parse(new[] { "--noOverheadEvaluation" }, new OutputLogger(Output)); + Assert.True(parsedConfiguration.isSuccess); + + foreach (var job in parsedConfiguration.config.GetJobs()) + { + Assert.False(job.Accuracy.EvaluateOverhead); + } + } } } From ce1b7491bde1ba897450a4d5ab7e0ee49ba67353 Mon Sep 17 00:00:00 2001 From: Johan von Tangen Sivertsen Date: Fri, 28 Oct 2022 15:44:44 +0200 Subject: [PATCH 35/58] Rename TargetCount to IterationCount in SimpleJob attribute (#2175) * Rename TargetCount to IterationCount in SimpleJob attribute * Lowercase I in iterationCount Co-authored-by: Johan Sivertsen --- docs/articles/configs/jobs.md | 6 ++--- docs/articles/faq.md | 2 +- docs/articles/guides/choosing-run-strategy.md | 2 +- docs/articles/overview.md | 2 +- docs/articles/samples/IntroExportJson.md | 6 ++--- docs/articles/samples/IntroMonitoring.md | 2 +- docs/articles/samples/IntroPercentiles.md | 6 ++--- .../BenchmarkDotNet.Samples/IntroColdStart.cs | 2 +- .../IntroMonitoring.cs | 2 +- .../IntroPercentiles.cs | 2 +- .../BenchmarkDotNet.Samples/IntroRatioSD.cs | 2 +- .../IntroSetupCleanupIteration.cs | 2 +- .../IntroSetupCleanupTarget.cs | 2 +- .../IntroStatisticalTesting.cs | 2 +- .../Attributes/Jobs/SimpleJobAttribute.cs | 22 +++++++++---------- .../Engines/EngineGeneralStage.cs | 8 +++---- 16 files changed, 35 insertions(+), 35 deletions(-) diff --git a/docs/articles/configs/jobs.md b/docs/articles/configs/jobs.md index dd52110a63..a6085e0bdb 100644 --- a/docs/articles/configs/jobs.md +++ b/docs/articles/configs/jobs.md @@ -61,11 +61,11 @@ In this category, you can specify how to benchmark each method. * `MinWarmupIterationCount`: Minimum count of warmup iterations that should be performed, the default value is 6 * `MaxWarmupIterationCount`: Maximum count of warmup iterations that should be performed, the default value is 50 -Usually, you shouldn't specify such characteristics like `LaunchCount`, `WarmupCount`, `TargetCount`, or `IterationTime` because BenchmarkDotNet has a smart algorithm to choose these values automatically based on received measurements. You can specify it for testing purposes or when you are damn sure that you know the right characteristics for your benchmark (when you set `TargetCount` = `20` you should understand why `20` is a good value for your case). +Usually, you shouldn't specify such characteristics like `LaunchCount`, `WarmupCount`, `IterationCount`, or `IterationTime` because BenchmarkDotNet has a smart algorithm to choose these values automatically based on received measurements. You can specify it for testing purposes or when you are damn sure that you know the right characteristics for your benchmark (when you set `IterationCount` = `20` you should understand why `20` is a good value for your case). ### Accuracy -If you want to change the accuracy level, you should use the following characteristics instead of manually adjusting values of `WarmupCount`, `TargetCount`, and so on. +If you want to change the accuracy level, you should use the following characteristics instead of manually adjusting values of `WarmupCount`, `IterationCount`, and so on. * `MaxRelativeError`, `MaxAbsoluteError`: Maximum acceptable error for a benchmark (by default, BenchmarkDotNet continue iterations until the actual error is less than the specified error). *In these two characteristics*, the error means half of 99.9% confidence interval. `MaxAbsoluteError` is an absolute `TimeInterval`; doesn't have a default value. `MaxRelativeError` defines max acceptable (`() / Mean`). * `MinIterationTime`: Minimum time of a single iteration. Unlike `Run.IterationTime`, this characteristic specifies only the lower limit. In case of need, BenchmarkDotNet can increase this value. @@ -154,7 +154,7 @@ You can also add new jobs via attributes. Examples: [DryJob] [ClrJob, CoreJob, MonoJob] [LegacyJitX86Job, LegacyJitX64Job, RyuJitX64Job] -[SimpleJob(RunStrategy.ColdStart, launchCount: 1, warmupCount: 5, targetCount: 5, id: "FastAndDirtyJob")] +[SimpleJob(RunStrategy.ColdStart, launchCount: 1, warmupCount: 5, iterationCount: 5, id: "FastAndDirtyJob")] public class MyBenchmarkClass ``` diff --git a/docs/articles/faq.md b/docs/articles/faq.md index e966e3795e..1fcfb13b1a 100644 --- a/docs/articles/faq.md +++ b/docs/articles/faq.md @@ -45,7 +45,7 @@ If you don't need such level of precision and just want to have a quick way to g For example, you can use the `SimpleJob` or `ShortRunJob` attributes: ```cs - [SimpleJob(launchCount: 1, warmupCount: 3, targetCount: 5, invocationCount:100, id: "QuickJob")] + [SimpleJob(launchCount: 1, warmupCount: 3, iterationCount: 5, invocationCount:100, id: "QuickJob")] [ShortRunJob] ``` diff --git a/docs/articles/guides/choosing-run-strategy.md b/docs/articles/guides/choosing-run-strategy.md index d399fe4a81..efe0dc12d4 100644 --- a/docs/articles/guides/choosing-run-strategy.md +++ b/docs/articles/guides/choosing-run-strategy.md @@ -21,7 +21,7 @@ A benchmark method should have a steady state. Of course, you can manually set all the characteristics. An example: ```cs -[SimpleJob(launchCount: 3, warmupCount: 10, targetCount: 30)] +[SimpleJob(launchCount: 3, warmupCount: 10, iterationCount: 30)] public class MyBenchmarkClass ``` diff --git a/docs/articles/overview.md b/docs/articles/overview.md index f921e4836b..4e6f2c3d9c 100644 --- a/docs/articles/overview.md +++ b/docs/articles/overview.md @@ -134,7 +134,7 @@ public class Md5VsSha256 Add(new Job(EnvMode.LegacyJitX86, EnvMode.Clr, RunMode.Dry) { Env = { Runtime = Runtime.Clr }, - Run = { LaunchCount = 3, WarmupCount = 5, TargetCount = 10 }, + Run = { LaunchCount = 3, WarmupCount = 5, IterationCount = 10 }, Accuracy = { MaxStdErrRelative = 0.01 } })); } diff --git a/docs/articles/samples/IntroExportJson.md b/docs/articles/samples/IntroExportJson.md index 9af24ed18b..8c35d62c5d 100644 --- a/docs/articles/samples/IntroExportJson.md +++ b/docs/articles/samples/IntroExportJson.md @@ -56,7 +56,7 @@ Example of `IntroJsonExport-report-brief.json`: "Runtime":"Host", "GcMode":"Host", "WarmupCount":"Auto", - "TargetCount":"Auto", + "IterationCount":"Auto", "LaunchCount":"Auto", "IterationTime":"Auto", "Affinity":"Auto" @@ -114,7 +114,7 @@ Example of `IntroJsonExport-report-brief.json`: "Runtime":"Host", "GcMode":"Host", "WarmupCount":"Auto", - "TargetCount":"Auto", + "IterationCount":"Auto", "LaunchCount":"Auto", "IterationTime":"Auto", "Affinity":"Auto" @@ -168,4 +168,4 @@ Example of `IntroJsonExport-report-brief.json`: * @docs.exporters * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroExportJson ---- \ No newline at end of file +--- diff --git a/docs/articles/samples/IntroMonitoring.md b/docs/articles/samples/IntroMonitoring.md index 5f2e256417..d33f1b0d27 100644 --- a/docs/articles/samples/IntroMonitoring.md +++ b/docs/articles/samples/IntroMonitoring.md @@ -16,7 +16,7 @@ It's a perfect mode for benchmarks which doesn't have a steady state and the per ### Usage ```cs -[SimpleJob(RunStrategy.Monitoring, launchCount: 10, warmupCount: 0, targetCount: 100)] +[SimpleJob(RunStrategy.Monitoring, launchCount: 10, warmupCount: 0, iterationCount: 100)] public class MyBenchmarkClass ``` diff --git a/docs/articles/samples/IntroPercentiles.md b/docs/articles/samples/IntroPercentiles.md index f5e5ddc7b4..103f9563d4 100644 --- a/docs/articles/samples/IntroPercentiles.md +++ b/docs/articles/samples/IntroPercentiles.md @@ -13,7 +13,7 @@ However, real-world scenarios often have so-called long tail distribution The percentiles allow to include the tail of distribution into the comparison. However, it requires some preparations steps. At first, you should have enough runs to count percentiles from. -The `TargetCount` in the config should be set to 10-20 runs at least. +The `IterationCount` in the config should be set to 10-20 runs at least. Second, the count of iterations for each run should not be very high, or the peak timings will be averaged. The `IterationTime = 25` works fine for most cases; @@ -47,7 +47,7 @@ Also, it's very easy to screw the results with incorrect setup. For example, the ```cs new Job { - TargetCount = 5, + IterationCount = 5, IterationTime = 500 } ``` @@ -69,4 +69,4 @@ completely hides the peak values: * @docs.statistics * The permanent link to this sample: @BenchmarkDotNet.Samples.IntroPercentiles ---- \ No newline at end of file +--- diff --git a/samples/BenchmarkDotNet.Samples/IntroColdStart.cs b/samples/BenchmarkDotNet.Samples/IntroColdStart.cs index 535e940bf1..3a8e8c9b96 100644 --- a/samples/BenchmarkDotNet.Samples/IntroColdStart.cs +++ b/samples/BenchmarkDotNet.Samples/IntroColdStart.cs @@ -5,7 +5,7 @@ namespace BenchmarkDotNet.Samples { - [SimpleJob(RunStrategy.ColdStart, targetCount: 5)] + [SimpleJob(RunStrategy.ColdStart, iterationCount: 5)] [MinColumn, MaxColumn, MeanColumn, MedianColumn] public class IntroColdStart { diff --git a/samples/BenchmarkDotNet.Samples/IntroMonitoring.cs b/samples/BenchmarkDotNet.Samples/IntroMonitoring.cs index 2265e393bc..92ca8558b6 100644 --- a/samples/BenchmarkDotNet.Samples/IntroMonitoring.cs +++ b/samples/BenchmarkDotNet.Samples/IntroMonitoring.cs @@ -5,7 +5,7 @@ namespace BenchmarkDotNet.Samples { - [SimpleJob(RunStrategy.Monitoring, targetCount: 10, id: "MonitoringJob")] + [SimpleJob(RunStrategy.Monitoring, iterationCount: 10, id: "MonitoringJob")] [MinColumn, Q1Column, Q3Column, MaxColumn] public class IntroMonitoring { diff --git a/samples/BenchmarkDotNet.Samples/IntroPercentiles.cs b/samples/BenchmarkDotNet.Samples/IntroPercentiles.cs index 0b9cf5aa8d..d6886e62ba 100644 --- a/samples/BenchmarkDotNet.Samples/IntroPercentiles.cs +++ b/samples/BenchmarkDotNet.Samples/IntroPercentiles.cs @@ -10,7 +10,7 @@ namespace BenchmarkDotNet.Samples // Using percentiles for adequate timings representation [Config(typeof(Config))] [SimpleJob(RunStrategy.ColdStart, launchCount: 4, - warmupCount: 3, targetCount: 20, id: "MyJob")] + warmupCount: 3, iterationCount: 20, id: "MyJob")] public class IntroPercentiles { // To share between runs. diff --git a/samples/BenchmarkDotNet.Samples/IntroRatioSD.cs b/samples/BenchmarkDotNet.Samples/IntroRatioSD.cs index 86b32f9976..bc617529f5 100644 --- a/samples/BenchmarkDotNet.Samples/IntroRatioSD.cs +++ b/samples/BenchmarkDotNet.Samples/IntroRatioSD.cs @@ -8,7 +8,7 @@ namespace BenchmarkDotNet.Samples // Don't remove outliers [Outliers(OutlierMode.DontRemove)] // Skip jitting, pilot, warmup; measure 10 iterations - [SimpleJob(RunStrategy.Monitoring, targetCount: 10, invocationCount: 1)] + [SimpleJob(RunStrategy.Monitoring, iterationCount: 10, invocationCount: 1)] public class IntroRatioSD { private int counter; diff --git a/samples/BenchmarkDotNet.Samples/IntroSetupCleanupIteration.cs b/samples/BenchmarkDotNet.Samples/IntroSetupCleanupIteration.cs index aa4e666f7e..3ef8fbad9b 100644 --- a/samples/BenchmarkDotNet.Samples/IntroSetupCleanupIteration.cs +++ b/samples/BenchmarkDotNet.Samples/IntroSetupCleanupIteration.cs @@ -5,7 +5,7 @@ namespace BenchmarkDotNet.Samples { [SimpleJob(RunStrategy.Monitoring, launchCount: 1, - warmupCount: 2, targetCount: 3)] + warmupCount: 2, iterationCount: 3)] public class IntroSetupCleanupIteration { private int setupCounter; diff --git a/samples/BenchmarkDotNet.Samples/IntroSetupCleanupTarget.cs b/samples/BenchmarkDotNet.Samples/IntroSetupCleanupTarget.cs index f684fe6408..b62c28a052 100644 --- a/samples/BenchmarkDotNet.Samples/IntroSetupCleanupTarget.cs +++ b/samples/BenchmarkDotNet.Samples/IntroSetupCleanupTarget.cs @@ -5,7 +5,7 @@ namespace BenchmarkDotNet.Samples { [SimpleJob(RunStrategy.Monitoring, launchCount: 0, - warmupCount: 0, targetCount: 1)] + warmupCount: 0, iterationCount: 1)] public class IntroSetupCleanupTarget { [GlobalSetup(Target = nameof(BenchmarkA))] diff --git a/samples/BenchmarkDotNet.Samples/IntroStatisticalTesting.cs b/samples/BenchmarkDotNet.Samples/IntroStatisticalTesting.cs index f0045a6178..bbdf85aad5 100644 --- a/samples/BenchmarkDotNet.Samples/IntroStatisticalTesting.cs +++ b/samples/BenchmarkDotNet.Samples/IntroStatisticalTesting.cs @@ -9,7 +9,7 @@ namespace BenchmarkDotNet.Samples [StatisticalTestColumn(StatisticalTestKind.MannWhitney, ThresholdUnit.Microseconds, 1, true)] [StatisticalTestColumn(StatisticalTestKind.Welch, ThresholdUnit.Ratio, 0.03, true)] [StatisticalTestColumn(StatisticalTestKind.MannWhitney, ThresholdUnit.Ratio, 0.03, true)] - [SimpleJob(warmupCount: 0, targetCount: 5)] + [SimpleJob(warmupCount: 0, iterationCount: 5)] public class IntroStatisticalTesting { [Benchmark] public void Sleep50() => Thread.Sleep(50); diff --git a/src/BenchmarkDotNet/Attributes/Jobs/SimpleJobAttribute.cs b/src/BenchmarkDotNet/Attributes/Jobs/SimpleJobAttribute.cs index a8cf4231da..e0b80cf482 100644 --- a/src/BenchmarkDotNet/Attributes/Jobs/SimpleJobAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/Jobs/SimpleJobAttribute.cs @@ -16,33 +16,33 @@ public class SimpleJobAttribute : JobConfigBaseAttribute public SimpleJobAttribute( int launchCount = DefaultValue, int warmupCount = DefaultValue, - int targetCount = DefaultValue, + int iterationCount = DefaultValue, int invocationCount = DefaultValue, string id = null, bool baseline = false - ) : base(CreateJob(id, launchCount, warmupCount, targetCount, invocationCount, null, baseline)) { } + ) : base(CreateJob(id, launchCount, warmupCount, iterationCount, invocationCount, null, baseline)) { } [PublicAPI] public SimpleJobAttribute( RunStrategy runStrategy, int launchCount = DefaultValue, int warmupCount = DefaultValue, - int targetCount = DefaultValue, + int iterationCount = DefaultValue, int invocationCount = DefaultValue, string id = null, bool baseline = false - ) : base(CreateJob(id, launchCount, warmupCount, targetCount, invocationCount, runStrategy, baseline)) { } + ) : base(CreateJob(id, launchCount, warmupCount, iterationCount, invocationCount, runStrategy, baseline)) { } [PublicAPI] public SimpleJobAttribute( RuntimeMoniker runtimeMoniker, int launchCount = DefaultValue, int warmupCount = DefaultValue, - int targetCount = DefaultValue, + int iterationCount = DefaultValue, int invocationCount = DefaultValue, string id = null, bool baseline = false - ) : base(CreateJob(id, launchCount, warmupCount, targetCount, invocationCount, null, baseline, runtimeMoniker)) { } + ) : base(CreateJob(id, launchCount, warmupCount, iterationCount, invocationCount, null, baseline, runtimeMoniker)) { } [PublicAPI] public SimpleJobAttribute( @@ -50,13 +50,13 @@ public SimpleJobAttribute( RuntimeMoniker runtimeMoniker, int launchCount = DefaultValue, int warmupCount = DefaultValue, - int targetCount = DefaultValue, + int iterationCount = DefaultValue, int invocationCount = DefaultValue, string id = null, bool baseline = false - ) : base(CreateJob(id, launchCount, warmupCount, targetCount, invocationCount, runStrategy, baseline, runtimeMoniker)) { } + ) : base(CreateJob(id, launchCount, warmupCount, iterationCount, invocationCount, runStrategy, baseline, runtimeMoniker)) { } - private static Job CreateJob(string id, int launchCount, int warmupCount, int targetCount, int invocationCount, RunStrategy? runStrategy, + private static Job CreateJob(string id, int launchCount, int warmupCount, int iterationCount, int invocationCount, RunStrategy? runStrategy, bool baseline, RuntimeMoniker runtimeMoniker = RuntimeMoniker.HostProcess) { var job = new Job(id); @@ -74,9 +74,9 @@ private static Job CreateJob(string id, int launchCount, int warmupCount, int ta manualValuesCount++; } - if (targetCount != DefaultValue) + if (iterationCount != DefaultValue) { - job.Run.IterationCount = targetCount; + job.Run.IterationCount = iterationCount; manualValuesCount++; } if (invocationCount != DefaultValue) diff --git a/src/BenchmarkDotNet/Engines/EngineGeneralStage.cs b/src/BenchmarkDotNet/Engines/EngineGeneralStage.cs index 8b82448048..cecd779899 100644 --- a/src/BenchmarkDotNet/Engines/EngineGeneralStage.cs +++ b/src/BenchmarkDotNet/Engines/EngineGeneralStage.cs @@ -14,7 +14,7 @@ public class EngineActualStage : EngineStage private const double MaxOverheadRelativeError = 0.05; private const int DefaultWorkloadCount = 10; - private readonly int? targetCount; + private readonly int? iterationCount; private readonly double maxRelativeError; private readonly TimeInterval? maxAbsoluteError; private readonly OutlierMode outlierMode; @@ -23,7 +23,7 @@ public class EngineActualStage : EngineStage public EngineActualStage(IEngine engine) : base(engine) { - targetCount = engine.TargetJob.ResolveValueAsNullable(RunMode.IterationCountCharacteristic); + iterationCount = engine.TargetJob.ResolveValueAsNullable(RunMode.IterationCountCharacteristic); maxRelativeError = engine.TargetJob.ResolveValue(AccuracyMode.MaxRelativeErrorCharacteristic, engine.Resolver); maxAbsoluteError = engine.TargetJob.ResolveValueAsNullable(AccuracyMode.MaxAbsoluteErrorCharacteristic); outlierMode = engine.TargetJob.ResolveValue(AccuracyMode.OutlierModeCharacteristic, engine.Resolver); @@ -38,9 +38,9 @@ public IReadOnlyList RunWorkload(long invokeCount, int unrollFactor => Run(invokeCount, IterationMode.Workload, false, unrollFactor, forceSpecific); internal IReadOnlyList Run(long invokeCount, IterationMode iterationMode, bool runAuto, int unrollFactor, bool forceSpecific = false) - => (runAuto || targetCount == null) && !forceSpecific + => (runAuto || iterationCount == null) && !forceSpecific ? RunAuto(invokeCount, iterationMode, unrollFactor) - : RunSpecific(invokeCount, iterationMode, targetCount ?? DefaultWorkloadCount, unrollFactor); + : RunSpecific(invokeCount, iterationMode, iterationCount ?? DefaultWorkloadCount, unrollFactor); private List RunAuto(long invokeCount, IterationMode iterationMode, int unrollFactor) { From 18c6ffdfc25ce9c94ce5e099ac659c08852789b8 Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Fri, 28 Oct 2022 10:22:20 -0700 Subject: [PATCH 36/58] Fix #1521 (#2176) --- src/BenchmarkDotNet/Jobs/JobComparer.cs | 10 +++++++--- src/BenchmarkDotNet/Validators/BaselineValidator.cs | 3 ++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/BenchmarkDotNet/Jobs/JobComparer.cs b/src/BenchmarkDotNet/Jobs/JobComparer.cs index 2dc307b332..0a0c287523 100644 --- a/src/BenchmarkDotNet/Jobs/JobComparer.cs +++ b/src/BenchmarkDotNet/Jobs/JobComparer.cs @@ -4,9 +4,9 @@ namespace BenchmarkDotNet.Jobs { - internal class JobComparer : IComparer + internal class JobComparer : IComparer, IEqualityComparer { - public static readonly IComparer Instance = new JobComparer(); + public static readonly JobComparer Instance = new JobComparer(); public int Compare(Job x, Job y) { @@ -19,7 +19,7 @@ public int Compare(Job x, Job y) if (y == null) return 1; - if (x.GetType()!=y.GetType()) + if (x.GetType() != y.GetType()) throw new InvalidOperationException($"The type of xJob ({x.GetType()}) != type of yJob ({y.GetType()})"); var presenter = CharacteristicPresenter.DefaultPresenter; @@ -48,5 +48,9 @@ public int Compare(Job x, Job y) return 0; } + + public bool Equals(Job x, Job y) => Compare(x, y) == 0; + + public int GetHashCode(Job obj) => obj.Id.GetHashCode(); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Validators/BaselineValidator.cs b/src/BenchmarkDotNet/Validators/BaselineValidator.cs index 085710e3a5..2bb8fea85c 100644 --- a/src/BenchmarkDotNet/Validators/BaselineValidator.cs +++ b/src/BenchmarkDotNet/Validators/BaselineValidator.cs @@ -1,6 +1,7 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; +using BenchmarkDotNet.Jobs; namespace BenchmarkDotNet.Validators { @@ -26,7 +27,7 @@ public IEnumerable Validate(ValidationParameters input) { var benchmarks = allBenchmarks.Where((benchmark, index) => benchmarkLogicalGroups[index] == logicalGroup).ToArray(); int methodBaselineCount = benchmarks.Select(b => b.Descriptor).Distinct().Count(it => it.Baseline); - int jobBaselineCount = benchmarks.Select(b => b.Job).Distinct().Count(it => it.Meta.Baseline); + int jobBaselineCount = benchmarks.Select(b => b.Job).Distinct(JobComparer.Instance).Count(it => it.Meta.Baseline); string className = benchmarks.First().Descriptor.Type.Name; if (methodBaselineCount > 1) From 4eb6d385c33cbaaeb0185bb8b8e7eb8c587597c7 Mon Sep 17 00:00:00 2001 From: Andrey Akinshin Date: Sat, 29 Oct 2022 22:09:20 +0400 Subject: [PATCH 37/58] Bump Build.csproj TFM to net6.0 netcoreapp3.1 is not supported on macOS Arm64 --- build/Build.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Build.csproj b/build/Build.csproj index cff0ed9937..4cb47e4f84 100644 --- a/build/Build.csproj +++ b/build/Build.csproj @@ -1,7 +1,7 @@ Exe - netcoreapp3.1 + net6.0 $(MSBuildProjectDirectory) From 279c96dd19c42422e5365508bb2d6a3258124890 Mon Sep 17 00:00:00 2001 From: Andrey Akinshin Date: Sat, 29 Oct 2022 23:37:45 +0400 Subject: [PATCH 38/58] Bump actions/checkout v2->v3 actions/checkout@v3 uses Node.js 12 which is deprecated --- .github/workflows/build.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index a4c28e093f..3e68fba968 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -8,7 +8,7 @@ jobs: build-windows: runs-on: windows-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Run shell: cmd run: | @@ -17,7 +17,7 @@ jobs: build-linux: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Clang uses: egor-tensin/setup-clang@v1 with: @@ -30,6 +30,6 @@ jobs: build-macos: runs-on: macOS-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Run run: ./build.sh \ No newline at end of file From a1c62cae153a4a237eef1ca63c308041e3e62f61 Mon Sep 17 00:00:00 2001 From: Andrey Akinshin Date: Mon, 31 Oct 2022 10:41:48 +0400 Subject: [PATCH 39/58] Bump actions/checkout v2->v3 for spellcheck.yml actions/checkout@v3 uses Node.js 12 which is deprecated --- .github/workflows/spellcheck.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index ec5ad83f7b..5ab4f3a77d 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -16,7 +16,7 @@ jobs: name: "Docs: Spellcheck" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 name: Check out the code - uses: actions/setup-node@v1 name: Setup node From 7ba989292ef03cfec57a13700a7e4cd5508776ed Mon Sep 17 00:00:00 2001 From: Andrey Akinshin Date: Mon, 31 Oct 2022 20:01:00 +0400 Subject: [PATCH 40/58] Automatically generate redirects in documentation (cherry picked from commit 0b0fa569fb0b3a249f87847114a582a782b18079) --- build/Program.cs | 39 ++- docs/_redirects/.gitignore | 1 + docs/_redirects/RedirectGenerator/.gitignore | 293 ++++++++++++++++++ docs/_redirects/RedirectGenerator/Program.cs | 55 ++++ .../RedirectGenerator.csproj | 10 + docs/_redirects/_redirects | 47 +++ 6 files changed, 444 insertions(+), 1 deletion(-) create mode 100644 docs/_redirects/.gitignore create mode 100644 docs/_redirects/RedirectGenerator/.gitignore create mode 100644 docs/_redirects/RedirectGenerator/Program.cs create mode 100644 docs/_redirects/RedirectGenerator/RedirectGenerator.csproj create mode 100644 docs/_redirects/_redirects diff --git a/build/Program.cs b/build/Program.cs index 03968e14f1..d159745f37 100644 --- a/build/Program.cs +++ b/build/Program.cs @@ -48,6 +48,11 @@ public class BuildContext : FrostingContext public DirectoryPath ChangeLogDirectory { get; } public DirectoryPath ChangeLogGenDirectory { get; } + + public DirectoryPath RedirectRootDirectory { get; } + public DirectoryPath RedirectProjectDirectory { get; } + public DirectoryPath RedirectSourceDirectory { get; } + public DirectoryPath RedirectTargetDirectory { get; } public FilePath SolutionFile { get; } public FilePath UnitTestsProjectFile { get; } @@ -81,6 +86,11 @@ public BuildContext(ICakeContext context) ChangeLogDirectory = RootDirectory.Combine("docs").Combine("changelog"); ChangeLogGenDirectory = RootDirectory.Combine("docs").Combine("_changelog"); + + RedirectRootDirectory = RootDirectory.Combine("docs").Combine("_redirects"); + RedirectProjectDirectory = RedirectRootDirectory.Combine("RedirectGenerator"); + RedirectSourceDirectory = RedirectRootDirectory.Combine("redirects"); + RedirectTargetDirectory = RootDirectory.Combine("docs").Combine("_site"); SolutionFile = RootDirectory.CombineWithFilePath("BenchmarkDotNet.sln"); UnitTestsProjectFile = RootDirectory.Combine("tests").Combine("BenchmarkDotNet.Tests") @@ -133,7 +143,7 @@ public void RunTests(FilePath projectFile, string alias, string tfm) public void DocfxChangelogDownload(string version, string versionPrevious, string lastCommit = "") { this.Information("DocfxChangelogDownload: " + version); - // Required environment variables: GITHIB_PRODUCT, GITHUB_TOKEN + // Required environment variables: GITHUB_PRODUCT, GITHUB_TOKEN var changeLogBuilderDirectory = ChangeLogGenDirectory.Combine("ChangeLogBuilder"); var changeLogBuilderProjectFile = changeLogBuilderDirectory.CombineWithFilePath("ChangeLogBuilder.csproj"); this.DotNetRun(changeLogBuilderProjectFile.FullPath, @@ -199,6 +209,21 @@ public void RunDocfx(FilePath docfxJson, string args = "") else this.StartProcess(DocfxExeFile.FullPath, new ProcessSettings { Arguments = docfxJson + " " + args }); } + + public void GenerateRedirects() + { + var redirectProjectFile = RedirectProjectDirectory.CombineWithFilePath("RedirectGenerator.csproj"); + this.Information(redirectProjectFile.FullPath); + this.DotNetBuild(redirectProjectFile.FullPath); + this.DotNetRun(redirectProjectFile.FullPath, new DotNetRunSettings + { + WorkingDirectory = RedirectProjectDirectory, + }); + + this.Information(RedirectTargetDirectory); + this.EnsureDirectoryExists(RedirectTargetDirectory); + this.CopyFiles(RedirectSourceDirectory + "/**/*", RedirectTargetDirectory, true); + } } public static class DocumentationHelper @@ -462,6 +487,15 @@ public override void Run(BuildContext context) } } +[TaskName("DocFX_Generate_Redirects")] +public class DocfxGenerateRedirectsTask : FrostingTask +{ + public override void Run(BuildContext context) + { + context.GenerateRedirects(); + } +} + // In order to work around xref issues in DocFx, BenchmarkDotNet and BenchmarkDotNet.Annotations must be build // before running the DocFX_Build target. However, including a dependency on BuildTask here may have unwanted // side effects (CleanTask). @@ -474,6 +508,7 @@ public class DocfxChangelogBuildTask : FrostingTask public override void Run(BuildContext context) { context.RunDocfx(context.DocfxJsonFile); + context.GenerateRedirects(); } } @@ -484,6 +519,8 @@ public class DocfxChangelogServeTask : FrostingTask { public override void Run(BuildContext context) { + context.RunDocfx(context.DocfxJsonFile); + context.GenerateRedirects(); context.RunDocfx(context.DocfxJsonFile, "--serve"); } } diff --git a/docs/_redirects/.gitignore b/docs/_redirects/.gitignore new file mode 100644 index 0000000000..0db051f89d --- /dev/null +++ b/docs/_redirects/.gitignore @@ -0,0 +1 @@ +redirects \ No newline at end of file diff --git a/docs/_redirects/RedirectGenerator/.gitignore b/docs/_redirects/RedirectGenerator/.gitignore new file mode 100644 index 0000000000..f96d8c2e72 --- /dev/null +++ b/docs/_redirects/RedirectGenerator/.gitignore @@ -0,0 +1,293 @@ +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. +## +## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore + +# User-specific files +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ + +# Visual Studio 2015 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUNIT +*.VisualState.xml +TestResult.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# .NET Core +project.lock.json +project.fragment.lock.json +artifacts/ +**/Properties/launchSettings.json + +*_i.c +*_p.c +*_i.h +*.ilk +*.meta +*.obj +*.pch +*.pdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*.log +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# JustCode is a .NET coding add-in +.JustCode + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# Visual Studio code coverage results +*.coverage +*.coveragexml + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# TODO: Comment the next line if you want to checkin your web deploy settings +# but database connection strings (with potential passwords) will be unencrypted +*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +*.snupkg + +# The packages folder can be ignored because of Package Restore +**/packages/* +# except build/, which is used as an MSBuild target. +!**/packages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/packages/repositories.config +# NuGet v3's project.json files produces more ignorable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +orleans.codegen.cs + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm + +# SQL Server files +*.mdf +*.ldf +*.ndf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat +node_modules/ + +# Typescript v1 declaration files +typings/ + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) +*.vbw + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# JetBrains Rider +.idea/ +*.sln.iml + +# CodeRush +.cr/ + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc + +# Cake - Uncomment if you are using it +# tools/** +# !tools/packages.config + +# Telerik's JustMock configuration file +*.jmconfig + +# BizTalk build output +*.btp.cs +*.btm.cs +*.odx.cs +*.xsd.cs + +# ChangeLogBuilder +*.md \ No newline at end of file diff --git a/docs/_redirects/RedirectGenerator/Program.cs b/docs/_redirects/RedirectGenerator/Program.cs new file mode 100644 index 0000000000..9b32dbb06e --- /dev/null +++ b/docs/_redirects/RedirectGenerator/Program.cs @@ -0,0 +1,55 @@ +try +{ + var root = System.Reflection.Assembly.GetExecutingAssembly().Location; + while (root != null && Path.GetPathRoot(root) != root && Path.GetFileName(root) != "_redirects") + root = Directory.GetParent(root)?.FullName; + if (root == null || Path.GetFileName(root) != "_redirects") + { + Console.Error.WriteLine("Can't find the _redirects folders"); + return; + } + + var redirectsDirectory = Path.Combine(root, "redirects"); + if (Directory.Exists(redirectsDirectory)) + Directory.Delete(redirectsDirectory, true); + Directory.CreateDirectory(redirectsDirectory); + + var redirects = File.ReadAllLines(Path.Combine(root, "_redirects")) + .Select(line => line.Split(' ')) + .Select(parts => (source: parts[0], target: parts[1])) + .ToList(); + + bool EnsureDirectoryExists(string? fullDirectoryName) + { + if (fullDirectoryName == null) + return false; + if (Directory.Exists(fullDirectoryName)) + return true; + if (!EnsureDirectoryExists(Directory.GetParent(fullDirectoryName)?.FullName)) + return false; + Directory.CreateDirectory(fullDirectoryName); + return true; + } + + foreach (var (source, target) in redirects) + { + var fileName = source.StartsWith("/") || source.StartsWith("\\") ? source[1..] : source; + var fullFileName = Path.Combine(redirectsDirectory, fileName); + var content = + $"" + + $"" + + $"" + + $"{target}" + + $"" + + $"" + + $"" + + $"" + + $""; + EnsureDirectoryExists(Path.GetDirectoryName(fullFileName)); + File.WriteAllText(fullFileName, content); + } +} +catch (Exception e) +{ + Console.Error.WriteLine(e); +} \ No newline at end of file diff --git a/docs/_redirects/RedirectGenerator/RedirectGenerator.csproj b/docs/_redirects/RedirectGenerator/RedirectGenerator.csproj new file mode 100644 index 0000000000..40c60dd4c8 --- /dev/null +++ b/docs/_redirects/RedirectGenerator/RedirectGenerator.csproj @@ -0,0 +1,10 @@ + + + + Exe + net6.0 + enable + enable + + + diff --git a/docs/_redirects/_redirects b/docs/_redirects/_redirects new file mode 100644 index 0000000000..3103530863 --- /dev/null +++ b/docs/_redirects/_redirects @@ -0,0 +1,47 @@ +/Advancedfeatures.htm /articles/overview.html +/HowItWorks.htm /Internals/HowItWorks.htm +/RulesOfBenchmarking.htm /Guides/GoodPractices.htm +/Overview.htm /articles/overview.html +/Configs.htm /articles/configs/configs.html +/HowToRun.htm /Guides/HowToRun.htm +/index.htm /index.html +/Contributing.htm /articles/contributing/building.html +/Team.htm /articles/team.html +/Internals.htm /articles/guides/how-it-works.html +/Guides.htm /articles/guides/getting-started.html +/FAQ.htm /articles/faq.html +/NuGet.htm /Guides/NuGet.htm +/license.htm /articles/license.html +/GettingStarted.htm /Guides/GettingStarted.htm +/Advanced/Baseline.htm /articles/features/baselines.html +/Advanced/STAThread.htm /articles/samples/IntroStaThread.html +/Advanced/EnvironmentVariables.htm /articles/samples/IntroEnvVars.html +/Advanced/Params.htm /articles/features/parameterization.html +/Advanced/CustomizingMono.htm /articles/guides/customizing-runtime.html +/Advanced/CustomMonoPath.htm /articles/guides/customizing-runtime.html +/Advanced/Arguments.htm /articles/features/parameterization.html +/Advanced/Percentiles.htm /articles/features/statistics.html +/Advanced/SetupAndCleanup.htm /articles/features/setup-and-cleanup.html +/Contributing/Debugging.htm /articles/contributing/debugging.html +/Contributing/Disassembler.htm /articles/contributing/disassembler.html +/Contributing/RunningTests.htm /articles/contributing/running-tests.html +/Contributing/Miscellaneous.htm /articles/contributing/miscellaneous.html +/Contributing/Development.htm /articles/contributing/development.html +/Contributing/Building.htm /articles/contributing/building.html +/Internals/HowItWorks.htm /articles/guides/how-it-works.html +/Guides/GoodPractices.htm /articles/guides/good-practices.html +/Guides/HowToRun.htm /articles/guides/how-to-run.html +/Guides/NuGet.htm /articles/guides/nuget.html +/Guides/ChoosingRunStrategy.htm /articles/guides/choosing-run-strategy.html +/Guides/GettingStarted.htm /articles/guides/getting-started.html +/Configs/Columns.htm /articles/configs/columns.html +/Configs/Configs.htm /articles/configs/configs.html +/Configs/Diagnosers.htm /articles/configs/diagnosers.html +/Configs/Exporters.htm /articles/configs/exporters.html +/Configs/Filters.htm /articles/configs/filters.html +/Configs/OrderProvider.htm /articles/configs/orderers.html +/Configs/Analyzers.htm /articles/configs/analysers.html +/Configs/Toolchains.htm /articles/configs/toolchains.html +/Configs/Loggers.htm /articles/configs/loggers.html +/Configs/Validators.htm /articles/configs/validators.html +/Configs/Jobs.htm /articles/configs/jobs.html From d55c47edff3962b642d4bcfe477e5494fe65165e Mon Sep 17 00:00:00 2001 From: Andrey Akinshin Date: Mon, 31 Oct 2022 20:29:02 +0400 Subject: [PATCH 41/58] Remove old redirect files in documentation --- docs/redirects/Advanced/Arguments.htm | 3 --- docs/redirects/Advanced/Baseline.htm | 3 --- docs/redirects/Advanced/CustomMonoPath.htm | 3 --- docs/redirects/Advanced/CustomizingMono.htm | 3 --- docs/redirects/Advanced/EnvironmentVariables.htm | 3 --- docs/redirects/Advanced/Params.htm | 3 --- docs/redirects/Advanced/Percentiles.htm | 3 --- docs/redirects/Advanced/STAThread.htm | 3 --- docs/redirects/Advanced/SetupAndCleanup.htm | 3 --- docs/redirects/Advancedfeatures.htm | 3 --- docs/redirects/Configs.htm | 3 --- docs/redirects/Configs/Analyzers.htm | 3 --- docs/redirects/Configs/Columns.htm | 3 --- docs/redirects/Configs/Configs.htm | 3 --- docs/redirects/Configs/Diagnosers.htm | 3 --- docs/redirects/Configs/Exporters.htm | 3 --- docs/redirects/Configs/Filters.htm | 3 --- docs/redirects/Configs/Jobs.htm | 3 --- docs/redirects/Configs/Loggers.htm | 3 --- docs/redirects/Configs/OrderProvider.htm | 3 --- docs/redirects/Configs/Toolchains.htm | 3 --- docs/redirects/Configs/Validators.htm | 3 --- docs/redirects/Contributing.htm | 3 --- docs/redirects/Contributing/Building.htm | 3 --- docs/redirects/Contributing/Debugging.htm | 3 --- docs/redirects/Contributing/Development.htm | 3 --- docs/redirects/Contributing/Disassembler.htm | 3 --- docs/redirects/Contributing/Miscellaneous.htm | 3 --- docs/redirects/Contributing/RunningTests.htm | 3 --- docs/redirects/FAQ.htm | 3 --- docs/redirects/GettingStarted.htm | 3 --- docs/redirects/Guides.htm | 3 --- docs/redirects/Guides/ChoosingRunStrategy.htm | 3 --- docs/redirects/Guides/GettingStarted.htm | 3 --- docs/redirects/Guides/GoodPractices.htm | 3 --- docs/redirects/Guides/HowToRun.htm | 3 --- docs/redirects/Guides/NuGet.htm | 3 --- docs/redirects/HowItWorks.htm | 3 --- docs/redirects/HowToRun.htm | 3 --- docs/redirects/Internals.htm | 3 --- docs/redirects/Internals/HowItWorks.htm | 3 --- docs/redirects/NuGet.htm | 3 --- docs/redirects/Overview.htm | 3 --- docs/redirects/RulesOfBenchmarking.htm | 3 --- docs/redirects/Team.htm | 3 --- docs/redirects/index.htm | 3 --- docs/redirects/license.htm | 3 --- 47 files changed, 141 deletions(-) delete mode 100644 docs/redirects/Advanced/Arguments.htm delete mode 100644 docs/redirects/Advanced/Baseline.htm delete mode 100644 docs/redirects/Advanced/CustomMonoPath.htm delete mode 100644 docs/redirects/Advanced/CustomizingMono.htm delete mode 100644 docs/redirects/Advanced/EnvironmentVariables.htm delete mode 100644 docs/redirects/Advanced/Params.htm delete mode 100644 docs/redirects/Advanced/Percentiles.htm delete mode 100644 docs/redirects/Advanced/STAThread.htm delete mode 100644 docs/redirects/Advanced/SetupAndCleanup.htm delete mode 100644 docs/redirects/Advancedfeatures.htm delete mode 100644 docs/redirects/Configs.htm delete mode 100644 docs/redirects/Configs/Analyzers.htm delete mode 100644 docs/redirects/Configs/Columns.htm delete mode 100644 docs/redirects/Configs/Configs.htm delete mode 100644 docs/redirects/Configs/Diagnosers.htm delete mode 100644 docs/redirects/Configs/Exporters.htm delete mode 100644 docs/redirects/Configs/Filters.htm delete mode 100644 docs/redirects/Configs/Jobs.htm delete mode 100644 docs/redirects/Configs/Loggers.htm delete mode 100644 docs/redirects/Configs/OrderProvider.htm delete mode 100644 docs/redirects/Configs/Toolchains.htm delete mode 100644 docs/redirects/Configs/Validators.htm delete mode 100644 docs/redirects/Contributing.htm delete mode 100644 docs/redirects/Contributing/Building.htm delete mode 100644 docs/redirects/Contributing/Debugging.htm delete mode 100644 docs/redirects/Contributing/Development.htm delete mode 100644 docs/redirects/Contributing/Disassembler.htm delete mode 100644 docs/redirects/Contributing/Miscellaneous.htm delete mode 100644 docs/redirects/Contributing/RunningTests.htm delete mode 100644 docs/redirects/FAQ.htm delete mode 100644 docs/redirects/GettingStarted.htm delete mode 100644 docs/redirects/Guides.htm delete mode 100644 docs/redirects/Guides/ChoosingRunStrategy.htm delete mode 100644 docs/redirects/Guides/GettingStarted.htm delete mode 100644 docs/redirects/Guides/GoodPractices.htm delete mode 100644 docs/redirects/Guides/HowToRun.htm delete mode 100644 docs/redirects/Guides/NuGet.htm delete mode 100644 docs/redirects/HowItWorks.htm delete mode 100644 docs/redirects/HowToRun.htm delete mode 100644 docs/redirects/Internals.htm delete mode 100644 docs/redirects/Internals/HowItWorks.htm delete mode 100644 docs/redirects/NuGet.htm delete mode 100644 docs/redirects/Overview.htm delete mode 100644 docs/redirects/RulesOfBenchmarking.htm delete mode 100644 docs/redirects/Team.htm delete mode 100644 docs/redirects/index.htm delete mode 100644 docs/redirects/license.htm diff --git a/docs/redirects/Advanced/Arguments.htm b/docs/redirects/Advanced/Arguments.htm deleted file mode 100644 index f2107e4e9a..0000000000 --- a/docs/redirects/Advanced/Arguments.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Advanced/Baseline.htm b/docs/redirects/Advanced/Baseline.htm deleted file mode 100644 index ec24135e74..0000000000 --- a/docs/redirects/Advanced/Baseline.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Advanced/CustomMonoPath.htm b/docs/redirects/Advanced/CustomMonoPath.htm deleted file mode 100644 index 4563e4cc66..0000000000 --- a/docs/redirects/Advanced/CustomMonoPath.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Advanced/CustomizingMono.htm b/docs/redirects/Advanced/CustomizingMono.htm deleted file mode 100644 index 4563e4cc66..0000000000 --- a/docs/redirects/Advanced/CustomizingMono.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Advanced/EnvironmentVariables.htm b/docs/redirects/Advanced/EnvironmentVariables.htm deleted file mode 100644 index 8ecf1af1af..0000000000 --- a/docs/redirects/Advanced/EnvironmentVariables.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Advanced/Params.htm b/docs/redirects/Advanced/Params.htm deleted file mode 100644 index f2107e4e9a..0000000000 --- a/docs/redirects/Advanced/Params.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Advanced/Percentiles.htm b/docs/redirects/Advanced/Percentiles.htm deleted file mode 100644 index 817df98207..0000000000 --- a/docs/redirects/Advanced/Percentiles.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Advanced/STAThread.htm b/docs/redirects/Advanced/STAThread.htm deleted file mode 100644 index ea0c1a8fb8..0000000000 --- a/docs/redirects/Advanced/STAThread.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Advanced/SetupAndCleanup.htm b/docs/redirects/Advanced/SetupAndCleanup.htm deleted file mode 100644 index 1e6305f5ed..0000000000 --- a/docs/redirects/Advanced/SetupAndCleanup.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Advancedfeatures.htm b/docs/redirects/Advancedfeatures.htm deleted file mode 100644 index b0c0898511..0000000000 --- a/docs/redirects/Advancedfeatures.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Configs.htm b/docs/redirects/Configs.htm deleted file mode 100644 index 911799c811..0000000000 --- a/docs/redirects/Configs.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Configs/Analyzers.htm b/docs/redirects/Configs/Analyzers.htm deleted file mode 100644 index f2ee745a43..0000000000 --- a/docs/redirects/Configs/Analyzers.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Configs/Columns.htm b/docs/redirects/Configs/Columns.htm deleted file mode 100644 index ba224eb2ed..0000000000 --- a/docs/redirects/Configs/Columns.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Configs/Configs.htm b/docs/redirects/Configs/Configs.htm deleted file mode 100644 index 911799c811..0000000000 --- a/docs/redirects/Configs/Configs.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Configs/Diagnosers.htm b/docs/redirects/Configs/Diagnosers.htm deleted file mode 100644 index 66a606cb4e..0000000000 --- a/docs/redirects/Configs/Diagnosers.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Configs/Exporters.htm b/docs/redirects/Configs/Exporters.htm deleted file mode 100644 index 41f451b4ce..0000000000 --- a/docs/redirects/Configs/Exporters.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Configs/Filters.htm b/docs/redirects/Configs/Filters.htm deleted file mode 100644 index ba527eff53..0000000000 --- a/docs/redirects/Configs/Filters.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Configs/Jobs.htm b/docs/redirects/Configs/Jobs.htm deleted file mode 100644 index 0eabbb284d..0000000000 --- a/docs/redirects/Configs/Jobs.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Configs/Loggers.htm b/docs/redirects/Configs/Loggers.htm deleted file mode 100644 index 3094635203..0000000000 --- a/docs/redirects/Configs/Loggers.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Configs/OrderProvider.htm b/docs/redirects/Configs/OrderProvider.htm deleted file mode 100644 index 5c0a34233d..0000000000 --- a/docs/redirects/Configs/OrderProvider.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Configs/Toolchains.htm b/docs/redirects/Configs/Toolchains.htm deleted file mode 100644 index 3986cd8c86..0000000000 --- a/docs/redirects/Configs/Toolchains.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Configs/Validators.htm b/docs/redirects/Configs/Validators.htm deleted file mode 100644 index 9470e5f094..0000000000 --- a/docs/redirects/Configs/Validators.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Contributing.htm b/docs/redirects/Contributing.htm deleted file mode 100644 index da570270c6..0000000000 --- a/docs/redirects/Contributing.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Contributing/Building.htm b/docs/redirects/Contributing/Building.htm deleted file mode 100644 index da570270c6..0000000000 --- a/docs/redirects/Contributing/Building.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Contributing/Debugging.htm b/docs/redirects/Contributing/Debugging.htm deleted file mode 100644 index ebbbf96740..0000000000 --- a/docs/redirects/Contributing/Debugging.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Contributing/Development.htm b/docs/redirects/Contributing/Development.htm deleted file mode 100644 index e82f4c0312..0000000000 --- a/docs/redirects/Contributing/Development.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Contributing/Disassembler.htm b/docs/redirects/Contributing/Disassembler.htm deleted file mode 100644 index a5aec745df..0000000000 --- a/docs/redirects/Contributing/Disassembler.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Contributing/Miscellaneous.htm b/docs/redirects/Contributing/Miscellaneous.htm deleted file mode 100644 index 8aa2a4d9ec..0000000000 --- a/docs/redirects/Contributing/Miscellaneous.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Contributing/RunningTests.htm b/docs/redirects/Contributing/RunningTests.htm deleted file mode 100644 index 8b6571e774..0000000000 --- a/docs/redirects/Contributing/RunningTests.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/FAQ.htm b/docs/redirects/FAQ.htm deleted file mode 100644 index 4a4d3fdf1d..0000000000 --- a/docs/redirects/FAQ.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/GettingStarted.htm b/docs/redirects/GettingStarted.htm deleted file mode 100644 index 8123772361..0000000000 --- a/docs/redirects/GettingStarted.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Guides.htm b/docs/redirects/Guides.htm deleted file mode 100644 index deae7df60b..0000000000 --- a/docs/redirects/Guides.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Guides/ChoosingRunStrategy.htm b/docs/redirects/Guides/ChoosingRunStrategy.htm deleted file mode 100644 index 5eca11d1f8..0000000000 --- a/docs/redirects/Guides/ChoosingRunStrategy.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Guides/GettingStarted.htm b/docs/redirects/Guides/GettingStarted.htm deleted file mode 100644 index deae7df60b..0000000000 --- a/docs/redirects/Guides/GettingStarted.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Guides/GoodPractices.htm b/docs/redirects/Guides/GoodPractices.htm deleted file mode 100644 index 88ceab97f1..0000000000 --- a/docs/redirects/Guides/GoodPractices.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Guides/HowToRun.htm b/docs/redirects/Guides/HowToRun.htm deleted file mode 100644 index c401eafdaa..0000000000 --- a/docs/redirects/Guides/HowToRun.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Guides/NuGet.htm b/docs/redirects/Guides/NuGet.htm deleted file mode 100644 index db36d593fe..0000000000 --- a/docs/redirects/Guides/NuGet.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/HowItWorks.htm b/docs/redirects/HowItWorks.htm deleted file mode 100644 index 3239dbefc9..0000000000 --- a/docs/redirects/HowItWorks.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/HowToRun.htm b/docs/redirects/HowToRun.htm deleted file mode 100644 index 929d19ca65..0000000000 --- a/docs/redirects/HowToRun.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Internals.htm b/docs/redirects/Internals.htm deleted file mode 100644 index 9a6cc60662..0000000000 --- a/docs/redirects/Internals.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Internals/HowItWorks.htm b/docs/redirects/Internals/HowItWorks.htm deleted file mode 100644 index 9a6cc60662..0000000000 --- a/docs/redirects/Internals/HowItWorks.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/NuGet.htm b/docs/redirects/NuGet.htm deleted file mode 100644 index 8987934b64..0000000000 --- a/docs/redirects/NuGet.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Overview.htm b/docs/redirects/Overview.htm deleted file mode 100644 index b0c0898511..0000000000 --- a/docs/redirects/Overview.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/RulesOfBenchmarking.htm b/docs/redirects/RulesOfBenchmarking.htm deleted file mode 100644 index 0a5c312489..0000000000 --- a/docs/redirects/RulesOfBenchmarking.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/Team.htm b/docs/redirects/Team.htm deleted file mode 100644 index b9c24b7aa4..0000000000 --- a/docs/redirects/Team.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/index.htm b/docs/redirects/index.htm deleted file mode 100644 index 29028036e2..0000000000 --- a/docs/redirects/index.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file diff --git a/docs/redirects/license.htm b/docs/redirects/license.htm deleted file mode 100644 index 439786b90e..0000000000 --- a/docs/redirects/license.htm +++ /dev/null @@ -1,3 +0,0 @@ - \ No newline at end of file From b6d329d8f69dcf058791677a86c7b8277f1e0662 Mon Sep 17 00:00:00 2001 From: Andrey Akinshin Date: Mon, 31 Oct 2022 20:34:26 +0400 Subject: [PATCH 42/58] Bump System.Drawing.Common in BenchmarkDotNet.Samples.csproj: 4.5.1->4.7.2 --- samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj b/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj index 9386dcf6bb..0693554d6f 100644 --- a/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj +++ b/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj @@ -17,7 +17,7 @@ - + From ee81250d051aab59ec015518dc7a450569d9db4b Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Tue, 1 Nov 2022 12:35:19 -0700 Subject: [PATCH 43/58] Remove allowMultiple=true from column attributes (#2183) * Remove allowMultiple=true * Remove redundant attributes --- .../Attributes/Columns/BaselineColumnAttribute.cs | 4 +--- .../Attributes/Columns/LogicalGroupColumnAttribute.cs | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/BenchmarkDotNet/Attributes/Columns/BaselineColumnAttribute.cs b/src/BenchmarkDotNet/Attributes/Columns/BaselineColumnAttribute.cs index b24a00e396..984f612eb0 100644 --- a/src/BenchmarkDotNet/Attributes/Columns/BaselineColumnAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/Columns/BaselineColumnAttribute.cs @@ -1,9 +1,7 @@ -using System; -using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Columns; namespace BenchmarkDotNet.Attributes { - [AttributeUsage(AttributeTargets.Class | AttributeTargets.Assembly, AllowMultiple = true)] public class BaselineColumnAttribute : ColumnConfigBaseAttribute { public BaselineColumnAttribute() : base(BaselineColumn.Default) { } diff --git a/src/BenchmarkDotNet/Attributes/Columns/LogicalGroupColumnAttribute.cs b/src/BenchmarkDotNet/Attributes/Columns/LogicalGroupColumnAttribute.cs index b4335e683c..cd5c2d033a 100644 --- a/src/BenchmarkDotNet/Attributes/Columns/LogicalGroupColumnAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/Columns/LogicalGroupColumnAttribute.cs @@ -1,9 +1,7 @@ -using System; -using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Columns; namespace BenchmarkDotNet.Attributes { - [AttributeUsage(AttributeTargets.Class | AttributeTargets.Assembly, AllowMultiple = true)] public class LogicalGroupColumnAttribute : ColumnConfigBaseAttribute { public LogicalGroupColumnAttribute() : base(LogicalGroupColumn.Default) { } From ddd2f31c9d1b6eaf6ac40d63b782977623ca74d8 Mon Sep 17 00:00:00 2001 From: leonvandermeer <64834803+leonvandermeer@users.noreply.github.com> Date: Tue, 1 Nov 2022 21:15:19 +0100 Subject: [PATCH 44/58] Fix a threading issue in ExceptionDiagnoser #1736 (#2182) When a Benchmark spawns multiple threads that throw Exceptions, FirstChanceException event handler is called concurrently. It must count Exceptions in a thread-safe way. Note: do not use lock keyword because that will influence Monitor lock contention count of ThreadingDiagnoser. --- .../Engines/ExceptionsStats.cs | 7 +++++-- .../ExceptionDiagnoserTests.cs | 20 ++++++++++++++++++- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/BenchmarkDotNet/Engines/ExceptionsStats.cs b/src/BenchmarkDotNet/Engines/ExceptionsStats.cs index 35aa387f97..0f8772f9fa 100644 --- a/src/BenchmarkDotNet/Engines/ExceptionsStats.cs +++ b/src/BenchmarkDotNet/Engines/ExceptionsStats.cs @@ -1,5 +1,6 @@ using System; using System.Runtime.ExceptionServices; +using System.Threading; namespace BenchmarkDotNet.Engines { @@ -7,7 +8,9 @@ internal class ExceptionsStats { internal const string ResultsLinePrefix = "// Exceptions: "; - internal ulong ExceptionsCount { get; private set; } + private long exceptionsCount; + + internal long ExceptionsCount { get => exceptionsCount; } public void StartListening() { @@ -21,7 +24,7 @@ public void Stop() private void OnFirstChanceException(object sender, FirstChanceExceptionEventArgs e) { - ExceptionsCount++; + Interlocked.Increment(ref exceptionsCount); } public static string ToOutputLine(double exceptionCount) => $"{ResultsLinePrefix} {exceptionCount}"; diff --git a/tests/BenchmarkDotNet.IntegrationTests/ExceptionDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/ExceptionDiagnoserTests.cs index 1fbf86afe6..71e4ba29bb 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ExceptionDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ExceptionDiagnoserTests.cs @@ -8,6 +8,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading.Tasks; using Xunit; namespace BenchmarkDotNet.IntegrationTests @@ -24,7 +25,8 @@ public void ExceptionCountIsAccurate() AssertStats(summary, new Dictionary { { nameof(ExceptionCount.DoNothing), ("ExceptionFrequency", 0.0) }, - { nameof(ExceptionCount.ThrowOneException), ("ExceptionFrequency", 1.0) } + { nameof(ExceptionCount.ThrowOneException), ("ExceptionFrequency", 1.0) }, + { nameof(ExceptionCount.ThrowFromMultipleThreads), ("ExceptionFrequency", 100.0) } }); } @@ -42,6 +44,22 @@ public void ThrowOneException() [Benchmark] public void DoNothing() { } + + [Benchmark] + public async Task ThrowFromMultipleThreads() + { + void ThrowMultipleExceptions() + { + for (int i = 0; i < 10; i++) + { + ThrowOneException(); + } + } + + var tasks = Enumerable.Range(1, 10).Select( + i => Task.Run(ThrowMultipleExceptions)); + await Task.WhenAll(tasks); + } } private IConfig CreateConfig() From 276f1add338e6a4c49053dc4809474d9ad9b91f8 Mon Sep 17 00:00:00 2001 From: Andrey Akinshin Date: Wed, 2 Nov 2022 12:05:44 +0400 Subject: [PATCH 45/58] Fix EngineStage.Run for warmupCount=0 (fixes #2185) (#2186) --- src/BenchmarkDotNet/Engines/EngineStage.cs | 6 +++ .../JobTests.cs | 40 +++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 tests/BenchmarkDotNet.IntegrationTests/JobTests.cs diff --git a/src/BenchmarkDotNet/Engines/EngineStage.cs b/src/BenchmarkDotNet/Engines/EngineStage.cs index 67c46267fc..2c3b825394 100644 --- a/src/BenchmarkDotNet/Engines/EngineStage.cs +++ b/src/BenchmarkDotNet/Engines/EngineStage.cs @@ -23,6 +23,12 @@ protected Measurement RunIteration(IterationMode mode, IterationStage stage, int internal List Run(IStoppingCriteria criteria, long invokeCount, IterationMode mode, IterationStage stage, int unrollFactor) { var measurements = new List(criteria.MaxIterationCount); + if (criteria.Evaluate(measurements).IsFinished) + { + WriteLine(); + return measurements; + } + int iterationCounter = 0; while (true) { diff --git a/tests/BenchmarkDotNet.IntegrationTests/JobTests.cs b/tests/BenchmarkDotNet.IntegrationTests/JobTests.cs new file mode 100644 index 0000000000..b3c8f1580b --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/JobTests.cs @@ -0,0 +1,40 @@ +using System.Linq; +using System.Threading; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Reports; +using Xunit; +using Xunit.Abstractions; + +namespace BenchmarkDotNet.IntegrationTests +{ + public class JobTests : BenchmarkTestExecutor + { + public JobTests(ITestOutputHelper output) : base(output) { } + + [Fact] + public void ZeroWarmupCountIsApplied() + { + var job = Job.Default + .WithEvaluateOverhead(false) + .WithWarmupCount(0) + .WithIterationCount(1) + .WithInvocationCount(1) + .WithUnrollFactor(1); + var config = DefaultConfig.Instance.AddJob(job).WithOptions(ConfigOptions.DisableOptimizationsValidator); + var summary = CanExecute(config); + var report = summary.Reports.Single(); + int workloadWarmupCount = report.AllMeasurements + .Count(m => m.Is(IterationMode.Workload, IterationStage.Warmup)); + Assert.Equal(0, workloadWarmupCount); + } + + public class ZeroWarmupBench + { + [Benchmark] + public void Foo() => Thread.Sleep(10); + } + } +} \ No newline at end of file From 681a6384f5ea62c4a88c8fd50b13e6c992b10c98 Mon Sep 17 00:00:00 2001 From: leonvandermeer <64834803+leonvandermeer@users.noreply.github.com> Date: Wed, 2 Nov 2022 13:59:49 +0100 Subject: [PATCH 46/58] Fix #2167 - Give main page a title. Then the tab displays 'Home | BenchmarkDotNet' (#2181) --- docs/index.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/index.md b/docs/index.md index 73a353f28c..c730572b08 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,3 +1,6 @@ +--- +title: Home +---

![](logo/logo-wide.png) From 00f69361f1dda1b398407b07c8f4d6af9c783f50 Mon Sep 17 00:00:00 2001 From: Andrey Akinshin Date: Wed, 2 Nov 2022 17:49:37 +0400 Subject: [PATCH 47/58] Fix race in AsyncProcessOutputReader (#2180) The process.WaitForExit call doesn't guarantee that the last calls of ProcessOnOutputDataReceived and ProcessOnErrorDataReceived are processed. The subsequent call of reader.StopRead detaches the *DataReceived events, which leaves the output/error queues incomplete. This race leads to flakiness of various tests (e.g., AllSetupAndCleanupMethodRunsForSpecificBenchmark). --- .../Loggers/AsyncProcessOutputReader.cs | 52 +++++++++++++------ 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/src/BenchmarkDotNet/Loggers/AsyncProcessOutputReader.cs b/src/BenchmarkDotNet/Loggers/AsyncProcessOutputReader.cs index 958d11138b..786ae45d5e 100644 --- a/src/BenchmarkDotNet/Loggers/AsyncProcessOutputReader.cs +++ b/src/BenchmarkDotNet/Loggers/AsyncProcessOutputReader.cs @@ -10,9 +10,12 @@ namespace BenchmarkDotNet.Loggers internal class AsyncProcessOutputReader : IDisposable { private readonly Process process; - private readonly ConcurrentQueue output, error; - private readonly bool logOutput, readStandardError; private readonly ILogger logger; + private readonly bool logOutput, readStandardError; + + private static readonly TimeSpan FinishEventTimeout = TimeSpan.FromMilliseconds(500); + private readonly AutoResetEvent outputFinishEvent, errorFinishEvent; + private readonly ConcurrentQueue output, error; private long status; @@ -28,6 +31,8 @@ internal AsyncProcessOutputReader(Process process, bool logOutput = false, ILogg this.process = process; output = new ConcurrentQueue(); error = new ConcurrentQueue(); + outputFinishEvent = new AutoResetEvent(false); + errorFinishEvent = new AutoResetEvent(false); status = (long)Status.Created; this.logOutput = logOutput; this.logger = logger; @@ -39,6 +44,9 @@ public void Dispose() Interlocked.Exchange(ref status, (long)Status.Disposed); Detach(); + + outputFinishEvent.Dispose(); + errorFinishEvent.Dispose(); } internal void BeginRead() @@ -72,6 +80,10 @@ internal void StopRead() if (Interlocked.CompareExchange(ref status, (long)Status.Stopped, (long)Status.Started) != (long)Status.Started) throw new InvalidOperationException("Only a started reader can be stopped"); + outputFinishEvent.WaitOne(FinishEventTimeout); + if (readStandardError) + errorFinishEvent.WaitOne(FinishEventTimeout); + Detach(); } @@ -103,34 +115,44 @@ private void Detach() private void ProcessOnOutputDataReceived(object sender, DataReceivedEventArgs e) { - if (!string.IsNullOrEmpty(e.Data)) + if (e.Data != null) { - output.Enqueue(e.Data); - - if (logOutput) + if (!string.IsNullOrEmpty(e.Data)) { - lock (this) // #2125 + output.Enqueue(e.Data); + + if (logOutput) { - logger.WriteLine(e.Data); + lock (this) // #2125 + { + logger.WriteLine(e.Data); + } } } } + else // 'e.Data == null' means EOF + outputFinishEvent.Set(); } private void ProcessOnErrorDataReceived(object sender, DataReceivedEventArgs e) { - if (!string.IsNullOrEmpty(e.Data)) + if (e.Data != null) { - error.Enqueue(e.Data); - - if (logOutput) + if (!string.IsNullOrEmpty(e.Data)) { - lock (this) // #2125 + error.Enqueue(e.Data); + + if (logOutput) { - logger.WriteLineError(e.Data); + lock (this) // #2125 + { + logger.WriteLineError(e.Data); + } } } } + else // 'e.Data == null' means EOF + errorFinishEvent.Set(); } private T ReturnIfStopped(Func getter) @@ -146,4 +168,4 @@ private enum Status : long Disposed } } -} +} \ No newline at end of file From 36e9985b86fde72a4c1bd8003566d8a108da741d Mon Sep 17 00:00:00 2001 From: Andrey Akinshin Date: Wed, 2 Nov 2022 18:47:33 +0400 Subject: [PATCH 48/58] Disable MemoryDiagnoserSupportsNativeAOT on osx-arm64 --- tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs index 5ae006765b..7e4ea811c7 100755 --- a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs @@ -9,6 +9,7 @@ using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Environments; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.IntegrationTests.Xunit; using BenchmarkDotNet.Jobs; @@ -69,6 +70,8 @@ public void MemoryDiagnoserSupportsNativeAOT() { if (ContinuousIntegration.IsAppVeyorOnWindows()) // too time consuming for AppVeyor (1h limit) return; + if (RuntimeInformation.GetCurrentPlatform() == Platform.Arm64 && RuntimeInformation.IsMacOSX()) + return; // Native compilation does not support targeting osx-arm64 yet. https://github.com/dotnet/corert/issues/4589 MemoryDiagnoserIsAccurate( NativeAotToolchain.CreateBuilder() From 9e759f9bc5f910f4816a4eb4753fa33db6e67b72 Mon Sep 17 00:00:00 2001 From: Andrey Akinshin Date: Wed, 2 Nov 2022 19:20:58 +0400 Subject: [PATCH 49/58] Disable ThreadingDiagnoserTests with ILCompiler 6.0.0-rc.1.21420.1 on osx-arm64 --- .../ThreadingDiagnoserTests.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests/ThreadingDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/ThreadingDiagnoserTests.cs index 645955a060..b30f962233 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ThreadingDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ThreadingDiagnoserTests.cs @@ -14,6 +14,8 @@ using System.Collections.Generic; using System.Linq; using System.Threading; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Portability; using Xunit; using Xunit.Abstractions; @@ -29,8 +31,10 @@ public static IEnumerable GetToolchains() { yield return new object[] { Job.Default.GetToolchain() }; + bool isOsxArm64 = RuntimeInformation.GetCurrentPlatform() == Platform.Arm64 && RuntimeInformation.IsMacOSX(); if (!ContinuousIntegration.IsGitHubActionsOnWindows() // no native dependencies - && !ContinuousIntegration.IsAppVeyorOnWindows()) // too time consuming for AppVeyor (1h limit) + && !ContinuousIntegration.IsAppVeyorOnWindows() // too time consuming for AppVeyor (1h limit) + && !isOsxArm64) // Native compilation does not support targeting osx-arm64 yet. https://github.com/dotnet/corert/issues/4589 { yield return new object[]{ NativeAotToolchain.CreateBuilder() .UseNuGet( From e75bdd8507b4c3042091dd7aee158ae0238b8d4d Mon Sep 17 00:00:00 2001 From: Andrey Akinshin Date: Thu, 3 Nov 2022 09:55:02 +0400 Subject: [PATCH 50/58] Engine.Run() should return the full list of performed measurements (fixes #2187) (#2188) * Engine.Run() should return the full list of performed measurements (fixes #2187) * Code review fixes --- src/BenchmarkDotNet/Engines/Engine.cs | 21 +++-- .../Engines/EnginePilotStage.cs | 38 +++++++-- .../Engines/EngineWarmupStage.cs | 6 +- src/BenchmarkDotNet/Engines/RunResults.cs | 51 ++++++++---- .../Toolchains/Results/ExecuteResult.cs | 2 +- .../CustomEngineTests.cs | 7 +- .../EngineTests.cs | 78 +++++++++++++++++++ .../InProcessEmitTest.cs | 6 +- .../InProcessTest.cs | 11 ++- .../JobTests.cs | 40 ---------- .../Engine/EnginePilotStageTests.cs | 4 +- .../Engine/EngineResultStageTests.cs | 2 +- 12 files changed, 186 insertions(+), 80 deletions(-) create mode 100644 tests/BenchmarkDotNet.IntegrationTests/EngineTests.cs delete mode 100644 tests/BenchmarkDotNet.IntegrationTests/JobTests.cs diff --git a/src/BenchmarkDotNet/Engines/Engine.cs b/src/BenchmarkDotNet/Engines/Engine.cs index 708a126b92..cdf8adcebc 100644 --- a/src/BenchmarkDotNet/Engines/Engine.cs +++ b/src/BenchmarkDotNet/Engines/Engine.cs @@ -41,6 +41,7 @@ public class Engine : IEngine private bool EvaluateOverhead { get; } private bool MemoryRandomization { get; } + private readonly List jittingMeasurements = new (10); private readonly EnginePilotStage pilotStage; private readonly EngineWarmupStage warmupStage; private readonly EngineActualStage actualStage; @@ -104,8 +105,10 @@ public void Dispose() public RunResults Run() { + var measurements = new List(); + measurements.AddRange(jittingMeasurements); + long invokeCount = TargetJob.ResolveValue(RunMode.InvocationCountCharacteristic, Resolver, 1); - IReadOnlyList idle = null; if (EngineEventSource.Log.IsEnabled()) EngineEventSource.Log.BenchmarkStart(BenchmarkName); @@ -114,21 +117,23 @@ public RunResults Run() { if (Strategy != RunStrategy.Monitoring) { - invokeCount = pilotStage.Run(); + var pilotStageResult = pilotStage.Run(); + invokeCount = pilotStageResult.PerfectInvocationCount; + measurements.AddRange(pilotStageResult.Measurements); if (EvaluateOverhead) { - warmupStage.RunOverhead(invokeCount, UnrollFactor); - idle = actualStage.RunOverhead(invokeCount, UnrollFactor); + measurements.AddRange(warmupStage.RunOverhead(invokeCount, UnrollFactor)); + measurements.AddRange(actualStage.RunOverhead(invokeCount, UnrollFactor)); } } - warmupStage.RunWorkload(invokeCount, UnrollFactor, Strategy); + measurements.AddRange(warmupStage.RunWorkload(invokeCount, UnrollFactor, Strategy)); } Host.BeforeMainRun(); - var main = actualStage.RunWorkload(invokeCount, UnrollFactor, forceSpecific: Strategy == RunStrategy.Monitoring); + measurements.AddRange(actualStage.RunWorkload(invokeCount, UnrollFactor, forceSpecific: Strategy == RunStrategy.Monitoring)); Host.AfterMainRun(); @@ -141,7 +146,7 @@ public RunResults Run() var outlierMode = TargetJob.ResolveValue(AccuracyMode.OutlierModeCharacteristic, Resolver); - return new RunResults(idle, main, outlierMode, workGcHasDone, threadingStats, exceptionFrequency); + return new RunResults(measurements, outlierMode, workGcHasDone, threadingStats, exceptionFrequency); } public Measurement RunIteration(IterationData data) @@ -183,6 +188,8 @@ public Measurement RunIteration(IterationData data) // Results var measurement = new Measurement(0, data.IterationMode, data.IterationStage, data.Index, totalOperations, clockSpan.GetNanoseconds()); WriteLine(measurement.ToString()); + if (measurement.IterationStage == IterationStage.Jitting) + jittingMeasurements.Add(measurement); Consume(stackMemory); diff --git a/src/BenchmarkDotNet/Engines/EnginePilotStage.cs b/src/BenchmarkDotNet/Engines/EnginePilotStage.cs index 9abda40774..1bf3e34d54 100644 --- a/src/BenchmarkDotNet/Engines/EnginePilotStage.cs +++ b/src/BenchmarkDotNet/Engines/EnginePilotStage.cs @@ -1,6 +1,9 @@ using System; +using System.Collections.Generic; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Reports; +using JetBrains.Annotations; using Perfolizer.Horology; namespace BenchmarkDotNet.Engines @@ -8,6 +11,25 @@ namespace BenchmarkDotNet.Engines // TODO: use clockResolution internal class EnginePilotStage : EngineStage { + public readonly struct PilotStageResult + { + public long PerfectInvocationCount { get; } + [NotNull] + public IReadOnlyList Measurements { get; } + + public PilotStageResult(long perfectInvocationCount, [NotNull] List measurements) + { + PerfectInvocationCount = perfectInvocationCount; + Measurements = measurements; + } + + public PilotStageResult(long perfectInvocationCount) + { + PerfectInvocationCount = perfectInvocationCount; + Measurements = Array.Empty(); + } + } + internal const long MaxInvokeCount = (long.MaxValue / 2 + 1) / 2; private readonly int unrollFactor; @@ -30,11 +52,11 @@ public EnginePilotStage(IEngine engine) : base(engine) } /// Perfect invocation count - public long Run() + public PilotStageResult Run() { // If InvocationCount is specified, pilot stage should be skipped if (TargetJob.HasValue(RunMode.InvocationCountCharacteristic)) - return TargetJob.Run.InvocationCount; + return new PilotStageResult(TargetJob.Run.InvocationCount); // Here we want to guess "perfect" amount of invocation return TargetJob.HasValue(RunMode.IterationTimeCharacteristic) @@ -45,15 +67,17 @@ public long Run() /// /// A case where we don't have specific iteration time. /// - private long RunAuto() + private PilotStageResult RunAuto() { long invokeCount = Autocorrect(minInvokeCount); + var measurements = new List(); int iterationCounter = 0; while (true) { iterationCounter++; var measurement = RunIteration(IterationMode.Workload, IterationStage.Pilot, iterationCounter, invokeCount, unrollFactor); + measurements.Add(measurement); double iterationTime = measurement.Nanoseconds; double operationError = 2.0 * resolution / invokeCount; // An operation error which has arisen due to the Chronometer precision @@ -75,15 +99,16 @@ private long RunAuto() } WriteLine(); - return invokeCount; + return new PilotStageResult(invokeCount, measurements); } /// /// A case where we have specific iteration time. /// - private long RunSpecific() + private PilotStageResult RunSpecific() { long invokeCount = Autocorrect(Engine.MinInvokeCount); + var measurements = new List(); int iterationCounter = 0; @@ -92,6 +117,7 @@ private long RunSpecific() { iterationCounter++; var measurement = RunIteration(IterationMode.Workload, IterationStage.Pilot, iterationCounter, invokeCount, unrollFactor); + measurements.Add(measurement); double actualIterationTime = measurement.Nanoseconds; long newInvokeCount = Autocorrect(Math.Max(minInvokeCount, (long)Math.Round(invokeCount * targetIterationTime / actualIterationTime))); @@ -105,7 +131,7 @@ private long RunSpecific() } WriteLine(); - return invokeCount; + return new PilotStageResult(invokeCount, measurements); } private long Autocorrect(long count) => (count + unrollFactor - 1) / unrollFactor * unrollFactor; diff --git a/src/BenchmarkDotNet/Engines/EngineWarmupStage.cs b/src/BenchmarkDotNet/Engines/EngineWarmupStage.cs index 0a4d841aa5..08ec7ec729 100644 --- a/src/BenchmarkDotNet/Engines/EngineWarmupStage.cs +++ b/src/BenchmarkDotNet/Engines/EngineWarmupStage.cs @@ -9,13 +9,13 @@ internal class EngineWarmupStage : EngineStage public EngineWarmupStage(IEngine engine) : base(engine) => this.engine = engine; - public void RunOverhead(long invokeCount, int unrollFactor) + public IReadOnlyList RunOverhead(long invokeCount, int unrollFactor) => Run(invokeCount, IterationMode.Overhead, unrollFactor, RunStrategy.Throughput); - public void RunWorkload(long invokeCount, int unrollFactor, RunStrategy runStrategy) + public IReadOnlyList RunWorkload(long invokeCount, int unrollFactor, RunStrategy runStrategy) => Run(invokeCount, IterationMode.Workload, unrollFactor, runStrategy); - internal List Run(long invokeCount, IterationMode iterationMode, int unrollFactor, RunStrategy runStrategy) + internal IReadOnlyList Run(long invokeCount, IterationMode iterationMode, int unrollFactor, RunStrategy runStrategy) { var criteria = DefaultStoppingCriteriaFactory.Instance.CreateWarmup(engine.TargetJob, engine.Resolver, iterationMode, runStrategy); return Run(criteria, invokeCount, iterationMode, IterationStage.Warmup, unrollFactor); diff --git a/src/BenchmarkDotNet/Engines/RunResults.cs b/src/BenchmarkDotNet/Engines/RunResults.cs index 5a7651f46a..ff49488707 100644 --- a/src/BenchmarkDotNet/Engines/RunResults.cs +++ b/src/BenchmarkDotNet/Engines/RunResults.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Mathematics; using BenchmarkDotNet.Reports; using JetBrains.Annotations; @@ -13,11 +14,20 @@ public struct RunResults { private readonly OutlierMode outlierMode; + [NotNull, PublicAPI] + public IReadOnlyList EngineMeasurements { get; } + [CanBeNull, PublicAPI] - public IReadOnlyList Overhead { get; } + public IReadOnlyList Overhead + => EngineMeasurements + .Where(m => m.Is(IterationMode.Overhead, IterationStage.Actual)) + .ToArray(); [NotNull, PublicAPI] - public IReadOnlyList Workload { get; } + public IReadOnlyList Workload + => EngineMeasurements + .Where(m => m.Is(IterationMode.Workload, IterationStage.Actual)) + .ToArray(); public GcStats GCStats { get; } @@ -25,27 +35,30 @@ public struct RunResults public double ExceptionFrequency { get; } - public RunResults([CanBeNull] IReadOnlyList overhead, - [NotNull] IReadOnlyList workload, - OutlierMode outlierMode, - GcStats gcStats, - ThreadingStats threadingStats, - double exceptionFrequency) + public RunResults([NotNull] IReadOnlyList engineMeasurements, + OutlierMode outlierMode, + GcStats gcStats, + ThreadingStats threadingStats, + double exceptionFrequency) { this.outlierMode = outlierMode; - Overhead = overhead; - Workload = workload; + EngineMeasurements = engineMeasurements; GCStats = gcStats; ThreadingStats = threadingStats; ExceptionFrequency = exceptionFrequency; } - public IEnumerable GetMeasurements() + public IEnumerable GetWorkloadResultMeasurements() { - double overhead = Overhead == null ? 0.0 : new Statistics(Overhead.Select(m => m.Nanoseconds)).Median; - var mainStats = new Statistics(Workload.Select(m => m.Nanoseconds)); + var overheadActualMeasurements = Overhead ?? Array.Empty(); + var workloadActualMeasurements = Workload; + if (workloadActualMeasurements.IsEmpty()) + yield break; + + double overhead = overheadActualMeasurements.IsEmpty() ? 0.0 : new Statistics(overheadActualMeasurements.Select(m => m.Nanoseconds)).Median; + var mainStats = new Statistics(workloadActualMeasurements.Select(m => m.Nanoseconds)); int resultIndex = 0; - foreach (var measurement in Workload) + foreach (var measurement in workloadActualMeasurements) { if (mainStats.IsActualOutlier(measurement.Nanoseconds, outlierMode)) continue; @@ -63,9 +76,17 @@ public IEnumerable GetMeasurements() } } + public IEnumerable GetAllMeasurements() + { + foreach (var measurement in EngineMeasurements) + yield return measurement; + foreach (var measurement in GetWorkloadResultMeasurements()) + yield return measurement; + } + public void Print(TextWriter outWriter) { - foreach (var measurement in GetMeasurements()) + foreach (var measurement in GetWorkloadResultMeasurements()) outWriter.WriteLine(measurement.ToString()); if (!GCStats.Equals(GcStats.Empty)) diff --git a/src/BenchmarkDotNet/Toolchains/Results/ExecuteResult.cs b/src/BenchmarkDotNet/Toolchains/Results/ExecuteResult.cs index dcd68000d8..76d9c41cd7 100644 --- a/src/BenchmarkDotNet/Toolchains/Results/ExecuteResult.cs +++ b/src/BenchmarkDotNet/Toolchains/Results/ExecuteResult.cs @@ -67,7 +67,7 @@ internal ExecuteResult(List measurements, GcStats gcStats, Threadin internal static ExecuteResult FromRunResults(RunResults runResults, int exitCode) => exitCode != 0 ? CreateFailed(exitCode) - : new ExecuteResult(runResults.GetMeasurements().ToList(), runResults.GCStats, runResults.ThreadingStats, runResults.ExceptionFrequency); + : new ExecuteResult(runResults.GetAllMeasurements().ToList(), runResults.GCStats, runResults.ThreadingStats, runResults.ExceptionFrequency); internal static ExecuteResult CreateFailed(int exitCode = -1) => new ExecuteResult(false, exitCode, default, Array.Empty(), Array.Empty(), Array.Empty(), 0); diff --git a/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs b/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs index 5f423c2595..43a7bac9ee 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs @@ -72,8 +72,11 @@ public RunResults Run() Console.WriteLine(EngineRunMessage); return new RunResults( - new List { new Measurement(1, IterationMode.Overhead, IterationStage.Actual, 1, 1, 1) }, - new List { new Measurement(1, IterationMode.Workload, IterationStage.Actual, 1, 1, 1) }, + new List + { + new (1, IterationMode.Overhead, IterationStage.Actual, 1, 1, 1), + new (1, IterationMode.Workload, IterationStage.Actual, 1, 1, 1) + }, OutlierMode.DontRemove, default, default, diff --git a/tests/BenchmarkDotNet.IntegrationTests/EngineTests.cs b/tests/BenchmarkDotNet.IntegrationTests/EngineTests.cs new file mode 100644 index 0000000000..624877f404 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/EngineTests.cs @@ -0,0 +1,78 @@ +using System.Linq; +using System.Threading; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Reports; +using Xunit; +using Xunit.Abstractions; + +namespace BenchmarkDotNet.IntegrationTests +{ + public class EngineTests : BenchmarkTestExecutor + { + public EngineTests(ITestOutputHelper output) : base(output) { } + + [Fact] + public void ZeroWarmupCountIsApplied() + { + var job = Job.InProcess + .WithEvaluateOverhead(false) + .WithWarmupCount(0) + .WithIterationCount(1) + .WithInvocationCount(1) + .WithUnrollFactor(1); + var config = DefaultConfig.Instance.AddJob(job).WithOptions(ConfigOptions.DisableOptimizationsValidator); + var summary = CanExecute(config); + var report = summary.Reports.Single(); + int workloadWarmupCount = report.AllMeasurements + .Count(m => m.Is(IterationMode.Workload, IterationStage.Warmup)); + Assert.Equal(0, workloadWarmupCount); + } + + [Fact] + public void AllMeasurementsArePerformedDefault() => AllMeasurementsArePerformed(Job.Default); + + [Fact] + public void AllMeasurementsArePerformedInProcess() => AllMeasurementsArePerformed(Job.InProcess); + + private void AllMeasurementsArePerformed(Job baseJob) + { + var job = baseJob + .WithWarmupCount(1) + .WithIterationCount(1) + .WithInvocationCount(1) + .WithUnrollFactor(1); + var config = DefaultConfig.Instance.AddJob(job).WithOptions(ConfigOptions.DisableOptimizationsValidator); + var summary = CanExecute(config); + var measurements = summary.Reports.Single().AllMeasurements; + + Output.WriteLine("*** AllMeasurements ***"); + foreach (var measurement in measurements) + Output.WriteLine(measurement.ToString()); + Output.WriteLine("-----"); + + void Check(IterationMode mode, IterationStage stage) + { + int count = measurements.Count(m => m.Is(mode, stage)); + Output.WriteLine($"Count({mode}{stage}) = {count}"); + Assert.True(count > 0, $"AllMeasurements don't contain {mode}{stage}"); + } + + Check(IterationMode.Overhead, IterationStage.Jitting); + Check(IterationMode.Workload, IterationStage.Jitting); + Check(IterationMode.Overhead, IterationStage.Warmup); + Check(IterationMode.Overhead, IterationStage.Actual); + Check(IterationMode.Workload, IterationStage.Warmup); + Check(IterationMode.Workload, IterationStage.Actual); + Check(IterationMode.Workload, IterationStage.Result); + } + + public class FooBench + { + [Benchmark] + public void Foo() => Thread.Sleep(10); + } + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs b/tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs index 227e6411ed..1d5619a27c 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs @@ -5,6 +5,7 @@ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Engines; using BenchmarkDotNet.IntegrationTests.InProcess.EmitTests; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; @@ -91,7 +92,10 @@ public void InProcessBenchmarkSimpleCasesReflectionEmitSupported() Assert.DoesNotContain("No benchmarks found", logger.GetLog()); // Operations + GlobalSetup + GlobalCleanup - long expectedCount = summary.Reports.SelectMany(r => r.AllMeasurements).Sum(m => m.Operations + 2); + long expectedCount = summary.Reports + .SelectMany(r => r.AllMeasurements) + .Where(m => m.IterationStage != IterationStage.Result) + .Sum(m => m.Operations + 2); Assert.Equal(expectedCount, BenchmarkAllCases.Counter); } finally diff --git a/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs b/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs index 6a2a0e2dd7..a5de2462f6 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs @@ -8,6 +8,7 @@ using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; @@ -213,7 +214,10 @@ public void InProcessBenchmarkAllCasesReflectionEmitSupported() Assert.DoesNotContain("No benchmarks found", logger.GetLog()); // Operations + GlobalSetup + GlobalCleanup - var expectedCount = summary.Reports.SelectMany(r => r.AllMeasurements).Sum(m => m.Operations + 2); + long expectedCount = summary.Reports + .SelectMany(r => r.AllMeasurements) + .Where(m => m.IterationStage != IterationStage.Result) + .Sum(m => m.Operations + 2); Assert.Equal(expectedCount, BenchmarkAllCases.Counter); } finally @@ -239,7 +243,10 @@ public void InProcessBenchmarkAllCasesDelegateCombineSupported() Assert.DoesNotContain("No benchmarks found", logger.GetLog()); // Operations + GlobalSetup + GlobalCleanup - var expectedCount = summary.Reports.SelectMany(r => r.AllMeasurements).Sum(m => m.Operations + 2); + long expectedCount = summary.Reports + .SelectMany(r => r.AllMeasurements) + .Where(m => m.IterationStage != IterationStage.Result) + .Sum(m => m.Operations + 2); Assert.Equal(expectedCount, BenchmarkAllCases.Counter); } finally diff --git a/tests/BenchmarkDotNet.IntegrationTests/JobTests.cs b/tests/BenchmarkDotNet.IntegrationTests/JobTests.cs deleted file mode 100644 index b3c8f1580b..0000000000 --- a/tests/BenchmarkDotNet.IntegrationTests/JobTests.cs +++ /dev/null @@ -1,40 +0,0 @@ -using System.Linq; -using System.Threading; -using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Configs; -using BenchmarkDotNet.Engines; -using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Reports; -using Xunit; -using Xunit.Abstractions; - -namespace BenchmarkDotNet.IntegrationTests -{ - public class JobTests : BenchmarkTestExecutor - { - public JobTests(ITestOutputHelper output) : base(output) { } - - [Fact] - public void ZeroWarmupCountIsApplied() - { - var job = Job.Default - .WithEvaluateOverhead(false) - .WithWarmupCount(0) - .WithIterationCount(1) - .WithInvocationCount(1) - .WithUnrollFactor(1); - var config = DefaultConfig.Instance.AddJob(job).WithOptions(ConfigOptions.DisableOptimizationsValidator); - var summary = CanExecute(config); - var report = summary.Reports.Single(); - int workloadWarmupCount = report.AllMeasurements - .Count(m => m.Is(IterationMode.Workload, IterationStage.Warmup)); - Assert.Equal(0, workloadWarmupCount); - } - - public class ZeroWarmupBench - { - [Benchmark] - public void Foo() => Thread.Sleep(10); - } - } -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Engine/EnginePilotStageTests.cs b/tests/BenchmarkDotNet.Tests/Engine/EnginePilotStageTests.cs index 19d419f716..d5b7b8c65c 100644 --- a/tests/BenchmarkDotNet.Tests/Engine/EnginePilotStageTests.cs +++ b/tests/BenchmarkDotNet.Tests/Engine/EnginePilotStageTests.cs @@ -47,7 +47,7 @@ private void AutoTest(Frequency clockFrequency, TimeInterval operationTime, doub Accuracy = { MaxRelativeError = maxRelativeError } }.Freeze(); var stage = CreateStage(job, data => data.InvokeCount * operationTime); - long invokeCount = stage.Run(); + long invokeCount = stage.Run().PerfectInvocationCount; output.WriteLine($"InvokeCount = {invokeCount} (Min= {minInvokeCount}, Max = {MaxPossibleInvokeCount})"); Assert.InRange(invokeCount, minInvokeCount, MaxPossibleInvokeCount); } @@ -60,7 +60,7 @@ private void SpecificTest(TimeInterval iterationTime, TimeInterval operationTime Run = { IterationTime = iterationTime } }.Freeze(); var stage = CreateStage(job, data => data.InvokeCount * operationTime); - long invokeCount = stage.Run(); + long invokeCount = stage.Run().PerfectInvocationCount; output.WriteLine($"InvokeCount = {invokeCount} (Min= {minInvokeCount}, Max = {maxInvokeCount})"); Assert.InRange(invokeCount, minInvokeCount, maxInvokeCount); } diff --git a/tests/BenchmarkDotNet.Tests/Engine/EngineResultStageTests.cs b/tests/BenchmarkDotNet.Tests/Engine/EngineResultStageTests.cs index 468ef87629..89c011e60c 100644 --- a/tests/BenchmarkDotNet.Tests/Engine/EngineResultStageTests.cs +++ b/tests/BenchmarkDotNet.Tests/Engine/EngineResultStageTests.cs @@ -31,7 +31,7 @@ public void OutliersTest() [AssertionMethod] private static void CheckResults(int expectedResultCount, List measurements, OutlierMode outlierMode) { - Assert.Equal(expectedResultCount, new RunResults(null, measurements, outlierMode, default, default, 0).GetMeasurements().Count()); + Assert.Equal(expectedResultCount, new RunResults(measurements, outlierMode, default, default, 0).GetWorkloadResultMeasurements().Count()); } private static void Add(List measurements, int time) From 28a8e78a721d685696dde3fdcfc1ba0f317e11c8 Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Thu, 3 Nov 2022 01:37:33 -0700 Subject: [PATCH 51/58] Simplify GetHashCode() (#2177) * Add package for netstandart2.0 * Simplify HashCode * Fix a bug: job.Infrastructure.Arguments is an array, so compare its values * Remove outdated suppressor * Revert "Fix a bug: job.Infrastructure.Arguments is an array, so compare its values" This reverts commit 2b5b6b47ea674fadb5a00ff4c66b1ae91b9fdbb7. * Revert "Simplify HashCode" This reverts commit 681ceab1dc22de7e29dc940abd99d80a40e5bf38. * Revert "Add package for netstandart2.0" This reverts commit 8f0e2e49e2b0ef327f841315927d15751a099082. * Mimic System.HashCode * Simplify GetHashCode() * Remove redundant * Fix bugs * Reorder * fix a bug: compare Argument by TextRepresentation, not by reference * Simplify * Add missing HashCode API * Suppress inspection --- src/BenchmarkDotNet/Analysers/Conclusion.cs | 12 +- .../Analysers/HideColumnsAnalyser.cs | 2 +- src/BenchmarkDotNet/Columns/SizeUnit.cs | 13 +- .../Disassemblers/ClrMdV2Disassembler.cs | 4 +- src/BenchmarkDotNet/Engines/GcStats.cs | 13 +- src/BenchmarkDotNet/Engines/ThreadingStats.cs | 11 +- .../Environments/BenchmarkEnvironmentInfo.cs | 1 - .../Environments/Runtimes/ClrRuntime.cs | 4 +- .../Runtimes/MonoAotLLVMRuntime.cs | 4 +- .../Environments/Runtimes/MonoRuntime.cs | 4 +- .../Environments/Runtimes/Runtime.cs | 4 +- .../Environments/Runtimes/WasmRuntime.cs | 4 +- src/BenchmarkDotNet/Helpers/HashCode.cs | 140 ++++++++++++++++++ src/BenchmarkDotNet/Jobs/Argument.cs | 26 +++- .../Jobs/EnvironmentVariable.cs | 8 +- src/BenchmarkDotNet/Jobs/GcMode.cs | 3 +- src/BenchmarkDotNet/Jobs/NugetReference.cs | 5 +- .../Jobs/NugetReferenceList.cs | 14 +- src/BenchmarkDotNet/Reports/SummaryStyle.cs | 27 ++-- .../Running/BenchmarkPartitioner.cs | 39 ++--- .../Toolchains/CsProj/CsProjGenerator.cs | 7 +- .../Validators/ReturnValueValidator.cs | 15 +- .../Validators/ValidationError.cs | 12 +- 23 files changed, 222 insertions(+), 150 deletions(-) create mode 100644 src/BenchmarkDotNet/Helpers/HashCode.cs diff --git a/src/BenchmarkDotNet/Analysers/Conclusion.cs b/src/BenchmarkDotNet/Analysers/Conclusion.cs index 435566746f..d5179134b5 100644 --- a/src/BenchmarkDotNet/Analysers/Conclusion.cs +++ b/src/BenchmarkDotNet/Analysers/Conclusion.cs @@ -64,15 +64,9 @@ public override bool Equals(object obj) public override int GetHashCode() { - unchecked - { - int hashCode = AnalyserId.GetHashCode(); - hashCode = (hashCode * 397) ^ (int) Kind; - hashCode = Mergeable - ? (hashCode * 397) ^ Message.GetHashCode() - : (hashCode * 397) ^ Report?.ToString().GetHashCode() ?? string.Empty.GetHashCode(); - return hashCode; - } + return Mergeable + ? HashCode.Combine(AnalyserId, Kind, Message) + : HashCode.Combine(AnalyserId, Kind, Report?.ToString()); } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Analysers/HideColumnsAnalyser.cs b/src/BenchmarkDotNet/Analysers/HideColumnsAnalyser.cs index 9db11cc7a7..8fce0c95dc 100644 --- a/src/BenchmarkDotNet/Analysers/HideColumnsAnalyser.cs +++ b/src/BenchmarkDotNet/Analysers/HideColumnsAnalyser.cs @@ -21,7 +21,7 @@ protected override IEnumerable AnalyseSummary(Summary summary) var columnNames = string.Join(", ", hiddenColumns.Select(c => c.OriginalColumn.ColumnName)); var message = $"Hidden columns: {columnNames}"; - yield return Conclusion.CreateHint(Id, message); + yield return CreateHint(message); } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Columns/SizeUnit.cs b/src/BenchmarkDotNet/Columns/SizeUnit.cs index 3f701069ac..b919fb7fa0 100644 --- a/src/BenchmarkDotNet/Columns/SizeUnit.cs +++ b/src/BenchmarkDotNet/Columns/SizeUnit.cs @@ -66,19 +66,10 @@ public override bool Equals(object obj) return Equals((SizeUnit) obj); } - public override int GetHashCode() - { - unchecked - { - int hashCode = (Name != null ? Name.GetHashCode() : 0); - hashCode = (hashCode * 397) ^ (Description != null ? Description.GetHashCode() : 0); - hashCode = (hashCode * 397) ^ ByteAmount.GetHashCode(); - return hashCode; - } - } + public override int GetHashCode() => HashCode.Combine(Name, Description, ByteAmount); public static bool operator ==(SizeUnit left, SizeUnit right) => Equals(left, right); public static bool operator !=(SizeUnit left, SizeUnit right) => !Equals(left, right); } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Disassemblers/ClrMdV2Disassembler.cs b/src/BenchmarkDotNet/Disassemblers/ClrMdV2Disassembler.cs index c2226c4082..4c824cc069 100644 --- a/src/BenchmarkDotNet/Disassemblers/ClrMdV2Disassembler.cs +++ b/src/BenchmarkDotNet/Disassemblers/ClrMdV2Disassembler.cs @@ -327,7 +327,7 @@ public bool Equals(Sharp x, Sharp y) return x.FilePath == y.FilePath && x.LineNumber == y.LineNumber; } - public int GetHashCode(Sharp obj) => obj.FilePath.GetHashCode() ^ obj.LineNumber; + public int GetHashCode(Sharp obj) => HashCode.Combine(obj.FilePath, obj.LineNumber); } } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/GcStats.cs b/src/BenchmarkDotNet/Engines/GcStats.cs index 1d3812173e..ae39c4718e 100644 --- a/src/BenchmarkDotNet/Engines/GcStats.cs +++ b/src/BenchmarkDotNet/Engines/GcStats.cs @@ -223,17 +223,6 @@ private static long CalculateAllocationQuantumSize() public override bool Equals(object obj) => obj is GcStats other && Equals(other); - public override int GetHashCode() - { - unchecked - { - int hashCode = Gen0Collections; - hashCode = (hashCode * 397) ^ Gen1Collections; - hashCode = (hashCode * 397) ^ Gen2Collections; - hashCode = (hashCode * 397) ^ AllocatedBytes.GetHashCode(); - hashCode = (hashCode * 397) ^ TotalOperations.GetHashCode(); - return hashCode; - } - } + public override int GetHashCode() => HashCode.Combine(Gen0Collections, Gen1Collections, Gen2Collections, AllocatedBytes, TotalOperations); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/ThreadingStats.cs b/src/BenchmarkDotNet/Engines/ThreadingStats.cs index 6afa5fec11..ad838613c6 100644 --- a/src/BenchmarkDotNet/Engines/ThreadingStats.cs +++ b/src/BenchmarkDotNet/Engines/ThreadingStats.cs @@ -86,15 +86,6 @@ private static Func CreateGetterDelegate(Type type, string propertyName) public override bool Equals(object obj) => obj is ThreadingStats other && Equals(other); - public override int GetHashCode() - { - unchecked - { - int hashCode = CompletedWorkItemCount.GetHashCode(); - hashCode = (hashCode * 397) ^ LockContentionCount.GetHashCode(); - hashCode = (hashCode * 397) ^ TotalOperations.GetHashCode(); - return hashCode; - } - } + public override int GetHashCode() => HashCode.Combine(CompletedWorkItemCount, LockContentionCount, TotalOperations); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Environments/BenchmarkEnvironmentInfo.cs b/src/BenchmarkDotNet/Environments/BenchmarkEnvironmentInfo.cs index 0c0b497abf..08fe7fc45e 100644 --- a/src/BenchmarkDotNet/Environments/BenchmarkEnvironmentInfo.cs +++ b/src/BenchmarkDotNet/Environments/BenchmarkEnvironmentInfo.cs @@ -70,7 +70,6 @@ internal string GetRuntimeInfo() return $"{RuntimeVersion}, {Architecture} {jitInfo}"; } - [SuppressMessage("ReSharper", "UnusedMember.Global")] // TODO: should be used or removed public static IEnumerable Validate(Job job) { if (job.Environment.Jit == Jit.RyuJit && !RuntimeInformation.HasRyuJit()) diff --git a/src/BenchmarkDotNet/Environments/Runtimes/ClrRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/ClrRuntime.cs index 1ce6d5e1eb..35460ce853 100644 --- a/src/BenchmarkDotNet/Environments/Runtimes/ClrRuntime.cs +++ b/src/BenchmarkDotNet/Environments/Runtimes/ClrRuntime.cs @@ -40,7 +40,7 @@ public static ClrRuntime CreateForLocalFullNetFrameworkBuild(string version) public bool Equals(ClrRuntime other) => other != null && base.Equals(other) && Version == other.Version; - public override int GetHashCode() => base.GetHashCode() ^ (Version?.GetHashCode() ?? 0); + public override int GetHashCode() => HashCode.Combine(base.GetHashCode(), Version); internal static ClrRuntime GetCurrentVersion() { @@ -69,4 +69,4 @@ internal static ClrRuntime GetCurrentVersion() } } } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Environments/Runtimes/MonoAotLLVMRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/MonoAotLLVMRuntime.cs index 61ff5bdc9f..e81e64f6c6 100644 --- a/src/BenchmarkDotNet/Environments/Runtimes/MonoAotLLVMRuntime.cs +++ b/src/BenchmarkDotNet/Environments/Runtimes/MonoAotLLVMRuntime.cs @@ -44,6 +44,6 @@ public bool Equals(MonoAotLLVMRuntime other) => other != null && base.Equals(other) && other.AOTCompilerPath == AOTCompilerPath; public override int GetHashCode() - => base.GetHashCode() ^ AOTCompilerPath.GetHashCode(); + => HashCode.Combine(base.GetHashCode(), AOTCompilerPath); } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Environments/Runtimes/MonoRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/MonoRuntime.cs index 36ef6e4976..ee9b348ec6 100644 --- a/src/BenchmarkDotNet/Environments/Runtimes/MonoRuntime.cs +++ b/src/BenchmarkDotNet/Environments/Runtimes/MonoRuntime.cs @@ -41,6 +41,6 @@ public bool Equals(MonoRuntime other) => base.Equals(other) && Name == other?.Name && CustomPath == other?.CustomPath && AotArgs == other?.AotArgs && MonoBclPath == other?.MonoBclPath; public override int GetHashCode() - => base.GetHashCode() ^ Name.GetHashCode() ^ (CustomPath?.GetHashCode() ?? 0) ^ (AotArgs?.GetHashCode() ?? 0) ^ (MonoBclPath?.GetHashCode() ?? 0); + => HashCode.Combine(base.GetHashCode(), Name, CustomPath, AotArgs, MonoBclPath); } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Environments/Runtimes/Runtime.cs b/src/BenchmarkDotNet/Environments/Runtimes/Runtime.cs index 9ebf5bed72..b695de321c 100644 --- a/src/BenchmarkDotNet/Environments/Runtimes/Runtime.cs +++ b/src/BenchmarkDotNet/Environments/Runtimes/Runtime.cs @@ -41,6 +41,6 @@ public bool Equals(Runtime other) public override bool Equals(object obj) => obj is Runtime other && Equals(other); - public override int GetHashCode() => Name.GetHashCode() ^ (int)RuntimeMoniker ^ MsBuildMoniker.GetHashCode(); + public override int GetHashCode() => HashCode.Combine(Name, MsBuildMoniker, RuntimeMoniker); } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Environments/Runtimes/WasmRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/WasmRuntime.cs index b267d77863..bb6c122700 100644 --- a/src/BenchmarkDotNet/Environments/Runtimes/WasmRuntime.cs +++ b/src/BenchmarkDotNet/Environments/Runtimes/WasmRuntime.cs @@ -46,6 +46,6 @@ public bool Equals(WasmRuntime other) => other != null && base.Equals(other) && other.JavaScriptEngine == JavaScriptEngine && other.JavaScriptEngineArguments == JavaScriptEngineArguments && other.Aot == Aot; public override int GetHashCode() - => base.GetHashCode() ^ (JavaScriptEngine?.GetHashCode() ?? 0) ^ (JavaScriptEngineArguments?.GetHashCode() ?? 0 ^ Aot.GetHashCode()); + => HashCode.Combine(base.GetHashCode(), JavaScriptEngine, JavaScriptEngineArguments, Aot); } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Helpers/HashCode.cs b/src/BenchmarkDotNet/Helpers/HashCode.cs new file mode 100644 index 0000000000..94a5858009 --- /dev/null +++ b/src/BenchmarkDotNet/Helpers/HashCode.cs @@ -0,0 +1,140 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Runtime.CompilerServices; + +// Mimics System.HashCode, which is missing in NetStandard2.0. +// Placed in root namespace to avoid ambiguous reference with System.HashCode + +// ReSharper disable once CheckNamespace +namespace BenchmarkDotNet +{ + internal struct HashCode + { + private int hashCode; + + public void Add(T value) + { + hashCode = Hash(hashCode, value); + } + + public void Add(T value, IEqualityComparer comparer) + { + hashCode = Hash(hashCode, value, comparer); + } + + public readonly int ToHashCode() => hashCode; + + public static int Combine(T1 value1) + { + int hashCode = 0; + hashCode = Hash(hashCode, value1); + return hashCode; + } + + public static int Combine(T1 value1, T2 value2) + { + int hashCode = 0; + hashCode = Hash(hashCode, value1); + hashCode = Hash(hashCode, value2); + return hashCode; + } + + public static int Combine(T1 value1, T2 value2, T3 value3) + { + int hashCode = 0; + hashCode = Hash(hashCode, value1); + hashCode = Hash(hashCode, value2); + hashCode = Hash(hashCode, value3); + return hashCode; + } + + public static int Combine(T1 value1, T2 value2, T3 value3, T4 value4) + { + int hashCode = 0; + hashCode = Hash(hashCode, value1); + hashCode = Hash(hashCode, value2); + hashCode = Hash(hashCode, value3); + hashCode = Hash(hashCode, value4); + return hashCode; + } + + public static int Combine(T1 value1, T2 value2, T3 value3, T4 value4, T5 value5) + { + int hashCode = 0; + hashCode = Hash(hashCode, value1); + hashCode = Hash(hashCode, value2); + hashCode = Hash(hashCode, value3); + hashCode = Hash(hashCode, value4); + hashCode = Hash(hashCode, value5); + return hashCode; + } + + public static int Combine(T1 value1, T2 value2, T3 value3, T4 value4, T5 value5, T6 value6) + { + int hashCode = 0; + hashCode = Hash(hashCode, value1); + hashCode = Hash(hashCode, value2); + hashCode = Hash(hashCode, value3); + hashCode = Hash(hashCode, value4); + hashCode = Hash(hashCode, value5); + hashCode = Hash(hashCode, value6); + return hashCode; + } + + public static int Combine(T1 value1, T2 value2, T3 value3, T4 value4, T5 value5, T6 value6, T7 value7) + { + int hashCode = 0; + hashCode = Hash(hashCode, value1); + hashCode = Hash(hashCode, value2); + hashCode = Hash(hashCode, value3); + hashCode = Hash(hashCode, value4); + hashCode = Hash(hashCode, value5); + hashCode = Hash(hashCode, value6); + hashCode = Hash(hashCode, value7); + return hashCode; + } + + public static int Combine(T1 value1, T2 value2, T3 value3, T4 value4, T5 value5, T6 value6, T7 value7, T8 value8) + { + int hashCode = 0; + hashCode = Hash(hashCode, value1); + hashCode = Hash(hashCode, value2); + hashCode = Hash(hashCode, value3); + hashCode = Hash(hashCode, value4); + hashCode = Hash(hashCode, value5); + hashCode = Hash(hashCode, value6); + hashCode = Hash(hashCode, value7); + hashCode = Hash(hashCode, value8); + return hashCode; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int Hash(int hashCode, T value) + { + unchecked + { + return (hashCode * 397) ^ (value?.GetHashCode() ?? 0); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int Hash(int hashCode, T value, IEqualityComparer comparer) + { + unchecked + { + return (hashCode * 397) ^ (value is null ? 0 : (comparer?.GetHashCode(value) ?? value.GetHashCode())); + } + } + +#pragma warning disable CS0809 // Obsolete member 'HashCode.GetHashCode()' overrides non-obsolete member 'object.GetHashCode()' + [Obsolete("HashCode is a mutable struct and should not be compared with other HashCodes. Use ToHashCode to retrieve the computed hash code.", error: true)] + [EditorBrowsable(EditorBrowsableState.Never)] + public override int GetHashCode() => throw new NotSupportedException(); + + [Obsolete("HashCode is a mutable struct and should not be compared with other HashCodes.", error: true)] + [EditorBrowsable(EditorBrowsableState.Never)] + public override bool Equals(object obj) => throw new NotSupportedException(); +#pragma warning restore CS0809 // Obsolete member 'HashCode.GetHashCode()' overrides non-obsolete member 'object.GetHashCode()' + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Jobs/Argument.cs b/src/BenchmarkDotNet/Jobs/Argument.cs index 0932ad9943..c86c83906a 100644 --- a/src/BenchmarkDotNet/Jobs/Argument.cs +++ b/src/BenchmarkDotNet/Jobs/Argument.cs @@ -3,12 +3,28 @@ namespace BenchmarkDotNet.Jobs { - public abstract class Argument + public abstract class Argument: IEquatable { - [PublicAPI] public string TextRepresentation { get; protected set; } + [PublicAPI] public string TextRepresentation { get; } + + protected Argument(string value) + { + TextRepresentation = value; + } // CharacteristicPresenters call ToString(), this is why we need this override public override string ToString() => TextRepresentation; + + public bool Equals(Argument other) + { + if (ReferenceEquals(null, other)) return false; + if (ReferenceEquals(this, other)) return true; + return TextRepresentation == other.TextRepresentation; + } + + public override bool Equals(object obj) => Equals(obj as Argument); + + public override int GetHashCode() => HashCode.Combine(TextRepresentation); } /// @@ -17,12 +33,10 @@ public abstract class Argument /// public class MonoArgument : Argument { - public MonoArgument(string value) + public MonoArgument(string value) : base(value) { if (value == "--llvm" || value == "--nollvm") throw new NotSupportedException("Please use job.Env.Jit to specify Jit in explicit way"); - - TextRepresentation = value; } } @@ -33,6 +47,6 @@ public MonoArgument(string value) [PublicAPI] public class MsBuildArgument : Argument { - public MsBuildArgument(string value) => TextRepresentation = value; + public MsBuildArgument(string value) : base(value) { } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Jobs/EnvironmentVariable.cs b/src/BenchmarkDotNet/Jobs/EnvironmentVariable.cs index 728ecee480..96ecaf9b1a 100644 --- a/src/BenchmarkDotNet/Jobs/EnvironmentVariable.cs +++ b/src/BenchmarkDotNet/Jobs/EnvironmentVariable.cs @@ -33,12 +33,6 @@ public override bool Equals(object obj) return Equals((EnvironmentVariable) obj); } - public override int GetHashCode() - { - unchecked - { - return (Key.GetHashCode() * 397) ^ Value.GetHashCode(); - } - } + public override int GetHashCode() => HashCode.Combine(Key, Value); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Jobs/GcMode.cs b/src/BenchmarkDotNet/Jobs/GcMode.cs index 9255423af6..79664e8c0c 100644 --- a/src/BenchmarkDotNet/Jobs/GcMode.cs +++ b/src/BenchmarkDotNet/Jobs/GcMode.cs @@ -123,7 +123,6 @@ public bool Equals(GcMode other) && other.Server == Server; public override int GetHashCode() - => AllowVeryLargeObjects.GetHashCode() ^ Concurrent.GetHashCode() ^ CpuGroups.GetHashCode() ^ Force.GetHashCode() ^ NoAffinitize.GetHashCode() ^ - RetainVm.GetHashCode() ^ Server.GetHashCode(); + => HashCode.Combine(AllowVeryLargeObjects, Concurrent, CpuGroups, Force, NoAffinitize, RetainVm, Server); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Jobs/NugetReference.cs b/src/BenchmarkDotNet/Jobs/NugetReference.cs index 8f4567c9a8..b69aae1ef3 100644 --- a/src/BenchmarkDotNet/Jobs/NugetReference.cs +++ b/src/BenchmarkDotNet/Jobs/NugetReference.cs @@ -43,10 +43,7 @@ public bool Equals(NuGetReference other) PackageVersion == other.PackageVersion; } - public override int GetHashCode() - { - return 557888800 + EqualityComparer.Default.GetHashCode(PackageName) + EqualityComparer.Default.GetHashCode(PackageVersion).GetHashCode(); - } + public override int GetHashCode() => HashCode.Combine(PackageName, PackageVersion); public override string ToString() => $"{PackageName}{(string.IsNullOrWhiteSpace(PackageVersion) ? string.Empty : $" {PackageVersion}")}"; diff --git a/src/BenchmarkDotNet/Jobs/NugetReferenceList.cs b/src/BenchmarkDotNet/Jobs/NugetReferenceList.cs index 5d384d29cf..13c727c28f 100644 --- a/src/BenchmarkDotNet/Jobs/NugetReferenceList.cs +++ b/src/BenchmarkDotNet/Jobs/NugetReferenceList.cs @@ -65,16 +65,10 @@ public override bool Equals(object obj) public override int GetHashCode() { - unchecked - { - int hashCode = 0; - foreach (var nuGetReference in references) - { - hashCode = hashCode * 397 + nuGetReference.GetHashCode(); - } - - return hashCode; - } + var hashCode = new HashCode(); + foreach (var reference in references) + hashCode.Add(reference); + return hashCode.ToHashCode(); } private class PackageNameComparer : IComparer diff --git a/src/BenchmarkDotNet/Reports/SummaryStyle.cs b/src/BenchmarkDotNet/Reports/SummaryStyle.cs index 4b92f2de8c..0143838a2d 100644 --- a/src/BenchmarkDotNet/Reports/SummaryStyle.cs +++ b/src/BenchmarkDotNet/Reports/SummaryStyle.cs @@ -80,24 +80,19 @@ public bool Equals(SummaryStyle other) public override bool Equals(object obj) => obj is SummaryStyle summary && Equals(summary); - public override int GetHashCode() - { - unchecked - { - int hashCode = PrintUnitsInHeader.GetHashCode(); - hashCode = (hashCode * 397) ^ PrintUnitsInContent.GetHashCode(); - hashCode = (hashCode * 397) ^ PrintZeroValuesInContent.GetHashCode(); - hashCode = (hashCode * 397) ^ (SizeUnit != null ? SizeUnit.GetHashCode() : 0); - hashCode = (hashCode * 397) ^ (CodeSizeUnit != null ? CodeSizeUnit.GetHashCode() : 0); - hashCode = (hashCode * 397) ^ (TimeUnit != null ? TimeUnit.GetHashCode() : 0); - hashCode = (hashCode * 397) ^ MaxParameterColumnWidth; - hashCode = (hashCode * 397) ^ RatioStyle.GetHashCode(); - return hashCode; - } - } + public override int GetHashCode() => + HashCode.Combine( + PrintUnitsInHeader, + PrintUnitsInContent, + PrintZeroValuesInContent, + SizeUnit, + CodeSizeUnit, + TimeUnit, + MaxParameterColumnWidth, + RatioStyle); public static bool operator ==(SummaryStyle left, SummaryStyle right) => Equals(left, right); public static bool operator !=(SummaryStyle left, SummaryStyle right) => !Equals(left, right); } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Running/BenchmarkPartitioner.cs b/src/BenchmarkDotNet/Running/BenchmarkPartitioner.cs index 861038e114..ada1b2b78e 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkPartitioner.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkPartitioner.cs @@ -67,30 +67,23 @@ public bool Equals(BenchmarkCase x, BenchmarkCase y) public int GetHashCode(BenchmarkCase obj) { + var hashCode = new HashCode(); + hashCode.Add(obj.GetToolchain()); + hashCode.Add(obj.GetRuntime()); + hashCode.Add(obj.Descriptor.Type.Assembly.Location); + hashCode.Add(obj.Descriptor.AdditionalLogic); + hashCode.Add(obj.Descriptor.WorkloadMethod.GetCustomAttributes(false).OfType().Any()); var job = obj.Job; - - int hashCode = obj.GetToolchain().GetHashCode(); - - hashCode ^= GetRuntime(job).GetType().MetadataToken; - - hashCode ^= obj.Descriptor.Type.Assembly.Location.GetHashCode(); - hashCode ^= (int)job.Environment.Jit; - hashCode ^= (int)job.Environment.Platform; - hashCode ^= job.Environment.LargeAddressAware.GetHashCode(); - hashCode ^= job.Environment.Gc.GetHashCode(); - - if (job.Infrastructure.BuildConfiguration != null) - hashCode ^= job.Infrastructure.BuildConfiguration.GetHashCode(); - if (job.Infrastructure.Arguments != null && job.Infrastructure.Arguments.Any()) - hashCode ^= job.Infrastructure.Arguments.GetHashCode(); - if (job.Infrastructure.NuGetReferences != null) - hashCode ^= job.Infrastructure.NuGetReferences.GetHashCode(); - if (!string.IsNullOrEmpty(obj.Descriptor.AdditionalLogic)) - hashCode ^= obj.Descriptor.AdditionalLogic.GetHashCode(); - - hashCode ^= obj.Descriptor.WorkloadMethod.GetCustomAttributes(false).OfType().Any().GetHashCode(); - - return hashCode; + hashCode.Add(job.Environment.Jit); + hashCode.Add(job.Environment.Platform); + hashCode.Add(job.Environment.LargeAddressAware); + hashCode.Add(job.Environment.Gc); + hashCode.Add(job.Infrastructure.BuildConfiguration); + foreach (var arg in job.Infrastructure.Arguments ?? Array.Empty()) + hashCode.Add(arg); + foreach (var reference in job.Infrastructure.NuGetReferences ?? Array.Empty()) + hashCode.Add(reference); + return hashCode.ToHashCode(); } private static Runtime GetRuntime(Job job) diff --git a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjGenerator.cs b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjGenerator.cs index 996355a6df..ce4811cb52 100644 --- a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjGenerator.cs @@ -183,9 +183,6 @@ public bool Equals(CsProjGenerator other) && PackagesPath == other.PackagesPath; public override int GetHashCode() - => TargetFrameworkMoniker.GetHashCode() - ^ (RuntimeFrameworkVersion?.GetHashCode() ?? 0) - ^ (CliPath?.GetHashCode() ?? 0) - ^ (PackagesPath?.GetHashCode() ?? 0); + => HashCode.Combine(TargetFrameworkMoniker, RuntimeFrameworkVersion, CliPath, PackagesPath); } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Validators/ReturnValueValidator.cs b/src/BenchmarkDotNet/Validators/ReturnValueValidator.cs index d017633257..f4b5b7b63d 100644 --- a/src/BenchmarkDotNet/Validators/ReturnValueValidator.cs +++ b/src/BenchmarkDotNet/Validators/ReturnValueValidator.cs @@ -83,18 +83,13 @@ public int GetHashCode(ParameterInstances obj) if (obj.Count == 0) return 0; - unchecked + var hashCode = new HashCode(); + foreach (var instance in obj.Items.OrderBy(i => i.Name)) { - int result = 0; - - foreach (var paramInstance in obj.Items.OrderBy(i => i.Name)) - { - result = (result * 397) ^ paramInstance.Name.GetHashCode(); - result = (result * 397) ^ (paramInstance.Value?.GetHashCode() ?? 0); - } - - return result; + hashCode.Add(instance.Name); + hashCode.Add(instance.Value); } + return hashCode.ToHashCode(); } } diff --git a/src/BenchmarkDotNet/Validators/ValidationError.cs b/src/BenchmarkDotNet/Validators/ValidationError.cs index 5cb7b63863..4f0522e4db 100644 --- a/src/BenchmarkDotNet/Validators/ValidationError.cs +++ b/src/BenchmarkDotNet/Validators/ValidationError.cs @@ -40,17 +40,7 @@ public override bool Equals(object obj) return Equals((ValidationError)obj); } - public override int GetHashCode() - { - unchecked - { - int hashCode = IsCritical.GetHashCode(); - hashCode = (hashCode * 397) ^ (Message != null ? Message.GetHashCode() : 0); - hashCode = (hashCode * 397) ^ (BenchmarkCase != null ? BenchmarkCase.GetHashCode() : 0); - return hashCode; - } - } - + public override int GetHashCode() => HashCode.Combine(IsCritical, Message, BenchmarkCase); public static bool operator ==(ValidationError left, ValidationError right) => Equals(left, right); From 36f9e7309630b99a86940d32862c1f6135d2bab3 Mon Sep 17 00:00:00 2001 From: Andrey Akinshin Date: Fri, 4 Nov 2022 15:27:20 +0400 Subject: [PATCH 52/58] Set net6.0 as the first TFM for BenchmarkDotNet.Tests --- tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj b/tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj index aa55557878..8dee6af89e 100755 --- a/tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj +++ b/tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj @@ -2,7 +2,7 @@ BenchmarkDotNet.Tests - net462;net6.0 + net6.0;net462 BenchmarkDotNet.Tests BenchmarkDotNet.Tests true From 761590eb1a0ab813e4c53ccfa8903608ae31cb12 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 10 Nov 2022 14:53:19 +0100 Subject: [PATCH 53/58] Add net8.0 support to all existing runtimes and toolchains (#2192) * add support for net8.0 * add support for NativeAOT .NET 8 * add support for MonoVM .NET 8 * add support for Mono AOT .NET 8 * add support for WASM .NET 8 * moniker name does not start with a dot * disable the CA1825 warning for samples and test projects --- NuGet.Config | 1 + .../BenchmarkDotNet.Samples.csproj | 2 +- .../Jobs/RuntimeMoniker.cs | 35 ++++++++++++++++--- .../ConsoleArguments/ConfigParser.cs | 13 +++++++ .../Environments/Runtimes/CoreRuntime.cs | 21 +++++------ .../Environments/Runtimes/MonoRuntime.cs | 1 + .../Environments/Runtimes/NativeAotRuntime.cs | 4 +++ .../Extensions/RuntimeMonikerExtensions.cs | 4 +++ .../Toolchains/CsProj/CsProjCoreToolchain.cs | 1 + .../DotNetCli/NetCoreAppSettings.cs | 17 ++++----- .../Toolchains/Mono/MonoToolchain.cs | 1 + .../NativeAot/NativeAotToolchain.cs | 8 +++++ .../Toolchains/ToolchainExtensions.cs | 9 +++++ .../BenchmarkDotNet.IntegrationTests.csproj | 2 +- .../BenchmarkDotNet.Tests.csproj | 2 +- .../ConfigParserTests.cs | 4 ++- 16 files changed, 98 insertions(+), 27 deletions(-) diff --git a/NuGet.Config b/NuGet.Config index b121982021..2b16f5257e 100644 --- a/NuGet.Config +++ b/NuGet.Config @@ -11,5 +11,6 @@ + diff --git a/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj b/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj index 0693554d6f..8d0d311d5b 100644 --- a/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj +++ b/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj @@ -10,7 +10,7 @@ false AnyCPU true - $(NoWarn);CA1018;CA5351 + $(NoWarn);CA1018;CA5351;CA1825 diff --git a/src/BenchmarkDotNet.Annotations/Jobs/RuntimeMoniker.cs b/src/BenchmarkDotNet.Annotations/Jobs/RuntimeMoniker.cs index 7b6d6830db..ca9b5a0772 100644 --- a/src/BenchmarkDotNet.Annotations/Jobs/RuntimeMoniker.cs +++ b/src/BenchmarkDotNet.Annotations/Jobs/RuntimeMoniker.cs @@ -100,6 +100,11 @@ public enum RuntimeMoniker ///

Net70, + /// + /// .NET 8.0 + /// + Net80, + /// /// NativeAOT compiled as net6.0 /// @@ -110,41 +115,56 @@ public enum RuntimeMoniker /// NativeAot70, + /// + /// NativeAOT compiled as net8.0 + /// + NativeAot80, + /// /// WebAssembly with default .Net version /// Wasm, /// - /// WebAssembly with .net5.0 + /// WebAssembly with net5.0 /// WasmNet50, /// - /// WebAssembly with .net6.0 + /// WebAssembly with net6.0 /// WasmNet60, /// - /// WebAssembly with .net7.0 + /// WebAssembly with net7.0 /// WasmNet70, + /// + /// WebAssembly with net8.0 + /// + WasmNet80, + /// /// Mono with the Ahead of Time LLVM Compiler backend /// MonoAOTLLVM, /// - /// Mono with the Ahead of Time LLVM Compiler backend and .net6.0 + /// Mono with the Ahead of Time LLVM Compiler backend and net6.0 /// MonoAOTLLVMNet60, /// - /// Mono with the Ahead of Time LLVM Compiler backend and .net7.0 + /// Mono with the Ahead of Time LLVM Compiler backend and net7.0 /// MonoAOTLLVMNet70, + /// + /// Mono with the Ahead of Time LLVM Compiler backend and net8.0 + /// + MonoAOTLLVMNet80, + /// /// .NET 6 using MonoVM (not CLR which is the default) /// @@ -154,5 +174,10 @@ public enum RuntimeMoniker /// .NET 7 using MonoVM (not CLR which is the default) /// Mono70, + + /// + /// .NET 8 using MonoVM (not CLR which is the default) + /// + Mono80, } } diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index 004639181c..e4b730aca3 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -394,6 +394,7 @@ private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, Comma case RuntimeMoniker.Net50: case RuntimeMoniker.Net60: case RuntimeMoniker.Net70: + case RuntimeMoniker.Net80: return baseJob .WithRuntime(runtimeMoniker.GetRuntime()) .WithToolchain(CsProjCoreToolchain.From(new NetCoreAppSettings(runtimeId, null, runtimeId, options.CliPath?.FullName, options.RestorePath?.FullName))); @@ -407,6 +408,9 @@ private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, Comma case RuntimeMoniker.NativeAot70: return CreateAotJob(baseJob, options, runtimeMoniker, "", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet7/nuget/v3/index.json"); + case RuntimeMoniker.NativeAot80: + return CreateAotJob(baseJob, options, runtimeMoniker, "", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet8/nuget/v3/index.json"); + case RuntimeMoniker.Wasm: return MakeWasmJob(baseJob, options, RuntimeInformation.IsNetCore ? CoreRuntime.GetCurrentVersion().MsBuildMoniker : "net5.0", runtimeMoniker); @@ -419,6 +423,9 @@ private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, Comma case RuntimeMoniker.WasmNet70: return MakeWasmJob(baseJob, options, "net7.0", runtimeMoniker); + case RuntimeMoniker.WasmNet80: + return MakeWasmJob(baseJob, options, "net8.0", runtimeMoniker); + case RuntimeMoniker.MonoAOTLLVM: return MakeMonoAOTLLVMJob(baseJob, options, RuntimeInformation.IsNetCore ? CoreRuntime.GetCurrentVersion().MsBuildMoniker : "net6.0"); @@ -428,12 +435,18 @@ private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, Comma case RuntimeMoniker.MonoAOTLLVMNet70: return MakeMonoAOTLLVMJob(baseJob, options, "net7.0"); + case RuntimeMoniker.MonoAOTLLVMNet80: + return MakeMonoAOTLLVMJob(baseJob, options, "net8.0"); + case RuntimeMoniker.Mono60: return MakeMonoJob(baseJob, options, MonoRuntime.Mono60); case RuntimeMoniker.Mono70: return MakeMonoJob(baseJob, options, MonoRuntime.Mono70); + case RuntimeMoniker.Mono80: + return MakeMonoJob(baseJob, options, MonoRuntime.Mono80); + default: throw new NotSupportedException($"Runtime {runtimeId} is not supported"); } diff --git a/src/BenchmarkDotNet/Environments/Runtimes/CoreRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/CoreRuntime.cs index 05f9172b79..d1e4f02708 100644 --- a/src/BenchmarkDotNet/Environments/Runtimes/CoreRuntime.cs +++ b/src/BenchmarkDotNet/Environments/Runtimes/CoreRuntime.cs @@ -11,16 +11,17 @@ namespace BenchmarkDotNet.Environments { public class CoreRuntime : Runtime { - public static readonly CoreRuntime Core20 = new CoreRuntime(RuntimeMoniker.NetCoreApp20, "netcoreapp2.0", ".NET Core 2.0"); - public static readonly CoreRuntime Core21 = new CoreRuntime(RuntimeMoniker.NetCoreApp21, "netcoreapp2.1", ".NET Core 2.1"); - public static readonly CoreRuntime Core22 = new CoreRuntime(RuntimeMoniker.NetCoreApp22, "netcoreapp2.2", ".NET Core 2.2"); - public static readonly CoreRuntime Core30 = new CoreRuntime(RuntimeMoniker.NetCoreApp30, "netcoreapp3.0", ".NET Core 3.0"); - public static readonly CoreRuntime Core31 = new CoreRuntime(RuntimeMoniker.NetCoreApp31, "netcoreapp3.1", ".NET Core 3.1"); - public static readonly CoreRuntime Core50 = new CoreRuntime(RuntimeMoniker.Net50, "net5.0", ".NET 5.0"); - public static readonly CoreRuntime Core60 = new CoreRuntime(RuntimeMoniker.Net60, "net6.0", ".NET 6.0"); - public static readonly CoreRuntime Core70 = new CoreRuntime(RuntimeMoniker.Net70, "net7.0", ".NET 7.0"); - - public static CoreRuntime Latest => Core70; // when dotnet/runtime branches for 8.0, this will need to get updated + public static readonly CoreRuntime Core20 = new (RuntimeMoniker.NetCoreApp20, "netcoreapp2.0", ".NET Core 2.0"); + public static readonly CoreRuntime Core21 = new (RuntimeMoniker.NetCoreApp21, "netcoreapp2.1", ".NET Core 2.1"); + public static readonly CoreRuntime Core22 = new (RuntimeMoniker.NetCoreApp22, "netcoreapp2.2", ".NET Core 2.2"); + public static readonly CoreRuntime Core30 = new (RuntimeMoniker.NetCoreApp30, "netcoreapp3.0", ".NET Core 3.0"); + public static readonly CoreRuntime Core31 = new (RuntimeMoniker.NetCoreApp31, "netcoreapp3.1", ".NET Core 3.1"); + public static readonly CoreRuntime Core50 = new (RuntimeMoniker.Net50, "net5.0", ".NET 5.0"); + public static readonly CoreRuntime Core60 = new (RuntimeMoniker.Net60, "net6.0", ".NET 6.0"); + public static readonly CoreRuntime Core70 = new (RuntimeMoniker.Net70, "net7.0", ".NET 7.0"); + public static readonly CoreRuntime Core80 = new (RuntimeMoniker.Net80, "net8.0", ".NET 8.0"); + + public static CoreRuntime Latest => Core80; // when dotnet/runtime branches for 9.0, this will need to get updated private CoreRuntime(RuntimeMoniker runtimeMoniker, string msBuildMoniker, string displayName) : base(runtimeMoniker, msBuildMoniker, displayName) diff --git a/src/BenchmarkDotNet/Environments/Runtimes/MonoRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/MonoRuntime.cs index ee9b348ec6..55fb9e6140 100644 --- a/src/BenchmarkDotNet/Environments/Runtimes/MonoRuntime.cs +++ b/src/BenchmarkDotNet/Environments/Runtimes/MonoRuntime.cs @@ -8,6 +8,7 @@ public class MonoRuntime : Runtime, IEquatable public static readonly MonoRuntime Default = new MonoRuntime("Mono"); public static readonly MonoRuntime Mono60 = new MonoRuntime("Mono with .NET 6.0", RuntimeMoniker.Mono60, "net6.0", isDotNetBuiltIn: true); public static readonly MonoRuntime Mono70 = new MonoRuntime("Mono with .NET 7.0", RuntimeMoniker.Mono70, "net7.0", isDotNetBuiltIn: true); + public static readonly MonoRuntime Mono80 = new MonoRuntime("Mono with .NET 8.0", RuntimeMoniker.Mono80, "net8.0", isDotNetBuiltIn: true); public string CustomPath { get; } diff --git a/src/BenchmarkDotNet/Environments/Runtimes/NativeAotRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/NativeAotRuntime.cs index 90a3887f11..063210b37c 100644 --- a/src/BenchmarkDotNet/Environments/Runtimes/NativeAotRuntime.cs +++ b/src/BenchmarkDotNet/Environments/Runtimes/NativeAotRuntime.cs @@ -14,6 +14,10 @@ public class NativeAotRuntime : Runtime /// NativeAOT compiled as net7.0 /// public static readonly NativeAotRuntime Net70 = new NativeAotRuntime(RuntimeMoniker.NativeAot70, "net7.0", "NativeAOT 7.0"); + /// + /// NativeAOT compiled as net8.0 + /// + public static readonly NativeAotRuntime Net80 = new NativeAotRuntime(RuntimeMoniker.NativeAot80, "net8.0", "NativeAOT 8.0"); public override bool IsAOT => true; diff --git a/src/BenchmarkDotNet/Extensions/RuntimeMonikerExtensions.cs b/src/BenchmarkDotNet/Extensions/RuntimeMonikerExtensions.cs index a8ba6c5632..60fa23df3e 100644 --- a/src/BenchmarkDotNet/Extensions/RuntimeMonikerExtensions.cs +++ b/src/BenchmarkDotNet/Extensions/RuntimeMonikerExtensions.cs @@ -43,12 +43,16 @@ internal static Runtime GetRuntime(this RuntimeMoniker runtimeMoniker) return CoreRuntime.Core60; case RuntimeMoniker.Net70: return CoreRuntime.Core70; + case RuntimeMoniker.Net80: + return CoreRuntime.Core80; case RuntimeMoniker.Mono: return MonoRuntime.Default; case RuntimeMoniker.NativeAot60: return NativeAotRuntime.Net60; case RuntimeMoniker.NativeAot70: return NativeAotRuntime.Net70; + case RuntimeMoniker.NativeAot80: + return NativeAotRuntime.Net80; case RuntimeMoniker.Mono60: return MonoRuntime.Mono60; case RuntimeMoniker.Mono70: diff --git a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjCoreToolchain.cs b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjCoreToolchain.cs index 9dbaacc68c..63ec4573b0 100644 --- a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjCoreToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjCoreToolchain.cs @@ -23,6 +23,7 @@ public class CsProjCoreToolchain : Toolchain, IEquatable [PublicAPI] public static readonly IToolchain NetCoreApp50 = From(NetCoreAppSettings.NetCoreApp50); [PublicAPI] public static readonly IToolchain NetCoreApp60 = From(NetCoreAppSettings.NetCoreApp60); [PublicAPI] public static readonly IToolchain NetCoreApp70 = From(NetCoreAppSettings.NetCoreApp70); + [PublicAPI] public static readonly IToolchain NetCoreApp80 = From(NetCoreAppSettings.NetCoreApp80); internal CsProjCoreToolchain(string name, IGenerator generator, IBuilder builder, IExecutor executor, string customDotNetCliPath) : base(name, generator, builder, executor) diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/NetCoreAppSettings.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/NetCoreAppSettings.cs index b4408546fb..50b1f7b614 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/NetCoreAppSettings.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/NetCoreAppSettings.cs @@ -9,14 +9,15 @@ namespace BenchmarkDotNet.Toolchains.DotNetCli [PublicAPI] public class NetCoreAppSettings { - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp20 = new NetCoreAppSettings("netcoreapp2.0", null, ".NET Core 2.0"); - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp21 = new NetCoreAppSettings("netcoreapp2.1", null, ".NET Core 2.1"); - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp22 = new NetCoreAppSettings("netcoreapp2.2", null, ".NET Core 2.2"); - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp30 = new NetCoreAppSettings("netcoreapp3.0", null, ".NET Core 3.0"); - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp31 = new NetCoreAppSettings("netcoreapp3.1", null, ".NET Core 3.1"); - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp50 = new NetCoreAppSettings("net5.0", null, ".NET 5.0"); - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp60 = new NetCoreAppSettings("net6.0", null, ".NET 6.0"); - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp70 = new NetCoreAppSettings("net7.0", null, ".NET 7.0"); + [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp20 = new ("netcoreapp2.0", null, ".NET Core 2.0"); + [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp21 = new ("netcoreapp2.1", null, ".NET Core 2.1"); + [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp22 = new ("netcoreapp2.2", null, ".NET Core 2.2"); + [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp30 = new ("netcoreapp3.0", null, ".NET Core 3.0"); + [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp31 = new ("netcoreapp3.1", null, ".NET Core 3.1"); + [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp50 = new ("net5.0", null, ".NET 5.0"); + [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp60 = new ("net6.0", null, ".NET 6.0"); + [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp70 = new ("net7.0", null, ".NET 7.0"); + [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp80 = new ("net8.0", null, ".NET 8.0"); /// /// diff --git a/src/BenchmarkDotNet/Toolchains/Mono/MonoToolchain.cs b/src/BenchmarkDotNet/Toolchains/Mono/MonoToolchain.cs index 28ea147c97..ba2ae2ac96 100644 --- a/src/BenchmarkDotNet/Toolchains/Mono/MonoToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/Mono/MonoToolchain.cs @@ -10,6 +10,7 @@ public class MonoToolchain : CsProjCoreToolchain, IEquatable { [PublicAPI] public static readonly IToolchain Mono60 = From(new NetCoreAppSettings("net6.0", null, "mono60")); [PublicAPI] public static readonly IToolchain Mono70 = From(new NetCoreAppSettings("net7.0", null, "mono70")); + [PublicAPI] public static readonly IToolchain Mono80 = From(new NetCoreAppSettings("net8.0", null, "mono80")); private MonoToolchain(string name, IGenerator generator, IBuilder builder, IExecutor executor, string customDotNetCliPath) : base(name, generator, builder, executor, customDotNetCliPath) diff --git a/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchain.cs b/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchain.cs index fb30e94a5a..c20cd2e3ca 100644 --- a/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchain.cs @@ -21,6 +21,14 @@ public class NativeAotToolchain : Toolchain .TargetFrameworkMoniker("net7.0") .ToToolchain(); + /// + /// compiled as net8.0, targets latest NativeAOT build from the .NET 8 feed: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet8/nuget/v3/index.json + /// + public static readonly IToolchain Net80 = CreateBuilder() + .UseNuGet("", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet8/nuget/v3/index.json") + .TargetFrameworkMoniker("net8.0") + .ToToolchain(); + internal NativeAotToolchain(string displayName, string ilCompilerVersion, string runtimeFrameworkVersion, string targetFrameworkMoniker, string runtimeIdentifier, diff --git a/src/BenchmarkDotNet/Toolchains/ToolchainExtensions.cs b/src/BenchmarkDotNet/Toolchains/ToolchainExtensions.cs index 6741f7b86e..c2af278c76 100644 --- a/src/BenchmarkDotNet/Toolchains/ToolchainExtensions.cs +++ b/src/BenchmarkDotNet/Toolchains/ToolchainExtensions.cs @@ -125,18 +125,27 @@ private static IToolchain GetToolchain(RuntimeMoniker runtimeMoniker) case RuntimeMoniker.Net70: return CsProjCoreToolchain.NetCoreApp70; + case RuntimeMoniker.Net80: + return CsProjCoreToolchain.NetCoreApp80; + case RuntimeMoniker.NativeAot60: return NativeAotToolchain.Net60; case RuntimeMoniker.NativeAot70: return NativeAotToolchain.Net70; + case RuntimeMoniker.NativeAot80: + return NativeAotToolchain.Net80; + case RuntimeMoniker.Mono60: return MonoToolchain.Mono60; case RuntimeMoniker.Mono70: return MonoToolchain.Mono70; + case RuntimeMoniker.Mono80: + return MonoToolchain.Mono80; + default: throw new ArgumentOutOfRangeException(nameof(runtimeMoniker), runtimeMoniker, "RuntimeMoniker not supported"); } diff --git a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj index 88b6b6605d..48e1f1a6e8 100755 --- a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj @@ -10,7 +10,7 @@ false AnyCPU true - $(NoWarn);CA2007 + $(NoWarn);CA2007;CA1825 true diff --git a/tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj b/tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj index 8dee6af89e..031a325884 100755 --- a/tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj +++ b/tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj @@ -7,7 +7,7 @@ BenchmarkDotNet.Tests true false - $(NoWarn);NU1701;1701;CA1018;CA2007 + $(NoWarn);NU1701;1701;CA1018;CA2007;CA1825 diff --git a/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs b/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs index 7276ee79e1..e3f32cede2 100644 --- a/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs +++ b/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs @@ -373,6 +373,7 @@ public void NetFrameworkMonikerParsedCorrectly(string tfm) [InlineData("net50")] [InlineData("net60")] [InlineData("net70")] + [InlineData("net80")] public void NetMonikersAreRecognizedAsNetCoreMonikers(string tfm) { var config = ConfigParser.Parse(new[] { "-r", tfm }, new OutputLogger(Output)).config; @@ -399,7 +400,7 @@ public void PlatformSpecificMonikersAreSupported(string msBuildMoniker) [Fact] public void CanCompareFewDifferentRuntimes() { - var config = ConfigParser.Parse(new[] { "--runtimes", "net462", "MONO", "netcoreapp3.0", "nativeaot6.0", "nativeAOT7.0"}, new OutputLogger(Output)).config; + var config = ConfigParser.Parse(new[] { "--runtimes", "net462", "MONO", "netcoreapp3.0", "nativeaot6.0", "nativeAOT7.0", "nativeAOT8.0" }, new OutputLogger(Output)).config; Assert.True(config.GetJobs().First().Meta.Baseline); // when the user provides multiple runtimes the first one should be marked as baseline Assert.Single(config.GetJobs().Where(job => job.Environment.Runtime is ClrRuntime clrRuntime && clrRuntime.MsBuildMoniker == "net462")); @@ -407,6 +408,7 @@ public void CanCompareFewDifferentRuntimes() Assert.Single(config.GetJobs().Where(job => job.Environment.Runtime is CoreRuntime coreRuntime && coreRuntime.MsBuildMoniker == "netcoreapp3.0" && coreRuntime.RuntimeMoniker == RuntimeMoniker.NetCoreApp30)); Assert.Single(config.GetJobs().Where(job => job.Environment.Runtime is NativeAotRuntime nativeAot && nativeAot.MsBuildMoniker == "net6.0" && nativeAot.RuntimeMoniker == RuntimeMoniker.NativeAot60)); Assert.Single(config.GetJobs().Where(job => job.Environment.Runtime is NativeAotRuntime nativeAot && nativeAot.MsBuildMoniker == "net7.0" && nativeAot.RuntimeMoniker == RuntimeMoniker.NativeAot70)); + Assert.Single(config.GetJobs().Where(job => job.Environment.Runtime is NativeAotRuntime nativeAot && nativeAot.MsBuildMoniker == "net8.0" && nativeAot.RuntimeMoniker == RuntimeMoniker.NativeAot80)); } [Theory] From 8b6159139280d094630db16efa4c7fc75e13c8a5 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 10 Nov 2022 17:00:14 +0100 Subject: [PATCH 54/58] use ImmutableConfig when doing apples-to-apples comparison (#2193) --- src/BenchmarkDotNet/Running/BenchmarkSwitcher.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/BenchmarkDotNet/Running/BenchmarkSwitcher.cs b/src/BenchmarkDotNet/Running/BenchmarkSwitcher.cs index d236491016..60ef98dbf2 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkSwitcher.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkSwitcher.cs @@ -114,7 +114,7 @@ internal IEnumerable RunWithDirtyAssemblyResolveHelper(string[] args, I if (effectiveConfig.Options.HasFlag(ConfigOptions.ApplesToApples)) { - return ApplesToApples(effectiveConfig, benchmarksToFilter, logger, options); + return ApplesToApples(ImmutableConfigBuilder.Create(effectiveConfig), benchmarksToFilter, logger, options); } var filteredBenchmarks = TypeFilter.Filter(effectiveConfig, benchmarksToFilter); @@ -140,7 +140,7 @@ private static void PrintList(ILogger nonNullLogger, IConfig effectiveConfig, IR printer.Print(testNames, nonNullLogger); } - private IEnumerable ApplesToApples(ManualConfig effectiveConfig, IReadOnlyList benchmarksToFilter, ILogger logger, CommandLineOptions options) + private IEnumerable ApplesToApples(ImmutableConfig effectiveConfig, IReadOnlyList benchmarksToFilter, ILogger logger, CommandLineOptions options) { var jobs = effectiveConfig.GetJobs().ToArray(); if (jobs.Length <= 1) From a4ab68f63f4185e26090ff96f41ceafef48a5d86 Mon Sep 17 00:00:00 2001 From: Richard Banks Date: Mon, 14 Nov 2022 20:15:22 +1100 Subject: [PATCH 55/58] Docs: Add note for ETW Profiling regarding Intel TDT and Windows Defender (#2196) --- docs/articles/features/etwprofiler.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/articles/features/etwprofiler.md b/docs/articles/features/etwprofiler.md index c5e56e2cbb..b142ffd042 100644 --- a/docs/articles/features/etwprofiler.md +++ b/docs/articles/features/etwprofiler.md @@ -30,6 +30,11 @@ What we have today comes with following limitations: true ``` +> [!NOTE] +> On certain machines [Intel TDT and Windows Defender](https://www.microsoft.com/en-us/security/blog/2021/04/26/defending-against-cryptojacking-with-microsoft-defender-for-endpoint-and-intel-tdt/) can cause CPU samples to be captured with no value. +> You can correct this problem by disabling the feature using `powershell.exe Set-MpPreference -DisableTDTFeature $true`. +> *WARNING:* Disabling security features will make your machine less secure; do so at your own risk. + ## How to use it? You need to install `BenchmarkDotNet.Diagnostics.Windows` package. From 9c32a878f2d7bef3314e2fd4ad7bfa1048915439 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 17 Nov 2022 22:22:56 +0100 Subject: [PATCH 56/58] fix resources leak (#2200) --- src/BenchmarkDotNet/Loggers/Broker.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/BenchmarkDotNet/Loggers/Broker.cs b/src/BenchmarkDotNet/Loggers/Broker.cs index e915d61eca..7e0a0825cf 100644 --- a/src/BenchmarkDotNet/Loggers/Broker.cs +++ b/src/BenchmarkDotNet/Loggers/Broker.cs @@ -55,7 +55,15 @@ internal void ProcessData() writer.WriteLine(Engine.Signals.Acknowledgment); - if (signal == HostSignal.AfterAll) + if (signal == HostSignal.BeforeAnythingElse) + { + // The client has connected, we no longer need to keep the local copy of client handle alive. + // This allows server to detect that child process is done and hence avoid resource leak. + // Full explanation: https://stackoverflow.com/a/39700027 + inputFromBenchmark.DisposeLocalCopyOfClientHandle(); + acknowledgments.DisposeLocalCopyOfClientHandle(); + } + else if (signal == HostSignal.AfterAll) { // we have received the last signal so we can stop reading from the pipe // if the process won't exit after this, its hung and needs to be killed From ad8e9b26ef22aa77abff2469f7668986e1449127 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Fri, 18 Nov 2022 23:14:39 +0100 Subject: [PATCH 57/58] fix WASM support (#2201) --- src/BenchmarkDotNet/Portability/RuntimeInformation.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BenchmarkDotNet/Portability/RuntimeInformation.cs b/src/BenchmarkDotNet/Portability/RuntimeInformation.cs index a630e2daf9..07c287b9f3 100644 --- a/src/BenchmarkDotNet/Portability/RuntimeInformation.cs +++ b/src/BenchmarkDotNet/Portability/RuntimeInformation.cs @@ -163,7 +163,7 @@ internal static CpuInfo GetCpuInfo() internal static string GetRuntimeVersion() { - if (IsMono) + if (IsMono && !IsWasm) { var monoRuntimeType = Type.GetType("Mono.Runtime"); var monoDisplayName = monoRuntimeType?.GetMethod("GetDisplayName", BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static); From ccbaf08b03b38d1d418756a9e528c60c9f4951d1 Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Mon, 21 Nov 2022 05:44:55 -0800 Subject: [PATCH 58/58] Remove duplicated jobs when creating immutable config (#2202) --- .../Configs/ImmutableConfigBuilder.cs | 2 +- .../Environments/BenchmarkEnvironmentInfo.cs | 2 +- .../Configs/ImmutableConfigTests.cs | 13 +++++++++++ .../JobRuntimePropertiesComparerTests.cs | 23 ------------------- 4 files changed, 15 insertions(+), 25 deletions(-) diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs index 677574bf86..56a1a6e8c4 100644 --- a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs +++ b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs @@ -215,7 +215,7 @@ private static ImmutableHashSet GetValidators(IEnumerable private static IReadOnlyList GetRunnableJobs(IEnumerable jobs) { - var unique = jobs.Distinct().ToArray(); + var unique = jobs.Distinct(JobComparer.Instance).ToArray(); var result = new List(); foreach (var standardJob in unique.Where(job => !job.Meta.IsMutator && !job.Meta.IsDefault)) diff --git a/src/BenchmarkDotNet/Environments/BenchmarkEnvironmentInfo.cs b/src/BenchmarkDotNet/Environments/BenchmarkEnvironmentInfo.cs index 08fe7fc45e..0bcee913f2 100644 --- a/src/BenchmarkDotNet/Environments/BenchmarkEnvironmentInfo.cs +++ b/src/BenchmarkDotNet/Environments/BenchmarkEnvironmentInfo.cs @@ -60,7 +60,7 @@ [PublicAPI] protected string GetConfigurationFlag() => Configuration == RuntimeI ? "" : Configuration; - [PublicAPI] protected string GetDebuggerFlag() => HasAttachedDebugger ? " [AttachedDebugger]" : ""; + [PublicAPI] protected string GetDebuggerFlag() => HasAttachedDebugger ? "[AttachedDebugger]" : ""; [PublicAPI] protected string GetGcServerFlag() => IsServerGC ? "Server" : "Workstation"; [PublicAPI] protected string GetGcConcurrentFlag() => IsConcurrentGC ? "Concurrent" : "Non-concurrent"; diff --git a/tests/BenchmarkDotNet.Tests/Configs/ImmutableConfigTests.cs b/tests/BenchmarkDotNet.Tests/Configs/ImmutableConfigTests.cs index 88bff1070e..6f18411877 100644 --- a/tests/BenchmarkDotNet.Tests/Configs/ImmutableConfigTests.cs +++ b/tests/BenchmarkDotNet.Tests/Configs/ImmutableConfigTests.cs @@ -19,6 +19,19 @@ namespace BenchmarkDotNet.Tests.Configs { public class ImmutableConfigTests { + [Fact] + public void DuplicateJobsAreExcluded() + { + var mutable = ManualConfig.CreateEmpty(); + + mutable.AddJob(new Job()); + mutable.AddJob(new Job()); + + var final = ImmutableConfigBuilder.Create(mutable); + + Assert.Single(final.GetJobs()); + } + [Fact] public void DuplicateColumnProvidersAreExcluded() { diff --git a/tests/BenchmarkDotNet.Tests/Running/JobRuntimePropertiesComparerTests.cs b/tests/BenchmarkDotNet.Tests/Running/JobRuntimePropertiesComparerTests.cs index 96e0bd2fc9..595d4a67b1 100644 --- a/tests/BenchmarkDotNet.Tests/Running/JobRuntimePropertiesComparerTests.cs +++ b/tests/BenchmarkDotNet.Tests/Running/JobRuntimePropertiesComparerTests.cs @@ -94,29 +94,6 @@ public void CustomClrBuildJobsAreGroupedByVersion() Assert.Equal(3 * 2, grouping.Count()); // (M1 + M2 + M3) * (Plain1 + Plain2) } - [Fact] - public void CustomNuGetJobsWithSamePackageVersionAreGroupedTogether() - { - var job1 = Job.Default.WithNuGet("AutoMapper", "7.0.1"); - var job2 = Job.Default.WithNuGet("AutoMapper", "7.0.1"); - - var config = ManualConfig.Create(DefaultConfig.Instance) - .AddJob(job1) - .AddJob(job2); - - var benchmarks1 = BenchmarkConverter.TypeToBenchmarks(typeof(Plain1), config); - var benchmarks2 = BenchmarkConverter.TypeToBenchmarks(typeof(Plain2), config); - - var grouped = benchmarks1.BenchmarksCases.Union(benchmarks2.BenchmarksCases) - .GroupBy(benchmark => benchmark, new BenchmarkPartitioner.BenchmarkRuntimePropertiesComparer()) - .ToArray(); - - Assert.Single(grouped); // 7.0.1 - - foreach (var grouping in grouped) - Assert.Equal(2 * 3 * 2, grouping.Count()); // ((job1 + job2) * (M1 + M2 + M3) * (Plain1 + Plain2) - } - [Fact] public void CustomNuGetJobsAreGroupedByPackageVersion() {