-
-
Notifications
You must be signed in to change notification settings - Fork 962
/
BenchmarkConverter.cs
280 lines (233 loc) · 12.8 KB
/
BenchmarkConverter.cs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Extensions;
using BenchmarkDotNet.Filters;
using BenchmarkDotNet.Jobs;
using BenchmarkDotNet.Order;
using BenchmarkDotNet.Parameters;
namespace BenchmarkDotNet.Running
{
public static partial class BenchmarkConverter
{
public static BenchmarkRunInfo TypeToBenchmarks(Type type, IConfig config = null)
{
var fullConfig = GetFullConfig(type, config);
var allMethods = type.GetMethods();
return MethodsToBenchmarksWithFullConfig(type, allMethods, fullConfig);
}
public static BenchmarkRunInfo MethodsToBenchmarks(Type containingType, MethodInfo[] benchmarkMethods, IConfig config = null)
{
var fullConfig = GetFullConfig(containingType, config);
return MethodsToBenchmarksWithFullConfig(containingType, benchmarkMethods, fullConfig);
}
private static BenchmarkRunInfo MethodsToBenchmarksWithFullConfig(Type containingType, MethodInfo[] benchmarkMethods, ReadOnlyConfig fullConfig)
{
if (fullConfig == null)
throw new ArgumentNullException(nameof(fullConfig));
var helperMethods = containingType.GetMethods(); // benchmarkMethods can be filtered, without Setups, look #564
var globalSetupMethods = GetAttributedMethods<GlobalSetupAttribute>(helperMethods, "GlobalSetup");
var globalCleanupMethods = GetAttributedMethods<GlobalCleanupAttribute>(helperMethods, "GlobalCleanup");
var iterationSetupMethods = GetAttributedMethods<IterationSetupAttribute>(helperMethods, "IterationSetup");
var iterationCleanupMethods = GetAttributedMethods<IterationCleanupAttribute>(helperMethods, "IterationCleanup");
var targetMethods = benchmarkMethods.Where(method => method.HasAttribute<BenchmarkAttribute>()).ToArray();
var parameterDefinitions = GetParameterDefinitions(containingType);
var parameterInstancesList = parameterDefinitions.Expand();
var rawJobs = fullConfig.GetJobs().ToArray();
if (rawJobs.IsEmpty())
rawJobs = new[] { Job.Default };
var jobs = rawJobs.Distinct().ToArray();
var targets = GetTargets(targetMethods, containingType, globalSetupMethods, globalCleanupMethods, iterationSetupMethods, iterationCleanupMethods).ToArray();
var benchmarks = (
from target in targets
from job in jobs
from parameterInstance in parameterInstancesList
select new Benchmark(target, job, parameterInstance)).ToArray();
var filters = fullConfig.GetFilters().ToList();
benchmarks = GetFilteredBenchmarks(benchmarks, filters);
var orderProvider = fullConfig.GetOrderProvider() ?? DefaultOrderProvider.Instance;
return new BenchmarkRunInfo(
orderProvider.GetExecutionOrder(benchmarks).ToArray(),
containingType,
fullConfig);
}
public static ReadOnlyConfig GetFullConfig(Type type, IConfig config)
{
config = config ?? DefaultConfig.Instance;
if (type != null)
{
var typeAttributes = type.GetTypeInfo().GetCustomAttributes(true).OfType<IConfigSource>();
var assemblyAttributes = type.GetTypeInfo().Assembly.GetCustomAttributes().OfType<IConfigSource>();
var allAttributes = typeAttributes.Concat(assemblyAttributes);
foreach (var configSource in allAttributes)
config = ManualConfig.Union(config, configSource.Config);
}
return config.AsReadOnly();
}
private static IEnumerable<Target> GetTargets(
MethodInfo[] targetMethods,
Type type,
Tuple<MethodInfo, TargetedAttribute>[] globalSetupMethods,
Tuple<MethodInfo, TargetedAttribute>[] globalCleanupMethods,
Tuple<MethodInfo, TargetedAttribute>[] iterationSetupMethods,
Tuple<MethodInfo, TargetedAttribute>[] iterationCleanupMethods)
{
return targetMethods
.Where(m => m.HasAttribute<BenchmarkAttribute>())
.Select(methodInfo => CreateTarget(type,
GetTargetedMatchingMethod(methodInfo, globalSetupMethods),
methodInfo,
GetTargetedMatchingMethod(methodInfo, globalCleanupMethods),
GetTargetedMatchingMethod(methodInfo, iterationSetupMethods),
GetTargetedMatchingMethod(methodInfo, iterationCleanupMethods),
methodInfo.ResolveAttribute<BenchmarkAttribute>(),
targetMethods));
}
private static MethodInfo GetTargetedMatchingMethod(MethodInfo benchmarkMethod, Tuple<MethodInfo, TargetedAttribute>[] methods)
{
foreach (var method in methods)
{
if (string.IsNullOrEmpty(method.Item2.Target))
return method.Item1;
var targets = method.Item2.Target.Split(',');
if (targets.Contains(benchmarkMethod.Name))
return method.Item1;
}
return null;
}
private static Tuple<MethodInfo, TargetedAttribute>[] GetAttributedMethods<T>(MethodInfo[] methods, string methodName) where T : TargetedAttribute
{
return methods.SelectMany(m => m.GetCustomAttributes<T>()
.Select(attr =>
{
AssertMethodHasCorrectSignature(methodName, m);
AssertMethodIsAccessible(methodName, m);
AssertMethodIsNotGeneric(methodName, m);
return new Tuple<MethodInfo, TargetedAttribute>(m, attr);
})).OrderByDescending(x => x.Item2.Target ?? "").ToArray();
}
private static Target CreateTarget(
Type type,
MethodInfo globalSetupMethod,
MethodInfo methodInfo,
MethodInfo globalCleanupMethod,
MethodInfo iterationSetupMethod,
MethodInfo iterationCleanupMethod,
BenchmarkAttribute attr,
MethodInfo[] targetMethods)
{
var target = new Target(
type,
methodInfo,
globalSetupMethod,
globalCleanupMethod,
iterationSetupMethod,
iterationCleanupMethod,
attr.Description,
baseline: attr.Baseline,
categories: GetCategories(methodInfo),
operationsPerInvoke: attr.OperationsPerInvoke,
methodIndex: Array.IndexOf(targetMethods, methodInfo));
AssertMethodHasCorrectSignature("Benchmark", methodInfo);
AssertMethodIsAccessible("Benchmark", methodInfo);
AssertMethodIsNotGeneric("Benchmark", methodInfo);
return target;
}
private static ParameterDefinitions GetParameterDefinitions(Type type)
{
const BindingFlags reflectionFlags = BindingFlags.Static | BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic;
var allParamsMembers = type.GetTypeMembersWithGivenAttribute<ParamsAttribute>(reflectionFlags);
var allParamsSourceMembers = type.GetTypeMembersWithGivenAttribute<ParamsSourceAttribute>(reflectionFlags);
var definitions = allParamsMembers
.Select(member =>
new ParameterDefinition(
member.Name,
member.IsStatic,
GetValidValues(member.Attribute.Values, member.ParameterType)))
.Concat(allParamsSourceMembers.Select(member =>
new ParameterDefinition(
member.Name,
member.IsStatic,
GetValidValuesForParamsSource(type, member.Attribute.Name))))
.ToArray();
return new ParameterDefinitions(definitions);
}
private static string[] GetCategories(MethodInfo method)
{
var attributes = new List<BenchmarkCategoryAttribute>();
attributes.AddRange(method.GetCustomAttributes(typeof(BenchmarkCategoryAttribute), false).OfType<BenchmarkCategoryAttribute>());
var type = method.DeclaringType;
if (type != null)
{
attributes.AddRange(type.GetTypeInfo().GetCustomAttributes(typeof(BenchmarkCategoryAttribute), false).OfType<BenchmarkCategoryAttribute>());
attributes.AddRange(type.GetTypeInfo().Assembly.GetCustomAttributes().OfType<BenchmarkCategoryAttribute>());
}
if (attributes.Count == 0)
return Array.Empty<string>();
return attributes.SelectMany(attr => attr.Categories).Distinct(StringComparer.OrdinalIgnoreCase).ToArray();
}
private static Benchmark[] GetFilteredBenchmarks(IList<Benchmark> benchmarks, IList<IFilter> filters)
{
return benchmarks.Where(benchmark => filters.All(filter => filter.Predicate(benchmark))).ToArray();
}
private static void AssertMethodHasCorrectSignature(string methodType, MethodInfo methodInfo)
{
if (methodInfo.GetParameters().Any())
throw new InvalidOperationException($"{methodType} method {methodInfo.Name} has incorrect signature.\nMethod shouldn't have any arguments.");
}
private static void AssertMethodIsAccessible(string methodType, MethodInfo methodInfo)
{
if (!methodInfo.IsPublic)
throw new InvalidOperationException($"{methodType} method {methodInfo.Name} has incorrect access modifiers.\nMethod must be public.");
var declaringType = methodInfo.DeclaringType;
while (declaringType != null)
{
if (!declaringType.GetTypeInfo().IsPublic && !declaringType.GetTypeInfo().IsNestedPublic)
throw new InvalidOperationException($"{declaringType.FullName} containing {methodType} method {methodInfo.Name} has incorrect access modifiers.\nDeclaring type must be public.");
declaringType = declaringType.DeclaringType;
}
}
private static void AssertMethodIsNotGeneric(string methodType, MethodInfo methodInfo)
{
if (methodInfo.IsGenericMethod)
throw new InvalidOperationException($"{methodType} method {methodInfo.Name} is generic.\nGeneric {methodType} methods are not supported.");
}
private static object[] GetValidValues(object[] values, Type parameterType)
{
if (values == null && parameterType.IsNullable())
{
return new object[] { null };
}
return values;
}
private static object[] GetValidValuesForParamsSource(Type parentType, string sourceName)
{
var paramsSourceMethod = parentType.GetAllMethods().SingleOrDefault(method => method.Name == sourceName && method.IsPublic);
if (paramsSourceMethod != default(MethodInfo))
return ToArray(
paramsSourceMethod.Invoke(paramsSourceMethod.IsStatic ? null : Activator.CreateInstance(parentType), null),
paramsSourceMethod,
parentType);
var paramsSourceProperty = parentType.GetAllProperties().SingleOrDefault(property => property.Name == sourceName && property.GetMethod.IsPublic);
if (paramsSourceProperty != default(PropertyInfo))
return ToArray(
paramsSourceProperty.GetValue(paramsSourceProperty.GetMethod.IsStatic ? null : Activator.CreateInstance(parentType)),
paramsSourceProperty,
parentType);
throw new InvalidOperationException($"{parentType.Name} has no public, accessible method/property called {sourceName}, unable to read values for [ParamsSource]");
}
private static object[] ToArray(object sourceValue, MemberInfo memberInfo, Type type)
{
if(!(sourceValue is IEnumerable collection))
throw new InvalidOperationException($"{memberInfo.Name} of type {type.Name} does not implement IEnumerable, unable to read values for [ParamsSource]");
var values = new List<object>();
foreach (var value in collection)
values.Add(value);
return values.ToArray();
}
}
}