Skip to content

Check BenchmarkRunner arguments #1911

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
YegorStepanov wants to merge 5 commits into from
+120 −10
Closed

Check BenchmarkRunner arguments #1911

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
4095706
Check arguments for BenchmarkRunner with no args
YegorStepanov Feb 2, 2022
7f5531f
Change to fast fail
YegorStepanov Feb 2, 2022
67bf7eb
Remove spaces
YegorStepanov Feb 2, 2022
ac22b52
Extract GetRunnableBenchmarks
YegorStepanov Aug 25, 2022
d9cb000
Rework args checking
YegorStepanov Aug 25, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions src/BenchmarkDotNet/Extensions/ReflectionExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,16 +138,21 @@ internal static Type[] GetRunnableBenchmarks(this Assembly assembly)
.ToArray();

internal static bool ContainsRunnableBenchmarks(this Type type)
{
return GetRunnableBenchmarks(type).Any();
}

internal static MethodInfo[] GetRunnableBenchmarks(this Type type)
{
var typeInfo = type.GetTypeInfo();

if (typeInfo.IsAbstract
|| typeInfo.IsSealed
|| typeInfo.IsNotPublic
|| typeInfo.IsGenericType && !IsRunnableGenericType(typeInfo))
return false;
return Array.Empty<MethodInfo>();

return typeInfo.GetBenchmarks().Any();
return typeInfo.GetBenchmarks();
}

private static MethodInfo[] GetBenchmarks(this TypeInfo typeInfo)
Expand Down Expand Up @@ -213,4 +218,4 @@ private static bool IsRunnableGenericType(TypeInfo typeInfo)

internal static bool IsLinqPad(this Assembly assembly) => assembly.FullName.IndexOf("LINQPAD", StringComparison.OrdinalIgnoreCase) >= 0;
}
}
}
119 changes: 112 additions & 7 deletions src/BenchmarkDotNet/Running/BenchmarkRunnerDirty.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,41 +27,54 @@ public static Summary Run<T>(IConfig config = null, string[] args = null)
[PublicAPI]
public static Summary Run(Type type, IConfig config = null, string[] args = null)
{
AssertNotNull(type, nameof(type));

using (DirtyAssemblyResolveHelper.Create())
return RunWithExceptionHandling(() => RunWithDirtyAssemblyResolveHelper(type, config, args));
}

[PublicAPI]
public static Summary[] Run(Type[] types, IConfig config = null, string[] args = null)
{
AssertNotNull(types, nameof(types));

using (DirtyAssemblyResolveHelper.Create())
return RunWithExceptionHandling(() => RunWithDirtyAssemblyResolveHelper(types, config, args));
}

[PublicAPI]
public static Summary Run(Type type, MethodInfo[] methods, IConfig config = null)
{
AssertNotNull(type, nameof(type));
AssertNotNull(methods, nameof(methods));

using (DirtyAssemblyResolveHelper.Create())
return RunWithExceptionHandling(() => RunWithDirtyAssemblyResolveHelper(type, methods, config));
}

[PublicAPI]
public static Summary[] Run(Assembly assembly, IConfig config = null, string[] args = null)
{
AssertNotNull(assembly, nameof(assembly));

using (DirtyAssemblyResolveHelper.Create())
return RunWithExceptionHandling(() => RunWithDirtyAssemblyResolveHelper(assembly, config, args));
}

[PublicAPI]
public static Summary Run(BenchmarkRunInfo benchmarkRunInfo)
{
AssertNotNull(benchmarkRunInfo, nameof(benchmarkRunInfo));

using (DirtyAssemblyResolveHelper.Create())
return RunWithExceptionHandling(() => RunWithDirtyAssemblyResolveHelper(new[] { benchmarkRunInfo }).Single());
}

[PublicAPI]
public static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos)
{
AssertNotNull(benchmarkRunInfos, nameof(benchmarkRunInfos));

using (DirtyAssemblyResolveHelper.Create())
return RunWithExceptionHandling(() => RunWithDirtyAssemblyResolveHelper(benchmarkRunInfos));
}
Expand Down Expand Up @@ -90,30 +103,50 @@ public static Summary RunSource(string source, IConfig config = null)

[MethodImpl(MethodImplOptions.NoInlining)]
private static Summary RunWithDirtyAssemblyResolveHelper(Type type, IConfig config, string[] args)
=> (args == null
? BenchmarkRunnerClean.Run(new[] { BenchmarkConverter.TypeToBenchmarks(type, config) })
: new BenchmarkSwitcher(new[] { type }).RunWithDirtyAssemblyResolveHelper(args, config, false))
{
Validate(type);

return (args == null
? BenchmarkRunnerClean.Run(new[] { BenchmarkConverter.TypeToBenchmarks(type, config) })
: new BenchmarkSwitcher(new[] { type }).RunWithDirtyAssemblyResolveHelper(args, config, false))
.Single();
}

[MethodImpl(MethodImplOptions.NoInlining)]
private static Summary RunWithDirtyAssemblyResolveHelper(Type type, MethodInfo[] methods, IConfig config = null)
=> BenchmarkRunnerClean.Run(new[] { BenchmarkConverter.MethodsToBenchmarks(type, methods, config) }).Single();
{
Validate(type, methods);

return BenchmarkRunnerClean.Run(new[] { BenchmarkConverter.MethodsToBenchmarks(type, methods, config) }).Single();
}

[MethodImpl(MethodImplOptions.NoInlining)]
private static Summary[] RunWithDirtyAssemblyResolveHelper(Assembly assembly, IConfig config, string[] args)
=> args == null
{
Validate(assembly);

return args == null
? BenchmarkRunnerClean.Run(assembly.GetRunnableBenchmarks().Select(type => BenchmarkConverter.TypeToBenchmarks(type, config)).ToArray())
: new BenchmarkSwitcher(assembly).RunWithDirtyAssemblyResolveHelper(args, config, false).ToArray();
}

[MethodImpl(MethodImplOptions.NoInlining)]
private static Summary[] RunWithDirtyAssemblyResolveHelper(Type[] types, IConfig config, string[] args)
=> args == null
{
Validate(types);

return args == null
? BenchmarkRunnerClean.Run(types.Select(type => BenchmarkConverter.TypeToBenchmarks(type, config)).ToArray())
: new BenchmarkSwitcher(types).RunWithDirtyAssemblyResolveHelper(args, config, false).ToArray();
}

[MethodImpl(MethodImplOptions.NoInlining)]
private static Summary[] RunWithDirtyAssemblyResolveHelper(BenchmarkRunInfo[] benchmarkRunInfos)
=> BenchmarkRunnerClean.Run(benchmarkRunInfos);
{
Validate(benchmarkRunInfos);

return BenchmarkRunnerClean.Run(benchmarkRunInfos);
}

[MethodImpl(MethodImplOptions.NoInlining)]
private static Summary RunUrlWithDirtyAssemblyResolveHelper(string url, IConfig config = null)
Expand Down Expand Up @@ -152,5 +185,77 @@ private static Summary[] RunWithExceptionHandling(Func<Summary[]> run)
return new[] { Summary.NothingToRun(e.Message, string.Empty, string.Empty) };
}
}

private static void AssertNotNull<T>(T instance, string paramName)
{
if (instance is null)
throw new ArgumentNullException(paramName);
}

private static void Validate(Type[] types)
{
if (types.IsEmpty())
throw new InvalidBenchmarkDeclarationException("No types provided.");

if (types.Any(t => t is null))
throw new InvalidBenchmarkDeclarationException("Null not allowed.");

var nonBenchmarkTypes = types.Where(t => !t.ContainsRunnableBenchmarks()).ToArray();
if (!nonBenchmarkTypes.IsEmpty())
{
var invalidNames = string.Join("\n", nonBenchmarkTypes.Select(type => $" {type.FullName}"));
throw new InvalidBenchmarkDeclarationException($"Invalid Types:\n{invalidNames}\nOnly public, non-generic (closed generic types with public parameterless ctors are supported), non-abstract, non-sealed, non-static types with public instance [Benchmark] method(s) are supported.");
}
}

private static void Validate(BenchmarkRunInfo[] benchmarkRunInfos)
{
if (benchmarkRunInfos.IsEmpty())
throw new InvalidBenchmarkDeclarationException($"No BenchmarkRunInfos provided.");

if (benchmarkRunInfos.Any(v => v is null))
throw new InvalidBenchmarkDeclarationException($"Null not allowed.");

foreach (var benchmarkRunInfo in benchmarkRunInfos)
{
if (benchmarkRunInfo.Config is null ||
benchmarkRunInfo.BenchmarksCases is null ||
benchmarkRunInfo.BenchmarksCases.IsEmpty() ||
benchmarkRunInfo.BenchmarksCases.Any(c => c is null))
throw new InvalidBenchmarkDeclarationException("BenchmarkRunInfo do not support null values.");
}
}

private static void Validate(Type type)
{
if (!type.ContainsRunnableBenchmarks())
throw new InvalidBenchmarkDeclarationException($"Type {type} is invalid. Only public, non-generic (closed generic types with public parameterless ctors are supported), non-abstract, non-sealed, non-static types with public instance [Benchmark] method(s) are supported.");
}

private static void Validate(Type type, MethodInfo[] methods)
{
if (!type.ContainsRunnableBenchmarks())
throw new InvalidBenchmarkDeclarationException($"Type {type} is invalid. Only public, non-generic (closed generic types with public parameterless ctors are supported), non-abstract, non-sealed, non-static types with public instance [Benchmark] method(s) are supported.");

if (methods.IsEmpty())
throw new InvalidBenchmarkDeclarationException($"No methods provided for {type}.");

if (methods.Any(m => m is null))
throw new InvalidBenchmarkDeclarationException($"Null not allowed.");

var benchmarkMethods = type.GetRunnableBenchmarks();
var invalidMethods = methods.Except(benchmarkMethods).ToArray();
if (!invalidMethods.IsEmpty())
{
var invalidNames = string.Join("\n", invalidMethods.Select(m => $" {m.ReflectedType?.FullName}.{m.Name}"));
Copy link
Contributor Author

@YegorStepanov YegorStepanov Aug 25, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe someone know, how to get "global" methods that return null for m.ReflectedType?

Module.GetMethods() returns empty array for BDN, BCL(string/object) modules.

MSDN says:

If the MemberInfo object is a global member (that is, if it was obtained from the Module.GetMethods method, which returns global methods on a module), the returned DeclaringType will be null.

Copy link
Member

@adamsitnik adamsitnik Oct 5, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

To check if given method belongs to the provided type you should be able to use method.DeclaringType

Copy link
Contributor Author

@YegorStepanov YegorStepanov Oct 6, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@adamsitnik DeclaringType returns "System.Object.ToString", but we need to display "MyNameSpace.Benchmark2.ToString".

If we pass the Benchmark1.ToString and Benchmark2.ToString methods, we get:

DeclaringType:

System.Object.ToString
System.Object.ToString

ReflectedType:

NS.Benchmark1.ToString
NS.Benchmark2.ToString

Copy link
Contributor Author

@YegorStepanov YegorStepanov Oct 24, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Found the answer when it returns null (this applies to DeclaringType as well):

It can be null if the property is defined in a module. In C# you cannot define such methods and properties without reflection (see PropertyBuilder). However, if you reference a VB.NET assembly, it can have such members.

throw new InvalidBenchmarkDeclarationException($"Invalid methods:\n{invalidNames}\nMethods must be of {type.FullName} type. Only public, non-generic (closed generic types with public parameterless ctors are supported), non-abstract, non-sealed, non-static types with public instance [Benchmark] method(s) are supported.");
}
}

private static void Validate(Assembly assembly)
{
if (assembly.GetRunnableBenchmarks().IsEmpty())
throw new InvalidBenchmarkDeclarationException("No benchmarks to choose from. Make sure you provided public non-sealed non-static types with public [Benchmark] methods.");
}
}
}