Skip to content

Commit 6f27090

Browse files
markplesmarkples
authored
Add some checks for merged test groups (#89521)
- CLRTest.Execute.Batch.targets - Fix CLRTestExecutionArguments (allows passing a filter argument to <mergedgroup>.cmd - Fix indenting in generated script - XUnitWrapperGenerator - Move some generated code into (shared) ITestInfo.cs - Also display exception on stdout on failure - Add check that projects in merged test groups don't contain entry points - Can be a sign of code that is expected to run but won't be - Or can be a [Fact]/etc-less entry point in a ReqProcIso project but would then need to be fixed if ReqProcIso were removed - Directory.Build.targets - Run XUnitWrapperGenerator on test projects in merged groups - Can't simply replace the conditional because we have XUnit-style tests outside of merged groups - Update tests to conform to above checks plus a bit of opportunistic cleanup
1 parent 91ba52c commit 6f27090

File tree

22 files changed

+321
-238
lines changed

22 files changed

+321
-238
lines changed

‎src/tests/Common/CLRTest.Execute.Batch.targets‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -164,8 +164,8 @@ IF NOT "%CLRTestExitCode%"=="%CLRTestExpectedExitCode%" (
164164
:TakeLock
165165
md %lockFolder%
166166
IF NOT "!ERRORLEVEL!"=="0" (
167-
timeout /t 10 /nobreak
168-
goto :TakeLock
167+
timeout /t 10 /nobreak
168+
goto :TakeLock
169169
)
170170
Exit /b 2
171171
@@ -403,15 +403,17 @@ IF /I [%1] == [-%(Identity)] set cond=1
403403
IF /I [%1] == [/%(Identity)] set cond=1
404404
IF %cond% EQU 1 (
405405
%(Command)
406-
shift
407-
IF /I [%(HasParam)] == [true] shift
408-
goto NextArg
406+
shift
407+
IF /I [%(HasParam)] == [true] shift
408+
goto NextArg
409409
)','
410410
')
411411
412+
:ExtraArgs
412413
if NOT "%1" == "" (
413-
set CLRTestExecutionArguments=%*
414-
goto :ArgsDone
414+
set CLRTestExecutionArguments=%CLRTestExecutionArguments% %1
415+
shift
416+
goto :ExtraArgs
415417
)
416418
417419
goto ArgsDone

‎src/tests/Common/XUnitWrapperGenerator/Descriptors.cs‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
4+
using System;
5+
using System.Collections.Generic;
6+
using System.Text;
7+
using Microsoft.CodeAnalysis;
8+
9+
namespace XUnitWrapperGenerator;
10+
11+
public static class Descriptors
12+
{
13+
public static readonly DiagnosticDescriptor XUWG1001 =
14+
new DiagnosticDescriptor(
15+
"XUW1001",
16+
"Projects in merged tests group should not have entry points",
17+
"Projects in merged tests group should not have entry points. Convert to Facts or Theories.",
18+
"XUnitWrapperGenerator",
19+
DiagnosticSeverity.Warning,
20+
isEnabledByDefault: true);
21+
}

‎src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -415,22 +415,18 @@ public CodeBuilder WrapTestExecutionWithReporting(CodeBuilder testExecutionExpre
415415
builder.AppendLine("try");
416416
using (builder.NewBracesScope())
417417
{
418-
builder.AppendLine($"System.Console.WriteLine(\"{{0:HH:mm:ss.fff}} Running test: {{1}}\", System.DateTime.Now, {test.TestNameExpression});");
418+
builder.AppendLine($"{_summaryLocalIdentifier}.ReportStartingTest({test.TestNameExpression}, System.Console.Out);");
419419
builder.AppendLine($"{_outputRecorderIdentifier}.ResetTestOutput();");
420420
builder.Append(testExecutionExpression);
421421

422422
builder.AppendLine($"{_summaryLocalIdentifier}.ReportPassedTest({test.TestNameExpression}, \"{test.ContainingType}\", @\"{test.Method}\","
423-
+ $" stopwatch.Elapsed - testStart, {_outputRecorderIdentifier}.GetTestOutput(), tempLogSw, statsCsvSw);");
424-
425-
builder.AppendLine($"System.Console.WriteLine(\"{{0:HH:mm:ss.fff}} Passed test: {{1}}\", System.DateTime.Now, {test.TestNameExpression});");
423+
+ $" stopwatch.Elapsed - testStart, {_outputRecorderIdentifier}.GetTestOutput(), System.Console.Out, tempLogSw, statsCsvSw);");
426424
}
427425
builder.AppendLine("catch (System.Exception ex)");
428426
using (builder.NewBracesScope())
429427
{
430428
builder.AppendLine($"{_summaryLocalIdentifier}.ReportFailedTest({test.TestNameExpression}, \"{test.ContainingType}\", @\"{test.Method}\","
431-
+ $" stopwatch.Elapsed - testStart, ex, {_outputRecorderIdentifier}.GetTestOutput(), tempLogSw, statsCsvSw);");
432-
433-
builder.AppendLine($"System.Console.WriteLine(\"{{0:HH:mm:ss.fff}} Failed test: {{1}}\", System.DateTime.Now, {test.TestNameExpression});");
429+
+ $" stopwatch.Elapsed - testStart, ex, {_outputRecorderIdentifier}.GetTestOutput(), System.Console.Out, tempLogSw, statsCsvSw);");
434430
}
435431
}
436432
builder.AppendLine("else");

‎src/tests/Common/XUnitWrapperGenerator/OptionsHelper.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ namespace XUnitWrapperGenerator;
66

77
public static class OptionsHelper
88
{
9+
private const string InMergedTestDirectoryOption = "build_property.InMergedTestDirectory";
910
private const string IsMergedTestRunnerAssemblyOption = "build_property.IsMergedTestRunnerAssembly";
1011
private const string PriorityOption = "build_property.Priority";
1112
private const string RuntimeFlavorOption = "build_property.RuntimeFlavor";
@@ -29,6 +30,8 @@ private static bool GetBoolOption(this AnalyzerConfigOptions options, string key
2930
? result : 0;
3031
}
3132

33+
internal static bool InMergedTestDirectory(this AnalyzerConfigOptions options) => options.GetBoolOption(InMergedTestDirectoryOption);
34+
3235
internal static bool IsMergedTestRunnerAssembly(this AnalyzerConfigOptions options) => options.GetBoolOption(IsMergedTestRunnerAssemblyOption);
3336

3437
internal static int? Priority(this AnalyzerConfigOptions options) => options.GetIntOption(PriorityOption);

‎src/tests/Common/XUnitWrapperGenerator/RoslynUtils.cs‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
4+
using System;
5+
using System.Collections.Generic;
6+
using System.Linq;
7+
using System.Text;
8+
using System.Threading;
9+
using Microsoft.CodeAnalysis;
10+
using Microsoft.CodeAnalysis.CSharp;
11+
using Microsoft.CodeAnalysis.CSharp.Syntax;
12+
13+
namespace XUnitWrapperGenerator
14+
{
15+
internal class RoslynUtils
16+
{
17+
/// <summary>
18+
/// Returns the Main method that would serve as the entry point of the assembly, ignoring
19+
/// whether the current target is an executable.
20+
/// </summary>
21+
/// <remarks>
22+
/// Replacement for CSharpCompilation.GetEntryPoint() which only works for executables.
23+
/// Replacement for its helpers that are internal.
24+
///
25+
/// Intended for the analyzer that is trying to find Main methods that won't be called in
26+
/// merged test groups. Ignores details such as SynthesizedSimpleProgramEntryPointSymbol.
27+
/// Ignores top-level statements as (1) in exes, they will generate an error for conflicting
28+
/// with the auto-generated main, and (2) in libs, they will generate an error for existing
29+
/// at all.
30+
/// </remarks>
31+
internal static IEnumerable<IMethodSymbol> GetPossibleEntryPoints(Compilation comp, CancellationToken cancellationToken)
32+
=> comp
33+
.GetSymbolsWithName(WellKnownMemberNames.EntryPointMethodName, SymbolFilter.Member)
34+
.OfType<IMethodSymbol>()
35+
.Where(m => m.IsStatic && !m.IsAbstract && !m.IsVirtual);
36+
}
37+
}

‎src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs‎

Lines changed: 70 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
using System.Collections.Generic;
66
using System.Collections.Immutable;
77
using System.Diagnostics;
8+
using System.Diagnostics.CodeAnalysis;
89
using System.IO;
910
using System.Linq;
1011
using System.Text;
@@ -15,6 +16,22 @@
1516

1617
namespace XUnitWrapperGenerator;
1718

19+
internal struct CompData
20+
{
21+
internal CompData(string assemblyName, IMethodSymbol? entryPoint, IEnumerable<IMethodSymbol> possibleEntryPoints, OutputKind outputKind)
22+
{
23+
AssemblyName = assemblyName;
24+
EntryPoint = entryPoint;
25+
PossibleEntryPoints = possibleEntryPoints;
26+
OutputKind = outputKind;
27+
}
28+
29+
public string AssemblyName { get; private set; }
30+
public IMethodSymbol? EntryPoint { get; private set; }
31+
public IEnumerable<IMethodSymbol> PossibleEntryPoints { get; private set; }
32+
public OutputKind OutputKind { get; private set; }
33+
}
34+
1835
[Generator]
1936
public sealed class XUnitWrapperGenerator : IIncrementalGenerator
2037
{
@@ -58,9 +75,11 @@ public void Initialize(IncrementalGeneratorInitializationContext context)
5875
return aliasMap.ToImmutable();
5976
}).WithComparer(new ImmutableDictionaryValueComparer<string, string>(EqualityComparer<string>.Default));
6077

61-
var assemblyName = context.CompilationProvider.Select((comp, ct) => comp.Assembly.MetadataName);
62-
63-
var alwaysWriteEntryPoint = context.CompilationProvider.Select((comp, ct) => comp.Options.OutputKind == OutputKind.ConsoleApplication && comp.GetEntryPoint(ct) is null);
78+
var compData = context.CompilationProvider.Select((comp, ct) => new CompData(
79+
assemblyName: comp.Assembly.MetadataName,
80+
entryPoint: comp.GetEntryPoint(ct),
81+
possibleEntryPoints: RoslynUtils.GetPossibleEntryPoints(comp, ct),
82+
outputKind: comp.Options.OutputKind));
6483

6584
var testsInSource =
6685
methodsInSource
@@ -112,40 +131,65 @@ public void Initialize(IncrementalGeneratorInitializationContext context)
112131
.Collect()
113132
.Combine(context.AnalyzerConfigOptionsProvider)
114133
.Combine(aliasMap)
115-
.Combine(assemblyName)
116-
.Combine(alwaysWriteEntryPoint),
134+
.Combine(compData),
117135
static (context, data) =>
118136
{
119-
var ((((methods, configOptions), aliasMap), assemblyName), alwaysWriteEntryPoint) = data;
137+
var (((methods, configOptions), aliasMap), compData) = data;
120138

121-
if (methods.Length == 0 && !alwaysWriteEntryPoint)
139+
bool inMergedTestDirectory = configOptions.GlobalOptions.InMergedTestDirectory();
140+
if (inMergedTestDirectory)
122141
{
123-
// If we have no test methods, assume that this project is not migrated to the new system yet
124-
// and that we shouldn't generate a no-op Main method.
125-
return;
142+
CheckNoEntryPoint(context, compData);
126143
}
127144

128-
bool isMergedTestRunnerAssembly = configOptions.GlobalOptions.IsMergedTestRunnerAssembly();
129-
configOptions.GlobalOptions.TryGetValue("build_property.TargetOS", out string? targetOS);
130-
131-
if (isMergedTestRunnerAssembly)
145+
if (compData.OutputKind != OutputKind.ConsoleApplication)
132146
{
133-
if (targetOS?.ToLowerInvariant() is "ios" or "iossimulator" or "tvos" or "tvossimulator" or "maccatalyst" or "android" or "browser")
134-
{
135-
context.AddSource("XHarnessRunner.g.cs", GenerateXHarnessTestRunner(methods, aliasMap, assemblyName));
136-
}
137-
else
138-
{
139-
context.AddSource("FullRunner.g.cs", GenerateFullTestRunner(methods, aliasMap, assemblyName));
140-
}
147+
return;
141148
}
142-
else
149+
150+
bool alwaysWriteEntryPoint = (compData.EntryPoint is null);
151+
if (methods.IsEmpty && !alwaysWriteEntryPoint)
143152
{
144-
context.AddSource("SimpleRunner.g.cs", GenerateStandaloneSimpleTestRunner(methods, aliasMap));
153+
// If we have no test methods, assume that this project is not migrated to the new system yet
154+
// and that we shouldn't generate a no-op Main method.
155+
return;
145156
}
157+
158+
AddRunnerSource(context, methods, configOptions, aliasMap, compData);
146159
});
147160
}
148161

162+
private static void AddRunnerSource(SourceProductionContext context, ImmutableArray<ITestInfo> methods, AnalyzerConfigOptionsProvider configOptions, ImmutableDictionary<string, string> aliasMap, CompData compData)
163+
{
164+
bool isMergedTestRunnerAssembly = configOptions.GlobalOptions.IsMergedTestRunnerAssembly();
165+
configOptions.GlobalOptions.TryGetValue("build_property.TargetOS", out string? targetOS);
166+
string assemblyName = compData.AssemblyName;
167+
168+
if (isMergedTestRunnerAssembly)
169+
{
170+
if (targetOS?.ToLowerInvariant() is "ios" or "iossimulator" or "tvos" or "tvossimulator" or "maccatalyst" or "android" or "browser")
171+
{
172+
context.AddSource("XHarnessRunner.g.cs", GenerateXHarnessTestRunner(methods, aliasMap, assemblyName));
173+
}
174+
else
175+
{
176+
context.AddSource("FullRunner.g.cs", GenerateFullTestRunner(methods, aliasMap, assemblyName));
177+
}
178+
}
179+
else
180+
{
181+
context.AddSource("SimpleRunner.g.cs", GenerateStandaloneSimpleTestRunner(methods, aliasMap));
182+
}
183+
}
184+
185+
private static void CheckNoEntryPoint(SourceProductionContext context, CompData compData)
186+
{
187+
foreach (IMethodSymbol entryPoint in compData.PossibleEntryPoints)
188+
{
189+
context.ReportDiagnostic(Diagnostic.Create(Descriptors.XUWG1001, entryPoint.Locations[0]));
190+
}
191+
}
192+
149193
private static void AppendAliasMap(CodeBuilder builder, ImmutableDictionary<string, string> aliasMap)
150194
{
151195
bool didOutput = false;
@@ -312,7 +356,7 @@ private static string GenerateXHarnessTestRunner(ImmutableArray<ITestInfo> testI
312356
builder.AppendLine("System.Collections.Generic.HashSet<string> testExclusionList = XUnitWrapperLibrary.TestFilter.LoadTestExclusionList();");
313357
builder.AppendLine($@"return await XHarnessRunnerLibrary.RunnerEntryPoint.RunTests(RunTests, ""{assemblyName}"", args.Length != 0 ? args[0] : null, testExclusionList);");
314358
}
315-
builder.AppendLine("catch(System.Exception ex)");
359+
builder.AppendLine("catch (System.Exception ex)");
316360
using (builder.NewBracesScope())
317361
{
318362
builder.AppendLine("System.Console.WriteLine(ex.ToString());");
@@ -435,7 +479,7 @@ private static string GenerateStandaloneSimpleTestRunner(ImmutableArray<ITestInf
435479
builder.Append(testInfo.GenerateTestExecution(reporter));
436480
}
437481
}
438-
builder.AppendLine("catch(System.Exception ex)");
482+
builder.AppendLine("catch (System.Exception ex)");
439483
using (builder.NewBracesScope())
440484
{
441485
builder.AppendLine("System.Console.WriteLine(ex.ToString());");

‎src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.props‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
<CompilerVisibleProperty Include="TargetArchitecture" />
77
<CompilerVisibleProperty Include="Priority" />
88
<!-- Properties that influence test harness generation -->
9+
<CompilerVisibleProperty Include="InMergedTestDirectory" />
910
<CompilerVisibleProperty Include="IsMergedTestRunnerAssembly" />
1011
<CompilerVisibleProperty Include="TestFilter" />
1112
<CompilerVisibleItemMetadata Include="AdditionalFiles" MetadataName="IsOutOfProcessTestAssembly" />

‎src/tests/Common/XUnitWrapperLibrary/TestSummary.cs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,11 +120,18 @@ public void WriteFooterToTempLog(StreamWriter tempLogSw)
120120
tempLogSw.WriteLine("</assembly>");
121121
}
122122

123+
public void ReportStartingTest(string name, TextWriter outTw)
124+
{
125+
outTw.WriteLine("{0:HH:mm:ss.fff} Running test: {1}", System.DateTime.Now, name);
126+
outTw.Flush();
127+
}
128+
123129
public void ReportPassedTest(string name,
124130
string containingTypeName,
125131
string methodName,
126132
TimeSpan duration,
127133
string output,
134+
TextWriter outTw,
128135
StreamWriter tempLogSw,
129136
StreamWriter statsCsvSw)
130137
{
@@ -133,8 +140,10 @@ public void ReportPassedTest(string name,
133140
var result = new TestResult(name, containingTypeName, methodName, duration, null, null, output);
134141
_testResults.Add(result);
135142

143+
outTw.WriteLine($"{0:HH:mm:ss.fff} Passed test: {1}", System.DateTime.Now, name);
136144
statsCsvSw.WriteLine($"{TotalTests},{PassedTests},{FailedTests},{SkippedTests}");
137145
tempLogSw.WriteLine(result.ToXmlString());
146+
outTw.Flush();
138147
statsCsvSw.Flush();
139148
tempLogSw.Flush();
140149
}
@@ -145,6 +154,7 @@ public void ReportFailedTest(string name,
145154
TimeSpan duration,
146155
Exception ex,
147156
string output,
157+
TextWriter outTw,
148158
StreamWriter tempLogSw,
149159
StreamWriter statsCsvSw)
150160
{
@@ -153,8 +163,11 @@ public void ReportFailedTest(string name,
153163
var result = new TestResult(name, containingTypeName, methodName, duration, ex, null, output);
154164
_testResults.Add(result);
155165

166+
outTw.WriteLine(ex);
167+
outTw.WriteLine("{0:HH:mm:ss.fff} Failed test: {1}", System.DateTime.Now, name);
156168
statsCsvSw.WriteLine($"{TotalTests},{PassedTests},{FailedTests},{SkippedTests}");
157169
tempLogSw.WriteLine(result.ToXmlString());
170+
outTw.Flush();
158171
statsCsvSw.Flush();
159172
tempLogSw.Flush();
160173
}

‎src/tests/Directory.Build.targets‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -510,7 +510,7 @@
510510
</ItemGroup>
511511
</Target>
512512

513-
<PropertyGroup Condition="'$(Language)' == 'C#' and ('$(BuildAsStandalone)' == 'true' or '$(RequiresProcessIsolation)' == 'true' or '$(IsMergedTestRunnerAssembly)' == 'true')">
513+
<PropertyGroup Condition="'$(Language)' == 'C#' and ('$(BuildAsStandalone)' == 'true' or '$(RequiresProcessIsolation)' == 'true' or '$(InMergedTestDirectory)' == 'true' or '$(IsMergedTestRunnerAssembly)' == 'true')">
514514
<ReferenceXUnitWrapperGenerator Condition="'$(ReferenceXUnitWrapperGenerator)' == ''">true</ReferenceXUnitWrapperGenerator>
515515
</PropertyGroup>
516516

0 commit comments

Comments
 (0)