Skip to content

Perf/generator #8212

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 18 commits into from
Mar 28, 2023
Merged

Perf/generator #8212

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
c7a95d3
Don't re-run declaration code if nothing has changed
chsienki Jan 19, 2023
3523147
Clean up test file
chsienki Jan 19, 2023
04f79ff
Split tag helper discovery into component and compilation discovery
chsienki Jan 20, 2023
6a4a589
Create a wrapper source generator engine that can selectively run phases
chsienki Jan 31, 2023
89660be
Ensure we don't re-use a rewritten syntax tree
chsienki Feb 2, 2023
7616424
Break output into multiple steps that can be largely skipped as needed
chsienki Feb 2, 2023
489f81c
Remove uneeded tag helper discovery from SG engine
chsienki Feb 3, 2023
ff6f5b4
Swap asserts
chsienki Feb 3, 2023
312ff75
Formatting
chsienki Feb 3, 2023
9422b03
PR feedback
chsienki Feb 3, 2023
ca2a9e5
Formatting
chsienki Feb 6, 2023
fb07d1c
PR Feedback
chsienki Feb 14, 2023
30c9815
Fix tests
chsienki Feb 22, 2023
baaddfa
Merge branch 'main' into perf/generator
chsienki Mar 28, 2023
46dbf1e
Use array builder pools
chsienki Mar 27, 2023
5cf0d50
Merge remote-tracking branch 'upstream/main' into perf/generator
chsienki Mar 28, 2023
c430fea
Fix merge
chsienki Mar 28, 2023
5ce316d
PR Feedback
chsienki Mar 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions ...ler/Microsoft.AspNetCore.Razor.Language/src/DefaultRazorTagHelperContextDiscoveryPhase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ internal sealed class DefaultRazorTagHelperContextDiscoveryPhase : RazorEnginePh
{
protected override void ExecuteCore(RazorCodeDocument codeDocument)
{
var syntaxTree = codeDocument.GetSyntaxTree();
var syntaxTree = codeDocument.GetPreTagHelperSyntaxTree() ?? codeDocument.GetSyntaxTree();
ThrowForMissingDocumentDependency(syntaxTree);

var descriptors = codeDocument.GetTagHelpers();
Expand Down Expand Up @@ -69,7 +69,6 @@ protected override void ExecuteCore(RazorCodeDocument codeDocument)

var context = TagHelperDocumentContext.Create(tagHelperPrefix, descriptors);
codeDocument.SetTagHelperContext(context);
codeDocument.SetPreTagHelperSyntaxTree(syntaxTree);
}

private static bool MatchesDirective(TagHelperDescriptor descriptor, string typePattern, string assemblyName)
Expand Down
21 changes: 12 additions & 9 deletions src/Compiler/Microsoft.AspNetCore.Razor.Language/src/DefaultRazorTagHelperRewritePhase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,24 +2,27 @@
// The .NET Foundation licenses this file to you under the MIT license.
#nullable enable

using System.Collections.Generic;
using Microsoft.AspNetCore.Razor.Language.Legacy;

namespace Microsoft.AspNetCore.Razor.Language;
internal sealed class DefaultRazorTagHelperRewritePhase : RazorEnginePhaseBase
{
protected override void ExecuteCore(RazorCodeDocument codeDocument)
{
var syntaxTree = codeDocument.GetPreTagHelperSyntaxTree();
var syntaxTree = codeDocument.GetPreTagHelperSyntaxTree() ?? codeDocument.GetSyntaxTree();
ThrowForMissingDocumentDependency(syntaxTree);

var context = codeDocument.GetTagHelperContext();
if (syntaxTree is null || context.TagHelpers.Count == 0)
if (context?.TagHelpers.Count > 0)
{
// No descriptors, no-op.
return;
var rewrittenSyntaxTree = TagHelperParseTreeRewriter.Rewrite(syntaxTree, context.Prefix, context.TagHelpers, out var usedHelpers);
codeDocument.SetSyntaxTree(rewrittenSyntaxTree);
codeDocument.SetReferencedTagHelpers(usedHelpers);
}
else
{
codeDocument.SetReferencedTagHelpers(new HashSet<TagHelperDescriptor>());
}

var rewrittenSyntaxTree = TagHelperParseTreeRewriter.Rewrite(syntaxTree, context.Prefix, context.TagHelpers, out var usedHelpers);

codeDocument.SetReferencedTagHelpers(usedHelpers);
codeDocument.SetSyntaxTree(rewrittenSyntaxTree);
}
}
10 changes: 3 additions & 7 deletions src/Compiler/Microsoft.NET.Sdk.Razor.SourceGenerators/RazorSourceGenerator.Helpers.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,8 +83,7 @@ private static RazorProjectEngine GetDiscoveryProjectEngine(
return discoveryProjectEngine;
}

private static RazorProjectEngine GetGenerationProjectEngine(
IReadOnlyList<TagHelperDescriptor> tagHelpers,
private static SourceGeneratorProjectEngine GetGenerationProjectEngine(
SourceGeneratorProjectItem item,
IEnumerable<SourceGeneratorProjectItem> imports,
RazorSourceGenerationOptions razorSourceGeneratorOptions)
Expand All @@ -96,7 +95,7 @@ private static RazorProjectEngine GetGenerationProjectEngine(
fileSystem.Add(import);
}

var projectEngine = RazorProjectEngine.Create(razorSourceGeneratorOptions.Configuration, fileSystem, b =>
var projectEngine = (DefaultRazorProjectEngine)RazorProjectEngine.Create(razorSourceGeneratorOptions.Configuration, fileSystem, b =>
{
b.Features.Add(new DefaultTypeNameFeature());
b.SetRootNamespace(razorSourceGeneratorOptions.RootNamespace);
Expand All @@ -107,16 +106,13 @@ private static RazorProjectEngine GetGenerationProjectEngine(
options.SupportLocalizedComponentNames = razorSourceGeneratorOptions.SupportLocalizedComponentNames;
}));

b.Features.Add(new StaticTagHelperFeature { TagHelpers = tagHelpers });
b.Features.Add(new DefaultTagHelperDescriptorProvider());

CompilerFeatures.Register(b);
RazorExtensions.Register(b);

b.SetCSharpLanguageVersion(razorSourceGeneratorOptions.CSharpLanguageVersion);
});

return projectEngine;
return new SourceGeneratorProjectEngine(projectEngine);
}
}
}
151 changes: 101 additions & 50 deletions src/Compiler/Microsoft.NET.Sdk.Razor.SourceGenerators/RazorSourceGenerator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,69 +66,80 @@ public void Initialize(IncrementalGeneratorInitializationContext context)
var generatedDeclarationCode = componentFiles
.Combine(importFiles.Collect())
.Combine(razorSourceGeneratorOptions)
.WithLambdaComparer((old, @new) => (old.Right.Equals(@new.Right) && old.Left.Left.Equals(@new.Left.Left) && old.Left.Right.SequenceEqual(@new.Left.Right)), (a) => a.GetHashCode())
.Select(static (pair, _) =>
{

var ((sourceItem, importFiles), razorSourceGeneratorOptions) = pair;
RazorSourceGeneratorEventSource.Log.GenerateDeclarationCodeStart(sourceItem.FilePath);
RazorSourceGeneratorEventSource.Log.GenerateDeclarationCodeStart(sourceItem.RelativePhysicalPath);

var projectEngine = GetDeclarationProjectEngine(sourceItem, importFiles, razorSourceGeneratorOptions);

var codeGen = projectEngine.Process(sourceItem);

var result = codeGen.GetCSharpDocument().GeneratedCode;

RazorSourceGeneratorEventSource.Log.GenerateDeclarationCodeStop(sourceItem.FilePath);
RazorSourceGeneratorEventSource.Log.GenerateDeclarationCodeStop(sourceItem.RelativePhysicalPath);

return result;
return (result, sourceItem.RelativePhysicalPath);
});

var generatedDeclarationSyntaxTrees = generatedDeclarationCode
.Combine(parseOptions)
.Select(static (pair, _) =>
{
var (generatedDeclarationCode, parseOptions) = pair;
return CSharpSyntaxTree.ParseText(generatedDeclarationCode, (CSharpParseOptions)parseOptions);
var ((generatedDeclarationCode, filePath), parseOptions) = pair;
return CSharpSyntaxTree.ParseText(generatedDeclarationCode, (CSharpParseOptions)parseOptions, filePath);
});

var tagHelpersFromCompilation = compilation
.Combine(generatedDeclarationSyntaxTrees.Collect())
var tagHelpersFromComponents = generatedDeclarationSyntaxTrees
Copy link
Member Author

@chsienki chsienki Feb 3, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This allows us to only look at razor files that have changed to get their updated tag helpers, rather than re-discovering everything anytime a single file changes.

.Combine(compilation)
.Combine(razorSourceGeneratorOptions)
.Select(static (pair, _) =>
.SelectMany(static (pair, ct) =>
{
RazorSourceGeneratorEventSource.Log.DiscoverTagHelpersFromCompilationStart();

var ((compilation, generatedDeclarationSyntaxTrees), razorSourceGeneratorOptions) = pair;
var ((generatedDeclarationSyntaxTree, compilation), razorSourceGeneratorOptions) = pair;
RazorSourceGeneratorEventSource.Log.DiscoverTagHelpersFromComponentStart(generatedDeclarationSyntaxTree.FilePath);

var tagHelperFeature = new StaticCompilationTagHelperFeature();
var discoveryProjectEngine = GetDiscoveryProjectEngine(compilation.References.ToImmutableArray(), tagHelperFeature);

var compilationWithDeclarations = compilation.AddSyntaxTrees(generatedDeclarationSyntaxTrees);
var compilationWithDeclarations = compilation.AddSyntaxTrees(generatedDeclarationSyntaxTree);

// try and find the specific class this component is declaring, falling back to the assembly if for any reason we can't
ISymbol targetSymbol = compilationWithDeclarations.Assembly;
Copy link
Member Author

@chsienki chsienki Feb 3, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We need to look at the whole compilation with the component added (as it might reference stuff from csharp), but there's no need to actually discover anything else from the compilation, so only search the component itself.

var classSyntax = generatedDeclarationSyntaxTree.GetRoot(ct).DescendantNodes().SingleOrDefault(n => n.IsKind(CodeAnalysis.CSharp.SyntaxKind.ClassDeclaration));
Copy link
Member

@jjonescz jjonescz Feb 3, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can there be nested classes in the declaration syntax tree? Then SingleOrDefault could throw.

Copy link
Member Author

@chsienki chsienki Feb 3, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Great catch. I've added a test, and fixed via patterns instead.

if (classSyntax is not null)
{
targetSymbol = compilationWithDeclarations.GetSemanticModel(generatedDeclarationSyntaxTree).GetDeclaredSymbol(classSyntax, ct) ?? targetSymbol;
}

tagHelperFeature.Compilation = compilationWithDeclarations;
tagHelperFeature.TargetSymbol = compilationWithDeclarations.Assembly;
tagHelperFeature.TargetSymbol = targetSymbol;

var result = (IList<TagHelperDescriptor>)tagHelperFeature.GetDescriptors();
RazorSourceGeneratorEventSource.Log.DiscoverTagHelpersFromCompilationStop();
RazorSourceGeneratorEventSource.Log.DiscoverTagHelpersFromComponentStop(generatedDeclarationSyntaxTree.FilePath);
return result;
})
.WithLambdaComparer(static (a, b) =>
});

var tagHelpersFromCompilation = compilation
Copy link
Member Author

@chsienki chsienki Feb 3, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We do a single compilation lookup here. If a user edits CSharp this runs without running any of the component stuff again. If the user edits a component, this no longer runs.

.Combine(razorSourceGeneratorOptions)
.Select(static (pair, _) =>
{
if (a.Count != b.Count)
{
return false;
}
RazorSourceGeneratorEventSource.Log.DiscoverTagHelpersFromCompilationStart();

for (var i = 0; i < a.Count; i++)
{
if (!a[i].Equals(b[i]))
{
return false;
}
}
var (compilation, razorSourceGeneratorOptions) = pair;

var tagHelperFeature = new StaticCompilationTagHelperFeature();
var discoveryProjectEngine = GetDiscoveryProjectEngine(compilation.References.ToImmutableArray(), tagHelperFeature);

return true;
}, getHashCode: static a => a.Count);
tagHelperFeature.Compilation = compilation;
tagHelperFeature.TargetSymbol = compilation.Assembly;

var result = (IList<TagHelperDescriptor>)tagHelperFeature.GetDescriptors();
RazorSourceGeneratorEventSource.Log.DiscoverTagHelpersFromCompilationStop();
return result;
});

var tagHelpersFromReferences = compilation
.Combine(razorSourceGeneratorOptions)
Expand Down Expand Up @@ -186,12 +197,13 @@ public void Initialize(IncrementalGeneratorInitializationContext context)
return (ICollection<TagHelperDescriptor>)descriptors;
});

var allTagHelpers = tagHelpersFromCompilation
var allTagHelpers = tagHelpersFromComponents.Collect()
.Combine(tagHelpersFromCompilation)
.Combine(tagHelpersFromReferences)
.Select(static (pair, _) =>
{
var (tagHelpersFromCompilation, tagHelpersFromReferences) = pair;
var count = tagHelpersFromCompilation.Count + tagHelpersFromReferences.Count;
var ((tagHelpersFromComponents, tagHelpersFromCompilation), tagHelpersFromReferences) = pair;
var count = tagHelpersFromCompilation.Count + tagHelpersFromReferences.Count+ tagHelpersFromComponents.Length;
if (count == 0)
{
return Array.Empty<TagHelperDescriptor>();
Expand All @@ -200,32 +212,67 @@ public void Initialize(IncrementalGeneratorInitializationContext context)
var allTagHelpers = new TagHelperDescriptor[count];
tagHelpersFromCompilation.CopyTo(allTagHelpers, 0);
tagHelpersFromReferences.CopyTo(allTagHelpers, tagHelpersFromCompilation.Count);
tagHelpersFromComponents.CopyTo(allTagHelpers, tagHelpersFromCompilation.Count + tagHelpersFromReferences.Count);

return allTagHelpers;
});

var generatedOutput = sourceItems
.Combine(importFiles.Collect())
.Combine(allTagHelpers)
.WithLambdaComparer((old, @new) => old.Left.Equals(@new.Left) && old.Right.SequenceEqual(@new.Right), (a) => GetHashCode())
.Combine(razorSourceGeneratorOptions)
.Select(static (pair, _) =>
{
var (((sourceItem, imports), allTagHelpers), razorSourceGeneratorOptions) = pair;

RazorSourceGeneratorEventSource.Log.RazorCodeGenerateStart(sourceItem.FilePath);

// Add a generated suffix so tools, such as coverlet, consider the file to be generated
var hintName = GetIdentifierFromPath(sourceItem.RelativePhysicalPath) + ".g.cs";

var projectEngine = GetGenerationProjectEngine(allTagHelpers, sourceItem, imports, razorSourceGeneratorOptions);

var codeDocument = projectEngine.Process(sourceItem);
var csharpDocument = codeDocument.GetCSharpDocument();

RazorSourceGeneratorEventSource.Log.RazorCodeGenerateStop(sourceItem.FilePath);
return (hintName, csharpDocument);
})
.WithLambdaComparer(static (a, b) =>
var ((sourceItem, imports), razorSourceGeneratorOptions) = pair;

RazorSourceGeneratorEventSource.Log.ParseRazorDocumentStart(sourceItem.RelativePhysicalPath);

var projectEngine = GetGenerationProjectEngine(sourceItem, imports, razorSourceGeneratorOptions);

var document = projectEngine.ProcessInitialParse(sourceItem);

RazorSourceGeneratorEventSource.Log.ParseRazorDocumentStop(sourceItem.RelativePhysicalPath);
return (projectEngine, sourceItem.RelativePhysicalPath, document);
})

// Add the tag helpers in, but ignore if they've changed or not, only reprocessing the actual document changed
.Combine(allTagHelpers)
Copy link
Member Author

@chsienki chsienki Feb 3, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the most impactful part of the change:

  • We re-write the document if it has changed, but we ignore the status of the tag helpers.
  • The first time through all documents will be changed, so we run with an up-to-date set of tag helpers.
  • Next time through, only documents that have changed will be re-written (using the latest tag helpers). Those that haven't changed are skipped (and might have incorrect tag helpers at this point).

Then, on line 253 we re-run again, not skipping the tag helper check:

  • (If neither tag helper nor document has changed, there is no work to do)
  • When we re-process we pass the checkIdempotency flag to the engine. This will see if the same taghelpers that were used last time are being passed this time and skip if so.

This effectively makes the two steps mutually exclusive: we either ran the first step, and thus the tag helpers are up to date and we skip doing anything here; or the document didn't change (so we skipped the first step) but the tag helpers are different meaning we need to re-write them.

.WithLambdaComparer((old, @new) => old.Left.Equals(@new.Left), (item) => item.GetHashCode())
.Select((pair, _) =>
{
var ((projectEngine, filePath, codeDocument), allTagHelpers) = pair;
RazorSourceGeneratorEventSource.Log.RewriteTagHelpersStart(filePath);

codeDocument = projectEngine.ProcessTagHelpers(codeDocument, allTagHelpers, checkForIdempotency: false);

RazorSourceGeneratorEventSource.Log.RewriteTagHelpersStop(filePath);
return (projectEngine, filePath, codeDocument);
})

// next we do a second parse, along with the helpers, but check for idempotency. If the tag helpers used on the previous parse match, the compiler can skip re-computing them
.Combine(allTagHelpers)
.Select((pair, _) => {

var ((projectEngine, filePath, document), allTagHelpers) = pair;
RazorSourceGeneratorEventSource.Log.CheckAndRewriteTagHelpersStart(filePath);

document = projectEngine.ProcessTagHelpers(document, allTagHelpers, checkForIdempotency: true);

RazorSourceGeneratorEventSource.Log.CheckAndRewriteTagHelpersStop(filePath);
return (projectEngine, filePath, document);
})

.Select((pair, _) =>
{
var (projectEngine, filePath, document) = pair;
RazorSourceGeneratorEventSource.Log.RazorCodeGenerateStart(filePath);
document = projectEngine.ProcessRemaining(document);
var csharpDocument = document.CodeDocument.GetCSharpDocument();

RazorSourceGeneratorEventSource.Log.RazorCodeGenerateStop(filePath);
return (filePath, csharpDocument);
})
.WithLambdaComparer(static (a, b) =>
{
if (a.csharpDocument.Diagnostics.Count > 0 || b.csharpDocument.Diagnostics.Count > 0)
{
Expand All @@ -238,8 +285,12 @@ public void Initialize(IncrementalGeneratorInitializationContext context)

context.RegisterSourceOutput(generatedOutput, static (context, pair) =>
{
var (hintName, csharpDocument) = pair;
RazorSourceGeneratorEventSource.Log.AddSyntaxTrees(hintName);
var (filePath, csharpDocument) = pair;

// Add a generated suffix so tools, such as coverlet, consider the file to be generated
var hintName = GetIdentifierFromPath(filePath) + ".g.cs";

RazorSourceGeneratorEventSource.Log.AddSyntaxTrees(hintName);
for (var i = 0; i < csharpDocument.Diagnostics.Count; i++)
{
var razorDiagnostic = csharpDocument.Diagnostics[i];
Expand Down
34 changes: 33 additions & 1 deletion src/Compiler/Microsoft.NET.Sdk.Razor.SourceGenerators/RazorSourceGeneratorEventSource.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,5 +59,37 @@ private RazorSourceGeneratorEventSource() { }
private const int GenerateDeclarationSyntaxTreeStopId = 14;
[Event(GenerateDeclarationSyntaxTreeStopId, Level = EventLevel.Informational)]
public void GenerateDeclarationSyntaxTreeStop() => WriteEvent(GenerateDeclarationSyntaxTreeStopId);
}

private const int DiscoverTagHelpersFromComponentStartId = 15;
[Event(DiscoverTagHelpersFromComponentStartId, Level = EventLevel.Informational)]
public void DiscoverTagHelpersFromComponentStart(string filePath) => WriteEvent(DiscoverTagHelpersFromComponentStartId, filePath);

private const int DiscoverTagHelpersFromComponentStopId = 16;
[Event(DiscoverTagHelpersFromComponentStopId, Level = EventLevel.Informational)]
public void DiscoverTagHelpersFromComponentStop(string filePath) => WriteEvent(DiscoverTagHelpersFromComponentStopId, filePath);

private const int ParseRazorDocumentStartId = 17;
[Event(ParseRazorDocumentStartId, Level = EventLevel.Informational)]
public void ParseRazorDocumentStart(string file) => WriteEvent(ParseRazorDocumentStartId, file);

private const int ParseRazorDocumentStopId = 18;
[Event(ParseRazorDocumentStopId, Level = EventLevel.Informational)]
public void ParseRazorDocumentStop(string file) => WriteEvent(ParseRazorDocumentStopId, file);

private const int RewriteTagHelpersStartId = 19;
[Event(RewriteTagHelpersStartId, Level = EventLevel.Informational)]
public void RewriteTagHelpersStart(string file) => WriteEvent(RewriteTagHelpersStartId, file);

private const int RewriteTagHelpersStopId = 20;
[Event(RewriteTagHelpersStopId, Level = EventLevel.Informational)]
public void RewriteTagHelpersStop(string file) => WriteEvent(RewriteTagHelpersStopId, file);

private const int CheckAndRewriteTagHelpersStartId = 21;
[Event(CheckAndRewriteTagHelpersStartId, Level = EventLevel.Informational)]
public void CheckAndRewriteTagHelpersStart(string file) => WriteEvent(CheckAndRewriteTagHelpersStartId, file);

private const int CheckAndRewriteTagHelpersStopId = 22;
[Event(CheckAndRewriteTagHelpersStopId, Level = EventLevel.Informational)]
public void CheckAndRewriteTagHelpersStop(string file) => WriteEvent(CheckAndRewriteTagHelpersStopId, file);
}
}
Loading