From 92da59ab87470e9e823c8ff3158014a8a2227ea7 Mon Sep 17 00:00:00 2001 From: Chris Sienkiewicz Date: Mon, 9 Aug 2021 12:12:14 -0700 Subject: [PATCH 01/11] Make JsonGenerator be an incremental generator --- eng/Versions.props | 6 +-- .../gen/JsonSourceGenerator.Emitter.cs | 12 ++--- .../gen/JsonSourceGenerator.Parser.cs | 19 ++++---- .../gen/JsonSourceGenerator.cs | 48 +++++++------------ 4 files changed, 38 insertions(+), 47 deletions(-) diff --git a/eng/Versions.props b/eng/Versions.props index 021b31aa65bcc6..02eefd820a2d48 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -41,9 +41,9 @@ - - 3.9.0 - 3.9.0 + + 4.0.0-3.final + 4.0.0-3.final diff --git a/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Emitter.cs b/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Emitter.cs index 9fb4433e743bde..ceae5d7a55296c 100644 --- a/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Emitter.cs +++ b/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Emitter.cs @@ -78,15 +78,15 @@ private sealed partial class Emitter defaultSeverity: DiagnosticSeverity.Warning, isEnabledByDefault: true); - private readonly GeneratorExecutionContext _executionContext; + private readonly SourceProductionContext _sourceProductionContext; private ContextGenerationSpec _currentContext = null!; private readonly SourceGenerationSpec _generationSpec = null!; - public Emitter(in GeneratorExecutionContext executionContext, SourceGenerationSpec generationSpec) + public Emitter(in SourceProductionContext sourceProductionContext, SourceGenerationSpec generationSpec) { - _executionContext = executionContext; + _sourceProductionContext = sourceProductionContext; _generationSpec = generationSpec; } @@ -165,7 +165,7 @@ namespace {@namespace} sb.AppendLine("}"); } - _executionContext.AddSource(fileName, SourceText.From(sb.ToString(), Encoding.UTF8)); + _sourceProductionContext.AddSource(fileName, SourceText.From(sb.ToString(), Encoding.UTF8)); } private void GenerateTypeInfo(TypeGenerationSpec typeGenerationSpec) @@ -242,7 +242,7 @@ private void GenerateTypeInfo(TypeGenerationSpec typeGenerationSpec) break; case ClassType.TypeUnsupportedBySourceGen: { - _executionContext.ReportDiagnostic( + _sourceProductionContext.ReportDiagnostic( Diagnostic.Create(TypeNotSupported, Location.None, new string[] { typeGenerationSpec.TypeRef })); return; } @@ -258,7 +258,7 @@ private void GenerateTypeInfo(TypeGenerationSpec typeGenerationSpec) } catch (ArgumentException) { - _executionContext.ReportDiagnostic(Diagnostic.Create(DuplicateTypeName, Location.None, new string[] { typeGenerationSpec.TypeInfoPropertyName })); + _sourceProductionContext.ReportDiagnostic(Diagnostic.Create(DuplicateTypeName, Location.None, new string[] { typeGenerationSpec.TypeInfoPropertyName })); } } diff --git a/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Parser.cs b/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Parser.cs index 33a60562a3a10e..2d38c15ddc0ce0 100644 --- a/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Parser.cs +++ b/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Parser.cs @@ -4,6 +4,7 @@ using System.Collections; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Collections.Immutable; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Linq; @@ -31,7 +32,8 @@ private sealed class Parser private const string JsonPropertyNameAttributeFullName = "System.Text.Json.Serialization.JsonPropertyNameAttribute"; private const string JsonPropertyOrderAttributeFullName = "System.Text.Json.Serialization.JsonPropertyOrderAttribute"; - private readonly GeneratorExecutionContext _executionContext; + private readonly Compilation _compilation; + private readonly SourceProductionContext _sourceProductionContext; private readonly MetadataLoadContextInternal _metadataLoadContext; private readonly Type _ilistOfTType; @@ -96,10 +98,11 @@ private sealed class Parser defaultSeverity: DiagnosticSeverity.Error, isEnabledByDefault: true); - public Parser(in GeneratorExecutionContext executionContext) + public Parser(Compilation compilation, in SourceProductionContext sourceProductionContext) { - _executionContext = executionContext; - _metadataLoadContext = new MetadataLoadContextInternal(executionContext.Compilation); + _compilation = compilation; + _sourceProductionContext = sourceProductionContext; + _metadataLoadContext = new MetadataLoadContextInternal(_compilation); _ilistOfTType = _metadataLoadContext.Resolve(SpecialType.System_Collections_Generic_IList_T); _icollectionOfTType = _metadataLoadContext.Resolve(SpecialType.System_Collections_Generic_ICollection_T); @@ -138,9 +141,9 @@ public Parser(in GeneratorExecutionContext executionContext) PopulateKnownTypes(); } - public SourceGenerationSpec? GetGenerationSpec(List classDeclarationSyntaxList) + public SourceGenerationSpec? GetGenerationSpec(ImmutableArray classDeclarationSyntaxList) { - Compilation compilation = _executionContext.Compilation; + Compilation compilation = _compilation; INamedTypeSymbol jsonSerializerContextSymbol = compilation.GetTypeByMetadataName("System.Text.Json.Serialization.JsonSerializerContext"); INamedTypeSymbol jsonSerializableAttributeSymbol = compilation.GetTypeByMetadataName("System.Text.Json.Serialization.JsonSerializableAttribute"); INamedTypeSymbol jsonSourceGenerationOptionsAttributeSymbol = compilation.GetTypeByMetadataName("System.Text.Json.Serialization.JsonSourceGenerationOptionsAttribute"); @@ -198,7 +201,7 @@ public Parser(in GeneratorExecutionContext executionContext) if (!TryGetClassDeclarationList(contextTypeSymbol, out List classDeclarationList)) { // Class or one of its containing types is not partial so we can't add to it. - _executionContext.ReportDiagnostic(Diagnostic.Create(ContextClassesMustBePartial, Location.None, new string[] { contextTypeSymbol.Name })); + _sourceProductionContext.ReportDiagnostic(Diagnostic.Create(ContextClassesMustBePartial, Location.None, new string[] { contextTypeSymbol.Name })); continue; } @@ -729,7 +732,7 @@ private TypeGenerationSpec GetOrAddTypeGenerationSpec(Type type, JsonSourceGener if (!type.TryGetDeserializationConstructor(useDefaultCtorInAnnotatedStructs, out ConstructorInfo? constructor)) { classType = ClassType.TypeUnsupportedBySourceGen; - _executionContext.ReportDiagnostic(Diagnostic.Create(MultipleJsonConstructorAttribute, Location.None, new string[] { $"{type}" })); + _sourceProductionContext.ReportDiagnostic(Diagnostic.Create(MultipleJsonConstructorAttribute, Location.None, new string[] { $"{type}" })); } else { diff --git a/src/libraries/System.Text.Json/gen/JsonSourceGenerator.cs b/src/libraries/System.Text.Json/gen/JsonSourceGenerator.cs index 84212e8c19c78f..54b76c7e061ba1 100644 --- a/src/libraries/System.Text.Json/gen/JsonSourceGenerator.cs +++ b/src/libraries/System.Text.Json/gen/JsonSourceGenerator.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Collections.Generic; +using System.Collections.Immutable; using System.Diagnostics; using System.Linq; using System.Reflection; @@ -16,22 +17,23 @@ namespace System.Text.Json.SourceGeneration /// Generates source code to optimize serialization and deserialization with JsonSerializer. /// [Generator] - public sealed partial class JsonSourceGenerator : ISourceGenerator + public sealed partial class JsonSourceGenerator : IIncrementalGenerator { - /// - /// Registers a syntax resolver to receive compilation units. - /// - /// - public void Initialize(GeneratorInitializationContext context) + private const string SystemTextJsonSourceGenerationName = "System.Text.Json.SourceGeneration"; + private const string IJsonOnSerializedFullName = "System.Text.Json.Serialization.IJsonOnSerialized"; + private const string IJsonOnSerializingFullName = "System.Text.Json.Serialization.IJsonOnSerializing"; + + + public void Initialize(IncrementalGeneratorInitializationContext context) { - context.RegisterForSyntaxNotifications(() => new SyntaxReceiver()); + var classDeclarations = context.SyntaxProvider.CreateSyntaxProvider((s, _) => s is ClassDeclarationSyntax, (s, _) => (ClassDeclarationSyntax)s.Node); + + var compilationAndClasses = context.CompilationProvider.Combine(classDeclarations.Collect()); + + context.RegisterSourceOutput(compilationAndClasses, (spc, source) => Execute(source.Left, source.Right, spc)); } - /// - /// Generates source code to optimize serialization and deserialization with JsonSerializer. - /// - /// - public void Execute(GeneratorExecutionContext executionContext) + private void Execute(Compilation compilation, ImmutableArray contextClasses, SourceProductionContext context) { #if LAUNCH_DEBUGGER if (!Diagnostics.Debugger.IsAttached) @@ -39,36 +41,22 @@ public void Execute(GeneratorExecutionContext executionContext) Diagnostics.Debugger.Launch(); } #endif - SyntaxReceiver receiver = (SyntaxReceiver)executionContext.SyntaxReceiver; - List? contextClasses = receiver.ClassDeclarationSyntaxList; - if (contextClasses == null) + if (contextClasses.IsDefaultOrEmpty) { return; } - Parser parser = new(executionContext); - SourceGenerationSpec? spec = parser.GetGenerationSpec(receiver.ClassDeclarationSyntaxList); + Parser parser = new(compilation, context); + SourceGenerationSpec? spec = parser.GetGenerationSpec(contextClasses); if (spec != null) { _rootTypes = spec.ContextGenerationSpecList[0].RootSerializableTypes; - Emitter emitter = new(executionContext, spec); + Emitter emitter = new(context, spec); emitter.Emit(); } } - private sealed class SyntaxReceiver : ISyntaxReceiver - { - public List? ClassDeclarationSyntaxList { get; private set; } - - public void OnVisitSyntaxNode(SyntaxNode syntaxNode) - { - if (syntaxNode is ClassDeclarationSyntax { AttributeLists.Count: > 0, BaseList.Types.Count: > 0 } cds) - { - (ClassDeclarationSyntaxList ??= new List()).Add(cds); - } - } - } /// /// Helper for unit tests. From bfa08f6aa631e41afa48e2343d2f5b9192c18303 Mon Sep 17 00:00:00 2001 From: Chris Sienkiewicz Date: Thu, 19 Aug 2021 10:49:55 -0700 Subject: [PATCH 02/11] Improve incrementalism by doing less work when not applicable --- .../gen/JsonSourceGenerator.Parser.cs | 33 ++++++++++++++++++- .../gen/JsonSourceGenerator.cs | 7 ++-- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Parser.cs b/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Parser.cs index 2d38c15ddc0ce0..c36037c54b3e52 100644 --- a/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Parser.cs +++ b/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Parser.cs @@ -45,7 +45,7 @@ private sealed class Parser private readonly Type? _dictionaryType; private readonly Type? _idictionaryOfTKeyTValueType; private readonly Type? _ireadonlyDictionaryType; - private readonly Type? _isetType; + private readonly Type? _isetType; private readonly Type? _stackOfTType; private readonly Type? _queueOfTType; private readonly Type? _concurrentStackType; @@ -403,6 +403,37 @@ private static bool TryGetClassDeclarationList(INamedTypeSymbol typeSymbol, [Not return typeGenerationSpec; } + internal static bool IsSyntaxTargetForGeneration(SyntaxNode node) => node is ClassDeclarationSyntax { AttributeLists: { Count: > 0 }, BaseList: { Types : {Count : 0 } } }; + + internal static ClassDeclarationSyntax? IsSemanticTargetForGeneration(GeneratorSyntaxContext context) + { + var classDeclarationSyntax = (ClassDeclarationSyntax)context.Node; + + foreach (AttributeListSyntax attributeListSyntax in classDeclarationSyntax.AttributeLists) + { + foreach (var attributeSyntax in attributeListSyntax.Attributes) + { + IMethodSymbol attributeSymbol = context.SemanticModel.GetSymbolInfo(attributeSyntax).Symbol as IMethodSymbol; + if (attributeSymbol == null) + { + continue; + } + + INamedTypeSymbol attributeContainingTypeSymbol = attributeSymbol.ContainingType; + var fullName = attributeContainingTypeSymbol.ToDisplayString(); + + if (fullName == "System.Text.Json.Serialization.JsonSerializableAttribute" + || fullName == "System.Text.Json.Serialization.JsonSourceGenerationOptionsAttribute") + { + return classDeclarationSyntax; + } + } + + } + + return null; + } + private static JsonSourceGenerationMode? GetJsonSourceGenerationModeEnumVal(SyntaxNode propertyValueMode) { IEnumerable enumTokens = propertyValueMode diff --git a/src/libraries/System.Text.Json/gen/JsonSourceGenerator.cs b/src/libraries/System.Text.Json/gen/JsonSourceGenerator.cs index 54b76c7e061ba1..dc23505488c2dd 100644 --- a/src/libraries/System.Text.Json/gen/JsonSourceGenerator.cs +++ b/src/libraries/System.Text.Json/gen/JsonSourceGenerator.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +//define LAUNCH_DEBUGGER using System.Collections.Generic; using System.Collections.Immutable; using System.Diagnostics; @@ -23,10 +24,12 @@ public sealed partial class JsonSourceGenerator : IIncrementalGenerator private const string IJsonOnSerializedFullName = "System.Text.Json.Serialization.IJsonOnSerialized"; private const string IJsonOnSerializingFullName = "System.Text.Json.Serialization.IJsonOnSerializing"; - public void Initialize(IncrementalGeneratorInitializationContext context) { - var classDeclarations = context.SyntaxProvider.CreateSyntaxProvider((s, _) => s is ClassDeclarationSyntax, (s, _) => (ClassDeclarationSyntax)s.Node); + + var classDeclarations = context.SyntaxProvider + .CreateSyntaxProvider((s, _) => Parser.IsSyntaxTargetForGeneration(s), (s, _) => Parser.IsSemanticTargetForGeneration(s)) + .Where(c => c is object); var compilationAndClasses = context.CompilationProvider.Combine(classDeclarations.Collect()); From f3e16e29ff0730580d6377352480fb95c6030295 Mon Sep 17 00:00:00 2001 From: Chris Sienkiewicz Date: Fri, 20 Aug 2021 10:39:39 -0700 Subject: [PATCH 03/11] Fix typo --- .../System.Text.Json/gen/JsonSourceGenerator.Parser.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Parser.cs b/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Parser.cs index c36037c54b3e52..8dc2aeece7606d 100644 --- a/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Parser.cs +++ b/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Parser.cs @@ -403,7 +403,7 @@ private static bool TryGetClassDeclarationList(INamedTypeSymbol typeSymbol, [Not return typeGenerationSpec; } - internal static bool IsSyntaxTargetForGeneration(SyntaxNode node) => node is ClassDeclarationSyntax { AttributeLists: { Count: > 0 }, BaseList: { Types : {Count : 0 } } }; + internal static bool IsSyntaxTargetForGeneration(SyntaxNode node) => node is ClassDeclarationSyntax { AttributeLists: { Count: > 0 }, BaseList: { Types : {Count : > 0 } } }; internal static ClassDeclarationSyntax? IsSemanticTargetForGeneration(GeneratorSyntaxContext context) { From 1e67b4a6800175a9fca9ea14d037a664fe9413fd Mon Sep 17 00:00:00 2001 From: Eric Erhardt Date: Wed, 25 Aug 2021 13:54:21 -0500 Subject: [PATCH 04/11] PR feedback --- .../System.Text.Json/gen/JsonSourceGenerator.Parser.cs | 9 ++++----- .../System.Text.Json/gen/JsonSourceGenerator.cs | 5 ++--- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Parser.cs b/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Parser.cs index 8dc2aeece7606d..39fb541cc8fd2c 100644 --- a/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Parser.cs +++ b/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Parser.cs @@ -405,13 +405,13 @@ private static bool TryGetClassDeclarationList(INamedTypeSymbol typeSymbol, [Not internal static bool IsSyntaxTargetForGeneration(SyntaxNode node) => node is ClassDeclarationSyntax { AttributeLists: { Count: > 0 }, BaseList: { Types : {Count : > 0 } } }; - internal static ClassDeclarationSyntax? IsSemanticTargetForGeneration(GeneratorSyntaxContext context) + internal static ClassDeclarationSyntax? GetSemanticTargetForGeneration(GeneratorSyntaxContext context) { var classDeclarationSyntax = (ClassDeclarationSyntax)context.Node; foreach (AttributeListSyntax attributeListSyntax in classDeclarationSyntax.AttributeLists) { - foreach (var attributeSyntax in attributeListSyntax.Attributes) + foreach (AttributeSyntax attributeSyntax in attributeListSyntax.Attributes) { IMethodSymbol attributeSymbol = context.SemanticModel.GetSymbolInfo(attributeSyntax).Symbol as IMethodSymbol; if (attributeSymbol == null) @@ -420,10 +420,9 @@ private static bool TryGetClassDeclarationList(INamedTypeSymbol typeSymbol, [Not } INamedTypeSymbol attributeContainingTypeSymbol = attributeSymbol.ContainingType; - var fullName = attributeContainingTypeSymbol.ToDisplayString(); + string fullName = attributeContainingTypeSymbol.ToDisplayString(); - if (fullName == "System.Text.Json.Serialization.JsonSerializableAttribute" - || fullName == "System.Text.Json.Serialization.JsonSourceGenerationOptionsAttribute") + if (fullName == "System.Text.Json.Serialization.JsonSerializableAttribute") { return classDeclarationSyntax; } diff --git a/src/libraries/System.Text.Json/gen/JsonSourceGenerator.cs b/src/libraries/System.Text.Json/gen/JsonSourceGenerator.cs index dc23505488c2dd..5a87f352fd4ae8 100644 --- a/src/libraries/System.Text.Json/gen/JsonSourceGenerator.cs +++ b/src/libraries/System.Text.Json/gen/JsonSourceGenerator.cs @@ -28,8 +28,8 @@ public void Initialize(IncrementalGeneratorInitializationContext context) { var classDeclarations = context.SyntaxProvider - .CreateSyntaxProvider((s, _) => Parser.IsSyntaxTargetForGeneration(s), (s, _) => Parser.IsSemanticTargetForGeneration(s)) - .Where(c => c is object); + .CreateSyntaxProvider(static (s, _) => Parser.IsSyntaxTargetForGeneration(s), static (s, _) => Parser.GetSemanticTargetForGeneration(s)) + .Where(static c => c is not null); var compilationAndClasses = context.CompilationProvider.Combine(classDeclarations.Collect()); @@ -60,7 +60,6 @@ private void Execute(Compilation compilation, ImmutableArray /// Helper for unit tests. /// From 0e7c57a315c9ef70ab0bb612fa8ddf5ce9eff3aa Mon Sep 17 00:00:00 2001 From: Eric Erhardt Date: Wed, 25 Aug 2021 13:54:51 -0500 Subject: [PATCH 05/11] Change SourceGeneration.UnitTests to SourceGeneration.Unit.Tests so it is built and executed in CI --- src/libraries/System.Text.Json/System.Text.Json.sln | 2 +- .../CompilationHelper.cs | 0 .../JsonSourceGeneratorDiagnosticsTests.cs | 0 .../JsonSourceGeneratorTests.cs | 0 .../System.Text.Json.SourceGeneration.Unit.Tests.csproj} | 0 .../TypeWrapperTests.cs | 0 6 files changed, 1 insertion(+), 1 deletion(-) rename src/libraries/System.Text.Json/tests/{System.Text.Json.SourceGeneration.UnitTests => System.Text.Json.SourceGeneration.Unit.Tests}/CompilationHelper.cs (100%) rename src/libraries/System.Text.Json/tests/{System.Text.Json.SourceGeneration.UnitTests => System.Text.Json.SourceGeneration.Unit.Tests}/JsonSourceGeneratorDiagnosticsTests.cs (100%) rename src/libraries/System.Text.Json/tests/{System.Text.Json.SourceGeneration.UnitTests => System.Text.Json.SourceGeneration.Unit.Tests}/JsonSourceGeneratorTests.cs (100%) rename src/libraries/System.Text.Json/tests/{System.Text.Json.SourceGeneration.UnitTests/System.Text.Json.SourceGeneration.UnitTests.csproj => System.Text.Json.SourceGeneration.Unit.Tests/System.Text.Json.SourceGeneration.Unit.Tests.csproj} (100%) rename src/libraries/System.Text.Json/tests/{System.Text.Json.SourceGeneration.UnitTests => System.Text.Json.SourceGeneration.Unit.Tests}/TypeWrapperTests.cs (100%) diff --git a/src/libraries/System.Text.Json/System.Text.Json.sln b/src/libraries/System.Text.Json/System.Text.Json.sln index f941a75d567cf4..49fd73ed4abfa5 100644 --- a/src/libraries/System.Text.Json/System.Text.Json.sln +++ b/src/libraries/System.Text.Json/System.Text.Json.sln @@ -31,7 +31,7 @@ Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "System.Text.Json.FSharp.Tes EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "System.Text.Json.SourceGeneration.Tests", "tests\System.Text.Json.SourceGeneration.Tests\System.Text.Json.SourceGeneration.Tests.csproj", "{33599A6C-F340-4E1B-9B4D-CB8946C22140}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "System.Text.Json.SourceGeneration.UnitTests", "tests\System.Text.Json.SourceGeneration.UnitTests\System.Text.Json.SourceGeneration.UnitTests.csproj", "{F6A18EB5-A8CC-4A39-9E85-5FA226019C3D}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "System.Text.Json.SourceGeneration.Unit.Tests", "tests\System.Text.Json.SourceGeneration.Unit.Tests\System.Text.Json.SourceGeneration.Unit.Tests.csproj", "{F6A18EB5-A8CC-4A39-9E85-5FA226019C3D}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "System.Text.Json.Tests", "tests\System.Text.Json.Tests\System.Text.Json.Tests.csproj", "{A0178BAA-A1AF-4C69-8E4A-A700A2723DDC}" EndProject diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.UnitTests/CompilationHelper.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/CompilationHelper.cs similarity index 100% rename from src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.UnitTests/CompilationHelper.cs rename to src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/CompilationHelper.cs diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.UnitTests/JsonSourceGeneratorDiagnosticsTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/JsonSourceGeneratorDiagnosticsTests.cs similarity index 100% rename from src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.UnitTests/JsonSourceGeneratorDiagnosticsTests.cs rename to src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/JsonSourceGeneratorDiagnosticsTests.cs diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.UnitTests/JsonSourceGeneratorTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/JsonSourceGeneratorTests.cs similarity index 100% rename from src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.UnitTests/JsonSourceGeneratorTests.cs rename to src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/JsonSourceGeneratorTests.cs diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.UnitTests/System.Text.Json.SourceGeneration.UnitTests.csproj b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/System.Text.Json.SourceGeneration.Unit.Tests.csproj similarity index 100% rename from src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.UnitTests/System.Text.Json.SourceGeneration.UnitTests.csproj rename to src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/System.Text.Json.SourceGeneration.Unit.Tests.csproj diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.UnitTests/TypeWrapperTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/TypeWrapperTests.cs similarity index 100% rename from src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.UnitTests/TypeWrapperTests.cs rename to src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/TypeWrapperTests.cs From 0d72cd7e50eca41684faa48e281e979204a85b26 Mon Sep 17 00:00:00 2001 From: Eric Erhardt Date: Wed, 25 Aug 2021 16:17:46 -0500 Subject: [PATCH 06/11] Get unit tests running after IIncrementalGenerator migration --- .../CompilationHelper.cs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/CompilationHelper.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/CompilationHelper.cs index 86a261f284e67a..f728ef0ab3b067 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/CompilationHelper.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/CompilationHelper.cs @@ -16,6 +16,11 @@ namespace System.Text.Json.SourceGeneration.UnitTests { public class CompilationHelper { + private static readonly CSharpParseOptions s_parseOptions = + new CSharpParseOptions(kind: SourceCodeKind.Regular, documentationMode: DocumentationMode.Parse) + // workaround https://github.com/dotnet/roslyn/pull/55866. We can remove "LangVersion=Preview" when we get a Roslyn build with that change. + .WithLanguageVersion(LanguageVersion.Preview); + public static Compilation CreateCompilation( string source, MetadataReference[] additionalReferences = null, @@ -55,18 +60,18 @@ public static Compilation CreateCompilation( return CSharpCompilation.Create( assemblyName, - syntaxTrees: new[] { CSharpSyntaxTree.ParseText(source) }, + syntaxTrees: new[] { CSharpSyntaxTree.ParseText(source, s_parseOptions) }, references: references.ToArray(), options: new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary) ); } - private static GeneratorDriver CreateDriver(Compilation compilation, params ISourceGenerator[] generators) + private static GeneratorDriver CreateDriver(Compilation compilation, IIncrementalGenerator[] generators) => CSharpGeneratorDriver.Create( - generators: ImmutableArray.Create(generators), - parseOptions: new CSharpParseOptions(kind: SourceCodeKind.Regular, documentationMode: DocumentationMode.Parse)); + generators: generators.Select(g => g.AsSourceGenerator()), + parseOptions: s_parseOptions); - public static Compilation RunGenerators(Compilation compilation, out ImmutableArray diagnostics, params ISourceGenerator[] generators) + public static Compilation RunGenerators(Compilation compilation, out ImmutableArray diagnostics, params IIncrementalGenerator[] generators) { CreateDriver(compilation, generators).RunGeneratorsAndUpdateCompilation(compilation, out Compilation outCompilation, out diagnostics); return outCompilation; From 7de17b15e4b9c216581073825ccc98f4961a4b01 Mon Sep 17 00:00:00 2001 From: Eric Erhardt Date: Wed, 25 Aug 2021 18:29:13 -0500 Subject: [PATCH 07/11] Fix duplicate file name tests by working around https://github.com/dotnet/roslyn/issues/54185. --- .../gen/JsonSourceGenerator.Emitter.cs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Emitter.cs b/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Emitter.cs index ceae5d7a55296c..038e7e72988d7d 100644 --- a/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Emitter.cs +++ b/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Emitter.cs @@ -84,6 +84,8 @@ private sealed partial class Emitter private readonly SourceGenerationSpec _generationSpec = null!; + private readonly HashSet _emittedPropertyFileNames = new(); + public Emitter(in SourceProductionContext sourceProductionContext, SourceGenerationSpec generationSpec) { _sourceProductionContext = sourceProductionContext; @@ -252,11 +254,14 @@ private void GenerateTypeInfo(TypeGenerationSpec typeGenerationSpec) } } - try + // Don't add a duplicate file, but instead raise a diagnostic to say the duplicate has been skipped. + // Workaround https://github.com/dotnet/roslyn/issues/54185 by keeping track of the file names we've used. + string propertyFileName = $"{_currentContext.ContextType.Name}.{typeGenerationSpec.TypeInfoPropertyName}.g.cs"; + if (_emittedPropertyFileNames.Add(propertyFileName)) { - AddSource($"{_currentContext.ContextType.Name}.{typeGenerationSpec.TypeInfoPropertyName}.g.cs", source); + AddSource(propertyFileName, source); } - catch (ArgumentException) + else { _sourceProductionContext.ReportDiagnostic(Diagnostic.Create(DuplicateTypeName, Location.None, new string[] { typeGenerationSpec.TypeInfoPropertyName })); } From 2333830482fb401194a9512f0bbd4ab2b900a41e Mon Sep 17 00:00:00 2001 From: Eric Erhardt Date: Thu, 26 Aug 2021 11:49:53 -0500 Subject: [PATCH 08/11] Fix unit tests now that they are running in CI against non-English languages. --- .../CompilationHelper.cs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/CompilationHelper.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/CompilationHelper.cs index f728ef0ab3b067..7d1858d5e47a91 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/CompilationHelper.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/CompilationHelper.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Collections.Immutable; +using System.Globalization; using System.IO; using System.Linq; using System.Reflection; @@ -272,7 +273,15 @@ internal static void CheckDiagnosticMessages(ImmutableArray diagnost Array.Sort(actualMessages); Array.Sort(expectedMessages); - Assert.Equal(expectedMessages, actualMessages); + if (CultureInfo.CurrentUICulture.Name.StartsWith("en", StringComparison.OrdinalIgnoreCase)) + { + Assert.Equal(expectedMessages, actualMessages); + } + else + { + // for non-English runs, just compare the number of messages are the same + Assert.Equal(expectedMessages.Length, actualMessages.Length); + } } } } From 85643afeb1cdee9d5c1afbf02fc008631a3feff9 Mon Sep 17 00:00:00 2001 From: Eric Erhardt Date: Thu, 26 Aug 2021 14:57:35 -0500 Subject: [PATCH 09/11] PR feedback --- .../System.Text.Json/gen/JsonSourceGenerator.cs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/libraries/System.Text.Json/gen/JsonSourceGenerator.cs b/src/libraries/System.Text.Json/gen/JsonSourceGenerator.cs index 5a87f352fd4ae8..e155488975622b 100644 --- a/src/libraries/System.Text.Json/gen/JsonSourceGenerator.cs +++ b/src/libraries/System.Text.Json/gen/JsonSourceGenerator.cs @@ -1,7 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -//define LAUNCH_DEBUGGER +//#define LAUNCH_DEBUGGER using System.Collections.Generic; using System.Collections.Immutable; using System.Diagnostics; @@ -20,20 +20,16 @@ namespace System.Text.Json.SourceGeneration [Generator] public sealed partial class JsonSourceGenerator : IIncrementalGenerator { - private const string SystemTextJsonSourceGenerationName = "System.Text.Json.SourceGeneration"; - private const string IJsonOnSerializedFullName = "System.Text.Json.Serialization.IJsonOnSerialized"; - private const string IJsonOnSerializingFullName = "System.Text.Json.Serialization.IJsonOnSerializing"; - public void Initialize(IncrementalGeneratorInitializationContext context) { - - var classDeclarations = context.SyntaxProvider + IncrementalValuesProvider classDeclarations = context.SyntaxProvider .CreateSyntaxProvider(static (s, _) => Parser.IsSyntaxTargetForGeneration(s), static (s, _) => Parser.GetSemanticTargetForGeneration(s)) .Where(static c => c is not null); - var compilationAndClasses = context.CompilationProvider.Combine(classDeclarations.Collect()); + IncrementalValueProvider<(Compilation, ImmutableArray)> compilationAndClasses = + context.CompilationProvider.Combine(classDeclarations.Collect()); - context.RegisterSourceOutput(compilationAndClasses, (spc, source) => Execute(source.Left, source.Right, spc)); + context.RegisterSourceOutput(compilationAndClasses, (spc, source) => Execute(source.Item1, source.Item2, spc)); } private void Execute(Compilation compilation, ImmutableArray contextClasses, SourceProductionContext context) From 1cc60137e9ce575eafdd9a94f6e43caa2664c4fe Mon Sep 17 00:00:00 2001 From: Eric Erhardt Date: Thu, 26 Aug 2021 18:21:25 -0500 Subject: [PATCH 10/11] Fix System.Text.Json.SourceGeneration.Unit.Tests on WASM --- .../System.Text.Json.SourceGeneration.Unit.Tests.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/System.Text.Json.SourceGeneration.Unit.Tests.csproj b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/System.Text.Json.SourceGeneration.Unit.Tests.csproj index fad3b73e1026df..ef8eb335c61e1c 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/System.Text.Json.SourceGeneration.Unit.Tests.csproj +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/System.Text.Json.SourceGeneration.Unit.Tests.csproj @@ -5,7 +5,7 @@ - + From 15c9106b4bd40e9c6e417f8f03b8cfe2f6b8df8e Mon Sep 17 00:00:00 2001 From: Eric Erhardt Date: Thu, 26 Aug 2021 19:49:52 -0500 Subject: [PATCH 11/11] Disable STJ.SourceGeneration.Unit.Tests on Browser --- src/libraries/tests.proj | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libraries/tests.proj b/src/libraries/tests.proj index f32deab2d7b92a..5a12aae1ce4532 100644 --- a/src/libraries/tests.proj +++ b/src/libraries/tests.proj @@ -227,6 +227,9 @@ + + +