diff --git a/lang/csharp/src/apache/codegen/Avro.codegen.csproj b/lang/csharp/src/apache/codegen/Avro.codegen.csproj index 371d2d77771..38b2c8df311 100644 --- a/lang/csharp/src/apache/codegen/Avro.codegen.csproj +++ b/lang/csharp/src/apache/codegen/Avro.codegen.csproj @@ -55,15 +55,6 @@ - - - - - - - - - diff --git a/lang/csharp/src/apache/codegen/AvroGen.cs b/lang/csharp/src/apache/codegen/AvroGen.cs index cb01671fe07..a3c9903d1d6 100644 --- a/lang/csharp/src/apache/codegen/AvroGen.cs +++ b/lang/csharp/src/apache/codegen/AvroGen.cs @@ -148,13 +148,9 @@ public static int GenProtocol(string infile, string outdir, try { string text = System.IO.File.ReadAllText(infile); - Protocol protocol = Protocol.Parse(text); CodeGen codegen = new CodeGen(); - codegen.AddProtocol(protocol); - - foreach (var entry in namespaceMapping) - codegen.NamespaceMapping[entry.Key] = entry.Value; + codegen.AddProtocol(text, namespaceMapping); codegen.GenerateCode(); codegen.WriteTypes(outdir); @@ -174,13 +170,8 @@ public static int GenSchema(string infile, string outdir, try { string text = System.IO.File.ReadAllText(infile); - Schema schema = Schema.Parse(text); - CodeGen codegen = new CodeGen(); - codegen.AddSchema(schema); - - foreach (var entry in namespaceMapping) - codegen.NamespaceMapping[entry.Key] = entry.Value; + codegen.AddSchema(text, namespaceMapping); codegen.GenerateCode(); codegen.WriteTypes(outdir); diff --git a/lang/csharp/src/apache/main/CodeGen/CodeGen.cs b/lang/csharp/src/apache/main/CodeGen/CodeGen.cs index 922bfe02fda..61bc2371d35 100644 --- a/lang/csharp/src/apache/main/CodeGen/CodeGen.cs +++ b/lang/csharp/src/apache/main/CodeGen/CodeGen.cs @@ -24,6 +24,7 @@ using System.Linq; using System.Reflection; using System.Text; +using System.Text.RegularExpressions; using Microsoft.CSharp; namespace Avro @@ -63,6 +64,7 @@ public class CodeGen /// /// The namespace mapping. /// + [Obsolete("NamespaceMapping is not used, use AddProtocol(string ...) or AddSchema(string ...) instead!")] public IDictionary NamespaceMapping { get; private set; } /// @@ -80,7 +82,6 @@ public CodeGen() { Schemas = new List(); Protocols = new List(); - NamespaceMapping = new Dictionary(); NamespaceLookup = new Dictionary(StringComparer.Ordinal); } @@ -103,6 +104,19 @@ public virtual void AddProtocol(Protocol protocol) Protocols.Add(protocol); } + /// + /// Parses and adds a protocol object to generate code for. + /// + /// The protocol. + /// namespace mapping key value pairs. + public virtual void AddProtocol(string protocolText, IEnumerable> namespaceMapping = null) + { + // Map namespaces + protocolText = ReplaceMappedNamespacesInSchema(protocolText, namespaceMapping); + Protocol protocol = Protocol.Parse(protocolText); + Protocols.Add(protocol); + } + /// /// Adds a schema object to generate code for. /// @@ -112,6 +126,19 @@ public virtual void AddSchema(Schema schema) Schemas.Add(schema); } + /// + /// Parses and adds a schema object to generate code for. + /// + /// schema object. + /// namespace mapping key value pairs. + public virtual void AddSchema(string schemaText, IEnumerable> namespaceMapping = null) + { + // Map namespaces + schemaText = ReplaceMappedNamespacesInSchema(schemaText, namespaceMapping); + Schema schema = Schema.Parse(schemaText); + Schemas.Add(schema); + } + /// /// Adds a namespace object for the given name into the dictionary if it doesn't exist yet. /// @@ -129,9 +156,7 @@ protected virtual CodeNamespace AddNamespace(string name) if (!NamespaceLookup.TryGetValue(name, out CodeNamespace ns)) { - ns = NamespaceMapping.TryGetValue(name, out string csharpNamespace) - ? new CodeNamespace(csharpNamespace) - : new CodeNamespace(CodeGenUtil.Instance.Mangle(name)); + ns = new CodeNamespace(CodeGenUtil.Instance.Mangle(name)); foreach (CodeNamespaceImport nci in CodeGenUtil.Instance.NamespaceImports) { @@ -1153,5 +1178,48 @@ public virtual void WriteTypes(string outputdir) } } } + + /// + /// Replace namespace(s) in schema or protocol definition. + /// + /// input schema or protocol definition. + /// namespace mappings object. + private static string ReplaceMappedNamespacesInSchema(string input, IEnumerable> namespaceMapping) + { + if (namespaceMapping == null || input == null) + return input; + + // Replace namespace in "namespace" definitions: + // "namespace": "originalnamespace" -> "namespace": "mappednamespace" + // "namespace": "originalnamespace.whatever" -> "namespace": "mappednamespace.whatever" + // Note: It keeps the original whitespaces + return Regex.Replace(input, @"""namespace""(\s*):(\s*)""([^""]*)""", m => + { + // m.Groups[1]: whitespaces before ':' + // m.Groups[2]: whitespaces after ':' + // m.Groups[3]: the namespace + + string ns = m.Groups[3].Value; + + foreach (var mapping in namespaceMapping) + { + // Full match + if (mapping.Key == ns) + { + ns = mapping.Value; + break; + } + else + // Partial match + if (ns.StartsWith($"{mapping.Key}.")) + { + ns = $"{mapping.Value}.{ns.Substring(mapping.Key.Length + 1)}"; + break; + } + } + + return $@"""namespace""{m.Groups[1].Value}:{m.Groups[2].Value}""{ns}"""; + }); + } } } diff --git a/lang/csharp/src/apache/test/AvroGen/AvroGenTests.cs b/lang/csharp/src/apache/test/AvroGen/AvroGenTests.cs index f4976d7b763..4baac664384 100644 --- a/lang/csharp/src/apache/test/AvroGen/AvroGenTests.cs +++ b/lang/csharp/src/apache/test/AvroGen/AvroGenTests.cs @@ -21,8 +21,6 @@ using System.Reflection; using System.Collections.Generic; using Microsoft.CodeAnalysis; -using Microsoft.CodeAnalysis.CSharp; -using Microsoft.CodeAnalysis.Emit; using NUnit.Framework; using Avro.Specific; @@ -263,6 +261,51 @@ class AvroGenTests ] }"; + // https://issues.apache.org/jira/browse/AVRO-2883 + private const string _schema_avro_2883 = @" +{ + ""type"" : ""record"", + ""name"" : ""TestModel"", + ""namespace"" : ""my.avro.ns"", + ""fields"" : [ { + ""name"" : ""eventType"", + ""type"" : { + ""type"" : ""enum"", + ""name"" : ""EventType"", + ""symbols"" : [ ""CREATE"", ""UPDATE"", ""DELETE"" ] + } +} ] +}"; + + // https://issues.apache.org/jira/browse/AVRO-3046 + private const string _schema_avro_3046 = @" +{ + ""type"": ""record"", + ""name"": ""ExampleRecord"", + ""namespace"": ""com.example"", + ""fields"": [ + { + ""name"": ""Id"", + ""type"": ""string"", + ""logicalType"": ""UUID"" + }, + { + ""name"": ""InnerRecord"", + ""type"": { + ""type"": ""record"", + ""name"": ""InnerRecord"", + ""fields"": [ + { + ""name"": ""Id"", + ""type"": ""string"", + ""logicalType"": ""UUID"" + } + ] + } + } + ] +}"; + private Assembly TestSchema( string schema, IEnumerable typeNamesToCheck = null, @@ -412,6 +455,18 @@ private Assembly TestSchema( { "org/apache/avro/codegentest/testdata/NullableLogicalTypesArray.cs" })] + [TestCase( + _schema_avro_2883, + new string[] + { + "my.avro.ns.TestModel", + "my.avro.ns.EventType", + }, + new string[] + { + "my/avro/ns/TestModel.cs", + "my/avro/ns/EventType.cs" + })] public void GenerateSchema(string schema, IEnumerable typeNamesToCheck, IEnumerable generatedFilesToCheck) { TestSchema(schema, typeNamesToCheck, generatedFilesToCheck: generatedFilesToCheck); @@ -428,6 +483,45 @@ public void GenerateSchema(string schema, IEnumerable typeNamesToCheck, { "org/apache/csharp/codegentest/testdata/NullableLogicalTypesArray.cs" })] + [TestCase( + _nestedLogicalTypesUnion, + "org.apache.avro.codegentest.testdata", "org.apache.csharp.codegentest.testdata", + new string[] + { + "org.apache.csharp.codegentest.testdata.NestedLogicalTypesUnion", + "org.apache.csharp.codegentest.testdata.RecordInUnion" + }, + new string[] + { + "org/apache/csharp/codegentest/testdata/NestedLogicalTypesUnion.cs", + "org/apache/csharp/codegentest/testdata/RecordInUnion.cs" + })] + [TestCase( + _schema_avro_2883, + "my.avro.ns", "my.csharp.ns", + new string[] + { + "my.csharp.ns.TestModel", + "my.csharp.ns.EventType", + }, + new string[] + { + "my/csharp/ns/TestModel.cs", + "my/csharp/ns/EventType.cs" + })] + [TestCase( + _schema_avro_3046, + "com.example", "Example", + new string[] + { + "Example.ExampleRecord", + "Example.InnerRecord", + }, + new string[] + { + "Example/ExampleRecord.cs", + "Example/InnerRecord.cs" + })] [TestCase( _nullableLogicalTypesArray, "org.apache.avro.codegentest.testdata", "org.apache.@return.@int", // Reserved keywords in namespace @@ -492,33 +586,6 @@ public void GenerateSchemaWithNamespaceMapping( TestSchema(schema, typeNamesToCheck, new Dictionary { { namespaceMappingFrom, namespaceMappingTo } }, generatedFilesToCheck); } - [TestCase( - _nestedLogicalTypesUnion, - "org.apache.avro.codegentest.testdata", "org.apache.csharp.codegentest.testdata", - new string[] - { - "org.apache.avro.codegentest.testdata.NestedLogicalTypesUnion", - "org.apache.avro.codegentest.testdata.RecordInUnion" - }, - new string[] - { - "org/apache/csharp/codegentest/testdata/NestedLogicalTypesUnion.cs", - "org/apache/csharp/codegentest/testdata/RecordInUnion.cs" - })] - public void GenerateSchemaWithNamespaceMapping_Bug_AVRO_2883( - string schema, - string namespaceMappingFrom, - string namespaceMappingTo, - IEnumerable typeNamesToCheck, - IEnumerable generatedFilesToCheck) - { - // !!! This is a bug which must be fixed - // !!! Once it is fixed, this test will fail and this test can be removed - // https://issues.apache.org/jira/browse/AVRO-2883 - // https://issues.apache.org/jira/browse/AVRO-3046 - Assert.Throws(() => TestSchema(schema, typeNamesToCheck, new Dictionary { { namespaceMappingFrom, namespaceMappingTo } }, generatedFilesToCheck)); - } - [TestCase(_logicalTypesWithCustomConversion, typeof(AvroTypeException))] [TestCase(_customConversionWithLogicalTypes, typeof(SchemaParseException))] [TestCase(_nestedLogicalTypesUnionFixedDecimal, typeof(SchemaParseException))] diff --git a/lang/csharp/src/apache/test/CodGen/CodeGenTest.cs b/lang/csharp/src/apache/test/CodGen/CodeGenTest.cs index 243d93e24c5..e514347206e 100644 --- a/lang/csharp/src/apache/test/CodGen/CodeGenTest.cs +++ b/lang/csharp/src/apache/test/CodGen/CodeGenTest.cs @@ -63,7 +63,7 @@ public void TestReservedKeywords() [TestCase("a.b.int", "a.b.@int")] [TestCase("int.long.while", "@int.@long.@while")] // Reserved keywords [TestCase("a.value.partial", "a.value.partial")] // Contextual keywords - [TestCase("a.value.b.int.c.while.longpartial", "a.value.b.@int.c.@while.longpartial")] // Rseserved and contextual keywords + [TestCase("a.value.b.int.c.while.longpartial", "a.value.b.@int.c.@while.longpartial")] // Reserved and contextual keywords public void TestMangleUnMangle(string input, string mangled) { // Mangle