diff --git a/lang/csharp/src/apache/codegen/AvroGen.cs b/lang/csharp/src/apache/codegen/AvroGen.cs index 349911754e9..cb01671fe07 100644 --- a/lang/csharp/src/apache/codegen/AvroGen.cs +++ b/lang/csharp/src/apache/codegen/AvroGen.cs @@ -1,4 +1,4 @@ -/** +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -17,13 +17,13 @@ */ using System; using System.Collections.Generic; -using System.Text; +using System.Linq; namespace Avro { - class AvroGen + public class AvroGenTool { - static int Main(string[] args) + public static int Main(string[] args) { // Print usage if no arguments provided if (args.Length == 0) @@ -33,7 +33,7 @@ static int Main(string[] args) } // Print usage if help requested - if (args[0] == "-h" || args[0] == "--help") + if (args.Contains("-h") || args.Contains("--help")) { Usage(); return 0; @@ -140,9 +140,9 @@ static void Usage() " The format is \"my.avro.namespace:my.csharp.namespace\".\n" + " May be specified multiple times to map multiple namespaces.\n", AppDomain.CurrentDomain.FriendlyName); - return; } - static int GenProtocol(string infile, string outdir, + + public static int GenProtocol(string infile, string outdir, IEnumerable> namespaceMapping) { try @@ -167,7 +167,8 @@ static int GenProtocol(string infile, string outdir, return 0; } - static int GenSchema(string infile, string outdir, + + public static int GenSchema(string infile, string outdir, IEnumerable> namespaceMapping) { try diff --git a/lang/csharp/src/apache/main/CodeGen/CodeGenUtil.cs b/lang/csharp/src/apache/main/CodeGen/CodeGenUtil.cs index 0ed10c18bb6..fd2823d1783 100644 --- a/lang/csharp/src/apache/main/CodeGen/CodeGenUtil.cs +++ b/lang/csharp/src/apache/main/CodeGen/CodeGenUtil.cs @@ -88,6 +88,9 @@ is regenerated ------------------------------------------------------------------------------"); // Visual Studio 2010 https://msdn.microsoft.com/en-us/library/x53a06bb.aspx + // Note: + // 1. Contextual keywords are not reserved keywords e.g. value, partial + // 2. __arglist, __makeref, __reftype, __refvalue are undocumented keywords, but recognised by the C# compiler ReservedKeywords = new HashSet() { "abstract","as", "base", "bool", "break", "byte", "case", "catch", "char", "checked", "class", "const", "continue", "decimal", "default", "delegate", "do", "double", "else", "enum", "event", @@ -96,7 +99,8 @@ is regenerated "null", "object", "operator", "out", "override", "params", "private", "protected", "public", "readonly", "ref", "return", "sbyte", "sealed", "short", "sizeof", "stackalloc", "static", "string", "struct", "switch", "this", "throw", "true", "try", "typeof", "uint", "ulong", - "unchecked", "unsafe", "ushort", "using", "virtual", "void", "volatile", "while", "value", "partial" }; + "unchecked", "unsafe", "ushort", "using", "virtual", "void", "volatile", "while", + "__arglist", "__makeref", "__reftype", "__refvalue" }; } /// diff --git a/lang/csharp/src/apache/test/Avro.test.csproj b/lang/csharp/src/apache/test/Avro.test.csproj index 6c359c81400..18b5dd8a31f 100644 --- a/lang/csharp/src/apache/test/Avro.test.csproj +++ b/lang/csharp/src/apache/test/Avro.test.csproj @@ -32,17 +32,17 @@ + + + - - - - + diff --git a/lang/csharp/src/apache/test/AvroGen/AvroGenHelper.cs b/lang/csharp/src/apache/test/AvroGen/AvroGenHelper.cs new file mode 100644 index 00000000000..53d7d4e0372 --- /dev/null +++ b/lang/csharp/src/apache/test/AvroGen/AvroGenHelper.cs @@ -0,0 +1,155 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.Emit; +using NUnit.Framework; + +namespace Avro.Test.AvroGen +{ + class AvroGenToolResult + { + public int ExitCode { get; set; } + public string[] StdOut { get; set; } + public string[] StdErr { get; set; } + } + + class AvroGenHelper + { + public static AvroGenToolResult RunAvroGenTool(params string[] args) + { + // Save stdout and stderr + TextWriter conOut = Console.Out; + TextWriter conErr = Console.Error; + + try + { + AvroGenToolResult result = new AvroGenToolResult(); + StringBuilder strBuilderOut = new StringBuilder(); + StringBuilder strBuilderErr = new StringBuilder(); + + using (StringWriter writerOut = new StringWriter(strBuilderOut)) + using (StringWriter writerErr = new StringWriter(strBuilderErr)) + { + writerOut.NewLine = "\n"; + writerErr.NewLine = "\n"; + + // Overwrite stdout and stderr to be able to capture console output + Console.SetOut(writerOut); + Console.SetError(writerErr); + + result.ExitCode = AvroGenTool.Main(args.ToArray()); + + writerOut.Flush(); + writerErr.Flush(); + + result.StdOut = strBuilderOut.Length == 0 ? Array.Empty() : strBuilderOut.ToString().Split(writerOut.NewLine); + result.StdErr = strBuilderErr.Length == 0 ? Array.Empty() : strBuilderErr.ToString().Split(writerErr.NewLine); + } + + return result; + } + finally + { + // Restore console + Console.SetOut(conOut); + Console.SetError(conErr); + } + } + + public static Assembly CompileCSharpFilesIntoLibrary(IEnumerable sourceFiles, string assemblyName = null, bool loadAssembly = true) + { + // Create random assembly name if not specified + if (assemblyName == null) + assemblyName = Path.GetRandomFileName(); + + // Base path to assemblies .NET assemblies + var assemblyPath = Path.GetDirectoryName(typeof(object).Assembly.Location); + + using (var compilerStream = new MemoryStream()) + { + List assemblies = new List() + { + typeof(object).Assembly.Location, + typeof(Schema).Assembly.Location, + Path.Combine(assemblyPath, "System.Runtime.dll"), + Path.Combine(assemblyPath, "netstandard.dll") + }; + + // Create compiler + CSharpCompilation compilation = CSharpCompilation + .Create(assemblyName) + .WithOptions(new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary)) + .AddReferences(assemblies.Select(path => MetadataReference.CreateFromFile(path))) + .AddSyntaxTrees(sourceFiles.Select(sourceFile => + { + string sourceText = System.IO.File.ReadAllText(sourceFile); + return CSharpSyntaxTree.ParseText(sourceText); + })); + + // Compile + EmitResult compilationResult = compilation.Emit(compilerStream); + + //Note: Comment the following out to analyze the compiler errors if needed + //if (!compilationResult.Success) + //{ + // foreach (Diagnostic diagnostic in compilationResult.Diagnostics) + // { + // if (diagnostic.IsWarningAsError || diagnostic.Severity == DiagnosticSeverity.Error) + // { + // TestContext.WriteLine($"{diagnostic.Id} - {diagnostic.GetMessage()} - {diagnostic.Location}"); + // } + // } + //} + + Assert.That(compilationResult.Success, Is.True); + + if (!loadAssembly) + { + return null; + } + + compilerStream.Seek(0, SeekOrigin.Begin); + return Assembly.Load(compilerStream.ToArray()); + } + } + + public static string CreateEmptyTemporyFolder(out string uniqueId, string path = null) + { + // Create unique id + uniqueId = Guid.NewGuid().ToString(); + + // Temporary folder name in working folder or the specified path + string tempFolder = Path.Combine(path ?? TestContext.CurrentContext.WorkDirectory, uniqueId); + + // Create folder + Directory.CreateDirectory(tempFolder); + + // Make sure it is empty + Assert.That(new DirectoryInfo(tempFolder), Is.Empty); + + return tempFolder; + } + } +} diff --git a/lang/csharp/src/apache/test/AvroGen/AvroGenTests.cs b/lang/csharp/src/apache/test/AvroGen/AvroGenTests.cs new file mode 100644 index 00000000000..f4976d7b763 --- /dev/null +++ b/lang/csharp/src/apache/test/AvroGen/AvroGenTests.cs @@ -0,0 +1,664 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +using System; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Collections.Generic; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.Emit; +using NUnit.Framework; +using Avro.Specific; + +namespace Avro.Test.AvroGen +{ + [TestFixture] + + class AvroGenTests + { + private const string _customConversionWithLogicalTypes = @" +{ + ""namespace"": ""org.apache.avro.codegentest.testdata"", + ""type"": ""record"", + ""name"": ""CustomConversionWithLogicalTypes"", + ""doc"" : ""Test custom conversion and logical types in generated Java classes"", + ""fields"": [ + { + ""name"": ""customEnum"", + ""type"": [""null"", { + ""namespace"": ""org.apache.avro.codegentest.testdata"", + ""name"": ""CustomAvroEnum"", + ""type"": ""enum"", + ""logicalType"": ""custom-enum"", + ""symbols"": [""ONE"", ""TWO"", ""THREE""] + }] + }] +} +"; + + private const string _logicalTypesWithCustomConversion = @" +{ +""namespace"": ""org.apache.avro.codegentest.testdata"", + ""type"": ""record"", + ""name"": ""LogicalTypesWithCustomConversion"", + ""doc"" : ""Test unions with logical types in generated Java classes"", + ""fields"": [ + {""name"": ""nullableCustomField"", ""type"": [""null"", {""type"": ""bytes"", ""logicalType"": ""decimal"", ""precision"": 9, ""scale"": 2}], ""default"": null}, + { ""name"": ""nonNullCustomField"", ""type"": { ""type"": ""bytes"", ""logicalType"": ""decimal"", ""precision"": 9, ""scale"": 2} }, + { ""name"": ""nullableFixedSizeString"", ""type"": [""null"", { ""type"": ""bytes"", ""logicalType"": ""fixed-size-string"", ""minLength"": 1, ""maxLength"": 50}], ""default"": null}, + { ""name"": ""nonNullFixedSizeString"", ""type"": { ""type"": ""bytes"", ""logicalType"": ""fixed-size-string"", ""minLength"": 1, ""maxLength"": 50} } + ] +} +"; + + private const string _logicalTypesWithDefaults = @" +{ +""namespace"": ""org.apache.avro.codegentest.testdata"", + ""type"": ""record"", + ""name"": ""LogicalTypesWithDefaults"", + ""doc"" : ""Test logical types and default values in generated Java classes"", + ""fields"": [ + {""name"": ""nullableDate"", ""type"": [{""type"": ""int"", ""logicalType"": ""date""}, ""null""], ""default"": 1234}, + { ""name"": ""nonNullDate"", ""type"": { ""type"": ""int"", ""logicalType"": ""date""}, ""default"": 1234} + ] +}"; + + private const string _nestedLogicalTypesArray = @" +{""namespace"": ""org.apache.avro.codegentest.testdata"", + ""type"": ""record"", + ""name"": ""NestedLogicalTypesArray"", + ""doc"" : ""Test nested types with logical types in generated Java classes"", + ""fields"": [ + { + ""name"": ""arrayOfRecords"", + ""type"": { + ""type"": ""array"", + ""items"": { + ""namespace"": ""org.apache.avro.codegentest.testdata"", + ""name"": ""RecordInArray"", + ""type"": ""record"", + ""fields"": [ + { + ""name"": ""nullableDateField"", + ""type"": [""null"", {""type"": ""int"", ""logicalType"": ""date""}] + } + ] + } + } + }] +} +"; + + private const string _nestedLogicalTypesMap = @" +{""namespace"": ""org.apache.avro.codegentest.testdata"", + ""type"": ""record"", + ""name"": ""NestedLogicalTypesMap"", + ""doc"" : ""Test nested types with logical types in generated Java classes"", + ""fields"": [ + { + ""name"": ""mapOfRecords"", + ""type"": { + ""type"": ""map"", + ""values"": { + ""namespace"": ""org.apache.avro.codegentest.testdata"", + ""name"": ""RecordInMap"", + ""type"": ""record"", + ""fields"": [ + { + ""name"": ""nullableDateField"", + ""type"": [""null"", {""type"": ""int"", ""logicalType"": ""date""}] + } + ] + } + } + }] +}"; + + private const string _nestedLogicalTypesRecord = @" +{""namespace"": ""org.apache.avro.codegentest.testdata"", + ""type"": ""record"", + ""name"": ""NestedLogicalTypesRecord"", + ""doc"" : ""Test nested types with logical types in generated Java classes"", + ""fields"": [ + { + ""name"": ""nestedRecord"", + ""type"": { + ""namespace"": ""org.apache.avro.codegentest.testdata"", + ""type"": ""record"", + ""name"": ""NestedRecord"", + ""fields"": [ + { + ""name"": ""nullableDateField"", + ""type"": [""null"", {""type"": ""int"", ""logicalType"": ""date""}] + } + ] + } + }] +}"; + + private const string _nestedLogicalTypesUnionFixedDecimal = @" +{""namespace"": ""org.apache.avro.codegentest.testdata"", + ""type"": ""record"", + ""name"": ""NestedLogicalTypesUnionFixedDecimal"", + ""doc"" : ""Test nested types with logical types in generated Java classes"", + ""fields"": [ + { + ""name"": ""unionOfFixedDecimal"", + ""type"": [""null"", { + ""namespace"": ""org.apache.avro.codegentest.testdata"", + ""name"": ""FixedInUnion"", + ""type"": ""fixed"", + ""size"": 12, + ""logicalType"": ""decimal"", + ""precision"": 28, + ""scale"": 15 + }] + }] +}"; + + private const string _nestedLogicalTypesUnion = @" +{""namespace"": ""org.apache.avro.codegentest.testdata"", + ""type"": ""record"", + ""name"": ""NestedLogicalTypesUnion"", + ""doc"" : ""Test nested types with logical types in generated Java classes"", + ""fields"": [ + { + ""name"": ""unionOfRecords"", + ""type"": [""null"", { + ""namespace"": ""org.apache.avro.codegentest.testdata"", + ""name"": ""RecordInUnion"", + ""type"": ""record"", + ""fields"": [ + { + ""name"": ""nullableDateField"", + ""type"": [""null"", {""type"": ""int"", ""logicalType"": ""date""}] + } + ] + }] + }] +}"; + + private const string _nestedSomeNamespaceRecord = @" +{""namespace"": ""org.apache.avro.codegentest.some"", + ""type"": ""record"", + ""name"": ""NestedSomeNamespaceRecord"", + ""doc"" : ""Test nested types with different namespace than the outer type"", + ""fields"": [ + { + ""name"": ""nestedRecord"", + ""type"": { + ""namespace"": ""org.apache.avro.codegentest.other"", + ""type"": ""record"", + ""name"": ""NestedOtherNamespaceRecord"", + ""fields"": [ + { + ""name"": ""someField"", + ""type"": ""int"" + } + ] + } + }] +}"; + + private const string _nullableLogicalTypesArray = @" +{""namespace"": ""org.apache.avro.codegentest.testdata"", + ""type"": ""record"", + ""name"": ""NullableLogicalTypesArray"", + ""doc"" : ""Test nested types with logical types in generated Java classes"", + ""fields"": [ + { + ""name"": ""arrayOfLogicalType"", + ""type"": { + ""type"": ""array"", + ""items"": [""null"", {""type"": ""int"", ""logicalType"": ""date""}] + } + }] +}"; + + private const string _nullableLogicalTypes = @" +{""namespace"": ""org.apache.avro.codegentest.testdata"", + ""type"": ""record"", + ""name"": ""NullableLogicalTypes"", + ""doc"" : ""Test unions with logical types in generated Java classes"", + ""fields"": [ + {""name"": ""nullableDate"", ""type"": [""null"", {""type"": ""int"", ""logicalType"": ""date""}], ""default"": null} + ] +}"; + + private const string _stringLogicalType = @" +{ + ""namespace"": ""org.apache.avro.codegentest.testdata"", + ""type"": ""record"", + ""name"": ""StringLogicalType"", + ""doc"": ""Test logical type applied to field of type string"", + ""fields"": [ + { + ""name"": ""someIdentifier"", + ""type"": { + ""type"": ""string"", + ""logicalType"": ""uuid"" + } +}, + { + ""name"": ""someJavaString"", + ""type"": ""string"", + ""doc"": ""Just to ensure no one removed String because this is the basis of this test"" + } + ] +}"; + + private Assembly TestSchema( + string schema, + IEnumerable typeNamesToCheck = null, + IEnumerable> namespaceMapping = null, + IEnumerable generatedFilesToCheck = null) + { + // Create temp folder + string outputDir = AvroGenHelper.CreateEmptyTemporyFolder(out string uniqueId); + + try + { + // Save schema + string schemaFileName = Path.Combine(outputDir, $"{uniqueId}.avsc"); + System.IO.File.WriteAllText(schemaFileName, schema); + + // Generate from schema file + Assert.That(AvroGenTool.GenSchema(schemaFileName, outputDir, namespaceMapping ?? new Dictionary()), Is.EqualTo(0)); + + // Check if all generated files exist + if (generatedFilesToCheck != null) + { + foreach (string generatedFile in generatedFilesToCheck) + { + Assert.That(new FileInfo(Path.Combine(outputDir, generatedFile)), Does.Exist); + } + } + + // Compile into netstandard library and load assembly + Assembly assembly = AvroGenHelper.CompileCSharpFilesIntoLibrary( + new DirectoryInfo(outputDir) + .EnumerateFiles("*.cs", SearchOption.AllDirectories) + .Select(fi => fi.FullName), + uniqueId); + + if (typeNamesToCheck != null) + { + // Check if the compiled code has the same number of types defined as the check list + Assert.That(typeNamesToCheck.Count(), Is.EqualTo(assembly.DefinedTypes.Count())); + + // Check if types available in compiled assembly + foreach (string typeName in typeNamesToCheck) + { + Type type = assembly.GetType(typeName); + Assert.That(type, Is.Not.Null); + + // Instantiate + object obj = Activator.CreateInstance(type); + Assert.That(obj, Is.Not.Null); + } + } + + return assembly; + } + finally + { + Directory.Delete(outputDir, true); + } + } + + [TestCase( + _logicalTypesWithDefaults, + new string[] + { + "org.apache.avro.codegentest.testdata.LogicalTypesWithDefaults" + }, + new string[] + { + "org/apache/avro/codegentest/testdata/LogicalTypesWithDefaults.cs" + })] + [TestCase( + _nestedLogicalTypesArray, + new string[] + { + "org.apache.avro.codegentest.testdata.NestedLogicalTypesArray", + "org.apache.avro.codegentest.testdata.RecordInArray" + }, + new string[] + { + "org/apache/avro/codegentest/testdata/NestedLogicalTypesArray.cs", + "org/apache/avro/codegentest/testdata/RecordInArray.cs" + })] + [TestCase( + _nestedLogicalTypesMap, + new string[] + { + "org.apache.avro.codegentest.testdata.NestedLogicalTypesMap", + "org.apache.avro.codegentest.testdata.RecordInMap" + }, + new string[] + { + "org/apache/avro/codegentest/testdata/NestedLogicalTypesMap.cs", + "org/apache/avro/codegentest/testdata/RecordInMap.cs" + })] + [TestCase( + _nestedLogicalTypesRecord, + new string[] + { + "org.apache.avro.codegentest.testdata.NestedLogicalTypesRecord", + "org.apache.avro.codegentest.testdata.NestedRecord" + }, + new string[] + { + "org/apache/avro/codegentest/testdata/NestedLogicalTypesRecord.cs", + "org/apache/avro/codegentest/testdata/NestedRecord.cs" + })] + [TestCase( + _nestedLogicalTypesUnion, + new string[] + { + "org.apache.avro.codegentest.testdata.NestedLogicalTypesUnion", + "org.apache.avro.codegentest.testdata.RecordInUnion" + }, + new string[] + { + "org/apache/avro/codegentest/testdata/NestedLogicalTypesUnion.cs", + "org/apache/avro/codegentest/testdata/RecordInUnion.cs" + })] + [TestCase( + _nestedSomeNamespaceRecord, + new string[] + { + "org.apache.avro.codegentest.some.NestedSomeNamespaceRecord", + "org.apache.avro.codegentest.other.NestedOtherNamespaceRecord" + }, + new string[] + { + "org/apache/avro/codegentest/some/NestedSomeNamespaceRecord.cs", + "org/apache/avro/codegentest/other/NestedOtherNamespaceRecord.cs" + })] + [TestCase( + _nullableLogicalTypes, + new string[] + { + "org.apache.avro.codegentest.testdata.NullableLogicalTypes" + }, + new string[] + { + "org/apache/avro/codegentest/testdata/NullableLogicalTypes.cs" + })] + [TestCase( + _nullableLogicalTypesArray, + new string[] + { + "org.apache.avro.codegentest.testdata.NullableLogicalTypesArray" + }, + new string[] + { + "org/apache/avro/codegentest/testdata/NullableLogicalTypesArray.cs" + })] + public void GenerateSchema(string schema, IEnumerable typeNamesToCheck, IEnumerable generatedFilesToCheck) + { + TestSchema(schema, typeNamesToCheck, generatedFilesToCheck: generatedFilesToCheck); + } + + [TestCase( + _nullableLogicalTypesArray, + "org.apache.avro.codegentest.testdata", "org.apache.csharp.codegentest.testdata", + new string[] + { + "org.apache.csharp.codegentest.testdata.NullableLogicalTypesArray" + }, + new string[] + { + "org/apache/csharp/codegentest/testdata/NullableLogicalTypesArray.cs" + })] + [TestCase( + _nullableLogicalTypesArray, + "org.apache.avro.codegentest.testdata", "org.apache.@return.@int", // Reserved keywords in namespace + new string[] + { + "org.apache.return.int.NullableLogicalTypesArray" + }, + new string[] + { + "org/apache/return/int/NullableLogicalTypesArray.cs" + })] + [TestCase( + _nullableLogicalTypesArray, + "org.apache.avro.codegentest.testdata", "org.apache.value.partial", // Contextual keywords in namespace + new string[] + { + "org.apache.value.partial.NullableLogicalTypesArray" + }, + new string[] + { + "org/apache/value/partial/NullableLogicalTypesArray.cs" + })] + [TestCase(@" +{ + ""type"": ""fixed"", + ""namespace"": ""com.base"", + ""name"": ""MD5"", + ""size"": 16 +}", + "com.base", "SchemaTest", + new string[] + { + "SchemaTest.MD5" + }, + new string[] + { + "SchemaTest/MD5.cs" + })] + [TestCase(@" +{ + ""type"": ""fixed"", + ""namespace"": ""com.base"", + ""name"": ""MD5"", + ""size"": 16 +}", + "miss", "SchemaTest", + new string[] + { + "com.base.MD5" + }, + new string[] + { + "com/base/MD5.cs" + })] + public void GenerateSchemaWithNamespaceMapping( + string schema, + string namespaceMappingFrom, + string namespaceMappingTo, + IEnumerable typeNamesToCheck, + IEnumerable generatedFilesToCheck) + { + TestSchema(schema, typeNamesToCheck, new Dictionary { { namespaceMappingFrom, namespaceMappingTo } }, generatedFilesToCheck); + } + + [TestCase( + _nestedLogicalTypesUnion, + "org.apache.avro.codegentest.testdata", "org.apache.csharp.codegentest.testdata", + new string[] + { + "org.apache.avro.codegentest.testdata.NestedLogicalTypesUnion", + "org.apache.avro.codegentest.testdata.RecordInUnion" + }, + new string[] + { + "org/apache/csharp/codegentest/testdata/NestedLogicalTypesUnion.cs", + "org/apache/csharp/codegentest/testdata/RecordInUnion.cs" + })] + public void GenerateSchemaWithNamespaceMapping_Bug_AVRO_2883( + string schema, + string namespaceMappingFrom, + string namespaceMappingTo, + IEnumerable typeNamesToCheck, + IEnumerable generatedFilesToCheck) + { + // !!! This is a bug which must be fixed + // !!! Once it is fixed, this test will fail and this test can be removed + // https://issues.apache.org/jira/browse/AVRO-2883 + // https://issues.apache.org/jira/browse/AVRO-3046 + Assert.Throws(() => TestSchema(schema, typeNamesToCheck, new Dictionary { { namespaceMappingFrom, namespaceMappingTo } }, generatedFilesToCheck)); + } + + [TestCase(_logicalTypesWithCustomConversion, typeof(AvroTypeException))] + [TestCase(_customConversionWithLogicalTypes, typeof(SchemaParseException))] + [TestCase(_nestedLogicalTypesUnionFixedDecimal, typeof(SchemaParseException))] + public void NotSupportedSchema(string schema, Type expectedException) + { + // Create temp folder + string outputDir = AvroGenHelper.CreateEmptyTemporyFolder(out string uniqueId); + + try + { + // Save schema + string schemaFileName = Path.Combine(outputDir, $"{uniqueId}.avsc"); + System.IO.File.WriteAllText(schemaFileName, schema); + + Assert.That(AvroGenTool.GenSchema(schemaFileName, outputDir, new Dictionary()), Is.EqualTo(1)); + } + finally + { + Directory.Delete(outputDir, true); + } + } + + [TestCase(@" +{ + ""type"" : ""record"", + ""name"" : ""ClassKeywords"", + ""namespace"" : ""com.base"", + ""fields"" : + [ + { ""name"" : ""int"", ""type"" : ""int"" }, + { ""name"" : ""base"", ""type"" : ""long"" }, + { ""name"" : ""event"", ""type"" : ""boolean"" }, + { ""name"" : ""foreach"", ""type"" : ""double"" }, + { ""name"" : ""bool"", ""type"" : ""float"" }, + { ""name"" : ""internal"", ""type"" : ""bytes"" }, + { ""name"" : ""while"", ""type"" : ""string"" }, + { ""name"" : ""return"", ""type"" : ""null"" }, + { ""name"" : ""enum"", ""type"" : { ""type"" : ""enum"", ""name"" : ""class"", ""symbols"" : [ ""Unknown"", ""A"", ""B"" ], ""default"" : ""Unknown"" } }, + { ""name"" : ""string"", ""type"" : { ""type"": ""fixed"", ""size"": 16, ""name"": ""static"" } } + ] +}", + new object[] { "com.base.ClassKeywords", typeof(int), typeof(long), typeof(bool), typeof(double), typeof(float), typeof(byte[]), typeof(string), typeof(object), "com.base.class", "com.base.static" })] + [TestCase(@" +{ + ""type"" : ""record"", + ""name"" : ""AvroNamespaceType"", + ""namespace"" : ""My.Avro"", + ""fields"" : + [ + { ""name"" : ""justenum"", ""type"" : { ""type"" : ""enum"", ""name"" : ""justenumEnum"", ""symbols"" : [ ""One"", ""Two"" ] } }, + ] +}", + new object[] { "My.Avro.AvroNamespaceType", "My.Avro.justenumEnum" })] + [TestCase(@" +{ + ""type"" : ""record"", + ""name"" : ""SchemaObject"", + ""namespace"" : ""schematest"", + ""fields"" : + [ + { ""name"" : ""myobject"", ""type"" : + [ + ""null"", + { ""type"" : ""array"", ""items"" : + [ + ""null"", + { ""type"" : ""enum"", ""name"" : ""MyEnum"", ""symbols"" : [ ""A"", ""B"" ] }, + { ""type"": ""fixed"", ""size"": 16, ""name"": ""MyFixed"" } + ] + } + ] + } + ] +}", + new object[] { "schematest.SchemaObject", typeof(IList) })] + [TestCase(@" +{ + ""type"" : ""record"", + ""name"" : ""LogicalTypes"", + ""namespace"" : ""schematest"", + ""fields"" : + [ + { ""name"" : ""nullibleguid"", ""type"" : [""null"", {""type"": ""string"", ""logicalType"": ""uuid"" } ]}, + { ""name"" : ""guid"", ""type"" : {""type"": ""string"", ""logicalType"": ""uuid"" } }, + { ""name"" : ""nullibletimestampmillis"", ""type"" : [""null"", {""type"": ""long"", ""logicalType"": ""timestamp-millis""}] }, + { ""name"" : ""timestampmillis"", ""type"" : {""type"": ""long"", ""logicalType"": ""timestamp-millis""} }, + { ""name"" : ""nullibiletimestampmicros"", ""type"" : [""null"", {""type"": ""long"", ""logicalType"": ""timestamp-micros""}] }, + { ""name"" : ""timestampmicros"", ""type"" : {""type"": ""long"", ""logicalType"": ""timestamp-micros""} }, + { ""name"" : ""nullibiletimemicros"", ""type"" : [""null"", {""type"": ""long"", ""logicalType"": ""time-micros""}] }, + { ""name"" : ""timemicros"", ""type"" : {""type"": ""long"", ""logicalType"": ""time-micros""} }, + { ""name"" : ""nullibiletimemillis"", ""type"" : [""null"", {""type"": ""int"", ""logicalType"": ""time-millis""}] }, + { ""name"" : ""timemillis"", ""type"" : {""type"": ""int"", ""logicalType"": ""time-millis""} }, + { ""name"" : ""nullibledecimal"", ""type"" : [""null"", {""type"": ""bytes"", ""logicalType"": ""decimal"", ""precision"": 4, ""scale"": 2}] }, + { ""name"" : ""decimal"", ""type"" : {""type"": ""bytes"", ""logicalType"": ""decimal"", ""precision"": 4, ""scale"": 2} } + ] +}", + new object[] { "schematest.LogicalTypes", typeof(Guid?), typeof(Guid), typeof(DateTime?), typeof(DateTime), typeof(DateTime?), typeof(DateTime), typeof(TimeSpan?), typeof(TimeSpan), typeof(TimeSpan?), typeof(TimeSpan), typeof(AvroDecimal?), typeof(AvroDecimal) })] + public void GenerateSchemaCheckFields(string schema, object[] result) + { + Assembly assembly = TestSchema(schema); + + // Instantiate object + Type type = assembly.GetType((string)result[0]); + Assert.That(type, Is.Not.Null); + + ISpecificRecord record = Activator.CreateInstance(type) as ISpecificRecord; + Assert.IsNotNull(record); + + // test type of each fields + for (int i = 1; i < result.Length; ++i) + { + object field = record.Get(i - 1); + Type stype; + if (result[i].GetType() == typeof(string)) + { + Type t = assembly.GetType((string)result[i]); + Assert.That(record, Is.Not.Null); + + object obj = Activator.CreateInstance(t); + Assert.That(obj, Is.Not.Null); + stype = obj.GetType(); + } + else + { + stype = (Type)result[i]; + } + if (!stype.IsValueType) + { + Assert.That(field, Is.Null); // can't test reference type, it will be null + } + else if (stype.IsValueType && field == null) + { + Assert.That(field, Is.Null); // nullable value type, so we can't get the type using GetType + } + else + { + Assert.That(field, Is.Not.Null); + Assert.That(field.GetType(), Is.EqualTo(stype)); + } + } + } + } +} diff --git a/lang/csharp/src/apache/test/AvroGen/AvroGenToolTests.cs b/lang/csharp/src/apache/test/AvroGen/AvroGenToolTests.cs new file mode 100644 index 00000000000..a5a46b4c4b0 --- /dev/null +++ b/lang/csharp/src/apache/test/AvroGen/AvroGenToolTests.cs @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +using System; +using System.IO; +using System.Linq; +using System.Text; +using NUnit.Framework; + +namespace Avro.Test.AvroGen +{ + [TestFixture] + + class AvroGenToolTests + { + [Test] + public void CommandLineNoArgs() + { + AvroGenToolResult result = AvroGenHelper.RunAvroGenTool(Array.Empty()); + + Assert.That(result.ExitCode, Is.EqualTo(1)); + Assert.That(result.StdOut, Is.Not.Empty); + Assert.That(result.StdErr, Is.Empty); + } + + [TestCase("-h")] + [TestCase("--help")] + [TestCase("--help", "-h")] + [TestCase("--help", "-s", "whatever.avsc", ".")] + [TestCase("-p", "whatever.avpr", ".", "-h")] + public void CommandLineHelp(params string[] args) + { + AvroGenToolResult result = AvroGenHelper.RunAvroGenTool(args); + + Assert.That(result.ExitCode, Is.EqualTo(0)); + Assert.That(result.StdOut, Is.Not.Empty); + Assert.That(result.StdErr, Is.Empty); + } + + [TestCase("-p")] + [TestCase("-s")] + [TestCase("-p", "whatever.avpr")] + [TestCase("-p", "whatever.avpr")] + [TestCase("-s", "whatever.avsc")] + [TestCase("whatever.avsc")] + [TestCase("whatever.avsc", ".")] + [TestCase(".")] + [TestCase("-s", "whatever.avsc", "--namespace")] + [TestCase("-s", "whatever.avsc", "--namespace", "org.apache")] + [TestCase("-s", "whatever.avsc", "--namespace", "org.apache:")] + [TestCase("-s", "whatever.avsc", ".", "whatever")] + public void CommandLineInvalidArgs(params string[] args) + { + AvroGenToolResult result = AvroGenHelper.RunAvroGenTool(args); + + Assert.That(result.ExitCode, Is.EqualTo(1)); + Assert.That(result.StdOut, Is.Not.Empty); + Assert.That(result.StdErr, Is.Not.Empty); + } + } +} diff --git a/lang/csharp/src/apache/test/CodGen/CodeGenTest.cs b/lang/csharp/src/apache/test/CodGen/CodeGenTest.cs index 3d4548520ee..243d93e24c5 100644 --- a/lang/csharp/src/apache/test/CodGen/CodeGenTest.cs +++ b/lang/csharp/src/apache/test/CodGen/CodeGenTest.cs @@ -17,194 +17,63 @@ */ using System; using System.Collections.Generic; -using System.IO; -using System.CodeDom.Compiler; -using Microsoft.CSharp; +using System.Linq; +using Microsoft.CodeAnalysis.CSharp; using NUnit.Framework; -using Avro.Specific; -namespace Avro.Test +namespace Avro.Test.CodeGen { [TestFixture] - - class CodeGenTest + class CodeGenTests { [Test] public void TestGetNullableTypeException() { - Assert.Throws(() => CodeGen.GetNullableType(null)); + Assert.Throws(() => Avro.CodeGen.GetNullableType(null)); } -#if !NETCOREAPP // System.CodeDom compilation not supported in .NET Core: https://github.com/dotnet/corefx/issues/12180 - [TestCase(@"{ -""type"" : ""record"", -""name"" : ""ClassKeywords"", -""namespace"" : ""com.base"", -""fields"" : - [ - { ""name"" : ""int"", ""type"" : ""int"" }, - { ""name"" : ""base"", ""type"" : ""long"" }, - { ""name"" : ""event"", ""type"" : ""boolean"" }, - { ""name"" : ""foreach"", ""type"" : ""double"" }, - { ""name"" : ""bool"", ""type"" : ""float"" }, - { ""name"" : ""internal"", ""type"" : ""bytes"" }, - { ""name"" : ""while"", ""type"" : ""string"" }, - { ""name"" : ""return"", ""type"" : ""null"" }, - { ""name"" : ""enum"", ""type"" : { ""type"" : ""enum"", ""name"" : ""class"", ""symbols"" : [ ""Unknown"", ""A"", ""B"" ], ""default"" : ""Unknown"" } }, - { ""name"" : ""string"", ""type"" : { ""type"": ""fixed"", ""size"": 16, ""name"": ""static"" } } - ] -} -", new object[] {"com.base.ClassKeywords", typeof(int), typeof(long), typeof(bool), typeof(double), typeof(float), typeof(byte[]), typeof(string),typeof(object),"com.base.class", "com.base.static"}, TestName = "TestCodeGen0")] - [TestCase(@"{ -""type"" : ""record"", -""name"" : ""AvroNamespaceType"", -""namespace"" : ""My.Avro"", -""fields"" : - [ - { ""name"" : ""justenum"", ""type"" : { ""type"" : ""enum"", ""name"" : ""justenumEnum"", ""symbols"" : [ ""One"", ""Two"" ] } }, - ] -} -", new object[] {"My.Avro.AvroNamespaceType", "My.Avro.justenumEnum"}, TestName = "TestCodeGen3 - Avro namespace conflict")] - [TestCase(@"{ -""type"" : ""record"", -""name"" : ""SchemaObject"", -""namespace"" : ""schematest"", -""fields"" : - [ - { ""name"" : ""myobject"", ""type"" : - [ - ""null"", - {""type"" : ""array"", ""items"" : [ ""null"", - { ""type"" : ""enum"", ""name"" : ""MyEnum"", ""symbols"" : [ ""A"", ""B"" ] }, - { ""type"": ""fixed"", ""size"": 16, ""name"": ""MyFixed"" } - ] - } - ] - } - ] -} -", new object[] { "schematest.SchemaObject", typeof(IList) }, TestName = "TestCodeGen1")] - [TestCase(@"{ - ""type"" : ""record"", - ""name"" : ""LogicalTypes"", - ""namespace"" : ""schematest"", - ""fields"" : - [ - { ""name"" : ""nullibleguid"", ""type"" : [""null"", {""type"": ""string"", ""logicalType"": ""uuid"" } ]}, - { ""name"" : ""guid"", ""type"" : {""type"": ""string"", ""logicalType"": ""uuid"" } }, - { ""name"" : ""nullibletimestampmillis"", ""type"" : [""null"", {""type"": ""long"", ""logicalType"": ""timestamp-millis""}] }, - { ""name"" : ""timestampmillis"", ""type"" : {""type"": ""long"", ""logicalType"": ""timestamp-millis""} }, - { ""name"" : ""nullibiletimestampmicros"", ""type"" : [""null"", {""type"": ""long"", ""logicalType"": ""timestamp-micros""}] }, - { ""name"" : ""timestampmicros"", ""type"" : {""type"": ""long"", ""logicalType"": ""timestamp-micros""} }, - { ""name"" : ""nullibiletimemicros"", ""type"" : [""null"", {""type"": ""long"", ""logicalType"": ""time-micros""}] }, - { ""name"" : ""timemicros"", ""type"" : {""type"": ""long"", ""logicalType"": ""time-micros""} }, - { ""name"" : ""nullibiletimemillis"", ""type"" : [""null"", {""type"": ""int"", ""logicalType"": ""time-millis""}] }, - { ""name"" : ""timemillis"", ""type"" : {""type"": ""int"", ""logicalType"": ""time-millis""} }, - { ""name"" : ""nullibledecimal"", ""type"" : [""null"", {""type"": ""bytes"", ""logicalType"": ""decimal"", ""precision"": 4, ""scale"": 2}] }, - { ""name"" : ""decimal"", ""type"" : {""type"": ""bytes"", ""logicalType"": ""decimal"", ""precision"": 4, ""scale"": 2} } - ] -} -", new object[] { "schematest.LogicalTypes", typeof(Guid?), typeof(Guid), typeof(DateTime?), typeof(DateTime), typeof(DateTime?), typeof(DateTime), typeof(TimeSpan?), typeof(TimeSpan), typeof(TimeSpan?), typeof(TimeSpan), typeof(AvroDecimal?), typeof(AvroDecimal) }, TestName = "TestCodeGen2 - Logical Types")] - public static void TestCodeGen(string str, object[] result) + [Test] + public void TestReservedKeywords() { - Schema schema = Schema.Parse(str); - - CompilerResults compres = GenerateSchema(schema); - - // instantiate object - ISpecificRecord rec = compres.CompiledAssembly.CreateInstance((string)result[0]) as ISpecificRecord; - Assert.IsNotNull(rec); + // https://github.com/dotnet/roslyn/blob/main/src/Compilers/CSharp/Portable/Syntax/SyntaxKindFacts.cs - // test type of each fields - for (int i = 1; i < result.Length; ++i) + // Check if all items in CodeGenUtil.Instance.ReservedKeywords are keywords + foreach (string keyword in CodeGenUtil.Instance.ReservedKeywords) { - object field = rec.Get(i - 1); - Type stype; - if (result[i].GetType() == typeof(string)) - { - object obj = compres.CompiledAssembly.CreateInstance((string)result[i]); - Assert.IsNotNull(obj); - stype = obj.GetType(); - } - else - stype = (Type)result[i]; - if (!stype.IsValueType) - Assert.IsNull(field); // can't test reference type, it will be null - else if (stype.IsValueType && field == null) - Assert.IsNull(field); // nullable value type, so we can't get the type using GetType - else - Assert.AreEqual(stype, field.GetType()); + Assert.That(SyntaxFacts.GetKeywordKind(keyword) != SyntaxKind.None, Is.True); } - } - [TestCase(@"{ -""type"": ""fixed"", -""namespace"": ""com.base"", -""name"": ""MD5"", -""size"": 16 -}", null, null, "com.base")] - [TestCase(@"{ -""type"": ""fixed"", -""namespace"": ""com.base"", -""name"": ""MD5"", -""size"": 16 -}", "com.base", "SchemaTest", "SchemaTest")] - [TestCase(@"{ -""type"": ""fixed"", -""namespace"": ""com.base"", -""name"": ""MD5"", -""size"": 16 -}", "miss", "SchemaTest", "com.base")] - public void TestCodeGenNamespaceMapping(string str, string avroNamespace, string csharpNamespace, - string expectedNamespace) - { - Schema schema = Schema.Parse(str); - - var codegen = new CodeGen(); - codegen.AddSchema(schema); - - if (avroNamespace != null && csharpNamespace != null) + // Check if all Roslyn defined keywords are in CodeGenUtil.Instance.ReservedKeywords + foreach (SyntaxKind keywordKind in SyntaxFacts.GetReservedKeywordKinds()) { - codegen.NamespaceMapping[avroNamespace] = csharpNamespace; + Assert.That(CodeGenUtil.Instance.ReservedKeywords, Does.Contain(SyntaxFacts.GetText(keywordKind))); } - var results = GenerateAssembly(codegen); - foreach(var type in results.CompiledAssembly.GetTypes()) - { - Assert.AreEqual(expectedNamespace, type.Namespace); - } + // If this test fails, CodeGenUtil.ReservedKeywords list must be updated. + // This might happen if newer version of C# language defines new reserved keywords. } - private static CompilerResults GenerateSchema(Schema schema) + [TestCase("a", "a")] + [TestCase("a.b", "a.b")] + [TestCase("a.b.c", "a.b.c")] + [TestCase("int", "@int")] + [TestCase("a.long.b", "a.@long.b")] + [TestCase("int.b.c", "@int.b.c")] + [TestCase("a.b.int", "a.b.@int")] + [TestCase("int.long.while", "@int.@long.@while")] // Reserved keywords + [TestCase("a.value.partial", "a.value.partial")] // Contextual keywords + [TestCase("a.value.b.int.c.while.longpartial", "a.value.b.@int.c.@while.longpartial")] // Rseserved and contextual keywords + public void TestMangleUnMangle(string input, string mangled) { - var codegen = new CodeGen(); - codegen.AddSchema(schema); - return GenerateAssembly(codegen); + // Mangle + Assert.That(CodeGenUtil.Instance.Mangle(input), Is.EqualTo(mangled)); + // Unmangle + Assert.That(CodeGenUtil.Instance.UnMangle(mangled), Is.EqualTo(input)); } - private static CompilerResults GenerateAssembly(CodeGen schema) - { - var compileUnit = schema.GenerateCode(); - - var comparam = new CompilerParameters(new string[] { "netstandard.dll" }); - comparam.ReferencedAssemblies.Add("System.dll"); - comparam.ReferencedAssemblies.Add(Path.Combine(TestContext.CurrentContext.TestDirectory, "Avro.dll")); - comparam.GenerateInMemory = true; - var ccp = new CSharpCodeProvider(); - var units = new[] { compileUnit }; - var compres = ccp.CompileAssemblyFromDom(comparam, units); - if (compres.Errors.Count > 0) - { - for (int i = 0; i < compres.Errors.Count; i++) - Console.WriteLine(compres.Errors[i]); - } - Assert.AreEqual(0, compres.Errors.Count); - return compres; - } -#endif [TestFixture] - public class CodeGenTestClass : CodeGen + public class CodeGenTestClass : Avro.CodeGen { [Test] public void TestGenerateNamesException() diff --git a/lang/csharp/versions.props b/lang/csharp/versions.props index 15cc410f3a7..aea7edb60c2 100644 --- a/lang/csharp/versions.props +++ b/lang/csharp/versions.props @@ -55,13 +55,15 @@ --> 0.13.1 - 17.0.0 - 17.0.0 - 4.0.1 + 17.1.0 + 17.1.0 + 4.1.0 + 4.1.0 + 4.1.0 6.0.0 - 17.0.0 + 17.1.0 3.13.2 - 3.14.0 + 3.15.0 4.2.1 1.1.118