ARROW-6972: [C#] Support for StructArrays

csharp/src/Apache.Arrow/Arrays/StructArray.cs

@@ -21,13 +21,13 @@ namespace Apache.Arrow
 {
     public class StructArray : Array
     {
-        private readonly List<Array> _fields;
+        private readonly IEnumerable<IArrowArray> _fields;
 
-        public IEnumerable<Array> Fields => _fields;
+        public IEnumerable<IArrowArray> Fields => _fields;
 
         public StructArray(
             IArrowType dataType, int length,
-            IEnumerable<Array> children,
+            IEnumerable<IArrowArray> children,
             ArrowBuffer nullBitmapBuffer, int nullCount = 0, int offset = 0)
         : this(new ArrayData(
             dataType, length, nullCount, offset, new[] { nullBitmapBuffer }, 

csharp/src/Apache.Arrow/Ipc/ArrowReaderImplementation.cs

@@ -147,10 +147,10 @@ private ArrayData LoadPrimitiveField(
         {
 
             ArrowBuffer nullArrowBuffer = BuildArrowBuffer(bodyData, recordBatchEnumerator.CurrentBuffer);
-            recordBatchEnumerator.MoveNextBuffer();
-            ArrowBuffer valueArrowBuffer = BuildArrowBuffer(bodyData, recordBatchEnumerator.CurrentBuffer);
-            recordBatchEnumerator.MoveNextBuffer();
-
+            if (!recordBatchEnumerator.MoveNextBuffer())
+            {
+                throw new Exception("Unable to move to the next buffer.");
+            }
 
             int fieldLength = (int)fieldNode.Length;
             int fieldNullCount = (int)fieldNode.NullCount;
@@ -165,8 +165,21 @@ private ArrayData LoadPrimitiveField(
                 throw new InvalidDataException("Null count length must be >= 0"); // TODO:Localize exception message
             }
 
-            ArrowBuffer[] arrowBuff = new[] { nullArrowBuffer, valueArrowBuffer };
-            ArrayData[] children = GetChildren(ref recordBatchEnumerator, field, bodyData);
+            ArrowBuffer[] arrowBuff = null;
+            ArrayData[] children = null;
+            if (field.DataType.TypeId == ArrowTypeId.Struct)
+            {
+                arrowBuff = new[] { nullArrowBuffer};
+                children = GetChildren(ref recordBatchEnumerator, field, bodyData);
+            }
+            else
+            {
+                ArrowBuffer valueArrowBuffer = BuildArrowBuffer(bodyData, recordBatchEnumerator.CurrentBuffer);
+                recordBatchEnumerator.MoveNextBuffer();
+
+                arrowBuff = new[] { nullArrowBuffer, valueArrowBuffer };
+                children = GetChildren(ref recordBatchEnumerator, field, bodyData);
+            }
 
             return new ArrayData(field.DataType, fieldLength, fieldNullCount, 0, arrowBuff, children);
         }
@@ -180,9 +193,15 @@ private ArrayData LoadVariableField(
         {
 
             ArrowBuffer nullArrowBuffer = BuildArrowBuffer(bodyData, recordBatchEnumerator.CurrentBuffer);
-            recordBatchEnumerator.MoveNextBuffer();
+            if (!recordBatchEnumerator.MoveNextBuffer())
+            {
+                throw new Exception("Unable to move to the next buffer.");
+            }
             ArrowBuffer offsetArrowBuffer = BuildArrowBuffer(bodyData, recordBatchEnumerator.CurrentBuffer);
-            recordBatchEnumerator.MoveNextBuffer();
+            if (!recordBatchEnumerator.MoveNextBuffer())
+            {
+                throw new Exception("Unable to move to the next buffer.");
+            }
             ArrowBuffer valueArrowBuffer = BuildArrowBuffer(bodyData, recordBatchEnumerator.CurrentBuffer);
             recordBatchEnumerator.MoveNextBuffer();
 

csharp/src/Apache.Arrow/Ipc/ArrowStreamWriter.cs

@@ -44,7 +44,8 @@ internal class ArrowRecordBatchFlatBufferBuilder :
             IArrowArrayVisitor<Date64Array>,
             IArrowArrayVisitor<ListArray>,
             IArrowArrayVisitor<StringArray>,
-            IArrowArrayVisitor<BinaryArray>
+            IArrowArrayVisitor<BinaryArray>,
+            IArrowArrayVisitor<StructArray>
         {
             public readonly struct Buffer
             {
@@ -102,6 +103,31 @@ public void Visit(BinaryArray array)
                 _buffers.Add(CreateBuffer(array.ValueBuffer));
             }
 
+            private void Visit(ArrayData[] children)
+            {
+                for (int i = 0; i < children.Length; i++)
+                {
+                    Visit(ArrowArrayFactory.BuildArray(children[i]));
+                }
+            }
+
+            public void Visit(StructArray array)
+            {
+                _buffers.Add(CreateBuffer(array.NullBitmapBuffer));
+                for (int i = 0; i < array.Data.Children.Length; i++)
+                {
+                    ArrayData childArray = array.Data.Children[i];
+                    if (childArray.Children != null)
+                    {
+                        Visit(childArray.Children);
+                    }
+                    for (int j = 0; j < childArray.Buffers.Length; j++)
+                    {
+                        _buffers.Add(CreateBuffer(childArray.Buffers[j]));
+                    }
+                }
+            }
+
             private void CreateBuffers(BooleanArray array)
             {
                 _buffers.Add(CreateBuffer(array.NullBitmapBuffer));

csharp/src/Apache.Arrow/Ipc/ArrowTypeFlatbufferBuilder.cs

@@ -30,7 +30,7 @@ public struct FieldType
             public readonly int Offset;
 
             public static FieldType Build<T>(Flatbuf.Type type, Offset<T> offset)
-                where T: struct =>
+                where T : struct =>
                 new FieldType(type, offset.Value);
 
             public FieldType(Flatbuf.Type type, int offset)
@@ -40,7 +40,7 @@ public FieldType(Flatbuf.Type type, int offset)
             }
         }
 
-        class TypeVisitor : 
+        class TypeVisitor :
             IArrowTypeVisitor<BooleanType>,
             IArrowTypeVisitor<Int8Type>,
             IArrowTypeVisitor<Int16Type>,
@@ -60,7 +60,8 @@ class TypeVisitor :
             IArrowTypeVisitor<BinaryType>,
             IArrowTypeVisitor<TimestampType>,
             IArrowTypeVisitor<ListType>,
-            IArrowTypeVisitor<UnionType>
+            IArrowTypeVisitor<UnionType>,
+            IArrowTypeVisitor<StructType>
         {
             private FlatBufferBuilder Builder { get; }
 
@@ -100,7 +101,7 @@ public void Visit(ListType type)
             {
                 Flatbuf.List.StartList(Builder);
                 Result = FieldType.Build(
-                    Flatbuf.Type.List, 
+                    Flatbuf.Type.List,
                     Flatbuf.List.EndList(Builder));
             }
 
@@ -118,14 +119,14 @@ public void Visit(StringType type)
             }
 
             public void Visit(TimestampType type)
-            {  
+            {
                 StringOffset timezoneStringOffset = default;
 
                 if (!string.IsNullOrWhiteSpace(type.Timezone))
                     timezoneStringOffset = Builder.CreateString(type.Timezone);
 
                 Result = FieldType.Build(
-                    Flatbuf.Type.Timestamp, 
+                    Flatbuf.Type.Timestamp,
                     Flatbuf.Timestamp.CreateTimestamp(Builder, ToFlatBuffer(type.Unit), timezoneStringOffset));
             }
 
@@ -171,6 +172,13 @@ public void Visit(Time64Type type)
                     Flatbuf.Time.CreateTime(Builder, ToFlatBuffer(type.Unit), 64));
             }
 
+            public void Visit(StructType type)
+            {
+                Flatbuf.Struct_.StartStruct_(Builder);
+                FieldType result = FieldType.Build(Flatbuf.Type.Struct_, Flatbuf.Struct_.EndStruct_(Builder));
+                Result = result;
+            }
+
             private void CreateIntType(NumberType type)
             {
                 Result = FieldType.Build(

csharp/src/Apache.Arrow/Ipc/MessageSerializer.cs

@@ -14,6 +14,7 @@
 // limitations under the License.
 
 using System;
+using System.Diagnostics;
 using System.IO;
 
 namespace Apache.Arrow.Ipc
@@ -57,16 +58,30 @@ internal static Schema GetSchema(Flatbuf.Schema schema)
             for (int i = 0; i < schema.FieldsLength; i++)
             {
                 Flatbuf.Field field = schema.Fields(i).GetValueOrDefault();
-
-                schemaBuilder.Field(
-                    new Field(field.Name, GetFieldArrowType(field), field.Nullable));
+                Field arrowField = FieldFromFlatbuffer(field);
+                schemaBuilder.Field(arrowField);
             }
 
             return schemaBuilder.Build();
         }
 
+        private static Field FieldFromFlatbuffer(Flatbuf.Field flatbufField)
+        {
+            System.Collections.Generic.List<Field> childFields = null;
+            if (flatbufField.ChildrenLength > 0)
+            {
+                childFields = new System.Collections.Generic.List<Field>();
+                for (int j = 0; j < flatbufField.ChildrenLength; j++)
+                {
+                    Flatbuf.Field? childFlatbufField = flatbufField.Children(j);
+                    Field childField = FieldFromFlatbuffer(childFlatbufField.Value);
+                    childFields.Add(childField);
+                }
+            }
+            return new Field(flatbufField.Name, GetFieldArrowType(flatbufField, childFields), flatbufField.Nullable);
+        }
 
-        private static Types.IArrowType GetFieldArrowType(Flatbuf.Field field)
+        private static Types.IArrowType GetFieldArrowType(Flatbuf.Field field, System.Collections.Generic.List<Field> childFields = null)
         {
             switch (field.TypeType)
             {
@@ -131,6 +146,9 @@ private static Types.IArrowType GetFieldArrowType(Flatbuf.Field field)
                         throw new InvalidDataException($"List type must have only one child.");
                     }
                     return new Types.ListType(GetFieldArrowType(field.Children(0).GetValueOrDefault()));
+                case Flatbuf.Type.Struct_:
+                    Debug.Assert(childFields != null);
+                    return new Types.StructType(childFields);
                 default:
                     throw new InvalidDataException($"Arrow primitive '{field.TypeType}' is unsupported.");
             }

csharp/src/Apache.Arrow/Types/StructType.cs

@@ -19,7 +19,7 @@
 
 namespace Apache.Arrow.Types
 {
-    public sealed class StructType : ArrowType
+    public sealed class StructType : NestedType
     {
         private readonly List<Field> _fields;
 
@@ -28,7 +28,7 @@ public sealed class StructType : ArrowType
 
         public IEnumerable<Field> Fields => _fields;
 
-        public StructType(IEnumerable<Field> fields)
+        public StructType(IReadOnlyList<Field> fields) : base(fields)
         {
             _fields = fields?.ToList();
         }

csharp/test/Apache.Arrow.Tests/ArrayBuilderTests.cs

@@ -73,7 +73,6 @@ public void StringArrayBuilderHandlesNullsAndEmptyStrings()
             Assert.Equal(string.Empty, stringArray.GetString(3));
         }
 
-
         [Fact]
         public void ListArrayBuilder()
         {

csharp/test/Apache.Arrow.Tests/ArrowReaderVerifier.cs

@@ -80,8 +80,8 @@ public void Visit(TimestampType actualType)
             {
                 Assert.IsAssignableFrom<TimestampType>(_expectedType);
 
-                var expectedType = (TimestampType) _expectedType;
-                
+                var expectedType = (TimestampType)_expectedType;
+
                 Assert.Equal(expectedType.Timezone, actualType.Timezone);
                 Assert.Equal(expectedType.Unit, actualType.Unit);
             }
@@ -149,7 +149,8 @@ private class ArrayComparer :
             IArrowArrayVisitor<Date64Array>,
             IArrowArrayVisitor<ListArray>,
             IArrowArrayVisitor<StringArray>,
-            IArrowArrayVisitor<BinaryArray>
+            IArrowArrayVisitor<BinaryArray>,
+            IArrowArrayVisitor<StructArray>
         {
             private readonly IArrowArray _expectedArray;
             private readonly ArrayTypeComparer _arrayTypeComparer;
@@ -179,11 +180,28 @@ public ArrayComparer(IArrowArray expectedArray)
             public void Visit(StringArray array) => CompareBinaryArrays<StringArray>(array);
 
             public void Visit(BinaryArray array) => CompareBinaryArrays<BinaryArray>(array);
+
+            public void Visit(StructArray array)
+            {
+                Assert.Equal(_expectedArray.Length, array.Length);
+                Assert.Equal(_expectedArray.NullCount, array.NullCount);
+                Assert.Equal(_expectedArray.Offset, array.Offset);
+                Assert.Equal(_expectedArray.Data.Children.Length, array.Data.Children.Length);
+
+                ArrayData data = array.Data;
+                for (int i = 0; i < data.Children.Length; i++)
+                {
+                    IArrowArray childArray = ArrowArrayFactory.BuildArray(data.Children[i]);
+                    IArrowArray expectedChildArray = ArrowArrayFactory.BuildArray(_expectedArray.Data.Children[i]);
+                    childArray.Accept(new ArrayComparer(expectedChildArray));
+                }
+            }
+
             public void Visit(FixedSizeBinaryType array) => throw new NotImplementedException();
             public void Visit(IArrowArray array) => throw new NotImplementedException();
 
             private void CompareBinaryArrays<T>(BinaryArray actualArray)
-                where T: IArrowArray
+                where T : IArrowArray
             {
                 Assert.IsAssignableFrom<T>(_expectedArray);
                 Assert.IsAssignableFrom<T>(actualArray);