cleanup PR

Author: Becca McHenry

src/Microsoft.Data.Analysis/VBufferDataFrameColumn.cs

@@ -71,13 +71,6 @@ protected internal override void Resize(long length)
             }
         }
 
-        /// <summary>
-        /// Indicates if the value at this <paramref name="index"/> is <see langword="null" />.
-        /// </summary>
-        /// <param name="index">The index to look up.</param>
-        /// <returns>A boolean value indicating the validity at this <paramref name="index"/>.</returns>
-        public bool IsValid(long index) => NullCount == 0;
-
         public void Append(VBuffer<T> value)
         {
             List<VBuffer<T>> lastBuffer = _vBuffers[_vBuffers.Count - 1];
@@ -87,8 +80,6 @@ public void Append(VBuffer<T> value)
                 _vBuffers.Add(lastBuffer);
             }
             lastBuffer.Add(value);
-            if (value.Length == 0) //TODO
-                _nullCount++;
             Length++;
         }
 
@@ -168,28 +159,50 @@ public IEnumerator<VBuffer<T>> GetEnumerator()
         protected override IEnumerator GetEnumeratorCore() => GetEnumerator();
 
         /// <inheritdoc/>
-        public override DataFrameColumn Sort(bool ascending = true) => throw new NotSupportedException();
-
-        /// <inheritdoc/>
-        public override DataFrameColumn Clone(DataFrameColumn mapIndices = null, bool invertMapIndices = false, long numberOfNullsToAppend = 0)
+        protected internal override void AddDataViewColumn(DataViewSchema.Builder builder)
         {
-            throw new NotImplementedException();
+            builder.AddColumn(Name, GetDataViewType());
         }
 
         /// <inheritdoc/>
-        public VBufferDataFrameColumn<T> FillNulls(VBuffer<T> value, bool inPlace = false)
+        protected internal override Delegate GetDataViewGetter(DataViewRowCursor cursor)
         {
-            throw new NotImplementedException();
+            return CreateValueGetterDelegate(cursor);
         }
 
-        public override DataFrameColumn Clamp<U>(U min, U max, bool inPlace = false) => throw new NotSupportedException();
+        private ValueGetter<VBuffer<T>> CreateValueGetterDelegate(DataViewRowCursor cursor) =>
+            (ref VBuffer<T> value) => value = this[cursor.Position];
 
-        public override DataFrameColumn Filter<U>(U min, U max) => throw new NotSupportedException();
+        public override Dictionary<long, ICollection<long>> GetGroupedOccurrences(DataFrameColumn other, out HashSet<long> otherColumnNullIndices)
+        {
+            return GetGroupedOccurrences<string>(other, out otherColumnNullIndices);
+        }
 
-        /// <inheritdoc/>
-        protected internal override void AddDataViewColumn(DataViewSchema.Builder builder)
+        protected internal override Delegate GetValueGetterUsingCursor(DataViewRowCursor cursor, DataViewSchema.Column schemaColumn)
         {
-            builder.AddColumn(Name, GetDataViewType());
+            return cursor.GetGetter<VBuffer<T>>(schemaColumn);
+        }
+
+        protected internal override void AddValueUsingCursor(DataViewRowCursor cursor, Delegate getter)
+        {
+            long row = cursor.Position;
+            VBuffer<T> value = default;
+            Debug.Assert(getter != null, "Excepted getter to be valid");
+
+            (getter as ValueGetter<VBuffer<T>>)(ref value);
+
+            if (Length > row)
+            {
+                this[row] = value;
+            }
+            else if (Length == row)
+            {
+                Append(value);
+            }
+            else
+            {
+                throw new IndexOutOfRangeException(nameof(row));
+            }
         }
 
         private static VectorDataViewType GetDataViewType()
@@ -249,58 +262,5 @@ private static VectorDataViewType GetDataViewType()
 
             throw new NotSupportedException();
         }
-
-        /// <inheritdoc/>
-        protected internal override Delegate GetDataViewGetter(DataViewRowCursor cursor)
-        {
-            return CreateValueGetterDelegate(cursor);
-        }
-
-        private ValueGetter<VBuffer<T>> CreateValueGetterDelegate(DataViewRowCursor cursor) =>
-            (ref VBuffer<T> value) => value = this[cursor.Position];
-
-
-        /// <inheritdoc/>
-        public override PrimitiveDataFrameColumn<bool> ElementwiseEquals<U>(U value)
-        {
-            throw new NotImplementedException();
-        }
-
-        public override Dictionary<long, ICollection<long>> GetGroupedOccurrences(DataFrameColumn other, out HashSet<long> otherColumnNullIndices)
-        {
-            return GetGroupedOccurrences<string>(other, out otherColumnNullIndices);
-        }
-
-        protected internal override Delegate GetValueGetterUsingCursor(DataViewRowCursor cursor, DataViewSchema.Column schemaColumn)
-        {
-            return cursor.GetGetter<VBuffer<T>>(schemaColumn);
-        }
-
-        IEnumerator<VBuffer<T>> IEnumerable<VBuffer<T>>.GetEnumerator()
-        {
-            throw new NotImplementedException();
-        }
-
-        protected internal override void AddValueUsingCursor(DataViewRowCursor cursor, Delegate getter)
-        {
-            long row = cursor.Position;
-            VBuffer<T> value = default;
-            Debug.Assert(getter != null, "Excepted getter to be valid");
-
-            (getter as ValueGetter<VBuffer<T>>)(ref value);
-
-            if (Length > row)
-            {
-                this[row] = value;
-            }
-            else if (Length == row)
-            {
-                Append(value);
-            }
-            else
-            {
-                throw new IndexOutOfRangeException(nameof(row));
-            }
-        }
     }
 }

src/Microsoft.ML.DataView/VectorType.cs

@@ -5,9 +5,7 @@
 using System;
 using System.Collections.Immutable;
 using System.Linq;
-using System.Runtime.CompilerServices;
 using System.Text;
-using System.Threading;
 using Microsoft.ML.Internal.DataView;
 using Microsoft.ML.Internal.Utilities;
 

test/Microsoft.Data.Analysis.Tests/DataFrameIDataViewTests.cs

@@ -9,23 +9,11 @@
 using Microsoft.ML;
 using Microsoft.ML.Data;
 using Xunit;
+using Microsoft.ML.Trainers;
+
 
 namespace Microsoft.Data.Analysis.Tests
 {
-    public class VectorInput
-    {
-        [LoadColumn(0, 1)]
-        [VectorType(2)]
-        public float[] FloatFeatures { get; set; }
-
-        [LoadColumn(3, 4)]
-        [VectorType(2)]
-        public Int32[] IntFeatures { get; set; }
-
-        [LoadColumn(5)]
-        public bool Label { get; set; }
-    }
-
     public partial class DataFrameIDataViewTests
     {
         [Fact]
@@ -365,25 +353,6 @@ private IDataView GetASampleIDataView()
             return data;
         }
 
-        private IDataView GetASampleIDataViewVBuffer()
-        {
-            var mlContext = new MLContext();
-
-            // Get a small dataset as an IEnumerable.
-            var enumerableOfData = new[]
-            {
-                new InputData() { Name = "Joey", FilterNext = false, Value = 1.0f },
-                new InputData() { Name = "Chandler", FilterNext = false , Value = 2.0f},
-                new InputData() { Name = "Ross", FilterNext = false , Value = 3.0f},
-                new InputData() { Name = "Monica", FilterNext = true , Value = 4.0f},
-                new InputData() { Name = "Rachel", FilterNext = true , Value = 5.0f},
-                new InputData() { Name = "Phoebe", FilterNext = false , Value = 6.0f},
-            };
-
-            IDataView data = mlContext.Data.LoadFromEnumerable(enumerableOfData);
-            return data;
-        }
-
         private void VerifyDataFrameColumnAndDataViewColumnValues<T>(string columnName, IDataView data, DataFrame df, int maxRows = -1)
         {
             int cc = 0;
@@ -467,35 +436,46 @@ public void TestDataFrameFromIDataView_MLData_SelectColumnsAndRows()
         }
 
         [Fact]
-        public void VBufferTest()
+        public void TestDataFrameFromIDataView_VBufferType()
         {
             var mlContext = new MLContext();
 
-            List<VectorInput> inputs = new List<VectorInput>()
+            var inputData = new[]
             {
-                new VectorInput()
-                {
-                    FloatFeatures = new float[] {33, 44},
-                    IntFeatures = new int[] {5, 6},
-                    Label = true
+                new {
+                    boolFeature = new bool[] {false, false},
+                    byteFeatures = new byte[] {0, 0},
+                    doubleFeatures = new double[] {0, 0},
+                    floatFeatures = new float[] {0, 0},
+                    intFeatures = new int[] {0, 0},
+                    longFeatures = new long[] {0, 0},
+                    sbyteFeatures = new sbyte[] {0, 0},
+                    shortFeatures = new short[] {0, 0},
+                    ushortFeatures = new ushort[] {0, 0},
+                    uintFeatures = new uint[] {0, 0},
+                    ulongFeatures = new ulong[] {0, 0},
                 },
-                new VectorInput()
-                {
-                    FloatFeatures = new float[] {55, 66},
-                    IntFeatures = new int[] {5, 6},
-                    Label = false
+                new {
+                    boolFeature = new bool[] {false, false},
+                    byteFeatures = new byte[] {0, 0},
+                    doubleFeatures = new double[] {0, 0},
+                    floatFeatures = new float[] {1, 1},
+                    intFeatures = new int[] {0, 0},
+                    longFeatures = new long[] {0, 0},
+                    sbyteFeatures = new sbyte[] {0, 0},
+                    shortFeatures = new short[] {0, 0},
+                    ushortFeatures = new ushort[] {0, 0},
+                    uintFeatures = new uint[] {0, 0},
+                    ulongFeatures = new ulong[] {0, 0},
                 }
             };
 
-            var data = mlContext.Data.LoadFromEnumerable<VectorInput>(inputs);
-
+            var data = mlContext.Data.LoadFromEnumerable(inputData);
             var df = data.ToDataFrame();
 
-            Assert.Equal(3, df.Columns.Count);
+            Assert.Equal(11, df.Columns.Count);
             Assert.Equal(2, df.Rows.Count);
-
-            var value = df[0, 0];
-            var a = df.Preview();
         }
     }
 }
+

test/Microsoft.ML.Core.Tests/UnitTests/TestVBuffer.cs

@@ -1098,7 +1098,7 @@ private static void GeneratePair(Random rgen, int len, out VBuffer<float> a, out
             const int cases = 8;
             Contracts.Assert(cases == Enum.GetValues(typeof(GenLogic)).Length);
             subcase = (GenLogic)rgen.Next(cases);
-            // VBufferEditor<float> bEditor;
+            VBufferEditor<float> bEditor;
             switch (subcase)
             {
                 case GenLogic.BothDense:
@@ -1116,21 +1116,21 @@ private static void GeneratePair(Random rgen, int len, out VBuffer<float> a, out
                 case GenLogic.BothSparseASameB:
                     GenerateVBuffer(rgen, len, rgen.Next(len), out a);
                     GenerateVBuffer(rgen, len, a.GetValues().Length, out b);
-                    /*bEditor = VBufferEditor.CreateFromBuffer(ref b);
+                    bEditor = VBufferEditor.CreateFromBuffer(ref b);
                     for (int i = 0; i < a.GetIndices().Length; ++i)
                         bEditor.Indices[i] = a.GetIndices()[i];
-                    b = bEditor.Commit();*/
+                    b = bEditor.Commit();
                     break;
                 case GenLogic.BothSparseASubsetB:
                 case GenLogic.BothSparseBSubsetA:
                     GenerateVBuffer(rgen, len, rgen.Next(len), out a);
                     GenerateVBuffer(rgen, a.GetValues().Length, rgen.Next(a.GetValues().Length), out b);
-                    /*bEditor = VBufferEditor.Create(ref b, len, b.GetValues().Length);
+                    bEditor = VBufferEditor.Create(ref b, len, b.GetValues().Length);
                     for (int i = 0; i < bEditor.Values.Length; ++i)
                         bEditor.Indices[i] = a.GetIndices()[bEditor.Indices[i]];
                     b = bEditor.Commit();
                     if (subcase == GenLogic.BothSparseASubsetB)
-                        Utils.Swap(ref a, ref b);*/
+                        Utils.Swap(ref a, ref b);
                     break;
                 case GenLogic.BothSparseAUnrelatedB:
                     GenerateVBuffer(rgen, len, rgen.Next(len), out a);
@@ -1143,14 +1143,14 @@ private static void GeneratePair(Random rgen, int len, out VBuffer<float> a, out
                     if (a.GetValues().Length != 0 && b.GetValues().Length != 0 && a.GetValues().Length != b.GetValues().Length)
                     {
                         var aEditor = VBufferEditor.CreateFromBuffer(ref a);
-                        /*bEditor = VBufferEditor.CreateFromBuffer(ref b);
+                        bEditor = VBufferEditor.CreateFromBuffer(ref b);
                         Utils.Shuffle(rgen, aEditor.Indices);
                         aEditor.Indices.Slice(boundary).CopyTo(bEditor.Indices);
 
                         GenericSpanSortHelper<int>.Sort(aEditor.Indices, 0, boundary);
                         GenericSpanSortHelper<int>.Sort(bEditor.Indices, 0, bEditor.Indices.Length);
                         a = aEditor.CommitTruncated(boundary);
-                        b = bEditor.Commit();*/
+                        b = bEditor.Commit();
                     }
                     if (rgen.Next(2) == 0)
                         Utils.Swap(ref a, ref b);