CollectionDataSource (train on top of memory collection instead of loading data from file)

src/Microsoft.ML.Api/DataViewConstructionUtils.cs

@@ -14,7 +14,7 @@ namespace Microsoft.ML.Runtime.Api
     /// <summary>
     /// A helper class to create data views based on the user-provided types.
     /// </summary>
-    internal static class DataViewConstructionUtils
+    public static class DataViewConstructionUtils
     {
         public static IDataView CreateFromList<TRow>(IHostEnvironment env, IList<TRow> data,
             SchemaDefinition schemaDefinition = null)

src/Microsoft.ML.Api/InternalSchemaDefinition.cs

@@ -16,7 +16,7 @@ namespace Microsoft.ML.Runtime.Api
     /// <summary>
     /// An internal class that holds the (already validated) mapping between a custom type and an IDataView schema.
     /// </summary>
-    internal sealed class InternalSchemaDefinition
+    public sealed class InternalSchemaDefinition
     {
         public readonly Column[] Columns;
 

src/Microsoft.ML/CSharpApi.cs

@@ -53,11 +53,22 @@ public Microsoft.ML.Data.TextLoader.Output Add(Microsoft.ML.Data.TextLoader inpu
                 return output;
             }
 
+            public Microsoft.ML.Data.DataViewReference.Output Add(Microsoft.ML.Data.DataViewReference input)
+            {
+                var output = new Microsoft.ML.Data.DataViewReference.Output();
+                Add(input, output);
+                return output;
+            }
+
             public void Add(Microsoft.ML.Data.TextLoader input, Microsoft.ML.Data.TextLoader.Output output)
             {
                 _jsonNodes.Add(Serialize("Data.TextLoader", input, output));
             }
 
+            public void Add(Microsoft.ML.Data.DataViewReference input, Microsoft.ML.Data.DataViewReference.Output output)
+            {
+                _jsonNodes.Add(Serialize("Data.DataViewReference", input, output));
+            }
             public Microsoft.ML.Models.AnomalyDetectionEvaluator.Output Add(Microsoft.ML.Models.AnomalyDetectionEvaluator input)
             {
                 var output = new Microsoft.ML.Models.AnomalyDetectionEvaluator.Output();
@@ -1311,6 +1322,23 @@ public sealed partial class TextLoader
             public string CustomSchema { get; set; }
 
 
+            public sealed class Output
+            {
+                /// <summary>
+                /// The resulting data view
+                /// </summary>
+                public Var<Microsoft.ML.Runtime.Data.IDataView> Data { get; set; } = new Var<Microsoft.ML.Runtime.Data.IDataView>();
+
+            }
+        }
+
+        public sealed partial class DataViewReference
+        {
+            /// <summary>
+            /// Location of the input file
+            /// </summary>
+            public Var<Microsoft.ML.Runtime.Data.IDataView> Data { get; set; } = new Var<Microsoft.ML.Runtime.Data.IDataView>();
+
             public sealed class Output
             {
                 /// <summary>

src/Microsoft.ML/MemoryCollection.cs

@@ -0,0 +1,66 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Microsoft.ML.Runtime;
+using Microsoft.ML.Runtime.Api;
+using Microsoft.ML.Runtime.Data;
+using Microsoft.ML.Runtime.EntryPoints;
+
+namespace Microsoft.ML
+{
+    public class MemoryCollection<TInput> : ILearningPipelineLoader
+        where TInput : class
+    {
+        public ILearningPipelineStep ApplyStep(ILearningPipelineStep previousStep, Experiment experiment)
+        {
+            Contracts.Assert(previousStep == null);
+            _dataViewEntryPoint = new Data.DataViewReference();
+            var importOutput = experiment.Add(_dataViewEntryPoint);
+            return new MemoryCollectionPipelineStep(importOutput.Data);
+        }
+
+        private readonly IList<TInput> _listCollection;
+        private readonly IEnumerable<TInput> _enumerableCollection;
+
+        private Data.DataViewReference _dataViewEntryPoint;
+        private IDataView _dataView;
+
+        public MemoryCollection(IList<TInput> collection)
+        {
+            //need validation at some point
+            _listCollection = collection;
+        }
+
+        public MemoryCollection(IEnumerable<TInput> collection)
+        {
+            //need validation at some point
+            _enumerableCollection = collection;
+        }
+
+        public void SetInput(IHostEnvironment env, Experiment experiment)
+        {
+            if (_listCollection!=null)
+            {
+                _dataView = DataViewConstructionUtils.CreateFromList(env, _listCollection);
+            }
+            if (_enumerableCollection!=null)
+            {
+                _dataView = DataViewConstructionUtils.CreateFromEnumerable(env, _listCollection);
+            }
+            env.CheckValue(_dataView, nameof(_dataView));
+            experiment.SetInput(_dataViewEntryPoint.Data, _dataView);
+        }
+
+        private class MemoryCollectionPipelineStep : ILearningPipelineDataStep
+        {
+            public MemoryCollectionPipelineStep(Var<IDataView> data)
+            {
+                Data = data;
+                Model = null;
+            }
+
+            public Var<IDataView> Data { get; }
+            public Var<ITransformModel> Model { get; }
+        }
+    }
+}

src/Microsoft.ML/Runtime/EntryPoints/DataViewReference.cs

@@ -0,0 +1,34 @@
+using Microsoft.ML.Data;
+using Microsoft.ML.Runtime;
+using Microsoft.ML.Runtime.CommandLine;
+using Microsoft.ML.Runtime.Data;
+using Microsoft.ML.Runtime.EntryPoints;
+
+[assembly: LoadableClass(typeof(void), typeof(InMemoryDataView), null, typeof(SignatureEntryPointModule), "InMemoryDataView")]
+namespace Microsoft.ML.Runtime.EntryPoints
+{
+    public class InMemoryDataView
+    {
+        public sealed class Input
+        {
+            [Argument(ArgumentType.Required, ShortName = "data", HelpText = "Pointer to IDataView in memory", SortOrder = 1)]
+            public IDataView Data;
+        }
+
+        public sealed class Output
+        {
+            [TlcModule.Output(Desc = "The resulting data view", SortOrder = 1)]
+            public IDataView Data;
+        }
+
+        [TlcModule.EntryPoint(Name = "Data.DataViewReference", Desc = "Pass dataview from memory to experiment")]
+        public static Output ImportData(IHostEnvironment env, Input input)
+        {
+            Contracts.CheckValue(env, nameof(env));
+            var host = env.Register("DataViewReference");
+            env.CheckValue(input, nameof(input));
+            EntryPointUtils.CheckInputArgs(host, input);
+            return new Output { Data = input.Data };
+        }
+    }
+}

test/Microsoft.ML.Tests/MemoryCollectionTests.cs

@@ -0,0 +1,176 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.ML;
+using Microsoft.ML.Runtime;
+using Microsoft.ML.Runtime.Api;
+using Microsoft.ML.Runtime.Data;
+using Microsoft.ML.TestFramework;
+using Microsoft.ML.Trainers;
+using Microsoft.ML.Transforms;
+using System;
+using System.Collections.Generic;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Microsoft.ML.EntryPoints.Tests
+{
+    public class MemoryCollectionTests : BaseTestClass
+    {
+        public MemoryCollectionTests(ITestOutputHelper output)
+            : base(output)
+        {
+
+        }
+
+        [Fact]
+        public void ConstructorDoesntThrow()
+        {
+            Assert.NotNull(new MemoryCollection<Input>(new List<Input>()));
+            Assert.NotNull(new MemoryCollection<Input>(new List<Input>() { new Input { Number1 = 1, String1 = "1" } }));
+            Assert.NotNull(new MemoryCollection<Input>(new Input[0]));
+            Assert.NotNull(new MemoryCollection<Input>(new Input[1] { new Input { Number1 = 1, String1 = "1" } }));
+            Assert.NotNull(new MemoryCollection<Input>(null));
+        }
+
+        [Fact]
+        public void CanSuccessfullyApplyATransform()
+        {
+            var collection = new MemoryCollection<Input>(new List<Input>() { new Input { Number1 = 1, String1 = "1" } });
+
+            using (var environment = new TlcEnvironment())
+            {
+                Experiment experiment = environment.CreateExperiment();
+                ILearningPipelineDataStep output = collection.ApplyStep(null, experiment) as ILearningPipelineDataStep;
+
+                Assert.NotNull(output.Data);
+                Assert.NotNull(output.Data.VarName);
+                Assert.Null(output.Model);
+            }
+        }
+
+        [Fact]
+        public void CanSuccessfullyEnumerated()
+        {
+            var collection = new MemoryCollection<Input>(new List<Input>() {
+                new Input { Number1 = 1, String1 = "1" },
+                new Input { Number1 = 2, String1 = "2" },
+                new Input { Number1 = 3, String1 = "3" }
+            });
+
+            using (var environment = new TlcEnvironment())
+            {
+                Experiment experiment = environment.CreateExperiment();
+                ILearningPipelineDataStep output = collection.ApplyStep(null, experiment) as ILearningPipelineDataStep;
+
+                experiment.Compile();
+                collection.SetInput(environment, experiment);
+                experiment.Run();
+
+                IDataView data = experiment.GetOutput(output.Data);
+                Assert.NotNull(data);
+
+                using (var cursor = data.GetRowCursor((a => true)))
+                {
+                    var IDGetter = cursor.GetGetter<float>(0);
+                    var TextGetter = cursor.GetGetter<DvText>(1);
+
+                    Assert.True(cursor.MoveNext());
+
+                    float ID = 0;
+                    IDGetter(ref ID);
+                    Assert.Equal(1, ID);
+
+                    DvText Text = new DvText();
+                    TextGetter(ref Text);
+                    Assert.Equal("1", Text.ToString());
+
+                    Assert.True(cursor.MoveNext());
+
+                    ID = 0;
+                    IDGetter(ref ID);
+                    Assert.Equal(2, ID);
+
+                    Text = new DvText();
+                    TextGetter(ref Text);
+                    Assert.Equal("2", Text.ToString());
+
+                    Assert.True(cursor.MoveNext());
+
+                    ID = 0;
+                    IDGetter(ref ID);
+                    Assert.Equal(3, ID);
+
+                    Text = new DvText();
+                    TextGetter(ref Text);
+                    Assert.Equal("3", Text.ToString());
+
+                    Assert.False(cursor.MoveNext());
+                }
+            }
+        }
+
+        [Fact]
+        public void CanTrain()
+        {
+            var pipeline = new LearningPipeline();
+            var collection = new MemoryCollection<IrisData>(new List<IrisData>() {
+                new IrisData { SepalLength = 1f, SepalWidth = 1f ,PetalLength=0.3f, PetalWidth=5.1f, Label=1},
+                new IrisData { SepalLength = 1f, SepalWidth = 1f ,PetalLength=0.3f, PetalWidth=5.1f, Label=1},
+                new IrisData { SepalLength = 1.2f, SepalWidth = 0.5f ,PetalLength=0.3f, PetalWidth=5.1f, Label=0}
+            });
+
+            pipeline.Add(collection);
+
+            pipeline.Add(new ColumnConcatenator(outputColumn: "Features",
+                "SepalLength", "SepalWidth", "PetalLength", "PetalWidth"));
+
+            pipeline.Add(new StochasticDualCoordinateAscentClassifier());
+            PredictionModel<IrisData, IrisPrediction> model = pipeline.Train<IrisData, IrisPrediction>();
+
+            IrisPrediction prediction = model.Predict(new IrisData()
+            {
+                SepalLength = 3.3f,
+                SepalWidth = 1.6f,
+                PetalLength = 0.2f,
+                PetalWidth = 5.1f,
+            });
+
+        }
+
+        public class Input
+        {
+            [Column("0")]
+            public float Number1;
+
+            [Column("1")]
+            public string String1;
+        }
+
+        public class IrisData
+        {
+            [Column("0")]
+            public float Label;
+
+            [Column("1")]
+            public float SepalLength;
+
+            [Column("2")]
+            public float SepalWidth;
+
+            [Column("3")]
+            public float PetalLength;
+
+            [Column("4")]
+            public float PetalWidth;
+        }
+
+        public class IrisPrediction
+        {
+            [ColumnName("Score")]
+            public float[] PredictedLabels;
+        }
+
+    }
+}