add pipelineitem for Ova

src/Microsoft.ML/Models/CrossValidator.cs

@@ -1,4 +1,8 @@
-using Microsoft.ML.Runtime;
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.ML.Runtime;
 using Microsoft.ML.Runtime.Api;
 using Microsoft.ML.Runtime.Data;
 using Microsoft.ML.Runtime.EntryPoints;

src/Microsoft.ML/Models/OneVersusAll.cs

@@ -0,0 +1,74 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using Microsoft.ML.Runtime;
+using Microsoft.ML.Runtime.Data;
+using Microsoft.ML.Runtime.EntryPoints;
+using static Microsoft.ML.Runtime.EntryPoints.CommonInputs;
+
+namespace Microsoft.ML.Models
+{
+    public sealed partial class OneVersusAll
+    {
+        /// <summary>
+        /// Create OneVersusAll multiclass trainer.
+        /// </summary>
+        /// <param name="trainer">Underlying binary trainer</param>
+        /// <param name="useProbabilities">"Use probabilities (vs. raw outputs) to identify top-score category</param>
+        public static ILearningPipelineItem With(ITrainerInputWithLabel trainer, bool useProbabilities = true)
+        {
+            return new OvaPipelineItem(trainer, useProbabilities);
+        }
+
+        private class OvaPipelineItem : ILearningPipelineItem
+        {
+            private Var<IDataView> _data;
+            private ITrainerInputWithLabel _trainer;
+            private bool _useProbabilities;
+
+            public OvaPipelineItem(ITrainerInputWithLabel trainer, bool useProbabilities)
+            {
+                _trainer = trainer;
+                _useProbabilities = useProbabilities;
+            }
+
+            public ILearningPipelineStep ApplyStep(ILearningPipelineStep previousStep, Experiment experiment)
+            {
+                using (var env = new TlcEnvironment())
+                {
+                    var subgraph = env.CreateExperiment();
+                    subgraph.Add(_trainer);
+                    var ova = new OneVersusAll();
+                    if (previousStep != null)
+                    {
+                        if (!(previousStep is ILearningPipelineDataStep dataStep))
+                        {
+                            throw new InvalidOperationException($"{ nameof(OneVersusAll)} only supports an { nameof(ILearningPipelineDataStep)} as an input.");
+                        }
+
+                        _data = dataStep.Data;
+                        ova.TrainingData = dataStep.Data;
+                        ova.UseProbabilities = _useProbabilities;
+                        ova.Nodes = subgraph;
+                    }
+                    Output output = experiment.Add(ova);
+                    return new OvaPipelineStep(output);
+                }
+            }
+
+            public Var<IDataView> GetInputData() => _data;
+        }
+
+        private class OvaPipelineStep : ILearningPipelinePredictorStep
+        {
+            public OvaPipelineStep(Output output)
+            {
+                Model = output.PredictorModel;
+            }
+
+            public Var<IPredictorModel> Model { get; }
+        }
+    }
+}

src/Microsoft.ML/Models/TrainTestEvaluator.cs

@@ -1,4 +1,8 @@
-using Microsoft.ML.Runtime;
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.ML.Runtime;
 using Microsoft.ML.Runtime.Api;
 using Microsoft.ML.Runtime.Data;
 using Microsoft.ML.Runtime.EntryPoints;
@@ -110,7 +114,7 @@ public TrainTestEvaluatorOutput<TInput, TOutput> TrainTestEvaluate<TInput, TOutp
                 Inputs.Data = firstTransform.GetInputData();
                 Outputs.PredictorModel = null;
                 Outputs.TransformModel = lastTransformModel;
-                var crossValidateOutput = experiment.Add(this);
+                var trainTestNodeOutput = experiment.Add(this);
                 experiment.Compile();
                 foreach (ILearningPipelineLoader loader in loaders)
                     loader.SetInput(environment, experiment);
@@ -124,35 +128,35 @@ public TrainTestEvaluatorOutput<TInput, TOutput> TrainTestEvaluate<TInput, TOutp
                 {
                     trainTestOutput.BinaryClassificationMetrics = BinaryClassificationMetrics.FromMetrics(
                         environment,
-                        experiment.GetOutput(crossValidateOutput.OverallMetrics),
-                        experiment.GetOutput(crossValidateOutput.ConfusionMatrix)).FirstOrDefault();
+                        experiment.GetOutput(trainTestNodeOutput.OverallMetrics),
+                        experiment.GetOutput(trainTestNodeOutput.ConfusionMatrix)).FirstOrDefault();
                 }
                 else if (Kind == MacroUtilsTrainerKinds.SignatureMultiClassClassifierTrainer)
                 {
                     trainTestOutput.ClassificationMetrics = ClassificationMetrics.FromMetrics(
                         environment,
-                        experiment.GetOutput(crossValidateOutput.OverallMetrics),
-                        experiment.GetOutput(crossValidateOutput.ConfusionMatrix)).FirstOrDefault();
+                        experiment.GetOutput(trainTestNodeOutput.OverallMetrics),
+                        experiment.GetOutput(trainTestNodeOutput.ConfusionMatrix)).FirstOrDefault();
                 }
                 else if (Kind == MacroUtilsTrainerKinds.SignatureRegressorTrainer)
                 {
                     trainTestOutput.RegressionMetrics = RegressionMetrics.FromOverallMetrics(
                         environment,
-                        experiment.GetOutput(crossValidateOutput.OverallMetrics)).FirstOrDefault();
+                        experiment.GetOutput(trainTestNodeOutput.OverallMetrics)).FirstOrDefault();
                 }
                 else if (Kind == MacroUtilsTrainerKinds.SignatureClusteringTrainer)
                 {
                     trainTestOutput.ClusterMetrics = ClusterMetrics.FromOverallMetrics(
                         environment,
-                        experiment.GetOutput(crossValidateOutput.OverallMetrics)).FirstOrDefault();
+                        experiment.GetOutput(trainTestNodeOutput.OverallMetrics)).FirstOrDefault();
                 }
                 else
                 {
                     //Implement metrics for ranking, clustering and anomaly detection.
                     throw Contracts.Except($"{Kind.ToString()} is not supported at the moment.");
                 }
 
-                ITransformModel model = experiment.GetOutput(crossValidateOutput.TransformModel);
+                ITransformModel model = experiment.GetOutput(trainTestNodeOutput.TransformModel);
                 BatchPredictionEngine<TInput, TOutput> predictor;
                 using (var memoryStream = new MemoryStream())
                 {

test/Microsoft.ML.Tests/Scenarios/IrisPlantClassificationTests.cs

@@ -136,6 +136,37 @@ public class IrisPrediction
             [ColumnName("Score")]
             public float[] PredictedLabels;
         }
+
+        [Fact]
+        public void TrainOva()
+        {
+            string dataPath = GetDataPath("iris.txt");
+
+            var pipeline = new LearningPipeline(seed: 1, conc: 1);
+            pipeline.Add(new TextLoader(dataPath).CreateFrom<IrisData>(useHeader: false));
+            pipeline.Add(new ColumnConcatenator(outputColumn: "Features",
+                "SepalLength", "SepalWidth", "PetalLength", "PetalWidth"));
+
+            pipeline.Add(OneVersusAll.With(new StochasticDualCoordinateAscentBinaryClassifier()));
+
+            var model = pipeline.Train<IrisData, IrisPrediction>();
+
+            var testData = new TextLoader(dataPath).CreateFrom<IrisData>(useHeader: false);
+            var evaluator = new ClassificationEvaluator();
+            ClassificationMetrics metrics = evaluator.Evaluate(model, testData);
+            CheckMetrics(metrics);
+
+            var trainTest = new TrainTestEvaluator() { Kind = MacroUtilsTrainerKinds.SignatureMultiClassClassifierTrainer }.TrainTestEvaluate<IrisData, IrisPrediction>(pipeline, testData);
+            CheckMetrics(trainTest.ClassificationMetrics);
+        }
+
+        private void CheckMetrics(ClassificationMetrics metrics)
+        {
+            Assert.Equal(.96, metrics.AccuracyMacro);
+            Assert.Equal(.96, metrics.AccuracyMicro, 2);
+            Assert.Equal(.19, metrics.LogLoss, 2);
+            Assert.InRange(metrics.LogLossReduction, 80, 84);
+        }
     }
 }