From f53e98fafa901f2d4b070da6a031ccd629e3176c Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Mon, 6 Jul 2020 18:24:22 -0700 Subject: [PATCH 01/12] Added ONNX export tests for other calibrators --- test/Microsoft.ML.Tests/OnnxConversionTest.cs | 61 +++++++++++++++++-- 1 file changed, 56 insertions(+), 5 deletions(-) diff --git a/test/Microsoft.ML.Tests/OnnxConversionTest.cs b/test/Microsoft.ML.Tests/OnnxConversionTest.cs index c91ce9b633..d03110a79c 100644 --- a/test/Microsoft.ML.Tests/OnnxConversionTest.cs +++ b/test/Microsoft.ML.Tests/OnnxConversionTest.cs @@ -261,8 +261,7 @@ public void TestVectorWhiteningOnnxConversionTest() Done(); } - [Fact] - public void PlattCalibratorOnnxConversionTest() + private (MLContext, IDataView, List>, EstimatorChain) GetEstimatorsForOnnxConversionTests() { var mlContext = new MLContext(seed: 1); string dataPath = GetDataPath("breast-cancer.txt"); @@ -289,6 +288,13 @@ public void PlattCalibratorOnnxConversionTest() var initialPipeline = mlContext.Transforms.ReplaceMissingValues("Features"). Append(mlContext.Transforms.NormalizeMinMax("Features")); + return (mlContext, dataView, estimators, initialPipeline); + } + + [Fact] + public void PlattCalibratorOnnxConversionTest() + { + var (mlContext, dataView, estimators, initialPipeline) = GetEstimatorsForOnnxConversionTests(); foreach (var estimator in estimators) { var pipeline = initialPipeline.Append(estimator).Append(mlContext.BinaryClassification.Calibrators.Platt()); @@ -299,7 +305,52 @@ public void PlattCalibratorOnnxConversionTest() Done(); } - class PlattModelInput + [Fact] + public void FixedPlattCalibratorOnnxConversionTest() + { + var (mlContext, dataView, estimators, initialPipeline) = GetEstimatorsForOnnxConversionTests(); + foreach (var estimator in estimators) + { + // Utilize FixedPlattCalibrator by defining slope and offset + var pipeline = initialPipeline.Append(estimator).Append(mlContext.BinaryClassification.Calibrators.Platt(slope: -1f, offset: -0.05f)); + var onnxFileName = $"{estimator}-WithFixedPlattCalibrator.onnx"; + + TestPipeline(pipeline, dataView, onnxFileName, new ColumnComparison[] { new ColumnComparison("Score", 3), new ColumnComparison("PredictedLabel"), new ColumnComparison("Probability", 3) }); + } + Done(); + } + + [Fact] + [Trait("Category", "SkipInCI")] + public void NaiveCalibratorOnnxConversionTest() + { + var (mlContext, dataView, estimators, initialPipeline) = GetEstimatorsForOnnxConversionTests(); + foreach (var estimator in estimators) + { + var pipeline = initialPipeline.Append(estimator).Append(mlContext.BinaryClassification.Calibrators.Naive()); + var onnxFileName = $"{estimator}-WithNaiveCalibrator.onnx"; + + TestPipeline(pipeline, dataView, onnxFileName, new ColumnComparison[] { new ColumnComparison("Score", 3), new ColumnComparison("PredictedLabel"), new ColumnComparison("Probability", 3) }); + } + Done(); + } + + [Fact] + [Trait("Category", "SkipInCI")] + public void IsotonicCalibratorOnnxConversionTest() + { + var (mlContext, dataView, estimators, initialPipeline) = GetEstimatorsForOnnxConversionTests(); + foreach (var estimator in estimators) + { + var pipeline = initialPipeline.Append(estimator).Append(mlContext.BinaryClassification.Calibrators.Isotonic()); + var onnxFileName = $"{estimator}-WithIsotonicCalibrator.onnx"; + + TestPipeline(pipeline, dataView, onnxFileName, new ColumnComparison[] { new ColumnComparison("Score", 3), new ColumnComparison("PredictedLabel"), new ColumnComparison("Probability", 3) }); + } + Done(); + } + + class ModelInput { public bool Label { get; set; } public float Score { get; set; } @@ -311,11 +362,11 @@ class PlattModelInput2 public float ScoreX { get; set; } } - static IEnumerable PlattGetData() + static IEnumerable PlattGetData() { for (int i = 0; i < 100; i++) { - yield return new PlattModelInput { Score = i, Label = i % 2 == 0 }; + yield return new ModelInput { Score = i, Label = i % 2 == 0 }; } } From b27d278708beed87d0e7131a58f7fe16abad3f3f Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Wed, 8 Jul 2020 01:44:06 -0700 Subject: [PATCH 02/12] Consolitated testing, started ONNX model conversion --- .../Prediction/Calibrator.cs | 73 ++++++++++- test/Microsoft.ML.Tests/OnnxConversionTest.cs | 116 +++++++++++++----- 2 files changed, 155 insertions(+), 34 deletions(-) diff --git a/src/Microsoft.ML.Data/Prediction/Calibrator.cs b/src/Microsoft.ML.Data/Prediction/Calibrator.cs index d1948c0cb4..d4b128f998 100644 --- a/src/Microsoft.ML.Data/Prediction/Calibrator.cs +++ b/src/Microsoft.ML.Data/Prediction/Calibrator.cs @@ -1158,7 +1158,7 @@ ICalibrator ICalibratorTrainer.FinishTraining(IChannel ch) /// /// The naive binning-based calibrator. /// - public sealed class NaiveCalibrator : ICalibrator, ICanSaveInBinaryFormat + public sealed class NaiveCalibrator : ICalibrator, ICanSaveInBinaryFormat, ISingleCanSaveOnnx { internal const string LoaderSignature = "NaiveCaliExec"; internal const string RegistrationName = "NaiveCalibrator"; @@ -1174,6 +1174,12 @@ private static VersionInfo GetVersionInfo() loaderAssemblyName: typeof(NaiveCalibrator).Assembly.FullName); } + /// + /// Bool required by the interface ISingleCanSaveOnnx, returns true if + /// and only if calibrator can be exported in ONNX. + /// + bool ICanSaveOnnx.CanSaveOnnx(OnnxContext ctx) => true; + private readonly IHost _host; /// The bin size. @@ -1280,6 +1286,48 @@ internal static int GetBinIdx(float output, float min, float binSize, int numBin return binIdx; } + bool ISingleCanSaveOnnx.SaveAsOnnx(OnnxContext ctx, string[] outputNames, string featureColumn) + { + _host.CheckValue(ctx, nameof(ctx)); + _host.CheckValue(outputNames, nameof(outputNames)); + _host.Check(Utils.Size(outputNames) == 2); + + const int minimumOpSetVersion = 9; + ctx.CheckOpSetVersion(minimumOpSetVersion, "NaiveCalibrator"); + + var binProbabilities = ctx.AddInitializer(_binProbs, new long[] { _binProbs.Length, 1 }, "binProbabilities"); + + string opType = "Sub"; + var minVar = ctx.AddInitializer((float)(Min), "Min"); + var subNodeOutput = ctx.AddIntermediateVariable(null, "subNodeOutput", true); + var node = ctx.CreateNode(opType, new[] { outputNames[0], minVar }, new[] { subNodeOutput }, ctx.GetNodeName(opType), ""); + + opType = "Div"; + var binSizeVar = ctx.AddInitializer((float)(BinSize), "BinSize"); + var binIndexOutput = ctx.AddIntermediateVariable(NumberDataViewType.Int32, "binIndexOutput", true); + node = ctx.CreateNode(opType, new[] { subNodeOutput, binSizeVar }, new[] { binIndexOutput }, ctx.GetNodeName(opType), ""); + + opType = "Cast"; + var castOutput = ctx.AddIntermediateVariable(BooleanDataViewType.Instance, "CastOutput"); + var castNode = ctx.CreateNode(opType, binIndexOutput, castOutput, ctx.GetNodeName(opType), ""); + var t = InternalDataKindExtensions.ToInternalDataKind(DataKind.Boolean).ToType(); + castNode.AddAttribute("to", t); + + opType = "Not"; + var notOutput = ctx.AddIntermediateVariable(BooleanDataViewType.Instance, "IsBinIndexZero"); + ctx.CreateNode(opType, castOutput, notOutput, ctx.GetNodeName(opType), ""); + + opType = "Cast"; + var castIsBinIndexToInt = ctx.AddIntermediateVariable(NumberDataViewType.Int32, "IsBinIndexAsInt"); + var castIsBinIndexToIntNode = ctx.CreateNode(opType, notOutput, castIsBinIndexToInt, ctx.GetNodeName(opType), ""); + var t1 = InternalDataKindExtensions.ToInternalDataKind(DataKind.Int32).ToType(); + castIsBinIndexToIntNode.AddAttribute("to", t1); + + var numBinsVar = ctx.AddInitializer((int)(BinSize), "NumBins"); + + // TO DO: Complete ONNX conversion. + return true; + } } /// @@ -1879,7 +1927,7 @@ public override ICalibrator CreateCalibrator(IChannel ch) /// [n], if x > [n] /// /// - public sealed class IsotonicCalibrator : ICalibrator, ICanSaveInBinaryFormat + public sealed class IsotonicCalibrator : ICalibrator, ICanSaveInBinaryFormat, ISingleCanSaveOnnx { internal const string LoaderSignature = "PAVCaliExec"; internal const string RegistrationName = "PAVCalibrator"; @@ -1914,6 +1962,12 @@ private static VersionInfo GetVersionInfo() /// public readonly ImmutableArray Values; + /// + /// Bool required by the interface ISingleCanSaveOnnx, returns true if + /// and only if calibrator can be exported in ONNX. + /// + bool ICanSaveOnnx.CanSaveOnnx(OnnxContext ctx) => true; + /// /// Initializes a new instance of . /// @@ -2070,7 +2124,20 @@ private float FindValue(float score) float t = (score - Maxes[pos - 1]) / (Mins[pos] - Maxes[pos - 1]); return Values[pos - 1] + t * (Values[pos] - Values[pos - 1]); } - } + + bool ISingleCanSaveOnnx.SaveAsOnnx(OnnxContext ctx, string[] outputNames, string featureColumn) + { + _host.CheckValue(ctx, nameof(ctx)); + _host.CheckValue(outputNames, nameof(outputNames)); + _host.Check(Utils.Size(outputNames) == 2); + + const int minimumOpSetVersion = 9; + ctx.CheckOpSetVersion(minimumOpSetVersion, "IsotonicCalibrator"); + + // TO DO: Complete ONNX conversion. + return true; + } + } internal static class Calibrate { diff --git a/test/Microsoft.ML.Tests/OnnxConversionTest.cs b/test/Microsoft.ML.Tests/OnnxConversionTest.cs index d03110a79c..bfd61d51c3 100644 --- a/test/Microsoft.ML.Tests/OnnxConversionTest.cs +++ b/test/Microsoft.ML.Tests/OnnxConversionTest.cs @@ -294,6 +294,7 @@ public void TestVectorWhiteningOnnxConversionTest() [Fact] public void PlattCalibratorOnnxConversionTest() { + // Step 1: Test calibrator with binary prediction trainer var (mlContext, dataView, estimators, initialPipeline) = GetEstimatorsForOnnxConversionTests(); foreach (var estimator in estimators) { @@ -302,21 +303,60 @@ public void PlattCalibratorOnnxConversionTest() TestPipeline(pipeline, dataView, onnxFileName, new ColumnComparison[] { new ColumnComparison("Score", 3), new ColumnComparison("PredictedLabel"), new ColumnComparison("Probability", 3) }); } + + // Step 2: Test calibrator without any binary prediction trainer + IDataView dataSoloCalibrator = mlContext.Data.LoadFromEnumerable(GetCalibratorTestData()); + + var pipelineSoloCalibrator = mlContext.BinaryClassification.Calibrators + .Platt(); + var onnxFileNameSoloCalibrator = $"{pipelineSoloCalibrator}-WithPlattCalibrator-Solo.onnx"; + + TestPipeline(pipelineSoloCalibrator, dataSoloCalibrator,onnxFileNameSoloCalibrator, new ColumnComparison[] { new ColumnComparison("Probability", 3) }); + + // Step 3: Test calibrator with a non-default Score column name and without any binary prediction trainer + IDataView dataSoloCalibratorNonStandard = mlContext.Data.LoadFromEnumerable(GetCalibratorTestDataNonStandard()); + + var pipelineSoloCalibratorNonStandard = mlContext.BinaryClassification.Calibrators + .Platt(scoreColumnName: "ScoreX"); + var onnxFileNameSoloCalibratorNonStandard = $"{pipelineSoloCalibratorNonStandard}-WithPlattCalibrator-Solo-NonStandard.onnx"; + + TestPipeline(pipelineSoloCalibratorNonStandard, dataSoloCalibratorNonStandard, onnxFileNameSoloCalibratorNonStandard, new ColumnComparison[] { new ColumnComparison("Probability", 3) }); + Done(); } [Fact] public void FixedPlattCalibratorOnnxConversionTest() { + // Below, FixedPlattCalibrator is utilized by defining slope and offset in Platt's constructor with sample values. + // Step 1: Test calibrator with binary prediction trainer var (mlContext, dataView, estimators, initialPipeline) = GetEstimatorsForOnnxConversionTests(); foreach (var estimator in estimators) { - // Utilize FixedPlattCalibrator by defining slope and offset var pipeline = initialPipeline.Append(estimator).Append(mlContext.BinaryClassification.Calibrators.Platt(slope: -1f, offset: -0.05f)); var onnxFileName = $"{estimator}-WithFixedPlattCalibrator.onnx"; TestPipeline(pipeline, dataView, onnxFileName, new ColumnComparison[] { new ColumnComparison("Score", 3), new ColumnComparison("PredictedLabel"), new ColumnComparison("Probability", 3) }); } + + // Step 2: Test calibrator without any binary prediction trainer + IDataView dataSoloCalibrator = mlContext.Data.LoadFromEnumerable(GetCalibratorTestData()); + + var pipelineSoloCalibrator = mlContext.BinaryClassification.Calibrators + .Platt(slope: -1f, offset: -0.05f); + var onnxFileNameSoloCalibrator = $"{pipelineSoloCalibrator}-WithFixedPlattCalibrator-Solo.onnx"; + + TestPipeline(pipelineSoloCalibrator, dataSoloCalibrator, onnxFileNameSoloCalibrator, new ColumnComparison[] { new ColumnComparison("Probability", 3) }); + + // Step 3: Test calibrator with a non-default Score column name and without any binary prediction trainer + IDataView dataSoloCalibratorNonStandard = mlContext.Data.LoadFromEnumerable(GetCalibratorTestDataNonStandard()); + + var pipelineSoloCalibratorNonStandard = mlContext.BinaryClassification.Calibrators + .Platt(scoreColumnName: "ScoreX", slope: -1f, offset: -0.05f); + var onnxFileNameSoloCalibratorNonStandard = $"{pipelineSoloCalibratorNonStandard}-WithFixedPlattCalibrator-Solo-NonStandard.onnx"; + + TestPipeline(pipelineSoloCalibratorNonStandard, dataSoloCalibratorNonStandard, onnxFileNameSoloCalibratorNonStandard, new ColumnComparison[] { new ColumnComparison("Probability", 3) }); + Done(); } @@ -324,6 +364,7 @@ public void FixedPlattCalibratorOnnxConversionTest() [Trait("Category", "SkipInCI")] public void NaiveCalibratorOnnxConversionTest() { + // Step 1: Test calibrator with binary prediction trainer var (mlContext, dataView, estimators, initialPipeline) = GetEstimatorsForOnnxConversionTests(); foreach (var estimator in estimators) { @@ -332,6 +373,25 @@ public void NaiveCalibratorOnnxConversionTest() TestPipeline(pipeline, dataView, onnxFileName, new ColumnComparison[] { new ColumnComparison("Score", 3), new ColumnComparison("PredictedLabel"), new ColumnComparison("Probability", 3) }); } + + // Step 2: Test calibrator without any binary prediction trainer + IDataView dataSoloCalibrator = mlContext.Data.LoadFromEnumerable(GetCalibratorTestData()); + + var pipelineSoloCalibrator = mlContext.BinaryClassification.Calibrators + .Naive(); + var onnxFileNameSoloCalibrator = $"{pipelineSoloCalibrator}-WithNaiveCalibrator-Solo.onnx"; + + TestPipeline(pipelineSoloCalibrator, dataSoloCalibrator, onnxFileNameSoloCalibrator, new ColumnComparison[] { new ColumnComparison("Probability", 3) }); + + // Step 3: Test calibrator with a non-default Score column name and without any binary prediction trainer + IDataView dataSoloCalibratorNonStandard = mlContext.Data.LoadFromEnumerable(GetCalibratorTestDataNonStandard()); + + var pipelineSoloCalibratorNonStandard = mlContext.BinaryClassification.Calibrators + .Naive(scoreColumnName: "ScoreX"); + var onnxFileNameSoloCalibratorNonStandard = $"{pipelineSoloCalibratorNonStandard}-WithNaiveCalibrator-Solo-NonStandard.onnx"; + + TestPipeline(pipelineSoloCalibratorNonStandard, dataSoloCalibratorNonStandard, onnxFileNameSoloCalibratorNonStandard, new ColumnComparison[] { new ColumnComparison("Probability", 3) }); + Done(); } @@ -339,6 +399,7 @@ public void NaiveCalibratorOnnxConversionTest() [Trait("Category", "SkipInCI")] public void IsotonicCalibratorOnnxConversionTest() { + // Step 1: Test calibrator with binary prediction trainer var (mlContext, dataView, estimators, initialPipeline) = GetEstimatorsForOnnxConversionTests(); foreach (var estimator in estimators) { @@ -347,6 +408,25 @@ public void IsotonicCalibratorOnnxConversionTest() TestPipeline(pipeline, dataView, onnxFileName, new ColumnComparison[] { new ColumnComparison("Score", 3), new ColumnComparison("PredictedLabel"), new ColumnComparison("Probability", 3) }); } + + // Step 2: Test calibrator without any binary prediction trainer + IDataView dataSoloCalibrator = mlContext.Data.LoadFromEnumerable(GetCalibratorTestData()); + + var pipelineSoloCalibrator = mlContext.BinaryClassification.Calibrators + .Isotonic(); + var onnxFileNameSoloCalibrator = $"{pipelineSoloCalibrator}-WithIsotonicCalibrator-Solo.onnx"; + + TestPipeline(pipelineSoloCalibrator, dataSoloCalibrator, onnxFileNameSoloCalibrator, new ColumnComparison[] { new ColumnComparison("Probability", 3) }); + + // Step 3: Test calibrator with a non-default Score column name and without any binary prediction trainer + IDataView dataSoloCalibratorNonStandard = mlContext.Data.LoadFromEnumerable(GetCalibratorTestDataNonStandard()); + + var pipelineSoloCalibratorNonStandard = mlContext.BinaryClassification.Calibrators + .Isotonic(scoreColumnName: "ScoreX"); + var onnxFileNameSoloCalibratorNonStandard = $"{pipelineSoloCalibratorNonStandard}-WithIsotonicCalibrator-Solo-NonStandard.onnx"; + + TestPipeline(pipelineSoloCalibratorNonStandard, dataSoloCalibratorNonStandard, onnxFileNameSoloCalibratorNonStandard, new ColumnComparison[] { new ColumnComparison("Probability", 3) }); + Done(); } @@ -356,13 +436,13 @@ class ModelInput public float Score { get; set; } } - class PlattModelInput2 + class ModelInputNonStandard { public bool Label { get; set; } public float ScoreX { get; set; } } - static IEnumerable PlattGetData() + static IEnumerable GetCalibratorTestData() { for (int i = 0; i < 100; i++) { @@ -370,40 +450,14 @@ static IEnumerable PlattGetData() } } - static IEnumerable PlattGetData2() + static IEnumerable GetCalibratorTestDataNonStandard() { for (int i = 0; i < 100; i++) { - yield return new PlattModelInput2 { ScoreX = i, Label = i % 2 == 0 }; + yield return new ModelInputNonStandard { ScoreX = i, Label = i % 2 == 0 }; } } - [Fact] - public void PlattCalibratorOnnxConversionTest2() - { - // Test PlattCalibrator without any binary prediction trainer - var mlContext = new MLContext(seed: 0); - - IDataView data = mlContext.Data.LoadFromEnumerable(PlattGetData()); - - var pipeline = mlContext.BinaryClassification.Calibrators - .Platt(); - var onnxFileName = $"{pipeline}.onnx"; - - TestPipeline(pipeline, data, onnxFileName, new ColumnComparison[] { new ColumnComparison("Probability", 3) }); - - // Test PlattCalibrator with a non-default Score column name, and without any binary prediction trainer - IDataView data2 = mlContext.Data.LoadFromEnumerable(PlattGetData2()); - - var pipeline2 = mlContext.BinaryClassification.Calibrators - .Platt(scoreColumnName: "ScoreX"); - var onnxFileName2 = $"{pipeline2}.onnx"; - - TestPipeline(pipeline2, data2, onnxFileName2, new ColumnComparison[] { new ColumnComparison("Probability", 3) }); - - Done(); - } - [Fact] public void TextNormalizingOnnxConversionTest() { From 0a9a115ae297c9ff371fb504d5e218d555465a79 Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Wed, 8 Jul 2020 22:12:22 -0700 Subject: [PATCH 03/12] Added ONNX export support for NaiveCalibrator --- .../Prediction/Calibrator.cs | 39 ++++++++----------- test/Microsoft.ML.Tests/OnnxConversionTest.cs | 16 ++++---- 2 files changed, 25 insertions(+), 30 deletions(-) diff --git a/src/Microsoft.ML.Data/Prediction/Calibrator.cs b/src/Microsoft.ML.Data/Prediction/Calibrator.cs index d4b128f998..57546a32b5 100644 --- a/src/Microsoft.ML.Data/Prediction/Calibrator.cs +++ b/src/Microsoft.ML.Data/Prediction/Calibrator.cs @@ -1295,37 +1295,32 @@ bool ISingleCanSaveOnnx.SaveAsOnnx(OnnxContext ctx, string[] outputNames, string const int minimumOpSetVersion = 9; ctx.CheckOpSetVersion(minimumOpSetVersion, "NaiveCalibrator"); - var binProbabilities = ctx.AddInitializer(_binProbs, new long[] { _binProbs.Length, 1 }, "binProbabilities"); - string opType = "Sub"; - var minVar = ctx.AddInitializer((float)(Min), "Min"); - var subNodeOutput = ctx.AddIntermediateVariable(null, "subNodeOutput", true); + var minVar = ctx.AddInitializer(Min, "Min"); + var subNodeOutput = ctx.AddIntermediateVariable(NumberDataViewType.Single, "subNodeOutput"); var node = ctx.CreateNode(opType, new[] { outputNames[0], minVar }, new[] { subNodeOutput }, ctx.GetNodeName(opType), ""); opType = "Div"; - var binSizeVar = ctx.AddInitializer((float)(BinSize), "BinSize"); - var binIndexOutput = ctx.AddIntermediateVariable(NumberDataViewType.Int32, "binIndexOutput", true); - node = ctx.CreateNode(opType, new[] { subNodeOutput, binSizeVar }, new[] { binIndexOutput }, ctx.GetNodeName(opType), ""); + var binSizeVar = ctx.AddInitializer(BinSize, "BinSize"); + var divNodeOutput = ctx.AddIntermediateVariable(NumberDataViewType.Single, "binIndexOutput"); + node = ctx.CreateNode(opType, new[] { subNodeOutput, binSizeVar }, new[] { divNodeOutput }, ctx.GetNodeName(opType), ""); opType = "Cast"; - var castOutput = ctx.AddIntermediateVariable(BooleanDataViewType.Instance, "CastOutput"); - var castNode = ctx.CreateNode(opType, binIndexOutput, castOutput, ctx.GetNodeName(opType), ""); - var t = InternalDataKindExtensions.ToInternalDataKind(DataKind.Boolean).ToType(); - castNode.AddAttribute("to", t); - - opType = "Not"; - var notOutput = ctx.AddIntermediateVariable(BooleanDataViewType.Instance, "IsBinIndexZero"); - ctx.CreateNode(opType, castOutput, notOutput, ctx.GetNodeName(opType), ""); + var castOutput = ctx.AddIntermediateVariable(NumberDataViewType.Int64, "CastOutput"); + node = ctx.CreateNode(opType, divNodeOutput, castOutput, ctx.GetNodeName(opType), ""); + var toTypeInt = InternalDataKindExtensions.ToInternalDataKind(DataKind.Int64).ToType(); + node.AddAttribute("to", toTypeInt); - opType = "Cast"; - var castIsBinIndexToInt = ctx.AddIntermediateVariable(NumberDataViewType.Int32, "IsBinIndexAsInt"); - var castIsBinIndexToIntNode = ctx.CreateNode(opType, notOutput, castIsBinIndexToInt, ctx.GetNodeName(opType), ""); - var t1 = InternalDataKindExtensions.ToInternalDataKind(DataKind.Int32).ToType(); - castIsBinIndexToIntNode.AddAttribute("to", t1); + opType = "Clip"; + var zeroVar = ctx.AddInitializer(0, "Zero"); + var numBinsMinusOneVar = ctx.AddInitializer(_binProbs.Length-1, "NumBinsMinusOne"); + var binIndexOutput = ctx.AddIntermediateVariable(NumberDataViewType.Int64, "binIndexOutput"); + node = ctx.CreateNode(opType, new[] { castOutput, zeroVar, numBinsMinusOneVar }, new[] { binIndexOutput }, ctx.GetNodeName(opType), ""); - var numBinsVar = ctx.AddInitializer((int)(BinSize), "NumBins"); + opType = "GatherElements"; + var binProbabilitiesVar = ctx.AddInitializer(_binProbs, new long[] { _binProbs.Length, 1 }, "binProbabilities"); + node = ctx.CreateNode(opType, new[] { binProbabilitiesVar, binIndexOutput }, new[] { outputNames[1] }, ctx.GetNodeName(opType), ""); - // TO DO: Complete ONNX conversion. return true; } } diff --git a/test/Microsoft.ML.Tests/OnnxConversionTest.cs b/test/Microsoft.ML.Tests/OnnxConversionTest.cs index bfd61d51c3..996d46b683 100644 --- a/test/Microsoft.ML.Tests/OnnxConversionTest.cs +++ b/test/Microsoft.ML.Tests/OnnxConversionTest.cs @@ -309,7 +309,7 @@ public void PlattCalibratorOnnxConversionTest() var pipelineSoloCalibrator = mlContext.BinaryClassification.Calibrators .Platt(); - var onnxFileNameSoloCalibrator = $"{pipelineSoloCalibrator}-WithPlattCalibrator-Solo.onnx"; + var onnxFileNameSoloCalibrator = $"{pipelineSoloCalibrator}-SoloCalibrator.onnx"; TestPipeline(pipelineSoloCalibrator, dataSoloCalibrator,onnxFileNameSoloCalibrator, new ColumnComparison[] { new ColumnComparison("Probability", 3) }); @@ -318,7 +318,7 @@ public void PlattCalibratorOnnxConversionTest() var pipelineSoloCalibratorNonStandard = mlContext.BinaryClassification.Calibrators .Platt(scoreColumnName: "ScoreX"); - var onnxFileNameSoloCalibratorNonStandard = $"{pipelineSoloCalibratorNonStandard}-WithPlattCalibrator-Solo-NonStandard.onnx"; + var onnxFileNameSoloCalibratorNonStandard = $"{pipelineSoloCalibratorNonStandard}-SoloCalibrator-NonStandard.onnx"; TestPipeline(pipelineSoloCalibratorNonStandard, dataSoloCalibratorNonStandard, onnxFileNameSoloCalibratorNonStandard, new ColumnComparison[] { new ColumnComparison("Probability", 3) }); @@ -344,7 +344,7 @@ public void FixedPlattCalibratorOnnxConversionTest() var pipelineSoloCalibrator = mlContext.BinaryClassification.Calibrators .Platt(slope: -1f, offset: -0.05f); - var onnxFileNameSoloCalibrator = $"{pipelineSoloCalibrator}-WithFixedPlattCalibrator-Solo.onnx"; + var onnxFileNameSoloCalibrator = $"{pipelineSoloCalibrator}-SoloCalibrator.onnx"; TestPipeline(pipelineSoloCalibrator, dataSoloCalibrator, onnxFileNameSoloCalibrator, new ColumnComparison[] { new ColumnComparison("Probability", 3) }); @@ -353,7 +353,7 @@ public void FixedPlattCalibratorOnnxConversionTest() var pipelineSoloCalibratorNonStandard = mlContext.BinaryClassification.Calibrators .Platt(scoreColumnName: "ScoreX", slope: -1f, offset: -0.05f); - var onnxFileNameSoloCalibratorNonStandard = $"{pipelineSoloCalibratorNonStandard}-WithFixedPlattCalibrator-Solo-NonStandard.onnx"; + var onnxFileNameSoloCalibratorNonStandard = $"{pipelineSoloCalibratorNonStandard}-SoloCalibrator-NonStandard.onnx"; TestPipeline(pipelineSoloCalibratorNonStandard, dataSoloCalibratorNonStandard, onnxFileNameSoloCalibratorNonStandard, new ColumnComparison[] { new ColumnComparison("Probability", 3) }); @@ -379,7 +379,7 @@ public void NaiveCalibratorOnnxConversionTest() var pipelineSoloCalibrator = mlContext.BinaryClassification.Calibrators .Naive(); - var onnxFileNameSoloCalibrator = $"{pipelineSoloCalibrator}-WithNaiveCalibrator-Solo.onnx"; + var onnxFileNameSoloCalibrator = $"{pipelineSoloCalibrator}-SoloCalibrator.onnx"; TestPipeline(pipelineSoloCalibrator, dataSoloCalibrator, onnxFileNameSoloCalibrator, new ColumnComparison[] { new ColumnComparison("Probability", 3) }); @@ -388,7 +388,7 @@ public void NaiveCalibratorOnnxConversionTest() var pipelineSoloCalibratorNonStandard = mlContext.BinaryClassification.Calibrators .Naive(scoreColumnName: "ScoreX"); - var onnxFileNameSoloCalibratorNonStandard = $"{pipelineSoloCalibratorNonStandard}-WithNaiveCalibrator-Solo-NonStandard.onnx"; + var onnxFileNameSoloCalibratorNonStandard = $"{pipelineSoloCalibratorNonStandard}-SoloCalibrator-NonStandard.onnx"; TestPipeline(pipelineSoloCalibratorNonStandard, dataSoloCalibratorNonStandard, onnxFileNameSoloCalibratorNonStandard, new ColumnComparison[] { new ColumnComparison("Probability", 3) }); @@ -414,7 +414,7 @@ public void IsotonicCalibratorOnnxConversionTest() var pipelineSoloCalibrator = mlContext.BinaryClassification.Calibrators .Isotonic(); - var onnxFileNameSoloCalibrator = $"{pipelineSoloCalibrator}-WithIsotonicCalibrator-Solo.onnx"; + var onnxFileNameSoloCalibrator = $"{pipelineSoloCalibrator}-SoloCalibrator.onnx"; TestPipeline(pipelineSoloCalibrator, dataSoloCalibrator, onnxFileNameSoloCalibrator, new ColumnComparison[] { new ColumnComparison("Probability", 3) }); @@ -423,7 +423,7 @@ public void IsotonicCalibratorOnnxConversionTest() var pipelineSoloCalibratorNonStandard = mlContext.BinaryClassification.Calibrators .Isotonic(scoreColumnName: "ScoreX"); - var onnxFileNameSoloCalibratorNonStandard = $"{pipelineSoloCalibratorNonStandard}-WithIsotonicCalibrator-Solo-NonStandard.onnx"; + var onnxFileNameSoloCalibratorNonStandard = $"{pipelineSoloCalibratorNonStandard}-SoloCalibrator-NonStandard.onnx"; TestPipeline(pipelineSoloCalibratorNonStandard, dataSoloCalibratorNonStandard, onnxFileNameSoloCalibratorNonStandard, new ColumnComparison[] { new ColumnComparison("Probability", 3) }); From ef769d6e3e77393813f1ed72d9b3d72e756e50e3 Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Wed, 8 Jul 2020 23:21:20 -0700 Subject: [PATCH 04/12] Enable NaiveCalibratorOnnxConversionTest --- test/Microsoft.ML.Tests/OnnxConversionTest.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/test/Microsoft.ML.Tests/OnnxConversionTest.cs b/test/Microsoft.ML.Tests/OnnxConversionTest.cs index 996d46b683..c2895eab84 100644 --- a/test/Microsoft.ML.Tests/OnnxConversionTest.cs +++ b/test/Microsoft.ML.Tests/OnnxConversionTest.cs @@ -361,7 +361,6 @@ public void FixedPlattCalibratorOnnxConversionTest() } [Fact] - [Trait("Category", "SkipInCI")] public void NaiveCalibratorOnnxConversionTest() { // Step 1: Test calibrator with binary prediction trainer From df4af9889d548e13c8e817bb92c219b79ce8757f Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Thu, 9 Jul 2020 01:13:53 -0700 Subject: [PATCH 05/12] Work on Isotonic Calibrator ONNX export support --- .../Prediction/Calibrator.cs | 103 +++++++++++++++++- 1 file changed, 100 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.ML.Data/Prediction/Calibrator.cs b/src/Microsoft.ML.Data/Prediction/Calibrator.cs index 57546a32b5..1bad779819 100644 --- a/src/Microsoft.ML.Data/Prediction/Calibrator.cs +++ b/src/Microsoft.ML.Data/Prediction/Calibrator.cs @@ -1306,7 +1306,7 @@ bool ISingleCanSaveOnnx.SaveAsOnnx(OnnxContext ctx, string[] outputNames, string node = ctx.CreateNode(opType, new[] { subNodeOutput, binSizeVar }, new[] { divNodeOutput }, ctx.GetNodeName(opType), ""); opType = "Cast"; - var castOutput = ctx.AddIntermediateVariable(NumberDataViewType.Int64, "CastOutput"); + var castOutput = ctx.AddIntermediateVariable(NumberDataViewType.Int64, "castOutput"); node = ctx.CreateNode(opType, divNodeOutput, castOutput, ctx.GetNodeName(opType), ""); var toTypeInt = InternalDataKindExtensions.ToInternalDataKind(DataKind.Int64).ToType(); node.AddAttribute("to", toTypeInt); @@ -1318,7 +1318,7 @@ bool ISingleCanSaveOnnx.SaveAsOnnx(OnnxContext ctx, string[] outputNames, string node = ctx.CreateNode(opType, new[] { castOutput, zeroVar, numBinsMinusOneVar }, new[] { binIndexOutput }, ctx.GetNodeName(opType), ""); opType = "GatherElements"; - var binProbabilitiesVar = ctx.AddInitializer(_binProbs, new long[] { _binProbs.Length, 1 }, "binProbabilities"); + var binProbabilitiesVar = ctx.AddInitializer(_binProbs, new long[] { _binProbs.Length, 1 }, "BinProbabilities"); node = ctx.CreateNode(opType, new[] { binProbabilitiesVar, binIndexOutput }, new[] { outputNames[1] }, ctx.GetNodeName(opType), ""); return true; @@ -2129,7 +2129,104 @@ bool ISingleCanSaveOnnx.SaveAsOnnx(OnnxContext ctx, string[] outputNames, string const int minimumOpSetVersion = 9; ctx.CheckOpSetVersion(minimumOpSetVersion, "IsotonicCalibrator"); - // TO DO: Complete ONNX conversion. + var minsLengthVar = ctx.AddInitializer(Mins.Length, "MinsLength"); + var minToReturnVar = ctx.AddInitializer((float)1e-15, "MinToReturn"); + var maxToReturnVar = ctx.AddInitializer((float)(1 - 1e-15), "MaxToReturn"); + var minsVar = ctx.AddInitializer(Mins, new long[] { Mins.Length, 1 }, "Mins"); + var maxesVar = ctx.AddInitializer(Maxes, new long[] { Maxes.Length, 1 }, "Maxes"); + var valuesVar = ctx.AddInitializer(Values, new long[] { Values.Length, 1 }, "Values"); + + //The isotonic regression optimization problem is defined by: + // min(sum(w_i, (y[i] - y_[i])^2)) + // subject to y_[i] <= y_[j] whenever X[i] <= X[j] (non-decreasing) + // and min(y_) = y_min, max(y_) = y_max + // where: + // *y[i] are inputs(real numbers) + // *y_[i] are fitted + // *X specifies the order.If X is non-decreasing then y_ is non-decreasing. + // *w[i] are optional strictly positive weights(default to 1.0) + + string opType = "PlaceHolder"; + var node = ctx.CreateNode(opType, new[] { "PlaceHolder" }, new[] { "PlaceHolder" }, ctx.GetNodeName(opType), ""); + + // Goal: Given output, calculate prob + + // 1st, implement if-then-else logic for (p = Mins.Length): + // If p == 0, Return 0 + // If score < Mins[0], Return prob = Values[0] + // If score > Maxes[p-1], Return prob = Values[0] + // Else, continue + + // To-do + + // 2nd, calculate pos, which is the index of the given score in the sorted Maxes + + // To-do + + // 3rd, calculate (score - Maxes[pos - 1]) / (Mins[pos] - Maxes[pos - 1]) + // score: outputNames[0] + + // placeholders for pos and pos-1, will be calculated above + var posVar = ctx.AddInitializer(1, "Pos"); + var posMinusOneVar = ctx.AddInitializer(0, "PosMinusOne"); + + opType = "GatherElements"; + var maxesPosMinusOneOutput = ctx.AddIntermediateVariable(NumberDataViewType.Single, "maxesPosMinusOneOutput"); + node = ctx.CreateNode(opType, new[] { maxesVar, posMinusOneVar }, new[] { maxesPosMinusOneOutput }, ctx.GetNodeName(opType), ""); + + opType = "GatherElements"; + var minsPosOutput = ctx.AddIntermediateVariable(NumberDataViewType.Single, "maxesPosMinusOneOutput"); + node = ctx.CreateNode(opType, new[] { minsVar, posVar }, new[] { minsPosOutput }, ctx.GetNodeName(opType), ""); + + opType = "Sub"; + var subNode1Output = ctx.AddIntermediateVariable(NumberDataViewType.Single, "subNodeUpperOutput"); + node = ctx.CreateNode(opType, new[] { outputNames[0], maxesPosMinusOneOutput }, new[] { subNode1Output }, ctx.GetNodeName(opType), ""); + + opType = "Sub"; + var subNode2Output = ctx.AddIntermediateVariable(NumberDataViewType.Single, "subNodeLowerOutput"); + node = ctx.CreateNode(opType, new[] { minsPosOutput, maxesPosMinusOneOutput }, new[] { subNode2Output }, ctx.GetNodeName(opType), ""); + + opType = "Div"; + var tNodeOutput = ctx.AddIntermediateVariable(NumberDataViewType.Single, "divNodeOutput"); + node = ctx.CreateNode(opType, new[] { subNode1Output, subNode2Output }, new[] { tNodeOutput }, ctx.GetNodeName(opType), ""); + + // 4th, if score >= Mins[pos], then prob = Values[pos] + + // To-do + + // 5th, calculate and return prob = Values[pos - 1] + t * (Values[pos] - Values[pos - 1]); + + opType = "GatherElements"; + var valuesPosMinusOneOutput = ctx.AddIntermediateVariable(NumberDataViewType.Single, "valuesPosMinusOneOutput"); + node = ctx.CreateNode(opType, new[] { valuesVar, posMinusOneVar }, new[] { valuesPosMinusOneOutput }, ctx.GetNodeName(opType), ""); + + opType = "GatherElements"; + var valuesPosOutput = ctx.AddIntermediateVariable(NumberDataViewType.Single, "valuesPosOutput"); + node = ctx.CreateNode(opType, new[] { valuesVar, posVar }, new[] { valuesPosOutput }, ctx.GetNodeName(opType), ""); + + opType = "Sub"; + var subNode3Output = ctx.AddIntermediateVariable(NumberDataViewType.Single, "subNodeLowerOutput"); + node = ctx.CreateNode(opType, new[] { valuesPosOutput, valuesPosMinusOneOutput }, new[] { subNode3Output }, ctx.GetNodeName(opType), ""); + + opType = "Mul"; + var mulNodeOutput = ctx.AddIntermediateVariable(NumberDataViewType.Single, "subNodeLowerOutput"); + node = ctx.CreateNode(opType, new[] { tNodeOutput, subNode3Output }, new[] { mulNodeOutput }, ctx.GetNodeName(opType), ""); + + opType = "Add"; + var probabilityNodeOutput = ctx.AddIntermediateVariable(NumberDataViewType.Single, "subNodeLowerOutput"); + node = ctx.CreateNode(opType, new[] { valuesPosMinusOneOutput, mulNodeOutput }, new[] { probabilityNodeOutput }, ctx.GetNodeName(opType), ""); + + // 6th, continue with logic + // if (prob < MinToReturn) + // return MinToReturn; + // if (prob > MaxToReturn) + // return MaxToReturn; + // return prob + + // To-do + opType = "Clip"; + node = ctx.CreateNode(opType, new[] { probabilityNodeOutput, probabilityNodeOutput, probabilityNodeOutput }, new[] { outputNames[1] }, ctx.GetNodeName(opType), ""); + return true; } } From 8211621b14b921c9530ceb460cf88c5ac03b0755 Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Thu, 9 Jul 2020 11:47:04 -0700 Subject: [PATCH 06/12] Removed Isotonic work from this PR --- .../Prediction/Calibrator.cs | 118 +----------------- test/Microsoft.ML.Tests/OnnxConversionTest.cs | 35 ------ 2 files changed, 1 insertion(+), 152 deletions(-) diff --git a/src/Microsoft.ML.Data/Prediction/Calibrator.cs b/src/Microsoft.ML.Data/Prediction/Calibrator.cs index 1bad779819..2138b3e2a4 100644 --- a/src/Microsoft.ML.Data/Prediction/Calibrator.cs +++ b/src/Microsoft.ML.Data/Prediction/Calibrator.cs @@ -1922,7 +1922,7 @@ public override ICalibrator CreateCalibrator(IChannel ch) /// [n], if x > [n] /// /// - public sealed class IsotonicCalibrator : ICalibrator, ICanSaveInBinaryFormat, ISingleCanSaveOnnx + public sealed class IsotonicCalibrator : ICalibrator, ICanSaveInBinaryFormat { internal const string LoaderSignature = "PAVCaliExec"; internal const string RegistrationName = "PAVCalibrator"; @@ -1957,12 +1957,6 @@ private static VersionInfo GetVersionInfo() /// public readonly ImmutableArray Values; - /// - /// Bool required by the interface ISingleCanSaveOnnx, returns true if - /// and only if calibrator can be exported in ONNX. - /// - bool ICanSaveOnnx.CanSaveOnnx(OnnxContext ctx) => true; - /// /// Initializes a new instance of . /// @@ -2119,116 +2113,6 @@ private float FindValue(float score) float t = (score - Maxes[pos - 1]) / (Mins[pos] - Maxes[pos - 1]); return Values[pos - 1] + t * (Values[pos] - Values[pos - 1]); } - - bool ISingleCanSaveOnnx.SaveAsOnnx(OnnxContext ctx, string[] outputNames, string featureColumn) - { - _host.CheckValue(ctx, nameof(ctx)); - _host.CheckValue(outputNames, nameof(outputNames)); - _host.Check(Utils.Size(outputNames) == 2); - - const int minimumOpSetVersion = 9; - ctx.CheckOpSetVersion(minimumOpSetVersion, "IsotonicCalibrator"); - - var minsLengthVar = ctx.AddInitializer(Mins.Length, "MinsLength"); - var minToReturnVar = ctx.AddInitializer((float)1e-15, "MinToReturn"); - var maxToReturnVar = ctx.AddInitializer((float)(1 - 1e-15), "MaxToReturn"); - var minsVar = ctx.AddInitializer(Mins, new long[] { Mins.Length, 1 }, "Mins"); - var maxesVar = ctx.AddInitializer(Maxes, new long[] { Maxes.Length, 1 }, "Maxes"); - var valuesVar = ctx.AddInitializer(Values, new long[] { Values.Length, 1 }, "Values"); - - //The isotonic regression optimization problem is defined by: - // min(sum(w_i, (y[i] - y_[i])^2)) - // subject to y_[i] <= y_[j] whenever X[i] <= X[j] (non-decreasing) - // and min(y_) = y_min, max(y_) = y_max - // where: - // *y[i] are inputs(real numbers) - // *y_[i] are fitted - // *X specifies the order.If X is non-decreasing then y_ is non-decreasing. - // *w[i] are optional strictly positive weights(default to 1.0) - - string opType = "PlaceHolder"; - var node = ctx.CreateNode(opType, new[] { "PlaceHolder" }, new[] { "PlaceHolder" }, ctx.GetNodeName(opType), ""); - - // Goal: Given output, calculate prob - - // 1st, implement if-then-else logic for (p = Mins.Length): - // If p == 0, Return 0 - // If score < Mins[0], Return prob = Values[0] - // If score > Maxes[p-1], Return prob = Values[0] - // Else, continue - - // To-do - - // 2nd, calculate pos, which is the index of the given score in the sorted Maxes - - // To-do - - // 3rd, calculate (score - Maxes[pos - 1]) / (Mins[pos] - Maxes[pos - 1]) - // score: outputNames[0] - - // placeholders for pos and pos-1, will be calculated above - var posVar = ctx.AddInitializer(1, "Pos"); - var posMinusOneVar = ctx.AddInitializer(0, "PosMinusOne"); - - opType = "GatherElements"; - var maxesPosMinusOneOutput = ctx.AddIntermediateVariable(NumberDataViewType.Single, "maxesPosMinusOneOutput"); - node = ctx.CreateNode(opType, new[] { maxesVar, posMinusOneVar }, new[] { maxesPosMinusOneOutput }, ctx.GetNodeName(opType), ""); - - opType = "GatherElements"; - var minsPosOutput = ctx.AddIntermediateVariable(NumberDataViewType.Single, "maxesPosMinusOneOutput"); - node = ctx.CreateNode(opType, new[] { minsVar, posVar }, new[] { minsPosOutput }, ctx.GetNodeName(opType), ""); - - opType = "Sub"; - var subNode1Output = ctx.AddIntermediateVariable(NumberDataViewType.Single, "subNodeUpperOutput"); - node = ctx.CreateNode(opType, new[] { outputNames[0], maxesPosMinusOneOutput }, new[] { subNode1Output }, ctx.GetNodeName(opType), ""); - - opType = "Sub"; - var subNode2Output = ctx.AddIntermediateVariable(NumberDataViewType.Single, "subNodeLowerOutput"); - node = ctx.CreateNode(opType, new[] { minsPosOutput, maxesPosMinusOneOutput }, new[] { subNode2Output }, ctx.GetNodeName(opType), ""); - - opType = "Div"; - var tNodeOutput = ctx.AddIntermediateVariable(NumberDataViewType.Single, "divNodeOutput"); - node = ctx.CreateNode(opType, new[] { subNode1Output, subNode2Output }, new[] { tNodeOutput }, ctx.GetNodeName(opType), ""); - - // 4th, if score >= Mins[pos], then prob = Values[pos] - - // To-do - - // 5th, calculate and return prob = Values[pos - 1] + t * (Values[pos] - Values[pos - 1]); - - opType = "GatherElements"; - var valuesPosMinusOneOutput = ctx.AddIntermediateVariable(NumberDataViewType.Single, "valuesPosMinusOneOutput"); - node = ctx.CreateNode(opType, new[] { valuesVar, posMinusOneVar }, new[] { valuesPosMinusOneOutput }, ctx.GetNodeName(opType), ""); - - opType = "GatherElements"; - var valuesPosOutput = ctx.AddIntermediateVariable(NumberDataViewType.Single, "valuesPosOutput"); - node = ctx.CreateNode(opType, new[] { valuesVar, posVar }, new[] { valuesPosOutput }, ctx.GetNodeName(opType), ""); - - opType = "Sub"; - var subNode3Output = ctx.AddIntermediateVariable(NumberDataViewType.Single, "subNodeLowerOutput"); - node = ctx.CreateNode(opType, new[] { valuesPosOutput, valuesPosMinusOneOutput }, new[] { subNode3Output }, ctx.GetNodeName(opType), ""); - - opType = "Mul"; - var mulNodeOutput = ctx.AddIntermediateVariable(NumberDataViewType.Single, "subNodeLowerOutput"); - node = ctx.CreateNode(opType, new[] { tNodeOutput, subNode3Output }, new[] { mulNodeOutput }, ctx.GetNodeName(opType), ""); - - opType = "Add"; - var probabilityNodeOutput = ctx.AddIntermediateVariable(NumberDataViewType.Single, "subNodeLowerOutput"); - node = ctx.CreateNode(opType, new[] { valuesPosMinusOneOutput, mulNodeOutput }, new[] { probabilityNodeOutput }, ctx.GetNodeName(opType), ""); - - // 6th, continue with logic - // if (prob < MinToReturn) - // return MinToReturn; - // if (prob > MaxToReturn) - // return MaxToReturn; - // return prob - - // To-do - opType = "Clip"; - node = ctx.CreateNode(opType, new[] { probabilityNodeOutput, probabilityNodeOutput, probabilityNodeOutput }, new[] { outputNames[1] }, ctx.GetNodeName(opType), ""); - - return true; - } } internal static class Calibrate diff --git a/test/Microsoft.ML.Tests/OnnxConversionTest.cs b/test/Microsoft.ML.Tests/OnnxConversionTest.cs index c2895eab84..626a925a6c 100644 --- a/test/Microsoft.ML.Tests/OnnxConversionTest.cs +++ b/test/Microsoft.ML.Tests/OnnxConversionTest.cs @@ -394,41 +394,6 @@ public void NaiveCalibratorOnnxConversionTest() Done(); } - [Fact] - [Trait("Category", "SkipInCI")] - public void IsotonicCalibratorOnnxConversionTest() - { - // Step 1: Test calibrator with binary prediction trainer - var (mlContext, dataView, estimators, initialPipeline) = GetEstimatorsForOnnxConversionTests(); - foreach (var estimator in estimators) - { - var pipeline = initialPipeline.Append(estimator).Append(mlContext.BinaryClassification.Calibrators.Isotonic()); - var onnxFileName = $"{estimator}-WithIsotonicCalibrator.onnx"; - - TestPipeline(pipeline, dataView, onnxFileName, new ColumnComparison[] { new ColumnComparison("Score", 3), new ColumnComparison("PredictedLabel"), new ColumnComparison("Probability", 3) }); - } - - // Step 2: Test calibrator without any binary prediction trainer - IDataView dataSoloCalibrator = mlContext.Data.LoadFromEnumerable(GetCalibratorTestData()); - - var pipelineSoloCalibrator = mlContext.BinaryClassification.Calibrators - .Isotonic(); - var onnxFileNameSoloCalibrator = $"{pipelineSoloCalibrator}-SoloCalibrator.onnx"; - - TestPipeline(pipelineSoloCalibrator, dataSoloCalibrator, onnxFileNameSoloCalibrator, new ColumnComparison[] { new ColumnComparison("Probability", 3) }); - - // Step 3: Test calibrator with a non-default Score column name and without any binary prediction trainer - IDataView dataSoloCalibratorNonStandard = mlContext.Data.LoadFromEnumerable(GetCalibratorTestDataNonStandard()); - - var pipelineSoloCalibratorNonStandard = mlContext.BinaryClassification.Calibrators - .Isotonic(scoreColumnName: "ScoreX"); - var onnxFileNameSoloCalibratorNonStandard = $"{pipelineSoloCalibratorNonStandard}-SoloCalibrator-NonStandard.onnx"; - - TestPipeline(pipelineSoloCalibratorNonStandard, dataSoloCalibratorNonStandard, onnxFileNameSoloCalibratorNonStandard, new ColumnComparison[] { new ColumnComparison("Probability", 3) }); - - Done(); - } - class ModelInput { public bool Label { get; set; } From a245b433f5ee4a6207a6a63ecc710e1340d85bef Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Thu, 9 Jul 2020 13:27:20 -0700 Subject: [PATCH 07/12] Nit correct spacing --- src/Microsoft.ML.Data/Prediction/Calibrator.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ML.Data/Prediction/Calibrator.cs b/src/Microsoft.ML.Data/Prediction/Calibrator.cs index 2138b3e2a4..c2615127a6 100644 --- a/src/Microsoft.ML.Data/Prediction/Calibrator.cs +++ b/src/Microsoft.ML.Data/Prediction/Calibrator.cs @@ -2113,7 +2113,7 @@ private float FindValue(float score) float t = (score - Maxes[pos - 1]) / (Mins[pos] - Maxes[pos - 1]); return Values[pos - 1] + t * (Values[pos] - Values[pos - 1]); } - } + } internal static class Calibrate { From 711a9a5acb2b599a8efacf76c47696396756bc1e Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Thu, 9 Jul 2020 21:04:42 -0700 Subject: [PATCH 08/12] Nit renaming to CalibratorInput(NonStandard) --- test/Microsoft.ML.Tests/OnnxConversionTest.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/Microsoft.ML.Tests/OnnxConversionTest.cs b/test/Microsoft.ML.Tests/OnnxConversionTest.cs index 626a925a6c..7bbf656997 100644 --- a/test/Microsoft.ML.Tests/OnnxConversionTest.cs +++ b/test/Microsoft.ML.Tests/OnnxConversionTest.cs @@ -394,31 +394,31 @@ public void NaiveCalibratorOnnxConversionTest() Done(); } - class ModelInput + class CalibratorInput { public bool Label { get; set; } public float Score { get; set; } } - class ModelInputNonStandard + class CalibratorInputNonStandard { public bool Label { get; set; } public float ScoreX { get; set; } } - static IEnumerable GetCalibratorTestData() + static IEnumerable GetCalibratorTestData() { for (int i = 0; i < 100; i++) { - yield return new ModelInput { Score = i, Label = i % 2 == 0 }; + yield return new CalibratorInput { Score = i, Label = i % 2 == 0 }; } } - static IEnumerable GetCalibratorTestDataNonStandard() + static IEnumerable GetCalibratorTestDataNonStandard() { for (int i = 0; i < 100; i++) { - yield return new ModelInputNonStandard { ScoreX = i, Label = i % 2 == 0 }; + yield return new CalibratorInputNonStandard { ScoreX = i, Label = i % 2 == 0 }; } } From 5acc93bb44580f79a1c00a469507880fffae5bef Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Fri, 10 Jul 2020 16:52:14 -0700 Subject: [PATCH 09/12] Organized tests --- .../Prediction/Calibrator.cs | 4 +- test/Microsoft.ML.Tests/OnnxConversionTest.cs | 100 +++++------------- 2 files changed, 28 insertions(+), 76 deletions(-) diff --git a/src/Microsoft.ML.Data/Prediction/Calibrator.cs b/src/Microsoft.ML.Data/Prediction/Calibrator.cs index c2615127a6..375e2a8fb1 100644 --- a/src/Microsoft.ML.Data/Prediction/Calibrator.cs +++ b/src/Microsoft.ML.Data/Prediction/Calibrator.cs @@ -1291,6 +1291,8 @@ bool ISingleCanSaveOnnx.SaveAsOnnx(OnnxContext ctx, string[] outputNames, string _host.CheckValue(ctx, nameof(ctx)); _host.CheckValue(outputNames, nameof(outputNames)); _host.Check(Utils.Size(outputNames) == 2); + // outputNames[0] refers to the name of the Score column, which is the input of this graph + // outputNames[1] refers to the name of the Probability column, which is the final output of this graph const int minimumOpSetVersion = 9; ctx.CheckOpSetVersion(minimumOpSetVersion, "NaiveCalibrator"); @@ -1308,7 +1310,7 @@ bool ISingleCanSaveOnnx.SaveAsOnnx(OnnxContext ctx, string[] outputNames, string opType = "Cast"; var castOutput = ctx.AddIntermediateVariable(NumberDataViewType.Int64, "castOutput"); node = ctx.CreateNode(opType, divNodeOutput, castOutput, ctx.GetNodeName(opType), ""); - var toTypeInt = InternalDataKindExtensions.ToInternalDataKind(DataKind.Int64).ToType(); + var toTypeInt = typeof(long); node.AddAttribute("to", toTypeInt); opType = "Clip"; diff --git a/test/Microsoft.ML.Tests/OnnxConversionTest.cs b/test/Microsoft.ML.Tests/OnnxConversionTest.cs index 7bbf656997..2f9c98cf0c 100644 --- a/test/Microsoft.ML.Tests/OnnxConversionTest.cs +++ b/test/Microsoft.ML.Tests/OnnxConversionTest.cs @@ -291,107 +291,57 @@ public void TestVectorWhiteningOnnxConversionTest() return (mlContext, dataView, estimators, initialPipeline); } - [Fact] - public void PlattCalibratorOnnxConversionTest() + private void CommonCalibratorOnnxConversionTest(MLContext mlContext, IDataView dataView, + List> estimators, EstimatorChain initialPipeline, + IEstimator calibrator, IEstimator calibratorNonStandard) { // Step 1: Test calibrator with binary prediction trainer - var (mlContext, dataView, estimators, initialPipeline) = GetEstimatorsForOnnxConversionTests(); foreach (var estimator in estimators) { - var pipeline = initialPipeline.Append(estimator).Append(mlContext.BinaryClassification.Calibrators.Platt()); - var onnxFileName = $"{estimator}-WithPlattCalibrator.onnx"; - - TestPipeline(pipeline, dataView, onnxFileName, new ColumnComparison[] { new ColumnComparison("Score", 3), new ColumnComparison("PredictedLabel"), new ColumnComparison("Probability", 3) }); + var pipelineEstimators = initialPipeline.Append(estimator).Append(calibrator); + var onnxFileName = $"{estimator}-With{calibrator}.onnx"; + TestPipeline(pipelineEstimators, dataView, onnxFileName, new ColumnComparison[] { new ColumnComparison("Score", 3), new ColumnComparison("PredictedLabel"), new ColumnComparison("Probability", 3) }); } // Step 2: Test calibrator without any binary prediction trainer IDataView dataSoloCalibrator = mlContext.Data.LoadFromEnumerable(GetCalibratorTestData()); - - var pipelineSoloCalibrator = mlContext.BinaryClassification.Calibrators - .Platt(); - var onnxFileNameSoloCalibrator = $"{pipelineSoloCalibrator}-SoloCalibrator.onnx"; - - TestPipeline(pipelineSoloCalibrator, dataSoloCalibrator,onnxFileNameSoloCalibrator, new ColumnComparison[] { new ColumnComparison("Probability", 3) }); + var onnxFileNameSoloCalibrator = $"{calibrator}-SoloCalibrator.onnx"; + TestPipeline(calibrator, dataSoloCalibrator, onnxFileNameSoloCalibrator, new ColumnComparison[] { new ColumnComparison("Probability", 3) }); // Step 3: Test calibrator with a non-default Score column name and without any binary prediction trainer IDataView dataSoloCalibratorNonStandard = mlContext.Data.LoadFromEnumerable(GetCalibratorTestDataNonStandard()); - - var pipelineSoloCalibratorNonStandard = mlContext.BinaryClassification.Calibrators - .Platt(scoreColumnName: "ScoreX"); - var onnxFileNameSoloCalibratorNonStandard = $"{pipelineSoloCalibratorNonStandard}-SoloCalibrator-NonStandard.onnx"; - - TestPipeline(pipelineSoloCalibratorNonStandard, dataSoloCalibratorNonStandard, onnxFileNameSoloCalibratorNonStandard, new ColumnComparison[] { new ColumnComparison("Probability", 3) }); + var onnxFileNameSoloCalibratorNonStandard = $"{calibratorNonStandard}-SoloCalibrator-NonStandard.onnx"; + TestPipeline(calibratorNonStandard, dataSoloCalibratorNonStandard, onnxFileNameSoloCalibratorNonStandard, new ColumnComparison[] { new ColumnComparison("Probability", 3) }); Done(); } + [Fact] + public void PlattCalibratorOnnxConversionTest() + { + var (mlContext, dataView, estimators, initialPipeline) = GetEstimatorsForOnnxConversionTests(); + CommonCalibratorOnnxConversionTest(mlContext, dataView, estimators, initialPipeline, + mlContext.BinaryClassification.Calibrators.Platt(), + mlContext.BinaryClassification.Calibrators.Platt(scoreColumnName: "ScoreX")); + } + [Fact] public void FixedPlattCalibratorOnnxConversionTest() { // Below, FixedPlattCalibrator is utilized by defining slope and offset in Platt's constructor with sample values. - // Step 1: Test calibrator with binary prediction trainer var (mlContext, dataView, estimators, initialPipeline) = GetEstimatorsForOnnxConversionTests(); - foreach (var estimator in estimators) - { - var pipeline = initialPipeline.Append(estimator).Append(mlContext.BinaryClassification.Calibrators.Platt(slope: -1f, offset: -0.05f)); - var onnxFileName = $"{estimator}-WithFixedPlattCalibrator.onnx"; - - TestPipeline(pipeline, dataView, onnxFileName, new ColumnComparison[] { new ColumnComparison("Score", 3), new ColumnComparison("PredictedLabel"), new ColumnComparison("Probability", 3) }); - } - - // Step 2: Test calibrator without any binary prediction trainer - IDataView dataSoloCalibrator = mlContext.Data.LoadFromEnumerable(GetCalibratorTestData()); - - var pipelineSoloCalibrator = mlContext.BinaryClassification.Calibrators - .Platt(slope: -1f, offset: -0.05f); - var onnxFileNameSoloCalibrator = $"{pipelineSoloCalibrator}-SoloCalibrator.onnx"; - - TestPipeline(pipelineSoloCalibrator, dataSoloCalibrator, onnxFileNameSoloCalibrator, new ColumnComparison[] { new ColumnComparison("Probability", 3) }); - - // Step 3: Test calibrator with a non-default Score column name and without any binary prediction trainer - IDataView dataSoloCalibratorNonStandard = mlContext.Data.LoadFromEnumerable(GetCalibratorTestDataNonStandard()); - - var pipelineSoloCalibratorNonStandard = mlContext.BinaryClassification.Calibrators - .Platt(scoreColumnName: "ScoreX", slope: -1f, offset: -0.05f); - var onnxFileNameSoloCalibratorNonStandard = $"{pipelineSoloCalibratorNonStandard}-SoloCalibrator-NonStandard.onnx"; - - TestPipeline(pipelineSoloCalibratorNonStandard, dataSoloCalibratorNonStandard, onnxFileNameSoloCalibratorNonStandard, new ColumnComparison[] { new ColumnComparison("Probability", 3) }); - - Done(); + CommonCalibratorOnnxConversionTest(mlContext, dataView, estimators, initialPipeline, + mlContext.BinaryClassification.Calibrators.Platt(slope: -1f, offset: -0.05f), + mlContext.BinaryClassification.Calibrators.Platt(slope: -1f, offset: -0.05f, scoreColumnName: "ScoreX")); } [Fact] public void NaiveCalibratorOnnxConversionTest() { - // Step 1: Test calibrator with binary prediction trainer var (mlContext, dataView, estimators, initialPipeline) = GetEstimatorsForOnnxConversionTests(); - foreach (var estimator in estimators) - { - var pipeline = initialPipeline.Append(estimator).Append(mlContext.BinaryClassification.Calibrators.Naive()); - var onnxFileName = $"{estimator}-WithNaiveCalibrator.onnx"; - - TestPipeline(pipeline, dataView, onnxFileName, new ColumnComparison[] { new ColumnComparison("Score", 3), new ColumnComparison("PredictedLabel"), new ColumnComparison("Probability", 3) }); - } - - // Step 2: Test calibrator without any binary prediction trainer - IDataView dataSoloCalibrator = mlContext.Data.LoadFromEnumerable(GetCalibratorTestData()); - - var pipelineSoloCalibrator = mlContext.BinaryClassification.Calibrators - .Naive(); - var onnxFileNameSoloCalibrator = $"{pipelineSoloCalibrator}-SoloCalibrator.onnx"; - - TestPipeline(pipelineSoloCalibrator, dataSoloCalibrator, onnxFileNameSoloCalibrator, new ColumnComparison[] { new ColumnComparison("Probability", 3) }); - - // Step 3: Test calibrator with a non-default Score column name and without any binary prediction trainer - IDataView dataSoloCalibratorNonStandard = mlContext.Data.LoadFromEnumerable(GetCalibratorTestDataNonStandard()); - - var pipelineSoloCalibratorNonStandard = mlContext.BinaryClassification.Calibrators - .Naive(scoreColumnName: "ScoreX"); - var onnxFileNameSoloCalibratorNonStandard = $"{pipelineSoloCalibratorNonStandard}-SoloCalibrator-NonStandard.onnx"; - - TestPipeline(pipelineSoloCalibratorNonStandard, dataSoloCalibratorNonStandard, onnxFileNameSoloCalibratorNonStandard, new ColumnComparison[] { new ColumnComparison("Probability", 3) }); - - Done(); + CommonCalibratorOnnxConversionTest(mlContext, dataView, estimators, initialPipeline, + mlContext.BinaryClassification.Calibrators.Naive(), + mlContext.BinaryClassification.Calibrators.Naive(scoreColumnName: "ScoreX")); } class CalibratorInput From 5d2e930f2ed9e4124bb5b3f128d6275b32cdabc2 Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Fri, 10 Jul 2020 16:56:27 -0700 Subject: [PATCH 10/12] Nit --- test/Microsoft.ML.Tests/OnnxConversionTest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.ML.Tests/OnnxConversionTest.cs b/test/Microsoft.ML.Tests/OnnxConversionTest.cs index 2f9c98cf0c..56e0635e7a 100644 --- a/test/Microsoft.ML.Tests/OnnxConversionTest.cs +++ b/test/Microsoft.ML.Tests/OnnxConversionTest.cs @@ -299,7 +299,7 @@ private void CommonCalibratorOnnxConversionTest(MLContext mlContext, IDataView d foreach (var estimator in estimators) { var pipelineEstimators = initialPipeline.Append(estimator).Append(calibrator); - var onnxFileName = $"{estimator}-With{calibrator}.onnx"; + var onnxFileName = $"{estimator}-With-{calibrator}.onnx"; TestPipeline(pipelineEstimators, dataView, onnxFileName, new ColumnComparison[] { new ColumnComparison("Score", 3), new ColumnComparison("PredictedLabel"), new ColumnComparison("Probability", 3) }); } From 3a8889bedfb4b43d40cf89c99b40a6fd459735df Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Fri, 10 Jul 2020 17:15:14 -0700 Subject: [PATCH 11/12] Clean-up initialization of vars for CommonCalibratorOnnxConversionTest --- test/Microsoft.ML.Tests/OnnxConversionTest.cs | 25 ++++++++----------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/test/Microsoft.ML.Tests/OnnxConversionTest.cs b/test/Microsoft.ML.Tests/OnnxConversionTest.cs index 56e0635e7a..f5bfd00c68 100644 --- a/test/Microsoft.ML.Tests/OnnxConversionTest.cs +++ b/test/Microsoft.ML.Tests/OnnxConversionTest.cs @@ -291,10 +291,11 @@ public void TestVectorWhiteningOnnxConversionTest() return (mlContext, dataView, estimators, initialPipeline); } - private void CommonCalibratorOnnxConversionTest(MLContext mlContext, IDataView dataView, - List> estimators, EstimatorChain initialPipeline, - IEstimator calibrator, IEstimator calibratorNonStandard) + private void CommonCalibratorOnnxConversionTest(IEstimator calibrator, IEstimator calibratorNonStandard) { + // Initialize variables needed for the ONNX conversion test + var (mlContext, dataView, estimators, initialPipeline) = GetEstimatorsForOnnxConversionTests(); + // Step 1: Test calibrator with binary prediction trainer foreach (var estimator in estimators) { @@ -319,29 +320,23 @@ private void CommonCalibratorOnnxConversionTest(MLContext mlContext, IDataView d [Fact] public void PlattCalibratorOnnxConversionTest() { - var (mlContext, dataView, estimators, initialPipeline) = GetEstimatorsForOnnxConversionTests(); - CommonCalibratorOnnxConversionTest(mlContext, dataView, estimators, initialPipeline, - mlContext.BinaryClassification.Calibrators.Platt(), - mlContext.BinaryClassification.Calibrators.Platt(scoreColumnName: "ScoreX")); + CommonCalibratorOnnxConversionTest(ML.BinaryClassification.Calibrators.Platt(), + ML.BinaryClassification.Calibrators.Platt(scoreColumnName: "ScoreX")); } [Fact] public void FixedPlattCalibratorOnnxConversionTest() { // Below, FixedPlattCalibrator is utilized by defining slope and offset in Platt's constructor with sample values. - var (mlContext, dataView, estimators, initialPipeline) = GetEstimatorsForOnnxConversionTests(); - CommonCalibratorOnnxConversionTest(mlContext, dataView, estimators, initialPipeline, - mlContext.BinaryClassification.Calibrators.Platt(slope: -1f, offset: -0.05f), - mlContext.BinaryClassification.Calibrators.Platt(slope: -1f, offset: -0.05f, scoreColumnName: "ScoreX")); + CommonCalibratorOnnxConversionTest(ML.BinaryClassification.Calibrators.Platt(slope: -1f, offset: -0.05f), + ML.BinaryClassification.Calibrators.Platt(slope: -1f, offset: -0.05f, scoreColumnName: "ScoreX")); } [Fact] public void NaiveCalibratorOnnxConversionTest() { - var (mlContext, dataView, estimators, initialPipeline) = GetEstimatorsForOnnxConversionTests(); - CommonCalibratorOnnxConversionTest(mlContext, dataView, estimators, initialPipeline, - mlContext.BinaryClassification.Calibrators.Naive(), - mlContext.BinaryClassification.Calibrators.Naive(scoreColumnName: "ScoreX")); + CommonCalibratorOnnxConversionTest(ML.BinaryClassification.Calibrators.Naive(), + ML.BinaryClassification.Calibrators.Naive(scoreColumnName: "ScoreX")); } class CalibratorInput From 25ce2daeaf6ff893bfc35b61794778c11d9b542d Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Fri, 10 Jul 2020 17:25:48 -0700 Subject: [PATCH 12/12] Removed MLContexts for ML --- test/Microsoft.ML.Tests/OnnxConversionTest.cs | 41 +++++++++---------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/test/Microsoft.ML.Tests/OnnxConversionTest.cs b/test/Microsoft.ML.Tests/OnnxConversionTest.cs index f5bfd00c68..f9c094a70b 100644 --- a/test/Microsoft.ML.Tests/OnnxConversionTest.cs +++ b/test/Microsoft.ML.Tests/OnnxConversionTest.cs @@ -261,40 +261,39 @@ public void TestVectorWhiteningOnnxConversionTest() Done(); } - private (MLContext, IDataView, List>, EstimatorChain) GetEstimatorsForOnnxConversionTests() + private (IDataView, List>, EstimatorChain) GetEstimatorsForOnnxConversionTests() { - var mlContext = new MLContext(seed: 1); string dataPath = GetDataPath("breast-cancer.txt"); // Now read the file (remember though, readers are lazy, so the actual reading will happen when the data is accessed). - var dataView = mlContext.Data.LoadFromTextFile(dataPath, separatorChar: '\t', hasHeader: true); + var dataView = ML.Data.LoadFromTextFile(dataPath, separatorChar: '\t', hasHeader: true); List> estimators = new List>() { - mlContext.BinaryClassification.Trainers.AveragedPerceptron(), - mlContext.BinaryClassification.Trainers.FastForest(), - mlContext.BinaryClassification.Trainers.FastTree(), - mlContext.BinaryClassification.Trainers.LbfgsLogisticRegression(), - mlContext.BinaryClassification.Trainers.LinearSvm(), - mlContext.BinaryClassification.Trainers.Prior(), - mlContext.BinaryClassification.Trainers.SdcaLogisticRegression(), - mlContext.BinaryClassification.Trainers.SdcaNonCalibrated(), - mlContext.BinaryClassification.Trainers.SgdCalibrated(), - mlContext.BinaryClassification.Trainers.SgdNonCalibrated(), - mlContext.BinaryClassification.Trainers.SymbolicSgdLogisticRegression(), + ML.BinaryClassification.Trainers.AveragedPerceptron(), + ML.BinaryClassification.Trainers.FastForest(), + ML.BinaryClassification.Trainers.FastTree(), + ML.BinaryClassification.Trainers.LbfgsLogisticRegression(), + ML.BinaryClassification.Trainers.LinearSvm(), + ML.BinaryClassification.Trainers.Prior(), + ML.BinaryClassification.Trainers.SdcaLogisticRegression(), + ML.BinaryClassification.Trainers.SdcaNonCalibrated(), + ML.BinaryClassification.Trainers.SgdCalibrated(), + ML.BinaryClassification.Trainers.SgdNonCalibrated(), + ML.BinaryClassification.Trainers.SymbolicSgdLogisticRegression(), }; if (Environment.Is64BitProcess) { - estimators.Add(mlContext.BinaryClassification.Trainers.LightGbm()); + estimators.Add(ML.BinaryClassification.Trainers.LightGbm()); } - var initialPipeline = mlContext.Transforms.ReplaceMissingValues("Features"). - Append(mlContext.Transforms.NormalizeMinMax("Features")); - return (mlContext, dataView, estimators, initialPipeline); + var initialPipeline = ML.Transforms.ReplaceMissingValues("Features"). + Append(ML.Transforms.NormalizeMinMax("Features")); + return (dataView, estimators, initialPipeline); } private void CommonCalibratorOnnxConversionTest(IEstimator calibrator, IEstimator calibratorNonStandard) { // Initialize variables needed for the ONNX conversion test - var (mlContext, dataView, estimators, initialPipeline) = GetEstimatorsForOnnxConversionTests(); + var (dataView, estimators, initialPipeline) = GetEstimatorsForOnnxConversionTests(); // Step 1: Test calibrator with binary prediction trainer foreach (var estimator in estimators) @@ -305,12 +304,12 @@ private void CommonCalibratorOnnxConversionTest(IEstimator calibra } // Step 2: Test calibrator without any binary prediction trainer - IDataView dataSoloCalibrator = mlContext.Data.LoadFromEnumerable(GetCalibratorTestData()); + IDataView dataSoloCalibrator = ML.Data.LoadFromEnumerable(GetCalibratorTestData()); var onnxFileNameSoloCalibrator = $"{calibrator}-SoloCalibrator.onnx"; TestPipeline(calibrator, dataSoloCalibrator, onnxFileNameSoloCalibrator, new ColumnComparison[] { new ColumnComparison("Probability", 3) }); // Step 3: Test calibrator with a non-default Score column name and without any binary prediction trainer - IDataView dataSoloCalibratorNonStandard = mlContext.Data.LoadFromEnumerable(GetCalibratorTestDataNonStandard()); + IDataView dataSoloCalibratorNonStandard = ML.Data.LoadFromEnumerable(GetCalibratorTestDataNonStandard()); var onnxFileNameSoloCalibratorNonStandard = $"{calibratorNonStandard}-SoloCalibrator-NonStandard.onnx"; TestPipeline(calibratorNonStandard, dataSoloCalibratorNonStandard, onnxFileNameSoloCalibratorNonStandard, new ColumnComparison[] { new ColumnComparison("Probability", 3) });