Enable TensorFlowTransform to work with pre-trained models that are not frozen

src/Microsoft.ML.TensorFlow/TensorFlow/Tensorflow.cs

@@ -1116,7 +1116,7 @@ public IEnumerable<DeviceAttributes> ListDevices(TFStatus status = null)
         /// here: https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/README.md
         /// </para>
         /// </remarks>
-        public TFSession FromSavedModel(TFSessionOptions sessionOptions, TFBuffer runOptions, string exportDir, string[] tags, TFGraph graph, TFBuffer metaGraphDef, TFStatus status = null)
+        public static TFSession FromSavedModel(TFSessionOptions sessionOptions, TFBuffer runOptions, string exportDir, string[] tags, TFGraph graph, TFBuffer metaGraphDef, TFStatus status = null)
         {
             if (graph == null)
                 throw new ArgumentNullException(nameof(graph));

src/Microsoft.ML/CSharpApi.cs

@@ -15785,9 +15785,14 @@ public sealed partial class TensorFlowScorer : Microsoft.ML.Runtime.EntryPoints.
 
 
             /// <summary>
-            /// This is the frozen protobuf model file. Please see https://www.tensorflow.org/mobile/prepare_models for more details.
+            /// TensorFlow model used by the transform. Please see https://www.tensorflow.org/mobile/prepare_models for more details.
             /// </summary>
-            public string ModelFile { get; set; }
+            public string Model { get; set; }
+
+            /// <summary>
+            /// Indicator for frozen models
+            /// </summary>
+            public bool IsFrozen { get; set; } = true;
 
             /// <summary>
             /// The names of the model inputs

test/BaselineOutput/Common/EntryPoints/core_manifest.json

@@ -21716,14 +21716,31 @@
       "ShortName": "TFTransform",
       "Inputs": [
         {
-          "Name": "ModelFile",
+          "Name": "Model",
           "Type": "String",
-          "Desc": "This is the frozen protobuf model file. Please see https://www.tensorflow.org/mobile/prepare_models for more details.",
+          "Desc": "TensorFlow model used by the transform. Please see https://www.tensorflow.org/mobile/prepare_models for more details.",
+          "Required": true,
+          "SortOrder": 0.0,
+          "IsNullable": false
+        },
+        {
+          "Name": "IsFrozen",
+          "Type": "Bool",
+          "Desc": "Indicator for frozen models",
           "Aliases": [
-            "model"
+            "frozen"
           ],
+          "Required": false,
+          "SortOrder": 1.0,
+          "IsNullable": false,
+          "Default": true
+        },
+        {
+          "Name": "Data",
+          "Type": "DataView",
+          "Desc": "Input dataset",
           "Required": true,
-          "SortOrder": 0.0,
+          "SortOrder": 1.0,
           "IsNullable": false
         },
         {
@@ -21737,15 +21754,7 @@
             "inputs"
           ],
           "Required": true,
-          "SortOrder": 1.0,
-          "IsNullable": false
-        },
-        {
-          "Name": "Data",
-          "Type": "DataView",
-          "Desc": "Input dataset",
-          "Required": true,
-          "SortOrder": 1.0,
+          "SortOrder": 2.0,
           "IsNullable": false
         },
         {
@@ -21759,7 +21768,7 @@
             "outputs"
           ],
           "Required": true,
-          "SortOrder": 2.0,
+          "SortOrder": 3.0,
           "IsNullable": false
         }
       ],

test/Microsoft.ML.Core.Tests/UnitTests/TestCSharpApi.cs

@@ -977,9 +977,9 @@ public void TestTensorFlowEntryPoint()
                 var tfTransformInput = new ML.Transforms.TensorFlowScorer
                 {
                     Data = importOutput.Data,
+                    Model = "mnist_model/frozen_saved_model.pb",
                     InputColumns = new[] { "Placeholder" },
                     OutputColumns = new[] { "Softmax" },
-                    ModelFile = "mnist_model/frozen_saved_model.pb"
                 };
                 var tfTransformOutput = experiment.Add(tfTransformInput);
 

test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs

@@ -3741,7 +3741,7 @@ public void EntryPointTensorFlowTransform()
                 new[]
                 {
                     @"'InputColumns': [ 'Placeholder' ],
-                      'ModelFile': 'mnist_model/frozen_saved_model.pb',
+                      'Model': 'mnist_model/frozen_saved_model.pb',
                       'OutputColumns': [ 'Softmax' ]"
                 });
         }

test/Microsoft.ML.Tests/Scenarios/PipelineApi/TrainSaveModelAndPredict.cs

@@ -3,8 +3,11 @@
 // See the LICENSE file in the project root for more information.
 
 using Microsoft.ML.Data;
+using Microsoft.ML.Scenarios;
 using Microsoft.ML.Trainers;
 using Microsoft.ML.Transforms;
+using Microsoft.ML.Transforms.TensorFlow;
+using System.IO;
 using Xunit;
 
 namespace Microsoft.ML.Tests.Scenarios.PipelineApi
@@ -38,5 +41,81 @@ public async void TrainSaveModelAndPredict()
             Assert.True(singlePrediction.Sentiment);
 
         }
+
+        [Fact]
+        public async void TensorFlowTransformTrainSaveModelAndPredict()
+        {
+            var imageHeight = 32;
+            var imageWidth = 32;
+            var model_location = "cifar_saved_model";
+            var dataFile = GetDataPath("images/images.tsv");
+            var imageFolder = Path.GetDirectoryName(dataFile);
+
+            var pipeline = new LearningPipeline(seed: 1);
+            pipeline.Add(new TextLoader(dataFile).CreateFrom<CifarData>(useHeader: false));
+            pipeline.Add(new ImageLoader(("ImagePath", "ImageReal"))
+            {
+                ImageFolder = imageFolder
+            });
+
+            pipeline.Add(new ImageResizer(("ImageReal", "ImageCropped"))
+            {
+                ImageHeight = imageHeight,
+                ImageWidth = imageWidth,
+                Resizing = ImageResizerTransformResizingKind.IsoCrop
+            });
+
+            pipeline.Add(new ImagePixelExtractor(("ImageCropped", "Input"))
+            {
+                UseAlpha = false,
+                InterleaveArgb = true
+            });
+
+            pipeline.Add(new TensorFlowScorer()
+            {
+                Model = model_location,
+                IsFrozen = false,
+                InputColumns = new[] { "Input" },
+                OutputColumns = new[] { "Output" },
+            });
+
+            pipeline.Add(new ColumnConcatenator(outputColumn: "Features", "Output"));
+            pipeline.Add(new TextToKeyConverter("Label"));
+            pipeline.Add(new StochasticDualCoordinateAscentClassifier());
+
+             TensorFlowUtils.Initialize();
+            var model = pipeline.Train<CifarData, CifarPrediction>();
+
+            var modelName = "tf_TrainSaveAndPredictdModel.zip";
+            DeleteOutputPath(modelName);
+            await model.WriteAsync(modelName);
+
+            var loadedModel = await PredictionModel.ReadAsync<CifarData, CifarPrediction>(modelName);
+
+            string[] scoreLabels;
+            loadedModel.TryGetScoreLabelNames(out scoreLabels);
+
+            Assert.NotNull(scoreLabels);
+            Assert.Equal(3, scoreLabels.Length);
+            Assert.Equal("banana", scoreLabels[0]);
+            Assert.Equal("hotdog", scoreLabels[1]);
+            Assert.Equal("tomato", scoreLabels[2]);
+
+            CifarPrediction prediction = loadedModel.Predict(new CifarData()
+            {
+                ImagePath = GetDataPath("images/banana.jpg")
+            });
+            Assert.Equal(1, prediction.PredictedLabels[0], 2);
+            Assert.Equal(0, prediction.PredictedLabels[1], 2);
+            Assert.Equal(0, prediction.PredictedLabels[2], 2);
+
+            prediction = loadedModel.Predict(new CifarData()
+            {
+                ImagePath = GetDataPath("images/hotdog.jpg")
+            });
+            Assert.Equal(0, prediction.PredictedLabels[0], 2);
+            Assert.Equal(1, prediction.PredictedLabels[1], 2);
+            Assert.Equal(0, prediction.PredictedLabels[2], 2);
+        }
     }
 }

test/Microsoft.ML.Tests/Scenarios/TensorflowTests.cs

@@ -49,7 +49,7 @@ public void TensorFlowTransformCifarLearningPipelineTest()
 
             pipeline.Add(new TensorFlowScorer()
             {
-                ModelFile = model_location,
+                Model = model_location,
                 InputColumns = new[] { "Input" },
                 OutputColumns = new[] { "Output" }
             });

test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs

@@ -41,7 +41,7 @@ public void TensorFlowTransformMatrixMultiplicationTest()
                                          b = new[] { 3.0f, 3.0f,
                                                      3.0f, 3.0f } } }));
 
-                var trans = TensorFlowTransform.Create(env, loader, model_location, "c", "a", "b");
+                var trans = TensorFlowTransform.Create(env, loader, model_location, new[] { "c" }, new[] { "a", "b" });
 
                 using (var cursor = trans.GetRowCursor(a => true))
                 {
@@ -202,7 +202,61 @@ public void TensorFlowTransformCifar()
                 }, cropped);
 
 
-                IDataView trans = TensorFlowTransform.Create(env, pixels, model_location, "Output", "Input");
+                IDataView trans = TensorFlowTransform.Create(env, pixels, model_location, new[] { "Output" }, new[] { "Input" });
+
+                trans.Schema.TryGetColumnIndex("Output", out int output);
+                using (var cursor = trans.GetRowCursor(col => col == output))
+                {
+                    var buffer = default(VBuffer<float>);
+                    var getter = cursor.GetGetter<VBuffer<float>>(output);
+                    var numRows = 0;
+                    while (cursor.MoveNext())
+                    {
+                        getter(ref buffer);
+                        Assert.Equal(10, buffer.Length);
+                        numRows += 1;
+                    }
+                    Assert.Equal(3, numRows);
+                }
+            }
+        }
+
+        [Fact]
+        public void TensorFlowTransformCifarSavedModel()
+        {
+            var model_location = "cifar_saved_model";
+
+            using (var env = new TlcEnvironment())
+            {
+                var imageHeight = 32;
+                var imageWidth = 32;
+                var dataFile = GetDataPath("images/images.tsv");
+                var imageFolder = Path.GetDirectoryName(dataFile);
+                var data = env.CreateLoader("Text{col=ImagePath:TX:0 col=Name:TX:1}", new MultiFileSource(dataFile));
+                var images = ImageLoaderTransform.Create(env, new ImageLoaderTransform.Arguments()
+                {
+                    Column = new ImageLoaderTransform.Column[1]
+                    {
+                        new ImageLoaderTransform.Column() { Source=  "ImagePath", Name="ImageReal" }
+                    },
+                    ImageFolder = imageFolder
+                }, data);
+                var cropped = ImageResizerTransform.Create(env, new ImageResizerTransform.Arguments()
+                {
+                    Column = new ImageResizerTransform.Column[1]{
+                        new ImageResizerTransform.Column() { Source = "ImageReal", Name= "ImageCropped", ImageHeight =imageHeight, ImageWidth = imageWidth, Resizing = ImageResizerTransform.ResizingKind.IsoCrop}
+                    }
+                }, images);
+
+                var pixels = ImagePixelExtractorTransform.Create(env, new ImagePixelExtractorTransform.Arguments()
+                {
+                    Column = new ImagePixelExtractorTransform.Column[1]{
+                        new ImagePixelExtractorTransform.Column() {  Source= "ImageCropped", Name = "Input", UseAlpha=false, InterleaveArgb=true}
+                    }
+                }, cropped);
+
+
+                IDataView trans = TensorFlowTransform.Create(env, pixels, model_location, new[] { "Output" }, new[] { "Input" }, false);
 
                 trans.Schema.TryGetColumnIndex("Output", out int output);
                 using (var cursor = trans.GetRowCursor(col => col == output))
@@ -259,7 +313,7 @@ public void TensorFlowTransformCifarInvalidShape()
                 var thrown = false;
                 try
                 {
-                    IDataView trans = TensorFlowTransform.Create(env, pixels, model_location, "Output", "Input");
+                    IDataView trans = TensorFlowTransform.Create(env, pixels, model_location, new[] { "Output" }, new[] { "Input" });
                 }
                 catch
                 {