Updated codegen to support object detection scenario.

src/Microsoft.ML.AutoML/TaskKind.cs

@@ -10,6 +10,7 @@ internal enum TaskKind
         MulticlassClassification,
         Regression,
         Recommendation,
+        ObjectDetection,
         Ranking
     }
 }

src/Microsoft.ML.CodeGenerator/CodeGenerator/CSharp/AzureCodeGenerator/AzureAttachConsoleAppCodeGenerator.cs

@@ -68,9 +68,10 @@ public AzureAttachConsoleAppCodeGenerator(Pipeline pipeline, ColumnInferenceResu
                     IncludeMklComponentsPackage = false,
                     IncludeLightGBMPackage = false,
                     IncludeFastTreePackage = false,
-                    IncludeImageTransformerPackage = _settings.IsImage,
+                    IncludeImageTransformerPackage = _settings.IsImage || _settings.IsObjectDetection,
                     IncludeImageClassificationPackage = false,
                     IncludeOnnxPackage = true,
+                    IncludeOnnxRuntime = _settings.IsObjectDetection,
                     IncludeResNet18Package = false,
                     IncludeRecommenderPackage = false,
                     StablePackageVersion = _settings.StablePackageVersion,
@@ -94,6 +95,7 @@ public AzureAttachConsoleAppCodeGenerator(Pipeline pipeline, ColumnInferenceResu
                     Separator = _columnInferenceResult.TextLoaderOptions.Separators.FirstOrDefault(),
                     Target = _settings.Target,
                     SampleData = sampleResult,
+                    IsObjectDetection = _settings.IsObjectDetection,
                 }.TransformText(),
                 Name = "Program.cs",
             };

src/Microsoft.ML.CodeGenerator/CodeGenerator/CSharp/AzureCodeGenerator/AzureAttachModelCodeGenerator.cs

@@ -25,10 +25,13 @@ internal class AzureAttachModelCodeGenerator : ICSharpProjectGenerator
         private readonly string _nameSpaceValue;
 
         public ICSharpFile ModelInputClass { get; private set; }
-        public ICSharpFile ModelOutputClass { get; private set; }
+        public ICSharpFile AzureImageModelOutputClass { get; private set; }
+        public ICSharpFile AzureObjectDetectionModelOutputClass { get; private set; }
         public ICSharpFile ModelProject { get; private set; }
         public ICSharpFile ConsumeModel { get; private set; }
         public ICSharpFile LabelMapping { get; private set; }
+        public ICSharpFile ImageLabelMapping { get; private set; }
+        public ICSharpFile ObjectDetectionConsumeModel { get; private set; }
         public string Name { get; set; }
 
         public AzureAttachModelCodeGenerator(Pipeline pipeline, ColumnInferenceResults columnInferenceResults, CodeGeneratorSettings options, string namespaceValue)
@@ -53,7 +56,7 @@ public AzureAttachModelCodeGenerator(Pipeline pipeline, ColumnInferenceResults c
             var labelType = _columnInferenceResult.TextLoaderOptions.Columns.Where(t => t.Name == _settings.LabelName).First().DataKind;
             Type labelTypeCsharp = Utils.GetCSharpType(labelType);
 
-            ModelOutputClass = new CSharpCodeFile()
+            AzureImageModelOutputClass = new CSharpCodeFile()
             {
                 File = new AzureImageModelOutputClass()
                 {
@@ -64,6 +67,17 @@ public AzureAttachModelCodeGenerator(Pipeline pipeline, ColumnInferenceResults c
                 Name = "ModelOutput.cs",
             };
 
+            AzureObjectDetectionModelOutputClass = new CSharpCodeFile()
+            {
+                File = new AzureObjectDetectionModelOutputClass()
+                {
+                    Namespace = _nameSpaceValue,
+                    Target = _settings.Target,
+                    Labels = _settings.ObjectLabel,
+                }.TransformText(),
+                Name = "ModelOutput.cs",
+            };
+
             ModelProject = new CSharpProjectFile()
             {
                 File = new ModelProject()
@@ -74,6 +88,7 @@ public AzureAttachModelCodeGenerator(Pipeline pipeline, ColumnInferenceResults c
                     IncludeLightGBMPackage = false,
                     IncludeMklComponentsPackage = false,
                     IncludeOnnxModel = true,
+                    IncludeOnnxRuntime = _settings.IsObjectDetection,
                     IncludeRecommenderPackage = false,
                     StablePackageVersion = _settings.StablePackageVersion,
                     UnstablePackageVersion = _settings.UnstablePackageVersion,
@@ -83,19 +98,6 @@ public AzureAttachModelCodeGenerator(Pipeline pipeline, ColumnInferenceResults c
                 Name = $"{ _settings.OutputName }.Model.csproj",
             };
 
-            LabelMapping = new CSharpCodeFile()
-            {
-                File = new LabelMapping()
-                {
-                    Target = _settings.Target,
-                    Namespace = _nameSpaceValue,
-                    LabelMappingInputLabelType = typeof(Int64).Name,
-                    PredictionLabelType = labelTypeCsharp.Name,
-                    TaskType = _settings.MlTask.ToString(),
-                }.TransformText(),
-                Name = "LabelMapping.cs",
-            };
-
             ConsumeModel = new CSharpCodeFile()
             {
                 File = new ConsumeModel()
@@ -105,6 +107,7 @@ public AzureAttachModelCodeGenerator(Pipeline pipeline, ColumnInferenceResults c
                     MLNetModelName = _settings.ModelName,
                     OnnxModelName = _settings.OnnxModelName,
                     IsAzureImage = _settings.IsAzureAttach && _settings.IsImage,
+                    IsAzureObjectDetection = _settings.IsObjectDetection && _settings.IsAzureAttach,
                 }.TransformText(),
                 Name = "ConsumeModel.cs",
             };
@@ -118,7 +121,16 @@ public ICSharpProject ToProject()
                 project = new CSharpProject()
                 {
                     ModelInputClass,
-                    ModelOutputClass,
+                    AzureImageModelOutputClass,
+                    ConsumeModel,
+                    ModelProject,
+                };
+            } else if(_settings.IsObjectDetection)
+            {
+                project = new CSharpProject()
+                {
+                    ModelInputClass,
+                    AzureObjectDetectionModelOutputClass,
                     ConsumeModel,
                     ModelProject,
                 };
@@ -128,10 +140,9 @@ public ICSharpProject ToProject()
                 project = new CSharpProject()
                 {
                     ModelInputClass,
-                    ModelOutputClass,
+                    AzureImageModelOutputClass,
                     ConsumeModel,
                     ModelProject,
-                    LabelMapping,
                 };
             }
             project.Name = Name;

src/Microsoft.ML.CodeGenerator/CodeGenerator/CSharp/CodeGeneratorSettings.cs

@@ -17,6 +17,7 @@ public CodeGeneratorSettings()
             Target = GenerateTarget.Cli;
             OnnxInputMapping = new Dictionary<string, ColumnMapping>();
             ClassificationLabel = new string[] { };
+            ObjectLabel = new string[] { };
         }
 
         public string LabelName { get; set; }
@@ -37,6 +38,8 @@ public CodeGeneratorSettings()
         /// </summary>
         public string[] ClassificationLabel { get; set; }
 
+        public string[] ObjectLabel { get; set; }
+
         public string OutputName { get; set; }
 
         public string OutputBaseDir { get; set; }
@@ -57,6 +60,8 @@ public CodeGeneratorSettings()
 
         public bool IsImage { get; set; }
 
+        public bool IsObjectDetection { get; set; }
+
         public IDictionary<string, ColumnMapping> OnnxInputMapping { get; set; }
 
         internal TaskKind MlTask { get; set; }

src/Microsoft.ML.CodeGenerator/CodeGenerator/CSharp/TransformGeneratorFactory.cs

@@ -20,7 +20,9 @@ internal enum SpecialTransformer
         ApplyOnnxModel = 0,
         ResizeImage = 1,
         ExtractPixel = 2,
+        ObjectDetectionResizeImage = 3,
     }
+
     internal static class TransformGeneratorFactory
     {
         internal static ITransformGenerator GetInstance(PipelineNode node)
@@ -78,7 +80,7 @@ internal static ITransformGenerator GetInstance(PipelineNode node)
                 }
             }
 
-            // For AzureAttach
+            // For  the AzureAttach
             if (Enum.TryParse(node.Name, out SpecialTransformer transformer))
             {
                 switch (transformer)
@@ -92,6 +94,9 @@ internal static ITransformGenerator GetInstance(PipelineNode node)
                     case SpecialTransformer.ApplyOnnxModel:
                         result = new ApplyOnnxModel(node);
                         break;
+                    case SpecialTransformer.ObjectDetectionResizeImage:
+                        result = new ObjectDetectionImageResizing(node);
+                        break;
                     default:
                         return null;
                 }

src/Microsoft.ML.CodeGenerator/CodeGenerator/CSharp/TransformGenerators.cs

@@ -400,6 +400,17 @@ public override string GenerateTransformer()
         }
     }
 
+    internal class ObjectDetectionImageResizing : TransformGeneratorBase
+    {
+        public ObjectDetectionImageResizing(PipelineNode node) : base(node) { }
+        internal override string MethodName => "ResizeImages";
+
+        public override string GenerateTransformer()
+        {
+            return @"ResizeImages(outputColumnName: ""ImageSource_featurized"", imageWidth: 800, imageHeight: 600, inputColumnName: ""ImageSource_featurized"")";
+        }
+    }
+
     internal class PixelExtract : TransformGeneratorBase
     {
         public PixelExtract(PipelineNode node) : base(node) { }

src/Microsoft.ML.CodeGenerator/Microsoft.ML.CodeGenerator.csproj

@@ -26,17 +26,13 @@
       <Generator>TextTemplatingFilePreprocessor</Generator>
       <LastGenOutput>AzureModelBuilder.cs</LastGenOutput>
     </None>
-    <None Update="Templates\Azure\Model\AzureImageModelOutputClass.tt">
-      <LastGenOutput>AzureImageModelOutputClass.cs</LastGenOutput>
+    <None Update="Templates\Azure\Model\AzureObjectDetectionModelOutputClass.tt">
+      <LastGenOutput>AzureObjectDetectionModelOutputClass.cs</LastGenOutput>
       <Generator>TextTemplatingFilePreprocessor</Generator>
     </None>
-    <None Update="Templates\Azure\Model\OnnxModelOutputClass.tt">
-      <Generator>TextTemplatingFilePreprocessor</Generator>
+    <None Update="Templates\Azure\Model\AzureImageModelOutputClass.tt">
       <LastGenOutput>AzureImageModelOutputClass.cs</LastGenOutput>
-    </None>
-    <None Update="Templates\Azure\Model\LabelMapping.tt">
       <Generator>TextTemplatingFilePreprocessor</Generator>
-      <LastGenOutput>LabelMapping.cs</LastGenOutput>
     </None>
     <None Update="Templates\Console\Annotation.ttinclude">
     </None>
@@ -80,8 +76,8 @@
       <AutoGen>True</AutoGen>
       <DependentUpon>AzureModelBuilder.tt</DependentUpon>
     </Compile>
-    <Compile Update="Templates\Azure\Model\OnnxModelOutputClass - Copy.cs">
-      <DependentUpon>OnnxModelOutputClass.tt</DependentUpon>
+    <Compile Update="Templates\Azure\Model\AzureImageModelOutputClass - Copy.cs">
+      <DependentUpon>AzureImageModelOutputClass.tt</DependentUpon>
       <DesignTime>True</DesignTime>
       <AutoGen>True</AutoGen>
     </Compile>
@@ -90,10 +86,15 @@
       <AutoGen>True</AutoGen>
       <DependentUpon>AzureImageModelOutputClass.tt</DependentUpon>
     </Compile>
-    <Compile Update="Templates\Azure\Model\LabelMapping.cs">
+    <Compile Update="Templates\Azure\Model\OnnxModelOutputClass - Copy.cs">
+      <DependentUpon>OnnxModelOutputClass.tt</DependentUpon>
+      <DesignTime>True</DesignTime>
+      <AutoGen>True</AutoGen>
+    </Compile>
+    <Compile Update="Templates\Azure\Model\AzureObjectDetectionModelOutputClass.cs">
       <DesignTime>True</DesignTime>
       <AutoGen>True</AutoGen>
-      <DependentUpon>LabelMapping.tt</DependentUpon>
+      <DependentUpon>AzureObjectDetectionModelOutputClass.tt</DependentUpon>
     </Compile>
     <Compile Update="Templates\Console\ConsumeModel.cs">
       <DesignTime>True</DesignTime>

src/Microsoft.ML.CodeGenerator/Templates/Azure/Model/LabelMapping.cs → src/Microsoft.ML.CodeGenerator/Templates/Azure/Model/AzureObjectDetectionModelOutputClass.cs

@@ -18,7 +18,7 @@ namespace Microsoft.ML.CodeGenerator.Templates.Azure.Model
     /// Class to produce the template output
     /// </summary>
     [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.TextTemplating", "16.0.0.0")]
-    internal partial class LabelMapping : LabelMappingBase
+    internal partial class AzureObjectDetectionModelOutputClass : AzureObjectDetectionModelOutputClassBase
     {
         /// <summary>
         /// Create the template output
@@ -30,72 +30,47 @@ public virtual string TransformText()
  } else if(Target == CSharp.GenerateTarget.ModelBuilder){ 
 MB_Annotation();
  } 
-            this.Write("\r\nusing Microsoft.ML.Data;\r\nusing Microsoft.ML.Transforms;\r\nusing System;\r\nusing " +
-                    "System.Linq;\r\n\r\nnamespace ");
+            this.Write("\r\nusing System;\r\nusing Microsoft.ML.Data;\r\nusing System.Collections.Generic;\r\nusi" +
+                    "ng System.Linq;\r\n\r\nnamespace ");
             this.Write(this.ToStringHelper.ToStringWithCulture(Namespace));
-            this.Write(@".Model
-{
-    [CustomMappingFactoryAttribute(nameof(LabelMapping))]
-    public class LabelMapping : CustomMappingFactory<LabelMappingInput, LabelMappingOutput>
-    {
-		// Custom mapping to determine the label with the highest probability
-        public static void Mapping(LabelMappingInput input, LabelMappingOutput output)
-        {
-");
-if("Boolean".Equals(PredictionLabelType)){ 
-            this.Write("\t\t\toutput.Prediction = input.label.GetValues().ToArray().First() == 1;\r\n");
- } else{ 
-            this.Write("            output.Prediction = input.label.GetValues().ToArray().First();\r\n");
- }
-            this.Write("\r\n");
-if("MulticlassClassification".Equals(TaskType)){ 
-            this.Write("            output.Score = input.probabilities.GetValues().ToArray();\r\n");
- } else {
-            this.Write("            output.Score = input.probabilities.GetValues().ToArray().First();\r\n");
- }
-            this.Write(@"        }
-		// Factory method called when loading the model to get the mapping operation
-        public override Action<LabelMappingInput, LabelMappingOutput> GetMapping()
-        {
-            return Mapping;
-        }
-    }
-    public class LabelMappingInput
-    {
-        [ColumnName(""label"")]
-        public VBuffer<Int64> label;
-
-        [ColumnName(""probabilities"")]
-        public VBuffer<float> probabilities;
-    }
-    public class LabelMappingOutput
-    {
-");
-if("BinaryClassification".Equals(TaskType)){ 
-            this.Write("        // ColumnName attribute is used to change the column name from\r\n\t\t// its " +
-                    "default value, which is the name of the field\r\n        [ColumnName(\"PredictedLab" +
-                    "el\")]\r\n        public bool Prediction { get; set; }\r\n");
- } if("MulticlassClassification".Equals(TaskType)){ 
-            this.Write("        // ColumnName attribute is used to change the column name from\r\n\t\t// its " +
-                    "default value, which is the name of the field\r\n        [ColumnName(\"PredictedLab" +
-                    "el\")]\r\n        public ");
-            this.Write(this.ToStringHelper.ToStringWithCulture(PredictionLabelType));
-            this.Write(" Prediction { get; set; }\r\n");
- }
-if("MulticlassClassification".Equals(TaskType)){ 
-            this.Write("        public float[] Score { get; set; }\r\n");
-}else{ 
-            this.Write("        public float Score { get; set; }\r\n");
+            this.Write(".Model\r\n{\r\n    public class ModelOutput\r\n    {\r\n        public string[] ObjectTag" +
+                    "s = new string[]{");
+foreach(var label in Labels){
+            this.Write("\"");
+            this.Write(this.ToStringHelper.ToStringWithCulture(label));
+            this.Write("\",");
 }
-            this.Write("    }\r\n}\r\n");
+            this.Write("};\r\n\r\n        [ColumnName(\"boxes\")]\r\n        public float[] Boxes { get; set; }\r\n" +
+                    "\r\n        [ColumnName(\"labels\")]\r\n        public long[] Labels { get; set; }\r\n\r\n" +
+                    "        [ColumnName(\"scores\")]\r\n        public float[] Scores { get; set; }\r\n\r\n " +
+                    "       private BoundingBox[] BoundingBoxes\r\n        {\r\n            get\r\n        " +
+                    "    {\r\n                var boundingBoxes = new List<BoundingBox>();\r\n\r\n         " +
+                    "       boundingBoxes = Enumerable.Range(0, this.Labels.Length)\r\n                " +
+                    "          .Select((index) =>\r\n                          {\r\n                     " +
+                    "         var boxes = this.Boxes;\r\n                              var scores = thi" +
+                    "s.Scores;\r\n                              var labels = this.Labels;\r\n\r\n          " +
+                    "                    return new BoundingBox()\r\n                              {\r\n " +
+                    "                                 Left = boxes[index * 4],\r\n                     " +
+                    "             Top = boxes[(index * 4) + 1],\r\n                                  Ri" +
+                    "ght = boxes[(index * 4) + 2],\r\n                                  Bottom = boxes[" +
+                    "(index * 4) + 3],\r\n                                  Score = scores[index],\r\n   " +
+                    "                               Label = this.ObjectTags[labels[index]],\r\n        " +
+                    "                      };\r\n                          }).ToList();\r\n              " +
+                    "  return boundingBoxes.ToArray();\r\n            }\r\n        }\r\n\r\n        public ov" +
+                    "erride string ToString()\r\n        {\r\n            return string.Join(\"\\n\", Boundi" +
+                    "ngBoxes.Select(x => x.ToString()));\r\n        }\r\n    }\r\n\r\n    public class Boundi" +
+                    "ngBox\r\n    {\r\n        public float Top;\r\n\r\n        public float Left;\r\n\r\n       " +
+                    " public float Right;\r\n\r\n        public float Bottom;\r\n\r\n        public string La" +
+                    "bel;\r\n\r\n        public float Score;\r\n\r\n        public override string ToString()" +
+                    "\r\n        {\r\n            return $\"Top: {this.Top}, Left: {this.Left}, Right: {th" +
+                    "is.Right}, Bottom: {this.Bottom}, Label: {this.Label}, Score: {this.Score}\";\r\n  " +
+                    "      }\r\n    }\r\n}\r\n");
             return this.GenerationEnvironment.ToString();
         }
 
 public string Namespace {get;set;}
 internal CSharp.GenerateTarget Target {get;set;}
-public string TaskType {get;set;}
-public string PredictionLabelType {get;set;}
-public string LabelMappingInputLabelType {get;set;}
+public string[] Labels {get; set;} = new string[0];
 
 
 void CLI_Annotation()
@@ -124,7 +99,7 @@ void MB_Annotation()
     /// Base class for this transformation
     /// </summary>
     [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.TextTemplating", "16.0.0.0")]
-    internal class LabelMappingBase
+    internal class AzureObjectDetectionModelOutputClassBase
     {
         #region Fields
         private global::System.Text.StringBuilder generationEnvironmentField;