Add option to execute only the last transform in TransformWrapper and have WordBagEstimator return transformer chain

docs/samples/Microsoft.ML.Samples/Program.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Reflection;
+using Samples.Dynamic;
 
 namespace Microsoft.ML.Samples
 {

src/Microsoft.ML.Data/DataLoadSave/LegacyCompositeDataLoader.cs

@@ -412,7 +412,7 @@ internal TransformerChain<ITransformer> GetTransformer()
                 }
                 else
                 {
-                    ITransformer transformer = new TransformWrapper(_host, transform.Transform);
+                    ITransformer transformer = new TransformWrapper(_host, transform.Transform, false, true);
                     result = result.Append(transformer);
                 }
             }

src/Microsoft.ML.Data/DataLoadSave/TransformWrapper.cs

@@ -29,23 +29,27 @@ internal sealed class TransformWrapper : ITransformer
         private readonly IDataView _xf;
         private readonly bool _allowSave;
         private readonly bool _isRowToRowMapper;
+        private readonly bool _useLastTransformOnly;
 
-        public TransformWrapper(IHostEnvironment env, IDataView xf, bool allowSave = false)
+        public TransformWrapper(IHostEnvironment env, IDataView xf, bool allowSave = false, bool useLastTransformOnly = false)
         {
             Contracts.CheckValue(env, nameof(env));
             _host = env.Register(nameof(TransformWrapper));
             _host.CheckValue(xf, nameof(xf));
             _xf = xf;
             _allowSave = allowSave;
             _isRowToRowMapper = IsChainRowToRowMapper(_xf);
+            _useLastTransformOnly = useLastTransformOnly;
         }
 
         public DataViewSchema GetOutputSchema(DataViewSchema inputSchema)
         {
             _host.CheckValue(inputSchema, nameof(inputSchema));
 
             var dv = new EmptyDataView(_host, inputSchema);
-            var output = ApplyTransformUtils.ApplyAllTransformsToData(_host, _xf, dv);
+            var output = _useLastTransformOnly ? ApplyTransformUtils.ApplyTransformToData(_host, (IDataTransform)_xf, dv) :
+                ApplyTransformUtils.ApplyAllTransformsToData(_host, _xf, dv);
+
             return output.Schema;
         }
 
@@ -115,7 +119,8 @@ private TransformWrapper(IHostEnvironment env, ModelLoadContext ctx)
             _isRowToRowMapper = IsChainRowToRowMapper(_xf);
         }
 
-        public IDataView Transform(IDataView input) => ApplyTransformUtils.ApplyAllTransformsToData(_host, _xf, input);
+        public IDataView Transform(IDataView input) => _useLastTransformOnly ? ApplyTransformUtils.ApplyTransformToData(_host, (IDataTransform)_xf, input) :
+            ApplyTransformUtils.ApplyAllTransformsToData(_host, _xf, input);
 
         private static bool IsChainRowToRowMapper(IDataView view)
         {
@@ -133,18 +138,29 @@ IRowToRowMapper ITransformer.GetRowToRowMapper(DataViewSchema inputSchema)
         {
             _host.CheckValue(inputSchema, nameof(inputSchema));
             var input = new EmptyDataView(_host, inputSchema);
-            var revMaps = new List<IRowToRowMapper>();
             IDataView chain;
-            for (chain = ApplyTransformUtils.ApplyAllTransformsToData(_host, _xf, input); chain is IDataTransform xf; chain = xf.Source)
+            if (_useLastTransformOnly)
+            {
+                chain = ApplyTransformUtils.ApplyTransformToData(_host, (IDataTransform)_xf, input);
+                return new CompositeRowToRowMapper(inputSchema, new[] { (IRowToRowMapper)chain });
+            }
+            else
             {
-                // Everything in the chain ought to be a row mapper.
-                _host.Assert(xf is IRowToRowMapper);
-                revMaps.Add((IRowToRowMapper)xf);
+                var revMaps = new List<IRowToRowMapper>();
+                for (chain = ApplyTransformUtils.ApplyAllTransformsToData(_host, _xf, input);
+                    chain is IDataTransform xf;
+                    chain = xf.Source)
+                {
+                    // Everything in the chain ought to be a row mapper.
+                    _host.Assert(xf is IRowToRowMapper);
+                    revMaps.Add((IRowToRowMapper)xf);
+                }
+
+                // The walkback should have ended at the input.
+                Contracts.Assert(chain == input);
+                revMaps.Reverse();
+                return new CompositeRowToRowMapper(inputSchema, revMaps.ToArray());
             }
-            // The walkback should have ended at the input.
-            Contracts.Assert(chain == input);
-            revMaps.Reverse();
-            return new CompositeRowToRowMapper(inputSchema, revMaps.ToArray());
         }
     }
 

src/Microsoft.ML.Transforms/Text/WordBagTransform.cs

@@ -95,7 +95,7 @@ internal sealed class Options : NgramExtractorTransform.ArgumentsBase
         internal const string Summary = "Produces a bag of counts of n-grams (sequences of consecutive words of length 1-n) in a given text. It does so by building "
             + "a dictionary of n-grams and using the id in the dictionary as the index in the bag.";
 
-        internal static IDataTransform Create(IHostEnvironment env, Options options, IDataView input)
+        internal static ITransformer CreateTransfomer(IHostEnvironment env, Options options, IDataView input)
         {
             Contracts.CheckValue(env, nameof(env));
             var h = env.Register(RegistrationName);
@@ -150,10 +150,16 @@ internal static IDataTransform Create(IHostEnvironment env, Options options, IDa
             }
 
             IDataView view = input;
-            view = NgramExtractionUtils.ApplyConcatOnSources(h, options.Columns, view);
-            view = new WordTokenizingEstimator(env, tokenizeColumns).Fit(view).Transform(view);
-            return NgramExtractorTransform.CreateDataTransform(h, extractorArgs, view);
+            ITransformer t0 = NgramExtractionUtils.ApplyConcatOnSources(h, options.Columns);
+            view = t0.Transform(view);
+            ITransformer t1 = new WordTokenizingEstimator(env, tokenizeColumns).Fit(view);
+            view = t1.Transform(view);
+            ITransformer t2 = NgramExtractorTransform.Create(h, extractorArgs, t1.Transform(view));
+            return new TransformerChain<ITransformer>(new[] { t0, t1, t2 });
         }
+
+        internal static IDataTransform Create(IHostEnvironment env, Options options, IDataView input) =>
+            (IDataTransform)CreateTransfomer(env, options, input).Transform(input);
     }
 
     /// <summary>
@@ -489,13 +495,11 @@ public ITransformer Create(IHostEnvironment env, IDataView input, ExtractorColum
 
     internal static class NgramExtractionUtils
     {
-        public static IDataView ApplyConcatOnSources(IHostEnvironment env, ManyToOneColumn[] columns, IDataView input)
+        public static ITransformer ApplyConcatOnSources(IHostEnvironment env, ManyToOneColumn[] columns)
         {
             Contracts.CheckValue(env, nameof(env));
             env.CheckValue(columns, nameof(columns));
-            env.CheckValue(input, nameof(input));
 
-            IDataView view = input;
             var concatColumns = new List<ColumnConcatenatingTransformer.ColumnOptions>();
             foreach (var col in columns)
             {
@@ -506,10 +510,11 @@ public static IDataView ApplyConcatOnSources(IHostEnvironment env, ManyToOneColu
                 if (col.Source.Length > 1)
                     concatColumns.Add(new ColumnConcatenatingTransformer.ColumnOptions(col.Name, col.Source));
             }
+
             if (concatColumns.Count > 0)
-                return new ColumnConcatenatingTransformer(env, concatColumns.ToArray()).Transform(view);
+                return new ColumnConcatenatingTransformer(env, concatColumns.ToArray());
 
-            return view;
+            return new TransformerChain<ITransformer>();
         }
 
         /// <summary>

src/Microsoft.ML.Transforms/Text/WordHashBagProducingTransform.cs

@@ -83,7 +83,7 @@ internal sealed class Options : NgramHashExtractingTransformer.ArgumentsBase
         internal const string Summary = "Produces a bag of counts of n-grams (sequences of consecutive words of length 1-n) in a given text. "
             + "It does so by hashing each n-gram and using the hash value as the index in the bag.";
 
-        internal static IDataTransform Create(IHostEnvironment env, Options options, IDataView input)
+        internal static ITransformer CreateTransformer(IHostEnvironment env, Options options, IDataView input)
         {
             Contracts.CheckValue(env, nameof(env));
             var h = env.Register(RegistrationName);
@@ -132,7 +132,7 @@ internal static IDataTransform Create(IHostEnvironment env, Options options, IDa
                     };
             }
 
-            view = new WordTokenizingEstimator(env, tokenizeColumns.ToArray()).Fit(view).Transform(view);
+            ITransformer t1 = new WordTokenizingEstimator(env, tokenizeColumns.ToArray()).Fit(view);
 
             var featurizeArgs =
                 new NgramHashExtractingTransformer.Options
@@ -147,11 +147,17 @@ internal static IDataTransform Create(IHostEnvironment env, Options options, IDa
                     MaximumNumberOfInverts = options.MaximumNumberOfInverts
                 };
 
-            view = NgramHashExtractingTransformer.Create(h, featurizeArgs, view).Transform(view);
+            view = t1.Transform(view);
+            ITransformer t2 = NgramHashExtractingTransformer.Create(h, featurizeArgs, t1.Transform(view));
 
             // Since we added columns with new names, we need to explicitly drop them before we return the IDataTransform.
-            return ColumnSelectingTransformer.CreateDrop(h, view, tmpColNames.ToArray()) as IDataTransform;
+            ITransformer t3 = new ColumnSelectingTransformer(env, null, tmpColNames.ToArray());
+
+            return new TransformerChain<ITransformer>(new[] { t1, t2, t3 });
         }
+
+        internal static IDataTransform Create(IHostEnvironment env, Options options, IDataView input) =>
+            (IDataTransform)CreateTransformer(env, options, input).Transform(input);
     }
 
     /// <summary>

src/Microsoft.ML.Transforms/Text/WrappedTextTransformers.cs

@@ -184,7 +184,7 @@ public ITransformer Fit(IDataView input)
                 Weighting = _weighting
             };
 
-            return new TransformWrapper(_host, WordBagBuildingTransformer.Create(_host, options, input), true);
+            return WordBagBuildingTransformer.CreateTransfomer(_host, options, input);
         }
 
         /// <summary>
@@ -365,7 +365,7 @@ public ITransformer Fit(IDataView input)
                 MaximumNumberOfInverts = _maximumNumberOfInverts
             };
 
-            return new TransformWrapper(_host, WordHashBagProducingTransformer.Create(_host, options, input), true);
+            return WordHashBagProducingTransformer.CreateTransformer(_host, options, input);
         }
 
         /// <summary>

test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs

@@ -981,7 +981,11 @@ public void EntryPointPipelineEnsembleText()
                         new WordHashBagProducingTransformer.Options()
                         {
                             Columns =
-                                new[] { new WordHashBagProducingTransformer.Column() { Name = "Features", Source = new[] { "Text" } }, }
+                                new[]
+                                {
+                                    new WordHashBagProducingTransformer.Column()
+                                        {Name = "Features", Source = new[] {"Text"}},
+                                }
                         },
                         data);
                 }

test/Microsoft.ML.Tests/Scenarios/WordBagTest.cs

@@ -0,0 +1,63 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Xunit;
+using System.Collections.Generic;
+using Microsoft.ML.Transforms.Text;
+
+namespace Microsoft.ML.Scenarios
+{
+    public partial class ScenariosTests
+    {
+        [Fact]
+        public static void WordBags()
+        {
+            var mlContext = new MLContext();
+            var samples = new List<TextData>()
+            {
+                new TextData(){ Text = "This is an example to compute bag-of-word features." },
+                new TextData(){ Text = "ML.NET's ProduceWordBags API produces bag-of-word features from input text." },
+                new TextData(){ Text = "It does so by first tokenizing text/string into words/tokens then " },
+                new TextData(){ Text = "computing n-grams and their neumeric values." },
+                new TextData(){ Text = "Each position in the output vector corresponds to a particular n-gram." },
+                new TextData(){ Text = "The value at each position corresponds to," },
+                new TextData(){ Text = "the number of times n-gram occured in the data (Tf), or" },
+                new TextData(){ Text = "the inverse of the number of documents contain the n-gram (Idf)," },
+                new TextData(){ Text = "or compute both and multipy together (Tf-Idf)." },
+            };
+
+            var dataview = mlContext.Data.LoadFromEnumerable(samples);
+            var textPipeline =
+                mlContext.Transforms.Text.ProduceWordBags("Text", "Text",
+                ngramLength: 3, useAllLengths: false, weighting: NgramExtractingEstimator.WeightingCriteria.Tf).Append(
+                mlContext.Transforms.Text.ProduceWordBags("Text2", "Text2",
+                    ngramLength: 3, useAllLengths: false, weighting: NgramExtractingEstimator.WeightingCriteria.Tf));
+
+
+            var textTransformer = textPipeline.Fit(dataview);
+            var transformedDataView = textTransformer.Transform(dataview);
+            var predictionEngine = mlContext.Model.CreatePredictionEngine<TextData, TransformedTextData>(textTransformer);
+            var prediction = predictionEngine.Predict(samples[0]);
+            Assert.Equal(prediction.Text, new float[] {
+                1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 });
+        }
+
+        private class TextData
+        {
+            public string Text { get; set; }
+#pragma warning disable 414
+            public string Text2 = "ABC";
+#pragma warning restore 414
+        }
+
+        private class TransformedTextData
+        {
+            public float[] Text { get; set; }
+        }
+    }
+
+}