Enable TextLoader to accept new lines in quoted fields

src/Microsoft.ML.Data/DataLoadSave/Text/TextLoader.cs

@@ -501,6 +501,13 @@ public class Options
             [Argument(ArgumentType.AtMostOnce, HelpText = "Use separate parsing threads?", ShortName = "threads", Hide = true)]
             public bool UseThreads = true;
 
+            /// <summary>
+            /// If true, new line characters are acceptable inside a quoted field, and thus one field can have multiple lines of text inside it
+            /// If <see cref="TextLoader.Options.AllowQuoting"/> is false, this option is ignored.
+            /// </summary>
+            [Argument(ArgumentType.AtMostOnce, HelpText = "Escape new line characters inside a quoted field? If AllowQuoting is false, this argument is ignored.", ShortName = "multilines", Hide = true)]
+            public bool ReadMultilines = false;
+
             /// <summary>
             /// File containing a header with feature names. If specified, the header defined in the data file is ignored regardless of <see cref="HasHeader"/>.
             /// </summary>
@@ -694,11 +701,11 @@ public Bindings(TextLoader parent, Column[] cols, IMultiStreamSource headerFile,
                     ch.Assert(0 <= inputSize & inputSize < SrcLim);
                     List<ReadOnlyMemory<char>> lines = null;
                     if (headerFile != null)
-                        Cursor.GetSomeLines(headerFile, 1, ref lines);
+                        Cursor.GetSomeLines(headerFile, 1, false, ref lines);
                     if (needInputSize && inputSize == 0)
-                        Cursor.GetSomeLines(dataSample, 100, ref lines);
+                        Cursor.GetSomeLines(dataSample, 100, parent._readMultilines,ref lines);
                     else if (headerFile == null && parent.HasHeader)
-                        Cursor.GetSomeLines(dataSample, 1, ref lines);
+                        Cursor.GetSomeLines(dataSample, 1, false, ref lines);
 
                     if (needInputSize && inputSize == 0)
                     {
@@ -1081,6 +1088,7 @@ private enum OptionFlags : uint
         private const int SrcLim = int.MaxValue;
 
         private readonly bool _useThreads;
+        private readonly bool _readMultilines;
         private readonly OptionFlags _flags;
         private readonly long _maxRows;
         // Input size is zero for unknown - determined by the data (including sparse rows).
@@ -1138,6 +1146,7 @@ internal TextLoader(IHostEnvironment env, Options options = null, IMultiStreamSo
             _host.Assert(Utils.Size(cols) > 0);
 
             _useThreads = options.UseThreads;
+            _readMultilines = options.AllowQuoting ? options.ReadMultilines : false;
 
             if (options.TrimWhitespace)
                 _flags |= OptionFlags.TrimWhitespace;
@@ -1350,6 +1359,8 @@ private TextLoader(IHost host, ModelLoadContext ctx)
 
             // REVIEW: Should we serialize this? It really isn't part of the data model.
             _useThreads = true;
+            // MYTODO: Should we serialize this? probably yes and also include it in Flags
+            _readMultilines = false;
 
             // *** Binary format ***
             // int: sizeof(Float)

src/Microsoft.ML.Data/DataLoadSave/Text/TextLoaderCursor.cs

@@ -4,6 +4,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
@@ -145,7 +146,7 @@ public static DataViewRowCursor Create(TextLoader parent, IMultiStreamSource fil
                 SetupCursor(parent, active, 0, out srcNeeded, out cthd);
                 Contracts.Assert(cthd > 0);
 
-                var reader = new LineReader(files, BatchSize, 100, parent.HasHeader, parent._maxRows, 1);
+                var reader = new LineReader(files, BatchSize, 100, parent.HasHeader, parent._readMultilines, parent._maxRows, 1);
                 var stats = new ParseStats(parent._host, 1);
                 return new Cursor(parent, stats, active, reader, srcNeeded, cthd);
             }
@@ -162,7 +163,7 @@ public static DataViewRowCursor[] CreateSet(TextLoader parent, IMultiStreamSourc
                 SetupCursor(parent, active, n, out srcNeeded, out cthd);
                 Contracts.Assert(cthd > 0);
 
-                var reader = new LineReader(files, BatchSize, 100, parent.HasHeader, parent._maxRows, cthd);
+                var reader = new LineReader(files, BatchSize, 100, parent.HasHeader, parent._readMultilines, parent._maxRows, cthd);
                 var stats = new ParseStats(parent._host, cthd);
                 if (cthd <= 1)
                     return new DataViewRowCursor[1] { new Cursor(parent, stats, active, reader, srcNeeded, 1) };
@@ -204,7 +205,7 @@ public override ValueGetter<DataViewRowId> GetIdGetter()
                     };
             }
 
-            public static void GetSomeLines(IMultiStreamSource source, int count, ref List<ReadOnlyMemory<char>> lines)
+            public static void GetSomeLines(IMultiStreamSource source, int count, bool readMultilines, ref List<ReadOnlyMemory<char>> lines)
             {
                 Contracts.AssertValue(source);
                 Contracts.Assert(count > 0);
@@ -214,7 +215,7 @@ public static void GetSomeLines(IMultiStreamSource source, int count, ref List<R
                     count = 2;
 
                 LineBatch batch;
-                var reader = new LineReader(source, count, 1, false, count, 1);
+                var reader = new LineReader(source, count, 1, false, readMultilines, count, 1);
                 try
                 {
                     batch = reader.GetBatch();
@@ -401,6 +402,7 @@ private sealed class LineReader
             {
                 private readonly long _limit;
                 private readonly bool _hasHeader;
+                private readonly bool _readMultilines;
                 private readonly int _batchSize;
                 private readonly IMultiStreamSource _files;
 
@@ -410,7 +412,7 @@ private sealed class LineReader
                 private Task _thdRead;
                 private volatile bool _abort;
 
-                public LineReader(IMultiStreamSource files, int batchSize, int bufSize, bool hasHeader, long limit, int cref)
+                public LineReader(IMultiStreamSource files, int batchSize, int bufSize, bool hasHeader, bool readMultilines, long limit, int cref)
                 {
                     // Note that files is allowed to be empty.
                     Contracts.AssertValue(files);
@@ -423,6 +425,7 @@ public LineReader(IMultiStreamSource files, int batchSize, int bufSize, bool has
                     _limit = limit;
                     _hasHeader = hasHeader;
                     _batchSize = batchSize;
+                    _readMultilines = readMultilines;
                     _files = files;
                     _cref = cref;
 
@@ -464,9 +467,69 @@ public LineBatch GetBatch()
                     throw Contracts.ExceptDecode(batch.Exception, "Stream reading encountered exception");
                 }
 
+                private static class MultiLineReader
+                {
+                    // When reading lines that contain quoted fields, the quoted fields can contain
+                    // '\n' so we we'll need to read multiple lines (multilines) to get all the fields
+                    // of a given row.
+                    public static string ReadMultiLine(TextReader sr, StringBuilder sb, bool ignoreHashLine)
+                    {
+                        string line;
+                        line = sr.ReadLine();
+
+                        // if it was an empty line or if we've reached the end of file (i.e. line = null)
+                        if (string.IsNullOrEmpty(line))
+                            return line;
+
+                        // In ML.NET we filter out lines beginning with // and # at the beginning of the file
+                        // Or lines beginning with // elsewhere in the file.
+                        // Thus, we don't care to check if there's a quoted multiline when the line begins with
+                        // these chars.
+                        if (line[0] == '/' && line[1] == '/')
+                            return line;
+                        if (ignoreHashLine && line[0] == '#')
+                            return line;
+
+                        // Get more lines until the number of quote characters is even
+                        // 2 consecutive quotes are considered scaped quotes
+                        long numOfQuotes = GetNumberOfChars(line, '"');
+                        if (numOfQuotes % 2 == 0)
+                            return line;
+
+                        sb.Clear();
+                        sb.Append(line);
+                        while (numOfQuotes % 2 != 0)
+                        {
+                            line = sr.ReadLine();
+
+                            if (line == null) // If we've reached the end of the file
+                                break; // MYTODO: This could happen if we have an invalid open quote which never closes so we reach the end of the file without properly closing the field, should we throw instead in this case?
+
+                            if (line.Length != 0)
+                                sb.Append(" "); // MYTODO: should we use instead a "\n" in here to separate lines?
+
+                            sb.Append(line);
+                            numOfQuotes += GetNumberOfChars(line, '"');
+                        }
+
+                        return sb.ToString();
+                    }
+
+                    public static int GetNumberOfChars(string line, char ch)
+                    {
+                        int count = 0;
+                        foreach (char c in line)
+                        {
+                            if (c == ch) count++;
+                        }
+                        return count;
+                    }
+                }
+
                 private void ThreadProc()
                 {
                     Contracts.Assert(_batchSize >= 2);
+                    var multilineSB = new StringBuilder();
 
                     try
                     {
@@ -487,7 +550,11 @@ private void ThreadProc()
                                     // REVIEW: Avoid allocating a string for every line. This would probably require
                                     // introducing a CharSpan type (similar to ReadOnlyMemory but based on char[] or StringBuilder)
                                     // and implementing all the necessary conversion functionality on it. See task 3871.
-                                    text = rdr.ReadLine();
+                                    if (_readMultilines)
+                                        text = MultiLineReader.ReadMultiLine(rdr, multilineSB, true);
+                                    else
+                                        text = rdr.ReadLine();
+
                                     if (text == null)
                                         goto LNext;
                                     line++;
@@ -514,7 +581,11 @@ private void ThreadProc()
                                     if (_abort)
                                         return;
 
-                                    text = rdr.ReadLine();
+                                    if (_readMultilines)
+                                        text = MultiLineReader.ReadMultiLine(rdr, multilineSB, true);
+                                    else
+                                        text = rdr.ReadLine();
+
                                     if (text == null)
                                     {
                                         // We're done with this file. Queue the last partial batch.

src/Microsoft.ML.Data/DataLoadSave/Text/TextLoaderParser.cs

@@ -1161,6 +1161,11 @@ private bool FetchNextField(ref ScanInfo scan, ReadOnlySpan<char> span)
                                 scan.QuotingError = true;
                                 break;
                             }
+
+                            // The logic below allow us to escape quotes (") inside quoted
+                            // fields by using doublo quotes (""). I.e. when the loader
+                            // encounters "" inside a quoted field, it will output only one "
+                            // and continue parsing the rest of the field.
                             if (span[ichCur] == '"')
                             {
                                 if (ichCur > ichRun)

src/Microsoft.ML.Data/DataLoadSave/Text/TextLoaderSaverCatalog.cs

@@ -64,7 +64,7 @@ public static TextLoader CreateTextLoader(this DataOperationsCatalog catalog,
                 HasHeader = hasHeader,
                 AllowQuoting = allowQuoting,
                 TrimWhitespace = trimWhitespace,
-                AllowSparse = allowSparse
+                AllowSparse = allowSparse,
             };
 
             return new TextLoader(CatalogUtils.GetEnvironment(catalog), options: options, dataSample: dataSample);

test/BaselineOutput/Common/EntryPoints/core_manifest.json

@@ -405,6 +405,18 @@
                 "IsNullable": false,
                 "Default": true
               },
+              {
+                "Name": "ReadMultilines",
+                "Type": "Bool",
+                "Desc": "Escape new line characters inside a quoted field? If AllowQuoting is false, this argument is ignored.",
+                "Aliases": [
+                  "multilines"
+                ],
+                "Required": false,
+                "SortOrder": 150.0,
+                "IsNullable": false,
+                "Default": false
+              },
               {
                 "Name": "HeaderFile",
                 "Type": "String",

test/BaselineOutput/Common/TextLoader/multiline.csv

@@ -0,0 +1,33 @@
+10
+11
+12
+13
+14
+15
+16
+17
+0
+19
+20
+this is a description
+this is a quoted description
+this is a multiline quoted description
+this has one"doublequote which should be escaped as a single quote
+this has "doublequotes" inside of it
+this is a multiline quoted description with "doublequotes" and empty new lines and escaped quotes inside of "it" //and this comment with // shouldn't be ignored since it is part of the "multiline"
+here is text after the empty line
+this is a line with an empty animal
+this is a line with an empty id
+
+this is the last row description
+dog
+cat
+bird
+dog
+cat
+bird
+dog
+
+bird
+dog
+cat

test/Microsoft.ML.Tests/TextLoaderTests.cs

@@ -660,7 +660,7 @@ public class ModelWithColumnNameAttribute
         }
     }
 
-    public class TextLoaderFromModelTests : BaseTestClass
+    public class TextLoaderFromModelTests : BaseTestBaseline
     {
         public TextLoaderFromModelTests(ITestOutputHelper output)
            : base(output)
@@ -920,5 +920,82 @@ public void TestLoadTextWithoutKeyTypeAttribute()
 
             Assert.Equal(expectedCount, data.Schema[1].Type.GetKeyCount());
         }
+
+        [Fact]
+        public void TestLoadTextWithEscapedNewLines()
+        {
+            var mlContext = new MLContext(seed: 1);
+            var dataPath = GetDataPath("multiline.csv");
+            var baselinePath = GetBaselinePath("TextLoader", "multiline.csv");
+            var options = new TextLoader.Options()
+            {
+                HasHeader = true,
+                Separator = ",",
+                AllowQuoting = true,
+                ReadMultilines = true,
+                Columns = new[]
+                {
+                    new TextLoader.Column("id", DataKind.Int32, 0),
+                    new TextLoader.Column("description", DataKind.String, 1),
+                    new TextLoader.Column("animal", DataKind.String, 2),
+                },
+            };
+
+            var data = mlContext.Data.LoadFromTextFile(dataPath, options);
+
+            // Get values from loaded dataview
+            var ids = new List<string>();
+            var descriptions = new List<string>();
+            var animals = new List<string>();
+            using(var curs = data.GetRowCursorForAllColumns())
+            {
+                var idGetter = curs.GetGetter<int>(data.Schema["id"]);
+                var descriptionGetter = curs.GetGetter<ReadOnlyMemory<char>>(data.Schema["description"]);
+                var animalGetter = curs.GetGetter<ReadOnlyMemory<char>>(data.Schema["animal"]);
+
+                int id = default;
+                ReadOnlyMemory<char> description = default;
+                ReadOnlyMemory<char> animal = default;
+
+                while(curs.MoveNext())
+                {
+                    idGetter(ref id);
+                    descriptionGetter(ref description);
+                    animalGetter(ref animal);
+
+                    ids.Add(id.ToString());
+                    descriptions.Add(description.ToString());
+                    animals.Add(animal.ToString());
+                }
+            }
+
+            const int numRows = 11;
+            Assert.Equal(numRows, ids.Count());
+            Assert.Equal(numRows, descriptions.Count());
+            Assert.Equal(numRows, animals.Count());
+
+            // Compare values with baseline file
+            string line;
+            using (var file = new StreamReader(baselinePath))
+            {
+                for(int i = 0; i < numRows; i++)
+                {
+                    line = file.ReadLine();
+                    Assert.Equal(ids[i], line);
+                }
+
+                for (int i = 0; i < numRows; i++)
+                {
+                    line = file.ReadLine();
+                    Assert.Equal(descriptions[i], line);
+                }
+
+                for (int i = 0; i < numRows; i++)
+                {
+                    line = file.ReadLine();
+                    Assert.Equal(animals[i], line);
+                }
+            }
+        }
     }
 }