Added decimal marker option in TextLoader

src/Microsoft.ML.Core/Utilities/DoubleParser.cs

@@ -527,6 +527,8 @@ private static bool TryParseCore(ReadOnlySpan<char> span, ref int ich, ref bool
             Contracts.Assert(num == 0);
             Contracts.Assert(exp == 0);
 
+            const char decimalMarker = '.';
+
             if (ich >= span.Length)
                 return false;
 
@@ -554,7 +556,7 @@ private static bool TryParseCore(ReadOnlySpan<char> span, ref int ich, ref bool
                         return false;
                     break;
 
-                case '.':
+                case decimalMarker:
                     goto LPoint;
 
                 // The common cases.
@@ -571,7 +573,7 @@ private static bool TryParseCore(ReadOnlySpan<char> span, ref int ich, ref bool
                     break;
             }
 
-            // Get digits before '.'
+            // Get digits before the decimal marker, which may be '.' or ','
             uint d;
             for (; ; )
             {
@@ -593,14 +595,14 @@ private static bool TryParseCore(ReadOnlySpan<char> span, ref int ich, ref bool
             }
             Contracts.Assert(i < span.Length);
 
-            if (span[i] != '.')
+            if (span[i] != decimalMarker)
                 goto LAfterDigits;
 
             LPoint:
             Contracts.Assert(i < span.Length);
-            Contracts.Assert(span[i] == '.');
+            Contracts.Assert(span[i] == decimalMarker);
 
-            // Get the digits after '.'
+            // Get the digits after the decimal marker, which may be '.' or ','
             for (; ; )
             {
                 if (++i >= span.Length)

src/Microsoft.ML.Data/DataLoadSave/Text/TextLoader.cs

@@ -474,6 +474,12 @@ public class Options
             [Argument(ArgumentType.AtMostOnce, Name = nameof(Separator), Visibility = ArgumentAttribute.VisibilityType.EntryPointsOnly, HelpText = "Source column separator.", ShortName = "sep")]
             public char[] Separators = new[] { Defaults.Separator };
 
+            /// <summary>
+            /// The character that should be used as the decimal marker.
+            /// </summary>
+            [Argument(ArgumentType.AtMostOnce, Name = "Decimal Marker", HelpText = "Character symbol used to separate the integer part from the fractional part of a number written in decimal form.", ShortName = "decimal")]
+            public char DecimalMarker = Defaults.DecimalMarker;
+
             /// <summary>
             /// Specifies the input columns that should be mapped to <see cref="IDataView"/> columns.
             /// </summary>
@@ -535,6 +541,7 @@ internal static class Defaults
             internal const bool AllowQuoting = false;
             internal const bool AllowSparse = false;
             internal const char Separator = '\t';
+            internal const char DecimalMarker = '.';
             internal const bool HasHeader = false;
             internal const bool TrimWhitespace = false;
             internal const bool ReadMultilines = false;
@@ -702,11 +709,11 @@ public Bindings(TextLoader parent, Column[] cols, IMultiStreamSource headerFile,
                     ch.Assert(0 <= inputSize & inputSize < SrcLim);
                     List<ReadOnlyMemory<char>> lines = null;
                     if (headerFile != null)
-                        Cursor.GetSomeLines(headerFile, 1, parent.ReadMultilines, parent._separators, ref lines);
+                        Cursor.GetSomeLines(headerFile, 1, parent.ReadMultilines, parent._separators, ref lines, parent._decimalMarker);
                     if (needInputSize && inputSize == 0)
-                        Cursor.GetSomeLines(dataSample, 100, parent.ReadMultilines, parent._separators, ref lines);
+                        Cursor.GetSomeLines(dataSample, 100, parent.ReadMultilines, parent._separators, ref lines, parent._decimalMarker);
                     else if (headerFile == null && parent.HasHeader)
-                        Cursor.GetSomeLines(dataSample, 1, parent.ReadMultilines, parent._separators, ref lines);
+                        Cursor.GetSomeLines(dataSample, 1, parent.ReadMultilines, parent._separators, ref lines, parent._decimalMarker);
 
                     if (needInputSize && inputSize == 0)
                     {
@@ -1063,7 +1070,8 @@ private static VersionInfo GetVersionInfo()
                 // verWrittenCur: 0x00010009, // Introduced _flags
                 //verWrittenCur: 0x0001000A, // Added ForceVector in Range
                 //verWrittenCur: 0x0001000B, // Header now retained if used and present
-                verWrittenCur: 0x0001000C, // Removed Min and Contiguous from KeyType, and added ReadMultilines flag to OptionFlags
+                //verWrittenCur: 0x0001000C, // Removed Min and Contiguous from KeyType, and added ReadMultilines flag to OptionFlags
+                verWrittenCur: 0x0001000D, // Added decimal marker option to allow for ',' to be a decimal marker
                 verReadableCur: 0x0001000A,
                 verWeCanReadBack: 0x00010009,
                 loaderSignature: LoaderSignature,
@@ -1094,6 +1102,7 @@ private enum OptionFlags : uint
         // Input size is zero for unknown - determined by the data (including sparse rows).
         private readonly int _inputSize;
         private readonly char[] _separators;
+        private readonly char _decimalMarker;
         private readonly Bindings _bindings;
 
         private readonly Parser _parser;
@@ -1210,6 +1219,9 @@ internal TextLoader(IHostEnvironment env, Options options = null, IMultiStreamSo
                 }
             }
 
+            if (options.DecimalMarker == ',' && _separators.Contains<char>(','))
+                throw _host.ExceptUserArg(nameof(Options.DecimalMarker), "Decimal marker and separator cannot be the same '{0}' character.", options.DecimalMarker);
+            _decimalMarker = options.DecimalMarker;
             _bindings = new Bindings(this, cols, headerFile, dataSample);
             _parser = new Parser(this);
         }
@@ -1373,6 +1385,7 @@ private TextLoader(IHost host, ModelLoadContext ctx)
             // int: inputSize: 0 for determined from data
             // int: number of separators
             // char[]: separators
+            // char: decimal marker
             // bindings
             int cbFloat = ctx.Reader.ReadInt32();
             host.CheckDecode(cbFloat == sizeof(float));
@@ -1397,6 +1410,11 @@ private TextLoader(IHost host, ModelLoadContext ctx)
             if (_separators.Contains(':'))
                 host.CheckDecode((_flags & OptionFlags.AllowSparse) == 0);
 
+            if (ctx.Header.ModelVerWritten >= 0x0001000D)
+            {
+                _decimalMarker = ctx.Reader.ReadChar();
+                host.CheckDecode(_decimalMarker == '.' || _decimalMarker == ',');
+            }
             _bindings = new Bindings(ctx, this);
             _parser = new Parser(this);
         }
@@ -1437,6 +1455,7 @@ void ICanSaveModel.Save(ModelSaveContext ctx)
             // int: inputSize: 0 for determined from data
             // int: number of separators
             // char[]: separators
+            // char: decimal marker
             // bindings
             ctx.Writer.Write(sizeof(float));
             ctx.Writer.Write(_maxRows);
@@ -1445,6 +1464,7 @@ void ICanSaveModel.Save(ModelSaveContext ctx)
             _host.Assert(0 <= _inputSize && _inputSize < SrcLim);
             ctx.Writer.Write(_inputSize);
             ctx.Writer.WriteCharArray(_separators);
+            ctx.Writer.Write(_decimalMarker);
 
             _bindings.Save(ctx);
         }
@@ -1473,12 +1493,14 @@ internal static TextLoader CreateTextLoader<TInput>(IHostEnvironment host,
            bool allowQuoting = Defaults.AllowQuoting,
            bool supportSparse = Defaults.AllowSparse,
            bool trimWhitespace = Defaults.TrimWhitespace,
-           IMultiStreamSource dataSample = null)
+           IMultiStreamSource dataSample = null,
+           char decimalMarker = Defaults.DecimalMarker)
         {
             Options options = new Options
             {
                 HasHeader = hasHeader,
                 Separators = new[] { separator },
+                DecimalMarker = decimalMarker,
                 AllowQuoting = allowQuoting,
                 AllowSparse = supportSparse,
                 TrimWhitespace = trimWhitespace

src/Microsoft.ML.Data/DataLoadSave/Text/TextLoaderCursor.cs

@@ -146,7 +146,7 @@ public static DataViewRowCursor Create(TextLoader parent, IMultiStreamSource fil
                 SetupCursor(parent, active, 0, out srcNeeded, out cthd);
                 Contracts.Assert(cthd > 0);
 
-                var reader = new LineReader(files, BatchSize, 100, parent.HasHeader, parent.ReadMultilines, parent._separators, parent._maxRows, 1);
+                var reader = new LineReader(files, BatchSize, 100, parent.HasHeader, parent.ReadMultilines, parent._separators, parent._maxRows, 1, parent._decimalMarker);
                 var stats = new ParseStats(parent._host, 1);
                 return new Cursor(parent, stats, active, reader, srcNeeded, cthd);
             }
@@ -163,7 +163,7 @@ public static DataViewRowCursor[] CreateSet(TextLoader parent, IMultiStreamSourc
                 SetupCursor(parent, active, n, out srcNeeded, out cthd);
                 Contracts.Assert(cthd > 0);
 
-                var reader = new LineReader(files, BatchSize, 100, parent.HasHeader, parent.ReadMultilines, parent._separators, parent._maxRows, cthd);
+                var reader = new LineReader(files, BatchSize, 100, parent.HasHeader, parent.ReadMultilines, parent._separators, parent._maxRows, cthd, parent._decimalMarker);
                 var stats = new ParseStats(parent._host, cthd);
                 if (cthd <= 1)
                     return new DataViewRowCursor[1] { new Cursor(parent, stats, active, reader, srcNeeded, 1) };
@@ -205,7 +205,7 @@ public override ValueGetter<DataViewRowId> GetIdGetter()
                     };
             }
 
-            public static void GetSomeLines(IMultiStreamSource source, int count, bool readMultilines, char[] separators, ref List<ReadOnlyMemory<char>> lines)
+            public static void GetSomeLines(IMultiStreamSource source, int count, bool readMultilines, char[] separators, ref List<ReadOnlyMemory<char>> lines, char decimalMarker)
             {
                 Contracts.AssertValue(source);
                 Contracts.Assert(count > 0);
@@ -215,7 +215,7 @@ public static void GetSomeLines(IMultiStreamSource source, int count, bool readM
                     count = 2;
 
                 LineBatch batch;
-                var reader = new LineReader(source, count, 1, false, readMultilines, separators, count, 1);
+                var reader = new LineReader(source, count, 1, false, readMultilines, separators, count, 1, decimalMarker);
                 try
                 {
                     batch = reader.GetBatch();
@@ -404,6 +404,7 @@ private sealed class LineReader
                 private readonly bool _hasHeader;
                 private readonly bool _readMultilines;
                 private readonly char[] _separators;
+                private readonly char _decimalMarker;
                 private readonly int _batchSize;
                 private readonly IMultiStreamSource _files;
 
@@ -413,7 +414,7 @@ private sealed class LineReader
                 private Task _thdRead;
                 private volatile bool _abort;
 
-                public LineReader(IMultiStreamSource files, int batchSize, int bufSize, bool hasHeader, bool readMultilines, char[] separators, long limit, int cref)
+                public LineReader(IMultiStreamSource files, int batchSize, int bufSize, bool hasHeader, bool readMultilines, char[] separators, long limit, int cref, char decimalMarker)
                 {
                     // Note that files is allowed to be empty.
                     Contracts.AssertValue(files);
@@ -430,6 +431,7 @@ public LineReader(IMultiStreamSource files, int batchSize, int bufSize, bool has
                     _separators = separators;
                     _files = files;
                     _cref = cref;
+                    _decimalMarker = decimalMarker;
 
                     _queue = new BlockingQueue<LineBatch>(bufSize);
                     _thdRead = Utils.RunOnBackgroundThreadAsync(ThreadProc);

src/Microsoft.ML.Data/DataLoadSave/Text/TextLoaderParser.cs

@@ -633,6 +633,7 @@ public void Clear()
             }
 
             private readonly char[] _separators;
+            private readonly char _decimalMarker;
             private readonly OptionFlags _flags;
             private readonly int _inputSize;
             private readonly ColInfo[] _infos;
@@ -683,6 +684,7 @@ public Parser(TextLoader parent)
                 }
 
                 _separators = parent._separators;
+                _decimalMarker = parent._decimalMarker;
                 _flags = parent._flags;
                 _inputSize = parent._inputSize;
                 Contracts.Assert(_inputSize >= 0);

test/BaselineOutput/Common/EntryPoints/core_manifest.json

@@ -369,6 +369,18 @@
                   "\t"
                 ]
               },
+              {
+                "Name": "Decimal Marker",
+                "Type": "Char",
+                "Desc": "Character symbol used to separate the integer part from the fractional part of a number written in decimal form.",
+                "Aliases": [
+                  "decimal"
+                ],
+                "Required": false,
+                "SortOrder": 150.0,
+                "IsNullable": false,
+                "Default": "."
+              },
               {
                 "Name": "TrimWhitespace",
                 "Type": "Bool",

test/Microsoft.ML.Tests/TextLoaderTests.cs

@@ -803,6 +803,41 @@ public void TestTextLoaderKeyTypeBackCompat()
             }
         }
 
+        [Fact]
+        public void TestCommaAsDecimalMarker()
+        {
+            string dataPath = GetDataPath("iris_decimal_marker_as_comma.txt");
+
+            // Create a new context for ML.NET operations. It can be used for exception tracking and logging, 
+            // as a catalog of available operations and as the source of randomness.
+            var mlContext = new MLContext(seed: 1);
+            var reader = new TextLoader(mlContext, new TextLoader.Options()
+            {
+                Columns = new[]
+                        {
+                            new TextLoader.Column("Label", DataKind.Single, 0),
+                            new TextLoader.Column("Features", DataKind.Single, new [] { new TextLoader.Range(1, 4) }),
+                        },
+                DecimalMarker = ','
+            });
+            // Data
+            var textData = reader.Load(GetDataPath(dataPath));
+            var data = mlContext.Data.Cache(mlContext.Transforms.Conversion.MapValueToKey("Label")
+                .Fit(textData).Transform(textData));
+
+            // Pipeline
+            var pipeline = mlContext.MulticlassClassification.Trainers.OneVersusAll(
+                mlContext.BinaryClassification.Trainers.LinearSvm(new Trainers.LinearSvmTrainer.Options { NumberOfIterations = 100 }),
+                useProbabilities: false);
+
+            var model = pipeline.Fit(data);
+            var predictions = model.Transform(data);
+
+            // Metrics
+            var metrics = mlContext.MulticlassClassification.Evaluate(predictions);
+            Assert.True(metrics.MicroAccuracy > 0.83);
+        }
+
         private class IrisNoFields
         {
         }