Merge branch 'main' into 6806_dataframe_append_issues

Author: michaelgsharp

src/Microsoft.Data.Analysis/DataFrame.IO.cs

@@ -98,18 +98,21 @@ private static Type MaxKind(Type a, Type b)
         /// <param name="guessRows">number of rows used to guess types</param>
         /// <param name="addIndexColumn">add one column with the row index</param>
         /// <param name="encoding">The character encoding. Defaults to UTF8 if not specified</param>
+        /// <param name="renameDuplicatedColumns">If set to true, columns with repeated names are auto-renamed.</param>
+        /// <param name="cultureInfo">culture info for formatting values</param>
         /// <returns>DataFrame</returns>
         public static DataFrame LoadCsv(string filename,
                                 char separator = ',', bool header = true,
                                 string[] columnNames = null, Type[] dataTypes = null,
                                 int numRows = -1, int guessRows = 10,
-                                bool addIndexColumn = false, Encoding encoding = null)
+                                bool addIndexColumn = false, Encoding encoding = null,
+                                bool renameDuplicatedColumns = false, CultureInfo cultureInfo = null)
         {
             using (Stream fileStream = new FileStream(filename, FileMode.Open))
             {
                 return LoadCsv(fileStream,
                                   separator: separator, header: header, columnNames: columnNames, dataTypes: dataTypes, numberOfRowsToRead: numRows,
-                                  guessRows: guessRows, addIndexColumn: addIndexColumn, encoding: encoding);
+                                  guessRows: guessRows, addIndexColumn: addIndexColumn, encoding: encoding, renameDuplicatedColumns: renameDuplicatedColumns, cultureInfo: cultureInfo);
             }
         }
 
@@ -351,8 +354,14 @@ private static DataFrame ReadCsvLinesIntoDataFrame(WrappedStreamReaderOrStringRe
                                 char separator = ',', bool header = true,
                                 string[] columnNames = null, Type[] dataTypes = null,
                                 long numberOfRowsToRead = -1, int guessRows = 10, bool addIndexColumn = false,
-                                bool renameDuplicatedColumns = false)
+                                bool renameDuplicatedColumns = false,
+                                CultureInfo cultureInfo = null)
         {
+            if (cultureInfo == null)
+            {
+                cultureInfo = CultureInfo.CurrentCulture;
+            }
+
             if (dataTypes == null && guessRows <= 0)
             {
                 throw new ArgumentException(string.Format(Strings.ExpectedEitherGuessRowsOrDataTypes, nameof(guessRows), nameof(dataTypes)));
@@ -452,7 +461,7 @@ private static DataFrame ReadCsvLinesIntoDataFrame(WrappedStreamReaderOrStringRe
                     }
                     else
                     {
-                        ret.Append(fields, inPlace: true);
+                        ret.Append(fields, inPlace: true, cultureInfo: cultureInfo);
                     }
                     ++rowline;
                 }
@@ -508,7 +517,6 @@ public TextReader GetTextReader()
                 }
 
             }
-
         }
 
         /// <summary>
@@ -522,14 +530,18 @@ public TextReader GetTextReader()
         /// <param name="numberOfRowsToRead">number of rows to read not including the header(if present)</param>
         /// <param name="guessRows">number of rows used to guess types</param>
         /// <param name="addIndexColumn">add one column with the row index</param>
+        /// <param name="renameDuplicatedColumns">If set to true, columns with repeated names are auto-renamed.</param>
+        /// <param name="cultureInfo">culture info for formatting values</param>
         /// <returns><see cref="DataFrame"/></returns>
         public static DataFrame LoadCsvFromString(string csvString,
                                 char separator = ',', bool header = true,
                                 string[] columnNames = null, Type[] dataTypes = null,
-                                long numberOfRowsToRead = -1, int guessRows = 10, bool addIndexColumn = false)
+                                long numberOfRowsToRead = -1, int guessRows = 10, bool addIndexColumn = false,
+                                bool renameDuplicatedColumns = false,
+                                CultureInfo cultureInfo = null)
         {
             WrappedStreamReaderOrStringReader wrappedStreamReaderOrStringReader = new WrappedStreamReaderOrStringReader(csvString);
-            return ReadCsvLinesIntoDataFrame(wrappedStreamReaderOrStringReader, separator, header, columnNames, dataTypes, numberOfRowsToRead, guessRows, addIndexColumn);
+            return ReadCsvLinesIntoDataFrame(wrappedStreamReaderOrStringReader, separator, header, columnNames, dataTypes, numberOfRowsToRead, guessRows, addIndexColumn, renameDuplicatedColumns, cultureInfo);
         }
 
         /// <summary>
@@ -545,12 +557,13 @@ public static DataFrame LoadCsvFromString(string csvString,
         /// <param name="addIndexColumn">add one column with the row index</param>
         /// <param name="encoding">The character encoding. Defaults to UTF8 if not specified</param>
         /// <param name="renameDuplicatedColumns">If set to true, columns with repeated names are auto-renamed.</param>
+        /// <param name="cultureInfo">culture info for formatting values</param>
         /// <returns><see cref="DataFrame"/></returns>
         public static DataFrame LoadCsv(Stream csvStream,
                                 char separator = ',', bool header = true,
                                 string[] columnNames = null, Type[] dataTypes = null,
                                 long numberOfRowsToRead = -1, int guessRows = 10, bool addIndexColumn = false,
-                                Encoding encoding = null, bool renameDuplicatedColumns = false)
+                                Encoding encoding = null, bool renameDuplicatedColumns = false, CultureInfo cultureInfo = null)
         {
             if (!csvStream.CanSeek)
             {
@@ -563,7 +576,7 @@ public static DataFrame LoadCsv(Stream csvStream,
             }
 
             WrappedStreamReaderOrStringReader wrappedStreamReaderOrStringReader = new WrappedStreamReaderOrStringReader(csvStream, encoding ?? Encoding.UTF8);
-            return ReadCsvLinesIntoDataFrame(wrappedStreamReaderOrStringReader, separator, header, columnNames, dataTypes, numberOfRowsToRead, guessRows, addIndexColumn, renameDuplicatedColumns);
+            return ReadCsvLinesIntoDataFrame(wrappedStreamReaderOrStringReader, separator, header, columnNames, dataTypes, numberOfRowsToRead, guessRows, addIndexColumn, renameDuplicatedColumns, cultureInfo);
         }
 
         /// <summary>

src/Microsoft.Data.Analysis/DataFrame.cs

@@ -5,6 +5,7 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Globalization;
 using System.Linq;
 using System.Text;
 
@@ -485,12 +486,13 @@ private void ResizeByOneAndAppend(DataFrameColumn column, object value)
         /// <remarks> Values are appended based on the column names</remarks>
         /// <param name="rows">The rows to be appended to this DataFrame </param> 
         /// <param name="inPlace">If set, appends <paramref name="rows"/> in place. Otherwise, a new DataFrame is returned with the <paramref name="rows"/> appended</param>
-        public DataFrame Append(IEnumerable<DataFrameRow> rows, bool inPlace = false)
+        /// <param name="cultureInfo">culture info for formatting values</param>
+        public DataFrame Append(IEnumerable<DataFrameRow> rows, bool inPlace = false, CultureInfo cultureInfo = null)
         {
             DataFrame ret = inPlace ? this : Clone();
             foreach (DataFrameRow row in rows)
             {
-                ret.Append(row.GetValues(), inPlace: true);
+                ret.Append(row.GetValues(), inPlace: true, cultureInfo: cultureInfo);
             }
             return ret;
         }
@@ -502,8 +504,14 @@ public DataFrame Append(IEnumerable<DataFrameRow> rows, bool inPlace = false)
         /// <remarks>If <paramref name="row"/> is null, a null value is appended to each column</remarks>
         /// <param name="row"></param> 
         /// <param name="inPlace">If set, appends a <paramref name="row"/> in place. Otherwise, a new DataFrame is returned with an appended <paramref name="row"/> </param>
-        public DataFrame Append(IEnumerable<object> row = null, bool inPlace = false)
+        /// <param name="cultureInfo">culture info for formatting values</param>
+        public DataFrame Append(IEnumerable<object> row = null, bool inPlace = false, CultureInfo cultureInfo = null)
         {
+            if (cultureInfo == null)
+            {
+                cultureInfo = CultureInfo.CurrentCulture;
+            }
+
             DataFrame ret = inPlace ? this : Clone();
             IEnumerator<DataFrameColumn> columnEnumerator = ret.Columns.GetEnumerator();
             bool columnMoveNext = columnEnumerator.MoveNext();
@@ -531,7 +539,7 @@ public DataFrame Append(IEnumerable<object> row = null, bool inPlace = false)
                     }
                     if (value != null)
                     {
-                        value = Convert.ChangeType(value, column.DataType);
+                        value = Convert.ChangeType(value, column.DataType, cultureInfo);
 
                         if (value is null)
                         {

src/Microsoft.ML.Core/Prediction/IPredictor.cs

@@ -2,62 +2,61 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-namespace Microsoft.ML
+namespace Microsoft.ML;
+
+/// <summary>
+/// Type of prediction task. Note that this is a legacy structure and usage of this should generally be
+/// discouraged in future projects. Its presence suggests that there are privileged and supported
+/// tasks, and anything outside of this is unsupported. This runs rather contrary to the idea of this
+/// being an expandable framework, and it is inappropriately limiting. For legacy pipelines based on
+/// <see cref="ITrainer"/> and <see cref="IPredictor"/> it is still useful, but for things based on
+/// the <see cref="IEstimator{TTransformer}"/> idiom, it is inappropriate.
+/// </summary>
+[BestFriend]
+internal enum PredictionKind
 {
-    /// <summary>
-    /// Type of prediction task. Note that this is a legacy structure and usage of this should generally be
-    /// discouraged in future projects. Its presence suggests that there are privileged and supported
-    /// tasks, and anything outside of this is unsupported. This runs rather contrary to the idea of this
-    /// being an expandable framework, and it is inappropriately limiting. For legacy pipelines based on
-    /// <see cref="ITrainer"/> and <see cref="IPredictor"/> it is still useful, but for things based on
-    /// the <see cref="IEstimator{TTransformer}"/> idiom, it is inappropriate.
-    /// </summary>
-    [BestFriend]
-    internal enum PredictionKind
-    {
-        Unknown = 0,
-        Custom = 1,
+    Unknown = 0,
+    Custom = 1,
 
-        BinaryClassification = 2,
-        MulticlassClassification = 3,
-        Regression = 4,
-        MultiOutputRegression = 5,
-        Ranking = 6,
-        Recommendation = 7,
-        AnomalyDetection = 8,
-        Clustering = 9,
-        SequenceClassification = 10,
+    BinaryClassification = 2,
+    MulticlassClassification = 3,
+    Regression = 4,
+    MultiOutputRegression = 5,
+    Ranking = 6,
+    Recommendation = 7,
+    AnomalyDetection = 8,
+    Clustering = 9,
+    SequenceClassification = 10,
 
-        // More to be added later.
-    }
+    // More to be added later.
+}
 
+/// <summary>
+/// Weakly typed version of IPredictor.
+/// </summary>
+[BestFriend]
+internal interface IPredictor
+{
     /// <summary>
-    /// Weakly typed version of IPredictor.
+    /// Return the type of prediction task.
     /// </summary>
-    [BestFriend]
-    internal interface IPredictor
-    {
-        /// <summary>
-        /// Return the type of prediction task.
-        /// </summary>
-        PredictionKind PredictionKind { get; }
-    }
+    PredictionKind PredictionKind { get; }
+}
 
-    /// <summary>
-    /// A predictor the produces values of the indicated type.
-    /// REVIEW: Determine whether this is just a temporary shim or long term solution.
-    /// </summary>
-    [BestFriend]
-    internal interface IPredictorProducing<out TResult> : IPredictor
-    {
-    }
+/// <summary>
+/// A predictor the produces values of the indicated type.
+/// REVIEW: Determine whether this is just a temporary shim or long term solution.
+/// </summary>
+[BestFriend]
+internal interface IPredictorProducing<out TResult> : IPredictor
+{
+}
 
-    /// <summary>
-    /// A predictor that produces values and distributions of the indicated types.
-    /// Note that from a public API perspective this is bad.
-    /// </summary>
-    [BestFriend]
-    internal interface IDistPredictorProducing<out TResult, out TResultDistribution> : IPredictorProducing<TResult>
-    {
-    }
+/// <summary>
+/// A predictor that produces values and distributions of the indicated types.
+/// Note that from a public API perspective this is bad.
+/// </summary>
+[BestFriend]
+internal interface IDistPredictorProducing<out TResult, out TResultDistribution> : IPredictorProducing<TResult>
+{
 }