ENH: ISO8601-compliant datetime string conversion in iterrows() and Series construction. (pandas-dev#19762)

Author: minggli

doc/source/whatsnew/v0.23.0.txt

@@ -766,7 +766,6 @@ Datetimelike
 - Bug in :func:`to_datetime` where passing an out-of-bounds datetime with ``errors='coerce'`` and ``utc=True`` would raise ``OutOfBoundsDatetime`` instead of parsing to ``NaT`` (:issue:`19612`)
 - Bug in :class:`DatetimeIndex` and :class:`TimedeltaIndex` addition and subtraction where name of the returned object was not always set consistently. (:issue:`19744`)
 - Bug in :class:`DatetimeIndex` and :class:`TimedeltaIndex` addition and subtraction where operations with numpy arrays raised ``TypeError`` (:issue:`19847`)
--
 
 Timedelta
 ^^^^^^^^^
@@ -918,6 +917,7 @@ Reshaping
 - :func:`Series.rename` now accepts ``axis`` as a kwarg (:issue:`18589`)
 - Comparisons between :class:`Series` and :class:`Index` would return a ``Series`` with an incorrect name, ignoring the ``Index``'s name attribute (:issue:`19582`)
 - Bug in :func:`qcut` where datetime and timedelta data with ``NaT`` present raised a ``ValueError`` (:issue:`19768`)
+- Bug in :func:`DataFrame.iterrows`, which would infers strings not compliant to `ISO8601 <https://en.wikipedia.org/wiki/ISO_8601>`_ to datetimes (:issue:`19671`)
 
 Other
 ^^^^^

pandas/core/dtypes/cast.py

@@ -904,16 +904,23 @@ def maybe_infer_to_datetimelike(value, convert_dates=False):
     def try_datetime(v):
         # safe coerce to datetime64
         try:
-            v = tslib.array_to_datetime(v, errors='raise')
+            # GH19671
+            v = tslib.array_to_datetime(v,
+                                        require_iso8601=True,
+                                        errors='raise')
         except ValueError:
 
             # we might have a sequence of the same-datetimes with tz's
             # if so coerce to a DatetimeIndex; if they are not the same,
-            # then these stay as object dtype
+            # then these stay as object dtype, xref GH19671
             try:
-                from pandas import to_datetime
-                return to_datetime(v)
-            except Exception:
+                from pandas._libs.tslibs import conversion
+                from pandas import DatetimeIndex
+
+                values, tz = conversion.datetime_to_datetime64(v)
+                return DatetimeIndex(values).tz_localize(
+                    'UTC').tz_convert(tz=tz)
+            except (ValueError, TypeError):
                 pass
 
         except Exception:

pandas/core/internals.py

@@ -2602,8 +2602,8 @@ def _maybe_coerce_values(self, values):
         """Input validation for values passed to __init__. Ensure that
         we have datetime64ns, coercing if necessary.
 
-        Parametetrs
-        -----------
+        Parameters
+        ----------
         values : array-like
             Must be convertible to datetime64
 

pandas/tests/dtypes/test_cast.py

@@ -301,6 +301,10 @@ def test_maybe_infer_to_datetimelike(self):
                                      [NaT, 'b', 1]]))
         assert result.size == 6
 
+        # GH19671
+        result = Series(['M1701', Timestamp('20130101')])
+        assert result.dtype.kind == 'O'
+
 
 class TestConvert(object):
 

pandas/tests/frame/test_api.py

@@ -15,7 +15,8 @@
 from numpy.random import randn
 import numpy as np
 
-from pandas import DataFrame, Series, date_range, timedelta_range, Categorical
+from pandas import (DataFrame, Series, date_range, timedelta_range,
+                    Categorical, SparseDataFrame)
 import pandas as pd
 
 from pandas.util.testing import (assert_almost_equal,
@@ -214,6 +215,18 @@ def test_iterrows(self):
             exp = self.mixed_frame.loc[k]
             self._assert_series_equal(v, exp)
 
+    def test_iterrows_iso8601(self):
+        # GH19671
+        if self.klass == SparseDataFrame:
+            pytest.xfail(reason='SparseBlock datetime type not implemented.')
+
+        s = self.klass(
+            {'non_iso8601': ['M1701', 'M1802', 'M1903', 'M2004'],
+             'iso8601': date_range('2000-01-01', periods=4, freq='M')})
+        for k, v in s.iterrows():
+            exp = s.loc[k]
+            self._assert_series_equal(v, exp)
+
     def test_itertuples(self):
         for i, tup in enumerate(self.frame.itertuples()):
             s = self.klass._constructor_sliced(tup[1:])