Proper boxing of scalars in DataFrame.to_dict

doc/source/whatsnew/v0.23.5.txt

@@ -42,7 +42,6 @@ Bug Fixes
 **Groupby/Resample/Rolling**
 
 - Bug in :meth:`DataFrame.resample` when resampling ``NaT`` in ``TimeDeltaIndex`` (:issue:`13223`).
--
 
 **Missing**
 
@@ -52,3 +51,4 @@ Bug Fixes
 **I/O**
 
 - Bug in :func:`read_csv` that caused it to raise ``OverflowError`` when trying to use 'inf' as ``na_value`` with integer index column (:issue:`17128`)
+- Bug in :meth:`DataFrame.to_dict` when the result dict contains non-Python scalars (:issue:`23753`)

pandas/core/frame.py

@@ -280,6 +280,7 @@
     Index(['value'], dtype='object')
 """
 
+
 # -----------------------------------------------------------------------
 # DataFrame class
 
@@ -1220,16 +1221,18 @@ def to_dict(self, orient='dict', into=dict):
         elif orient.lower().startswith('sp'):
             return into_c((('index', self.index.tolist()),
                            ('columns', self.columns.tolist()),
-                           ('data', lib.map_infer(self.values.ravel(),
-                                                  com.maybe_box_datetimelike)
-                            .reshape(self.values.shape).tolist())))
+                           ('data', [
+                               list(map(com.maybe_box_datetimelike, t))
+                               for t in self.itertuples(index=False)]
+                            )))
         elif orient.lower().startswith('s'):
             return into_c((k, com.maybe_box_datetimelike(v))
                           for k, v in compat.iteritems(self))
         elif orient.lower().startswith('r'):
-            return [into_c((k, com.maybe_box_datetimelike(v))
-                           for k, v in zip(self.columns, np.atleast_1d(row)))
-                    for row in self.values]
+            return [
+                into_c((k, com.maybe_box_datetimelike(v))
+                       for k, v in compat.iteritems(row._asdict()))
+                for row in self.itertuples(index=False)]
         elif orient.lower().startswith('i'):
             if not self.index.is_unique:
                 raise ValueError(
@@ -2653,6 +2656,7 @@ def _get_value(self, index, col, takeable=False):
             col = self.columns.get_loc(col)
             index = self.index.get_loc(index)
             return self._get_value(index, col, takeable=True)
+
     _get_value.__doc__ = get_value.__doc__
 
     def set_value(self, index, col, value, takeable=False):
@@ -2697,6 +2701,7 @@ def _set_value(self, index, col, value, takeable=False):
             self._item_cache.pop(col, None)
 
             return self
+
     _set_value.__doc__ = set_value.__doc__
 
     def _ixs(self, i, axis=0):
@@ -3160,6 +3165,7 @@ def select_dtypes(self, include=None, exclude=None):
         4   True  1.0
         5  False  2.0
         """
+
         def _get_info_slice(obj, indexer):
             """Slice the info axis of `obj` with `indexer`."""
             if not hasattr(obj, '_info_axis_number'):
@@ -6044,9 +6050,9 @@ def diff(self, periods=1, axis=0):
     # Function application
 
     def _gotitem(self,
-                 key,           # type: Union[str, List[str]]
-                 ndim,          # type: int
-                 subset=None    # type: Union[Series, DataFrame, None]
+                 key,  # type: Union[str, List[str]]
+                 ndim,  # type: int
+                 subset=None  # type: Union[Series, DataFrame, None]
                  ):
         # type: (...) -> Union[Series, DataFrame]
         """

pandas/tests/frame/test_convert_to.py

@@ -150,7 +150,7 @@ def test_to_records_index_name(self):
     def test_to_records_with_unicode_index(self):
         # GH13172
         # unicode_literals conflict with to_records
-        result = DataFrame([{u'a': u'x', u'b': 'y'}]).set_index(u'a')\
+        result = DataFrame([{u'a': u'x', u'b': 'y'}]).set_index(u'a') \
             .to_records()
         expected = np.rec.array([('x', 'y')], dtype=[('a', 'O'), ('b', 'O')])
         tm.assert_almost_equal(result, expected)
@@ -282,22 +282,35 @@ def test_to_records_datetimeindex_with_tz(self, tz):
         tm.assert_numpy_array_equal(result, expected)
 
     def test_to_dict_box_scalars(self):
-        # 14216
+        # 14216, 23753
         # make sure that we are boxing properly
-        d = {'a': [1], 'b': ['b']}
+        df = DataFrame({'a': [1, 2], 'b': [.1, .2]})
 
-        result = DataFrame(d).to_dict()
-        assert isinstance(list(result['a'])[0], (int, long))
-        assert isinstance(list(result['b'])[0], (int, long))
+        result = df.to_dict()
+        assert isinstance(result['a'][0], (int, long))
+        assert isinstance(result['b'][0], float)
 
-        result = DataFrame(d).to_dict(orient='records')
+        result = df.to_dict(orient='records')
+        assert isinstance(result[0]['a'], (int, long))
+        assert isinstance(result[0]['b'], float)
+
+        result = df.to_dict(orient='list')
+        assert isinstance(result['a'][0], (int, long))
+        assert isinstance(result['b'][0], float)
+
+        result = df.to_dict(orient='split')
+        assert isinstance(result['data'][0][0], (int, long))
+        assert isinstance(result['data'][0][1], float)
+
+        result = df.to_dict(orient='index')
         assert isinstance(result[0]['a'], (int, long))
+        assert isinstance(result[0]['b'], float)
 
     def test_frame_to_dict_tz(self):
         # GH18372 When converting to dict with orient='records' columns of
         # datetime that are tz-aware were not converted to required arrays
         data = [(datetime(2017, 11, 18, 21, 53, 0, 219225, tzinfo=pytz.utc),),
-                (datetime(2017, 11, 18, 22, 6, 30, 61810, tzinfo=pytz.utc,),)]
+                (datetime(2017, 11, 18, 22, 6, 30, 61810, tzinfo=pytz.utc, ),)]
         df = DataFrame(list(data), columns=["d", ])
 
         result = df.to_dict(orient='records')