From e903cd2361b43e35d596e8619ee7844bb5cb33bf Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Thu, 22 Jun 2017 15:51:41 +0900 Subject: [PATCH 1/2] Work around astype with columns in Pandas < 0.19.0 --- python/pyspark/sql/dataframe.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py index 0c0ead9c5df7f..2daf94d9f33ed 100644 --- a/python/pyspark/sql/dataframe.py +++ b/python/pyspark/sql/dataframe.py @@ -1725,11 +1725,14 @@ def toPandas(self): dtype = {} for field in self.schema: pandas_type = _to_corrected_pandas_type(field.dataType) - if (pandas_type): + if pandas_type is not None: dtype[field.name] = pandas_type - df = pd.DataFrame.from_records(self.collect(), columns=self.columns) - return df.astype(dtype, copy=False) + pdf = pd.DataFrame.from_records(self.collect(), columns=self.columns) + + for f, t in dtype.items(): + pdf[f] = pdf[f].astype(t) + return pdf ########################################################################################## # Pandas compatibility From 6702ad131fe0c982b38ae5a0d55e38a9bd604353 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Thu, 22 Jun 2017 15:58:40 +0900 Subject: [PATCH 2/2] No copy --- python/pyspark/sql/dataframe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py index 2daf94d9f33ed..e03a4c86a2ab2 100644 --- a/python/pyspark/sql/dataframe.py +++ b/python/pyspark/sql/dataframe.py @@ -1731,7 +1731,7 @@ def toPandas(self): pdf = pd.DataFrame.from_records(self.collect(), columns=self.columns) for f, t in dtype.items(): - pdf[f] = pdf[f].astype(t) + pdf[f] = pdf[f].astype(t, copy=False) return pdf ##########################################################################################