Skip to content

[SPARK-40606][PS][TEST] Eliminate to_pandas warnings in test #38042

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
zhengruifeng wants to merge 1 commit into from
Closed

[SPARK-40606][PS][TEST] Eliminate to_pandas warnings in test #38042

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion python/pyspark/pandas/tests/data_type_ops/test_binary_ops.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ def test_from_to_pandas(self):
data = [b"1", b"2", b"3"]
pser = pd.Series(data)
psser = ps.Series(data)
self.assert_eq(pser, psser.to_pandas())
self.assert_eq(pser, psser._to_pandas())
Copy link
Member

@srowen srowen Sep 29, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just for my info, what is the difference in the old and new method, anything functional?

Copy link
Contributor Author

@zhengruifeng zhengruifeng Sep 30, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the only difference between to_pandas() and _to_pandas() is the warning.

self.assert_eq(ps.from_pandas(pser), psser)

def test_isnull(self):
Expand Down
2 changes: 1 addition & 1 deletion python/pyspark/pandas/tests/data_type_ops/test_boolean_ops.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -732,7 +732,7 @@ def test_from_to_pandas(self):
data = [True, True, False, None]
pser = pd.Series(data, dtype="boolean")
psser = ps.Series(data, dtype="boolean")
self.check_extension(pser, psser.to_pandas())
self.check_extension(pser, psser._to_pandas())
self.check_extension(ps.from_pandas(pser), psser)

def test_isnull(self):
Expand Down
2 changes: 1 addition & 1 deletion python/pyspark/pandas/tests/data_type_ops/test_categorical_ops.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ def test_from_to_pandas(self):
data = [1, "x", "y"]
pser = pd.Series(data, dtype="category")
psser = ps.Series(data, dtype="category")
self.assert_eq(pser, psser.to_pandas())
self.assert_eq(pser, psser._to_pandas())
self.assert_eq(ps.from_pandas(pser), psser)

def test_isnull(self):
Expand Down
2 changes: 1 addition & 1 deletion python/pyspark/pandas/tests/data_type_ops/test_complex_ops.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,7 @@ def test_from_to_pandas(self):
pdf, psdf = self.array_pdf, self.array_psdf
for col in self.array_df_cols:
pser, psser = pdf[col], psdf[col]
self.assert_eq(pser, psser.to_pandas())
self.assert_eq(pser, psser._to_pandas())
self.assert_eq(ps.from_pandas(pser), psser)

def test_isnull(self):
Expand Down
2 changes: 1 addition & 1 deletion python/pyspark/pandas/tests/data_type_ops/test_date_ops.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ def test_from_to_pandas(self):
data = [datetime.date(1994, 1, 31), datetime.date(1994, 2, 1), datetime.date(1994, 2, 2)]
pser = pd.Series(data)
psser = ps.Series(data)
self.assert_eq(pser, psser.to_pandas())
self.assert_eq(pser, psser._to_pandas())
self.assert_eq(ps.from_pandas(pser), psser)

def test_isnull(self):
Expand Down
2 changes: 1 addition & 1 deletion python/pyspark/pandas/tests/data_type_ops/test_datetime_ops.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ def test_from_to_pandas(self):
data = pd.date_range("1994-1-31 10:30:15", periods=3, freq="M")
pser = pd.Series(data)
psser = ps.Series(data)
self.assert_eq(pser, psser.to_pandas())
self.assert_eq(pser, psser._to_pandas())
self.assert_eq(ps.from_pandas(pser), psser)

def test_isnull(self):
Expand Down
2 changes: 1 addition & 1 deletion python/pyspark/pandas/tests/data_type_ops/test_null_ops.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ def test_from_to_pandas(self):
data = [None, None, None]
pser = pd.Series(data)
psser = ps.Series(data)
self.assert_eq(pser, psser.to_pandas())
self.assert_eq(pser, psser._to_pandas())
self.assert_eq(ps.from_pandas(pser), psser)

def test_isnull(self):
Expand Down
6 changes: 3 additions & 3 deletions python/pyspark/pandas/tests/data_type_ops/test_num_ops.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -320,7 +320,7 @@ def test_from_to_pandas(self):
pdf, psdf = self.pdf, self.psdf
for col in self.numeric_df_cols:
pser, psser = pdf[col], psdf[col]
self.assert_eq(pser, psser.to_pandas())
self.assert_eq(pser, psser._to_pandas())
self.assert_eq(ps.from_pandas(pser), psser)

def test_isnull(self):
Expand Down Expand Up @@ -464,7 +464,7 @@ def intergral_extension_pser_psser_pairs(self):

def test_from_to_pandas(self):
for pser, psser in self.intergral_extension_pser_psser_pairs:
self.check_extension(pser, psser.to_pandas())
self.check_extension(pser, psser._to_pandas())
self.check_extension(ps.from_pandas(pser), psser)

def test_isnull(self):
Expand Down Expand Up @@ -607,7 +607,7 @@ def fractional_extension_pser_psser_pairs(self):

def test_from_to_pandas(self):
for pser, psser in self.fractional_extension_pser_psser_pairs:
self.check_extension(pser, psser.to_pandas())
self.check_extension(pser, psser._to_pandas())
self.check_extension(ps.from_pandas(pser), psser)

def test_isnull(self):
Expand Down
4 changes: 2 additions & 2 deletions python/pyspark/pandas/tests/data_type_ops/test_string_ops.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ def test_from_to_pandas(self):
data = ["x", "y", "z"]
pser = pd.Series(data)
psser = ps.Series(data)
self.assert_eq(pser, psser.to_pandas())
self.assert_eq(pser, psser._to_pandas())
self.assert_eq(ps.from_pandas(pser), psser)

def test_isnull(self):
Expand Down Expand Up @@ -275,7 +275,7 @@ def test_from_to_pandas(self):
data = ["x", "y", "z", None]
pser = pd.Series(data, dtype="string")
psser = ps.Series(data, dtype="string")
self.assert_eq(pser, psser.to_pandas())
self.assert_eq(pser, psser._to_pandas())
self.assert_eq(ps.from_pandas(pser), psser)

def test_isnull(self):
Expand Down
2 changes: 1 addition & 1 deletion python/pyspark/pandas/tests/data_type_ops/test_timedelta_ops.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ def test_from_to_pandas(self):
data = [timedelta(1), timedelta(microseconds=2)]
pser = pd.Series(data)
psser = ps.Series(data)
self.assert_eq(pser, psser.to_pandas())
self.assert_eq(pser, psser._to_pandas())
self.assert_eq(ps.from_pandas(pser), psser)

def test_isnull(self):
Expand Down
2 changes: 1 addition & 1 deletion python/pyspark/pandas/tests/data_type_ops/test_udt_ops.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ def test_from_to_pandas(self):
sparse_vector = SparseVector(len(sparse_values), sparse_values)
pser = pd.Series([sparse_vector])
psser = ps.Series([sparse_vector])
self.assert_eq(pser, psser.to_pandas())
self.assert_eq(pser, psser._to_pandas())
self.assert_eq(ps.from_pandas(pser), psser)

def test_isnull(self):
Expand Down
16 changes: 8 additions & 8 deletions python/pyspark/pandas/tests/indexes/test_base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -380,11 +380,11 @@ def test_index_unique(self):
# here the output is different than pandas in terms of order
expected = [0, 1, 3, 5, 6, 8, 9]

self.assert_eq(expected, sorted(psidx.unique().to_pandas()))
self.assert_eq(expected, sorted(psidx.unique(level=0).to_pandas()))
self.assert_eq(expected, sorted(psidx.unique()._to_pandas()))
self.assert_eq(expected, sorted(psidx.unique(level=0)._to_pandas()))

expected = [1, 2, 4, 6, 7, 9, 10]
self.assert_eq(expected, sorted((psidx + 1).unique().to_pandas()))
self.assert_eq(expected, sorted((psidx + 1).unique()._to_pandas()))

with self.assertRaisesRegex(IndexError, "Too many levels*"):
psidx.unique(level=1)
Expand Down Expand Up @@ -507,7 +507,7 @@ def test_multi_index_symmetric_difference(self):

self.assert_eq(
midx.symmetric_difference(midx_),
midx.to_pandas().symmetric_difference(midx_.to_pandas()),
midx._to_pandas().symmetric_difference(midx_._to_pandas()),
)

with self.assertRaisesRegex(NotImplementedError, "Doesn't support*"):
Expand Down Expand Up @@ -1356,15 +1356,15 @@ def test_monotonic(self):
psdf = ps.DataFrame({"a": [-5, -4, -3, -2, -1], "b": [1, 1, 1, 1, 1]})
psdf["b"] = None
psmidx = psdf.set_index(["a", "b"]).index
pmidx = psmidx.to_pandas()
pmidx = psmidx._to_pandas()
self.assert_eq(psmidx.is_monotonic_increasing, pmidx.is_monotonic_increasing)
self.assert_eq(psmidx.is_monotonic_decreasing, pmidx.is_monotonic_decreasing)

# For [(None, "e"), (None, "c"), (None, "b"), (None, "d"), (None, "a")]
psdf = ps.DataFrame({"a": [1, 1, 1, 1, 1], "b": ["e", "c", "b", "d", "a"]})
psdf["a"] = None
psmidx = psdf.set_index(["a", "b"]).index
pmidx = psmidx.to_pandas()
pmidx = psmidx._to_pandas()
self.assert_eq(psmidx.is_monotonic_increasing, pmidx.is_monotonic_increasing)
self.assert_eq(psmidx.is_monotonic_decreasing, pmidx.is_monotonic_decreasing)

Expand All @@ -1373,7 +1373,7 @@ def test_monotonic(self):
psdf["a"] = None
psdf["b"] = None
psmidx = psdf.set_index(["a", "b"]).index
pmidx = psmidx.to_pandas()
pmidx = psmidx._to_pandas()
self.assert_eq(psmidx.is_monotonic_increasing, pmidx.is_monotonic_increasing)
self.assert_eq(psmidx.is_monotonic_decreasing, pmidx.is_monotonic_decreasing)

Expand All @@ -1382,7 +1382,7 @@ def test_monotonic(self):
psdf["a"] = None
psdf["b"] = None
psmidx = psdf.set_index(["a", "b"]).index
pmidx = psmidx.to_pandas()
pmidx = psmidx._to_pandas()
self.assert_eq(psmidx.is_monotonic_increasing, pmidx.is_monotonic_increasing)
self.assert_eq(psmidx.is_monotonic_decreasing, pmidx.is_monotonic_decreasing)

Expand Down
2 changes: 1 addition & 1 deletion python/pyspark/pandas/tests/plot/test_frame_plot_plotly.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ def check_scatter_plot(pdf, psdf, x, y, c):

def test_pie_plot(self):
def check_pie_plot(psdf):
pdf = psdf.to_pandas()
pdf = psdf._to_pandas()
self.assertEqual(
psdf.plot(kind="pie", y=psdf.columns[0]),
express.pie(pdf, values="a", names=pdf.index),
Expand Down
2 changes: 1 addition & 1 deletion python/pyspark/pandas/tests/plot/test_series_plot_matplotlib.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ def psdf2(self):

@property
def pdf2(self):
return self.psdf2.to_pandas()
return self.psdf2._to_pandas()

@staticmethod
def plot_to_base64(ax):
Expand Down
6 changes: 3 additions & 3 deletions python/pyspark/pandas/tests/plot/test_series_plot_plotly.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ def psdf2(self):

@property
def pdf2(self):
return self.psdf2.to_pandas()
return self.psdf2._to_pandas()

def test_bar_plot(self):
pdf = self.pdf1
Expand Down Expand Up @@ -111,7 +111,7 @@ def test_area_plot(self):

def test_pie_plot(self):
psdf = self.psdf1
pdf = psdf.to_pandas()
pdf = psdf._to_pandas()
self.assertEqual(
psdf["a"].plot(kind="pie"),
express.pie(pdf, values=pdf.columns[0], names=pdf.index),
Expand All @@ -134,7 +134,7 @@ def test_pie_plot(self):
# },
# index=pd.MultiIndex.from_tuples([("x", "y")] * 11),
# )
# pdf = psdf.to_pandas()
# pdf = psdf._to_pandas()
# self.assertEqual(
# psdf["a"].plot(kind="pie"), express.pie(pdf, values=pdf.columns[0], names=pdf.index),
# )
Expand Down
Loading