Skip to content

[SPARK-42011][SPARK-42012][CONNECT][PYTHON][TESTS][FOLLOW-UP] Enable csv, orc tests in connect/test_parity_datasources.py #39581

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
techaddict wants to merge 3 commits into from
Closed

[SPARK-42011][SPARK-42012][CONNECT][PYTHON][TESTS][FOLLOW-UP] Enable csv, orc tests in connect/test_parity_datasources.py #39581

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
107d4c4
[SPARK-42011][SPARK-42012][CONNECT][PYTHON][TESTS][FOLLOWUP] Enable t…
techaddict Jan 15, 2023
aaf02e1
[SPARK-41921][CONNECT][TESTS][FOLLOW-UP] Enable doctests in connect/c…
techaddict Jan 15, 2023
aac9f23
Update test_parity_datasources.py
techaddict Jan 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions python/pyspark/sql/connect/column.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -439,9 +439,6 @@ def _test() -> None:
.getOrCreate()
)

# Spark Connect has a different string representation for Column.
del pyspark.sql.connect.column.Column.getItem.__doc__

# TODO(SPARK-41772): Enable pyspark.sql.connect.column.Column.withField doctest
del pyspark.sql.connect.column.Column.withField.__doc__

Expand Down
4 changes: 0 additions & 4 deletions python/pyspark/sql/connect/readwriter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -616,12 +616,8 @@ def _test() -> None:
globs = pyspark.sql.connect.readwriter.__dict__.copy()

# TODO(SPARK-41817): Support reading with schema
del pyspark.sql.connect.readwriter.DataFrameReader.load.__doc__
del pyspark.sql.connect.readwriter.DataFrameReader.option.__doc__
del pyspark.sql.connect.readwriter.DataFrameReader.text.__doc__
del pyspark.sql.connect.readwriter.DataFrameWriter.csv.__doc__
del pyspark.sql.connect.readwriter.DataFrameWriter.option.__doc__
del pyspark.sql.connect.readwriter.DataFrameWriter.text.__doc__
del pyspark.sql.connect.readwriter.DataFrameWriter.bucketBy.__doc__
del pyspark.sql.connect.readwriter.DataFrameWriter.sortBy.__doc__

Expand Down
2 changes: 1 addition & 1 deletion python/pyspark/sql/dataframe.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -506,7 +506,7 @@ def write(self) -> DataFrameWriter:
--------
>>> df = spark.createDataFrame([(2, "Alice"), (5, "Bob")], schema=["age", "name"])
>>> type(df.write)
<class 'pyspark.sql.readwriter.DataFrameWriter'>
<class '...readwriter.DataFrameWriter'>

Write the DataFrame as a table.

Expand Down
16 changes: 0 additions & 16 deletions python/pyspark/sql/tests/connect/test_parity_datasources.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,6 @@


class DataSourcesParityTests(DataSourcesTestsMixin, ReusedConnectTestCase):

# TODO(SPARK-42011): Implement DataFrameReader.csv
@unittest.skip("Fails in Spark Connect, should enable.")
def test_checking_csv_header(self):
super().test_checking_csv_header()

@unittest.skip("Spark Connect does not support RDD but the tests depend on them.")
def test_csv_sampling_ratio(self):
super().test_csv_sampling_ratio()
Expand All @@ -36,16 +30,6 @@ def test_csv_sampling_ratio(self):
def test_json_sampling_ratio(self):
super().test_json_sampling_ratio()

# TODO(SPARK-42011): Implement DataFrameReader.csv
@unittest.skip("Fails in Spark Connect, should enable.")
def test_multiline_csv(self):
super().test_multiline_csv()

# TODO(SPARK-42012): Implement DataFrameReader.orc
@unittest.skip("Fails in Spark Connect, should enable.")
def test_read_multiple_orc_file(self):
super().test_read_multiple_orc_file()


if __name__ == "__main__":
import unittest
Expand Down