From 9cfd67350670ef668781cf498597612713cba628 Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Tue, 28 Jun 2016 21:38:57 -0700 Subject: [PATCH 1/2] fix --- python/pyspark/sql/readwriter.py | 4 +++- .../scala/org/apache/spark/sql/DataFrameReader.scala | 12 ++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/python/pyspark/sql/readwriter.py b/python/pyspark/sql/readwriter.py index 3f28d7ad5051..73cdffaf411b 100644 --- a/python/pyspark/sql/readwriter.py +++ b/python/pyspark/sql/readwriter.py @@ -143,7 +143,9 @@ def load(self, path=None, format=None, schema=None, **options): if schema is not None: self.schema(schema) self.options(**options) - if path is not None: + if isinstance(path, basestring): + return self._df(self._jreader.load(path)) + elif path is not None: if type(path) != list: path = [path] return self._df(self._jreader.load(self._spark._sc._jvm.PythonUtils.toSeq(path))) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala index 35ba52278633..dbde158a9ec2 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala @@ -244,7 +244,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging { */ def json(path: String): DataFrame = { // This method ensures that calls that explicit need single argument works, see SPARK-16009 - json(Seq(path): _*) + option("path", path).json(Seq.empty: _*) } /** @@ -337,7 +337,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging { */ def csv(path: String): DataFrame = { // This method ensures that calls that explicit need single argument works, see SPARK-16009 - csv(Seq(path): _*) + option("path", path).csv(Seq.empty: _*) } /** @@ -406,7 +406,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging { */ def parquet(path: String): DataFrame = { // This method ensures that calls that explicit need single argument works, see SPARK-16009 - parquet(Seq(path): _*) + option("path", path).parquet(Seq.empty: _*) } /** @@ -434,7 +434,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging { */ def orc(path: String): DataFrame = { // This method ensures that calls that explicit need single argument works, see SPARK-16009 - orc(Seq(path): _*) + option("path", path).orc(Seq.empty: _*) } /** @@ -467,7 +467,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging { */ def text(path: String): DataFrame = { // This method ensures that calls that explicit need single argument works, see SPARK-16009 - text(Seq(path): _*) + option("path", path).text(Seq.empty: _*) } /** @@ -496,7 +496,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging { */ def textFile(path: String): Dataset[String] = { // This method ensures that calls that explicit need single argument works, see SPARK-16009 - textFile(Seq(path): _*) + option("path", path).textFile(Seq.empty: _*) } /** From f5100d646ffdf38464719344f47a94a285491c7a Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Wed, 29 Jun 2016 10:29:57 -0700 Subject: [PATCH 2/2] revert --- .../scala/org/apache/spark/sql/DataFrameReader.scala | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala index dbde158a9ec2..35ba52278633 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala @@ -244,7 +244,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging { */ def json(path: String): DataFrame = { // This method ensures that calls that explicit need single argument works, see SPARK-16009 - option("path", path).json(Seq.empty: _*) + json(Seq(path): _*) } /** @@ -337,7 +337,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging { */ def csv(path: String): DataFrame = { // This method ensures that calls that explicit need single argument works, see SPARK-16009 - option("path", path).csv(Seq.empty: _*) + csv(Seq(path): _*) } /** @@ -406,7 +406,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging { */ def parquet(path: String): DataFrame = { // This method ensures that calls that explicit need single argument works, see SPARK-16009 - option("path", path).parquet(Seq.empty: _*) + parquet(Seq(path): _*) } /** @@ -434,7 +434,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging { */ def orc(path: String): DataFrame = { // This method ensures that calls that explicit need single argument works, see SPARK-16009 - option("path", path).orc(Seq.empty: _*) + orc(Seq(path): _*) } /** @@ -467,7 +467,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging { */ def text(path: String): DataFrame = { // This method ensures that calls that explicit need single argument works, see SPARK-16009 - option("path", path).text(Seq.empty: _*) + text(Seq(path): _*) } /** @@ -496,7 +496,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging { */ def textFile(path: String): Dataset[String] = { // This method ensures that calls that explicit need single argument works, see SPARK-16009 - option("path", path).textFile(Seq.empty: _*) + textFile(Seq(path): _*) } /**