Skip to content

Commit ce52353

Browse files
committed
Adds new SQLContext.load() overload with user defined dynamic partition columns
1 parent 8d2ff71 commit ce52353

File tree

1 file changed

+32
-0
lines changed

1 file changed

+32
-0
lines changed

sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -781,6 +781,22 @@ class SQLContext(@transient val sparkContext: SparkContext)
781781
load(source, schema, options.toMap)
782782
}
783783

784+
/**
785+
* :: Experimental ::
786+
* (Java-specific) Returns the dataset specified by the given data source and
787+
* a set of options as a DataFrame, using the given schema as the schema of the DataFrame.
788+
*
789+
* @group genericdata
790+
*/
791+
@Experimental
792+
def load(
793+
source: String,
794+
schema: StructType,
795+
partitionColumns: Array[String],
796+
options: java.util.Map[String, String]): DataFrame = {
797+
load(source, schema, partitionColumns, options.toMap)
798+
}
799+
784800
/**
785801
* :: Experimental ::
786802
* (Scala-specific) Returns the dataset specified by the given data source and
@@ -796,6 +812,22 @@ class SQLContext(@transient val sparkContext: SparkContext)
796812
DataFrame(this, LogicalRelation(resolved.relation))
797813
}
798814

815+
/**
816+
* :: Experimental ::
817+
* (Scala-specific) Returns the dataset specified by the given data source and
818+
* a set of options as a DataFrame, using the given schema as the schema of the DataFrame.
819+
* @group genericdata
820+
*/
821+
@Experimental
822+
def load(
823+
source: String,
824+
schema: StructType,
825+
partitionColumns: Array[String],
826+
options: Map[String, String]): DataFrame = {
827+
val resolved = ResolvedDataSource(this, Some(schema), partitionColumns, source, options)
828+
DataFrame(this, LogicalRelation(resolved.relation))
829+
}
830+
799831
/**
800832
* :: Experimental ::
801833
* Creates an external table from the given path and returns the corresponding DataFrame.

0 commit comments

Comments
 (0)