-
Notifications
You must be signed in to change notification settings - Fork 29.3k
[SPARK-16401] [SQL] Data Source API: Enable Extending RelationProvider and CreatableRelationProvider without Extending SchemaRelationProvider #14075
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 1 commit
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -82,6 +82,34 @@ class DefaultSource | |
| } | ||
| } | ||
|
|
||
| /** Dummy provider with only RelationProvider and CreatableRelationProvider. */ | ||
| class DefaultSourceWithoutUserSpecifiedSchema | ||
| extends RelationProvider | ||
| with CreatableRelationProvider { | ||
|
|
||
| case class FakeRelation(sqlContext: SQLContext) extends BaseRelation { | ||
| override def schema: StructType = StructType(Seq(StructField("a", StringType))) | ||
| } | ||
|
|
||
| override def createRelation( | ||
| sqlContext: SQLContext, | ||
| parameters: Map[String, String]): BaseRelation = { | ||
| LastOptions.parameters = parameters | ||
| LastOptions.schema = None | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why assign values here? We don't check them in the test right?
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. That is from the another default source. It is useless in this test case. Let me clean them. Thanks! |
||
| FakeRelation(sqlContext) | ||
| } | ||
|
|
||
| override def createRelation( | ||
| sqlContext: SQLContext, | ||
| mode: SaveMode, | ||
| parameters: Map[String, String], | ||
| data: DataFrame): BaseRelation = { | ||
| LastOptions.parameters = parameters | ||
| LastOptions.schema = None | ||
| LastOptions.saveMode = mode | ||
| FakeRelation(sqlContext) | ||
| } | ||
| } | ||
|
|
||
| class DataFrameReaderWriterSuite extends QueryTest with SharedSQLContext with BeforeAndAfter { | ||
|
|
||
|
|
@@ -120,6 +148,15 @@ class DataFrameReaderWriterSuite extends QueryTest with SharedSQLContext with Be | |
| .save() | ||
| } | ||
|
|
||
| test("resolve default source without extending SchemaRelationProvider") { | ||
| spark.read | ||
| .format("org.apache.spark.sql.test.DefaultSourceWithoutUserSpecifiedSchema") | ||
| .load() | ||
| .write | ||
| .format("org.apache.spark.sql.test.DefaultSourceWithoutUserSpecifiedSchema") | ||
| .save() | ||
| } | ||
|
|
||
| test("resolve full class") { | ||
| spark.read | ||
| .format("org.apache.spark.sql.test.DefaultSource") | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
hmmm, is it a convention to assign parameters to
LastOptions? I think the test can work without it.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yeah, sure. Let me remove all these things. Thanks!