-
Notifications
You must be signed in to change notification settings - Fork 29.3k
[SPARK-20204][SQL][Followup] SQLConf should react to change in default timezone settings #17537
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 3 commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -147,6 +147,14 @@ private[spark] class TypedConfigBuilder[T]( | |
| } | ||
| } | ||
|
|
||
| /** Creates a [[ConfigEntry]] with a function has a default value */ | ||
| def createWithDefaultFunction(defaultFunc: () => T): ConfigEntry[T] = { | ||
| val entry = new ConfigEntryWithDefaultFunction[T](parent.key, defaultFunc, converter, | ||
| stringConverter, parent._doc, parent._public) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit |
||
| parent._onCreate.foreach(_ (entry)) | ||
| entry | ||
| } | ||
|
|
||
| /** | ||
| * Creates a [[ConfigEntry]] that has a default value. The default value is provided as a | ||
| * [[String]] and must be a valid value for the entry. | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -17,6 +17,7 @@ | |
|
|
||
| package org.apache.spark.internal.config | ||
|
|
||
| import java.util.TimeZone | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Remove it? |
||
| import java.util.concurrent.TimeUnit | ||
|
|
||
| import org.apache.spark.{SparkConf, SparkFunSuite} | ||
|
|
@@ -51,6 +52,8 @@ class ConfigEntrySuite extends SparkFunSuite { | |
| assert(conf.get(dConf) === 20.0) | ||
| } | ||
|
|
||
|
|
||
|
|
||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Nit: two more extra spaces |
||
| test("conf entry: boolean") { | ||
| val conf = new SparkConf() | ||
| val bConf = ConfigBuilder(testKey("boolean")).booleanConf.createWithDefault(false) | ||
|
|
@@ -251,4 +254,13 @@ class ConfigEntrySuite extends SparkFunSuite { | |
| .createWithDefault(null) | ||
| testEntryRef(nullConf, ref(nullConf)) | ||
| } | ||
|
|
||
| test("conf entry : default function") { | ||
| var data = 0 | ||
| val conf = new SparkConf() | ||
| val iConf = ConfigBuilder(testKey("int")).intConf.createWithDefaultFunction(() => data) | ||
| assert(conf.get(iConf) === 0) | ||
| data = 2 | ||
| assert(conf.get(iConf) === 2) | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -752,7 +752,7 @@ object SQLConf { | |
| buildConf("spark.sql.session.timeZone") | ||
| .doc("""The ID of session local timezone, e.g. "GMT", "America/Los_Angeles", etc.""") | ||
| .stringConf | ||
| .createWithDefault(TimeZone.getDefault().getID()) | ||
| .createWithDefaultFunction(() => TimeZone.getDefault().getID()) | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
|
|
||
| val WINDOW_EXEC_BUFFER_SPILL_THRESHOLD = | ||
| buildConf("spark.sql.windowExec.buffer.spill.threshold") | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Creates a [[ConfigEntry]] with a function has a default value->
Creates a [[ConfigEntry]] with a function to determine the default value