Skip to content

[SPARK-22452][SQL]Add getInt, getLong, getBoolean to DataSourceV2Options #19902

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
skambha wants to merge 5 commits into from
Closed

[SPARK-22452][SQL]Add getInt, getLong, getBoolean to DataSourceV2Options #19902

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
566b7a5
Add getInt, getLong, getBoolean to DataSourceV2Options and add unit t…
skambha Dec 5, 2017
bb01ff5
Fix formatting
skambha Dec 5, 2017
15e8ce5
Add methods that take defaultValue and return the primitive
skambha Dec 6, 2017
5de7962
Remove the version of the 3 methods that doesnt take a default value
skambha Dec 6, 2017
7d87edc
Add comments
skambha Dec 6, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions sql/core/src/main/java/org/apache/spark/sql/sources/v2/DataSourceV2Options.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,4 +49,35 @@ public DataSourceV2Options(Map<String, String> originalMap) {
public Optional<String> get(String key) {
return Optional.ofNullable(keyLowerCasedMap.get(toLowerCase(key)));
}

Copy link
Copy Markdown
Member

@gatorsmile gatorsmile Dec 6, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

/** Get a parameter as a boolean, falling back to a default if not set */

Copy link
Copy Markdown
Contributor Author

@skambha skambha Dec 6, 2017 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks. Is it ok if I modify the comment for the method a bit to this:

  /**
   * Returns the boolean value to which the specified key is mapped, 
   * or defaultValue if there is no mapping for the key.
   * The key match is case-insensitive
   */

/**
* Returns the boolean value to which the specified key is mapped,
* or defaultValue if there is no mapping for the key. The key match is case-insensitive
*/
public boolean getBoolean(String key, boolean defaultValue) {
String lcaseKey = toLowerCase(key);
return keyLowerCasedMap.containsKey(lcaseKey) ?
Boolean.parseBoolean(keyLowerCasedMap.get(lcaseKey)) : defaultValue;
}

/**
* Returns the integer value to which the specified key is mapped,
* or defaultValue if there is no mapping for the key. The key match is case-insensitive
*/
public int getInt(String key, int defaultValue) {
String lcaseKey = toLowerCase(key);
return keyLowerCasedMap.containsKey(lcaseKey) ?
Integer.parseInt(keyLowerCasedMap.get(lcaseKey)) : defaultValue;
}

/**
* Returns the long value to which the specified key is mapped,
* or defaultValue if there is no mapping for the key. The key match is case-insensitive
*/
public long getLong(String key, long defaultValue) {
String lcaseKey = toLowerCase(key);
return keyLowerCasedMap.containsKey(lcaseKey) ?
Long.parseLong(keyLowerCasedMap.get(lcaseKey)) : defaultValue;
}

}
31 changes: 31 additions & 0 deletions sql/core/src/test/scala/org/apache/spark/sql/sources/v2/DataSourceV2OptionsSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,4 +37,35 @@ class DataSourceV2OptionsSuite extends SparkFunSuite {
val options = new DataSourceV2Options(Map("foo" -> "bAr").asJava)
assert(options.get("foo").get == "bAr")
}

test("getInt") {
val options = new DataSourceV2Options(Map("numFOo" -> "1", "foo" -> "bar").asJava)
assert(options.getInt("numFOO", 10) == 1)
assert(options.getInt("numFOO2", 10) == 10)

intercept[NumberFormatException]{
options.getInt("foo", 1)
}
}

test("getBoolean") {
val options = new DataSourceV2Options(
Map("isFoo" -> "true", "isFOO2" -> "false", "foo" -> "bar").asJava)
assert(options.getBoolean("isFoo", false))
assert(!options.getBoolean("isFoo2", true))
assert(options.getBoolean("isBar", true))
assert(!options.getBoolean("isBar", false))
assert(!options.getBoolean("FOO", true))
}

test("getLong") {
val options = new DataSourceV2Options(Map("numFoo" -> "9223372036854775807",
"foo" -> "bar").asJava)
assert(options.getLong("numFOO", 0L) == 9223372036854775807L)
assert(options.getLong("numFoo2", -1L) == -1L)

intercept[NumberFormatException]{
options.getLong("foo", 0L)
}
}
}