Skip to content
Closed
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ private[hive] class SparkGetColumnsOperation(
*/
private def getColumnSize(typ: DataType): Option[Int] = typ match {
case dt @ (BooleanType | _: NumericType | DateType | TimestampType |
CalendarIntervalType | NullType) =>
CalendarIntervalType | NullType | YearMonthIntervalType | DayTimeIntervalType) =>

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Size of what does it return? CalendarIntervalType, YearMonthIntervalType, DayTimeIntervalType are returned as strings in rowSets.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

CalendarIntervalType return it's defaultSize (4 + 4 + 8 = 16).
It seems YearMonthIntervalType and DayTimeIntervalType should return defaultSize too.

Some(dt.defaultSize)
case CharType(n) => Some(n)
case StructType(fields) =>
Expand Down Expand Up @@ -171,8 +171,8 @@ private[hive] class SparkGetColumnsOperation(
case BooleanType => java.sql.Types.BOOLEAN
case ByteType => java.sql.Types.TINYINT
case ShortType => java.sql.Types.SMALLINT
case IntegerType => java.sql.Types.INTEGER
case LongType => java.sql.Types.BIGINT
case IntegerType | YearMonthIntervalType => java.sql.Types.INTEGER
case LongType | DayTimeIntervalType => java.sql.Types.BIGINT

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hmm, I am not sure that we should expose ANSI intervals as raw integers/longs via JDBC. I would consider strings (preferable) or java.time.Duration/Period. @cloud-fan @srielau WDYT?

@beliefer beliefer Apr 26, 2021

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good question! I have the confusion too.
cc @cloud-fan

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the metadata, where do we handle the data? e.g. if we want to return string or Duration/Period, where shall we instantiate string or Duration/Period values?

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

See the PR #32121. We should return strings. Since there is no appropriate type in java.sql.* for intervals. I believe we should return the same as for CalendarIntervalType - java.sql.Types.OTHER.

@beliefer Could you handle YearMonthIntervalType and DayTimeIntervalType separately, and return java.sql.Types.OTHER

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK

case FloatType => java.sql.Types.FLOAT
case DoubleType => java.sql.Types.DOUBLE
case _: DecimalType => java.sql.Types.DECIMAL
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -377,6 +377,60 @@ class SparkMetadataOperationSuite extends HiveThriftServer2TestBase {
}
}

test("SPARK-35085: Get columns operation should handle ANSI interval column properly") {
val viewName1 = "view_interval1"
val yearMonthDDL =
s"CREATE GLOBAL TEMP VIEW $viewName1 as select interval '1-1' year to month as i"

withJdbcStatement(viewName1) { statement =>
statement.execute(yearMonthDDL)
val data = statement.getConnection.getMetaData
val rowSet = data.getColumns("", "global_temp", viewName1, null)
while (rowSet.next()) {
assert(rowSet.getString("TABLE_CAT") === null)
assert(rowSet.getString("TABLE_SCHEM") === "global_temp")
assert(rowSet.getString("TABLE_NAME") === viewName1)
assert(rowSet.getString("COLUMN_NAME") === "i")
assert(rowSet.getInt("DATA_TYPE") === java.sql.Types.INTEGER)
assert(rowSet.getString("TYPE_NAME").equalsIgnoreCase(YearMonthIntervalType.sql))
assert(rowSet.getInt("COLUMN_SIZE") === YearMonthIntervalType.defaultSize)
assert(rowSet.getInt("DECIMAL_DIGITS") === 0)
assert(rowSet.getInt("NUM_PREC_RADIX") === 0)
assert(rowSet.getInt("NULLABLE") === 0)
assert(rowSet.getString("REMARKS") === "")
assert(rowSet.getInt("ORDINAL_POSITION") === 0)
assert(rowSet.getString("IS_NULLABLE") === "YES")
assert(rowSet.getString("IS_AUTO_INCREMENT") === "NO")
}
}

val viewName2 = "view_interval2"
val dayTimeDDL =
s"CREATE GLOBAL TEMP VIEW $viewName2 as select interval '1 2:3:4.001' day to second as i"

withJdbcStatement(viewName2) { statement =>
statement.execute(dayTimeDDL)
val data = statement.getConnection.getMetaData
val rowSet = data.getColumns("", "global_temp", viewName2, null)
while (rowSet.next()) {
assert(rowSet.getString("TABLE_CAT") === null)
assert(rowSet.getString("TABLE_SCHEM") === "global_temp")
assert(rowSet.getString("TABLE_NAME") === viewName2)
assert(rowSet.getString("COLUMN_NAME") === "i")
assert(rowSet.getInt("DATA_TYPE") === java.sql.Types.BIGINT)
assert(rowSet.getString("TYPE_NAME").equalsIgnoreCase(DayTimeIntervalType.sql))
assert(rowSet.getInt("COLUMN_SIZE") === DayTimeIntervalType.defaultSize)
assert(rowSet.getInt("DECIMAL_DIGITS") === 0)
assert(rowSet.getInt("NUM_PREC_RADIX") === 0)
assert(rowSet.getInt("NULLABLE") === 0)
assert(rowSet.getString("REMARKS") === "")
assert(rowSet.getInt("ORDINAL_POSITION") === 0)
assert(rowSet.getString("IS_NULLABLE") === "YES")
assert(rowSet.getString("IS_AUTO_INCREMENT") === "NO")
}
}
}

test("handling null in view for get columns operations") {
val viewName = "view_null"
val ddl = s"CREATE GLOBAL TEMP VIEW $viewName as select null as n"
Expand Down