From 23c8cf2489d77af8bd754705d6aec07544e36a85 Mon Sep 17 00:00:00 2001 From: gengjiaan Date: Mon, 26 Apr 2021 14:28:35 +0800 Subject: [PATCH 1/3] Get columns operation should handle ANSI interval column properly --- .../SparkGetColumnsOperation.scala | 6 +-- .../SparkMetadataOperationSuite.scala | 53 +++++++++++++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetColumnsOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetColumnsOperation.scala index 4ba6dbf00c0f6..fc0d9a8077d35 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetColumnsOperation.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetColumnsOperation.scala @@ -131,7 +131,7 @@ private[hive] class SparkGetColumnsOperation( */ private def getColumnSize(typ: DataType): Option[Int] = typ match { case dt @ (BooleanType | _: NumericType | DateType | TimestampType | - CalendarIntervalType | NullType) => + CalendarIntervalType | NullType | YearMonthIntervalType | DayTimeIntervalType) => Some(dt.defaultSize) case CharType(n) => Some(n) case StructType(fields) => @@ -171,8 +171,8 @@ private[hive] class SparkGetColumnsOperation( case BooleanType => java.sql.Types.BOOLEAN case ByteType => java.sql.Types.TINYINT case ShortType => java.sql.Types.SMALLINT - case IntegerType => java.sql.Types.INTEGER - case LongType => java.sql.Types.BIGINT + case IntegerType | YearMonthIntervalType => java.sql.Types.INTEGER + case LongType | DayTimeIntervalType => java.sql.Types.BIGINT case FloatType => java.sql.Types.FLOAT case DoubleType => java.sql.Types.DOUBLE case _: DecimalType => java.sql.Types.DECIMAL diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationSuite.scala index 897ea00975a05..531ec4de5fe2a 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationSuite.scala @@ -377,6 +377,59 @@ class SparkMetadataOperationSuite extends HiveThriftServer2TestBase { } } + test("SPARK-35085: Get columns operation should handle ANSI interval column properly") { + val viewName = "view_interval" + val yearMonthDDL = + s"CREATE GLOBAL TEMP VIEW $viewName as select interval '1-1' year to month as i" + + withJdbcStatement(viewName) { statement => + statement.execute(yearMonthDDL) + val data = statement.getConnection.getMetaData + val rowSet = data.getColumns("", "global_temp", viewName, null) + while (rowSet.next()) { + assert(rowSet.getString("TABLE_CAT") === null) + assert(rowSet.getString("TABLE_SCHEM") === "global_temp") + assert(rowSet.getString("TABLE_NAME") === viewName) + assert(rowSet.getString("COLUMN_NAME") === "i") + assert(rowSet.getInt("DATA_TYPE") === java.sql.Types.INTEGER) + assert(rowSet.getString("TYPE_NAME").equalsIgnoreCase(YearMonthIntervalType.sql)) + assert(rowSet.getInt("COLUMN_SIZE") === YearMonthIntervalType.defaultSize) + assert(rowSet.getInt("DECIMAL_DIGITS") === 0) + assert(rowSet.getInt("NUM_PREC_RADIX") === 0) + assert(rowSet.getInt("NULLABLE") === 0) + assert(rowSet.getString("REMARKS") === "") + assert(rowSet.getInt("ORDINAL_POSITION") === 0) + assert(rowSet.getString("IS_NULLABLE") === "YES") + assert(rowSet.getString("IS_AUTO_INCREMENT") === "NO") + } + } + + val dayTimeDDL = + s"CREATE GLOBAL TEMP VIEW $viewName as select interval '1 2:3:4.001' day to second as i" + + withJdbcStatement(viewName) { statement => + statement.execute(dayTimeDDL) + val data = statement.getConnection.getMetaData + val rowSet = data.getColumns("", "global_temp", viewName, null) + while (rowSet.next()) { + assert(rowSet.getString("TABLE_CAT") === null) + assert(rowSet.getString("TABLE_SCHEM") === "global_temp") + assert(rowSet.getString("TABLE_NAME") === viewName) + assert(rowSet.getString("COLUMN_NAME") === "i") + assert(rowSet.getInt("DATA_TYPE") === java.sql.Types.BIGINT) + assert(rowSet.getString("TYPE_NAME").equalsIgnoreCase(DayTimeIntervalType.sql)) + assert(rowSet.getInt("COLUMN_SIZE") === DayTimeIntervalType.defaultSize) + assert(rowSet.getInt("DECIMAL_DIGITS") === 0) + assert(rowSet.getInt("NUM_PREC_RADIX") === 0) + assert(rowSet.getInt("NULLABLE") === 0) + assert(rowSet.getString("REMARKS") === "") + assert(rowSet.getInt("ORDINAL_POSITION") === 0) + assert(rowSet.getString("IS_NULLABLE") === "YES") + assert(rowSet.getString("IS_AUTO_INCREMENT") === "NO") + } + } + } + test("handling null in view for get columns operations") { val viewName = "view_null" val ddl = s"CREATE GLOBAL TEMP VIEW $viewName as select null as n" From 211dbf2c28c91c6a6977ddf84306252e89946038 Mon Sep 17 00:00:00 2001 From: gengjiaan Date: Mon, 26 Apr 2021 15:17:49 +0800 Subject: [PATCH 2/3] test --- .../SparkMetadataOperationSuite.scala | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationSuite.scala index 531ec4de5fe2a..952ca56c5b2f4 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationSuite.scala @@ -378,18 +378,18 @@ class SparkMetadataOperationSuite extends HiveThriftServer2TestBase { } test("SPARK-35085: Get columns operation should handle ANSI interval column properly") { - val viewName = "view_interval" + val viewName1 = "view_interval1" val yearMonthDDL = - s"CREATE GLOBAL TEMP VIEW $viewName as select interval '1-1' year to month as i" + s"CREATE GLOBAL TEMP VIEW $viewName1 as select interval '1-1' year to month as i" - withJdbcStatement(viewName) { statement => + withJdbcStatement(viewName1) { statement => statement.execute(yearMonthDDL) val data = statement.getConnection.getMetaData - val rowSet = data.getColumns("", "global_temp", viewName, null) + val rowSet = data.getColumns("", "global_temp", viewName1, null) while (rowSet.next()) { assert(rowSet.getString("TABLE_CAT") === null) assert(rowSet.getString("TABLE_SCHEM") === "global_temp") - assert(rowSet.getString("TABLE_NAME") === viewName) + assert(rowSet.getString("TABLE_NAME") === viewName1) assert(rowSet.getString("COLUMN_NAME") === "i") assert(rowSet.getInt("DATA_TYPE") === java.sql.Types.INTEGER) assert(rowSet.getString("TYPE_NAME").equalsIgnoreCase(YearMonthIntervalType.sql)) @@ -404,17 +404,18 @@ class SparkMetadataOperationSuite extends HiveThriftServer2TestBase { } } + val viewName2 = "view_interval2" val dayTimeDDL = - s"CREATE GLOBAL TEMP VIEW $viewName as select interval '1 2:3:4.001' day to second as i" + s"CREATE GLOBAL TEMP VIEW $viewName2 as select interval '1 2:3:4.001' day to second as i" - withJdbcStatement(viewName) { statement => + withJdbcStatement(viewName2) { statement => statement.execute(dayTimeDDL) val data = statement.getConnection.getMetaData - val rowSet = data.getColumns("", "global_temp", viewName, null) + val rowSet = data.getColumns("", "global_temp", viewName2, null) while (rowSet.next()) { assert(rowSet.getString("TABLE_CAT") === null) assert(rowSet.getString("TABLE_SCHEM") === "global_temp") - assert(rowSet.getString("TABLE_NAME") === viewName) + assert(rowSet.getString("TABLE_NAME") === viewName2) assert(rowSet.getString("COLUMN_NAME") === "i") assert(rowSet.getInt("DATA_TYPE") === java.sql.Types.BIGINT) assert(rowSet.getString("TYPE_NAME").equalsIgnoreCase(DayTimeIntervalType.sql)) From 4c594bb749f67fa50042f626668a94fe151e40c2 Mon Sep 17 00:00:00 2001 From: beliefer Date: Wed, 28 Apr 2021 12:07:06 +0800 Subject: [PATCH 3/3] Update code --- .../sql/hive/thriftserver/SparkGetColumnsOperation.scala | 7 ++++--- .../hive/thriftserver/SparkMetadataOperationSuite.scala | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetColumnsOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetColumnsOperation.scala index fc0d9a8077d35..a354050d2fc47 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetColumnsOperation.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetColumnsOperation.scala @@ -171,8 +171,8 @@ private[hive] class SparkGetColumnsOperation( case BooleanType => java.sql.Types.BOOLEAN case ByteType => java.sql.Types.TINYINT case ShortType => java.sql.Types.SMALLINT - case IntegerType | YearMonthIntervalType => java.sql.Types.INTEGER - case LongType | DayTimeIntervalType => java.sql.Types.BIGINT + case IntegerType => java.sql.Types.INTEGER + case LongType => java.sql.Types.BIGINT case FloatType => java.sql.Types.FLOAT case DoubleType => java.sql.Types.DOUBLE case _: DecimalType => java.sql.Types.DECIMAL @@ -186,7 +186,8 @@ private[hive] class SparkGetColumnsOperation( case _: MapType => java.sql.Types.JAVA_OBJECT case _: StructType => java.sql.Types.STRUCT // Hive's year-month and day-time intervals are mapping to java.sql.Types.OTHER - case _: CalendarIntervalType => java.sql.Types.OTHER + case _: CalendarIntervalType | YearMonthIntervalType | DayTimeIntervalType => + java.sql.Types.OTHER case _ => throw new IllegalArgumentException(s"Unrecognized type name: ${typ.sql}") } diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationSuite.scala index 952ca56c5b2f4..bc2d413fdf1e9 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationSuite.scala @@ -391,7 +391,7 @@ class SparkMetadataOperationSuite extends HiveThriftServer2TestBase { assert(rowSet.getString("TABLE_SCHEM") === "global_temp") assert(rowSet.getString("TABLE_NAME") === viewName1) assert(rowSet.getString("COLUMN_NAME") === "i") - assert(rowSet.getInt("DATA_TYPE") === java.sql.Types.INTEGER) + assert(rowSet.getInt("DATA_TYPE") === java.sql.Types.OTHER) assert(rowSet.getString("TYPE_NAME").equalsIgnoreCase(YearMonthIntervalType.sql)) assert(rowSet.getInt("COLUMN_SIZE") === YearMonthIntervalType.defaultSize) assert(rowSet.getInt("DECIMAL_DIGITS") === 0) @@ -417,7 +417,7 @@ class SparkMetadataOperationSuite extends HiveThriftServer2TestBase { assert(rowSet.getString("TABLE_SCHEM") === "global_temp") assert(rowSet.getString("TABLE_NAME") === viewName2) assert(rowSet.getString("COLUMN_NAME") === "i") - assert(rowSet.getInt("DATA_TYPE") === java.sql.Types.BIGINT) + assert(rowSet.getInt("DATA_TYPE") === java.sql.Types.OTHER) assert(rowSet.getString("TYPE_NAME").equalsIgnoreCase(DayTimeIntervalType.sql)) assert(rowSet.getInt("COLUMN_SIZE") === DayTimeIntervalType.defaultSize) assert(rowSet.getInt("DECIMAL_DIGITS") === 0)