From c3a80e81b39ba07c7f8532def8b7694a4bd9f02d Mon Sep 17 00:00:00 2001 From: zouxxyy Date: Thu, 28 Dec 2023 16:32:02 +0800 Subject: [PATCH 1/3] [SPARK-39859][SQL] Fix describe extended (formatted) a column without col stats --- .../datasources/v2/DescribeColumnExec.scala | 2 +- .../command/v2/DescribeTableSuite.scala | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeColumnExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeColumnExec.scala index 61ccda3fc954..2683d8d547f0 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeColumnExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeColumnExec.scala @@ -53,7 +53,7 @@ case class DescribeColumnExec( read.newScanBuilder(CaseInsensitiveStringMap.empty()).build() match { case s: SupportsReportStatistics => val stats = s.estimateStatistics() - Some(stats.columnStats().get(FieldReference.column(column.name))) + Option(stats.columnStats().get(FieldReference.column(column.name))) case _ => None } case _ => None diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeTableSuite.scala index e2f2aee56115..8d3f40e20f2c 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeTableSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeTableSuite.scala @@ -175,4 +175,25 @@ class DescribeTableSuite extends command.DescribeTableSuiteBase Row("max_col_len", "NULL"))) } } + + test("describe extended (formatted) a column without col stats") { + withNamespaceAndTable("ns", "tbl") { tbl => + sql( + s""" + |CREATE TABLE $tbl + |(key INT COMMENT 'column_comment', col STRING) + |$defaultUsing""".stripMargin) + + val descriptionDf = sql(s"DESCRIBE TABLE EXTENDED $tbl key") + assert(descriptionDf.schema.map(field => (field.name, field.dataType)) === Seq( + ("info_name", StringType), + ("info_value", StringType))) + QueryTest.checkAnswer( + descriptionDf, + Seq( + Row("col_name", "key"), + Row("data_type", "int"), + Row("comment", "column_comment"))) + } + } } From 7ce0040f73a1c868dda61afd12b228ea33544c7d Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Thu, 28 Dec 2023 16:53:21 +0800 Subject: [PATCH 2/3] Update sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeTableSuite.scala --- .../spark/sql/execution/command/v2/DescribeTableSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeTableSuite.scala index 8d3f40e20f2c..951a47099a85 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeTableSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeTableSuite.scala @@ -176,7 +176,7 @@ class DescribeTableSuite extends command.DescribeTableSuiteBase } } - test("describe extended (formatted) a column without col stats") { + test("SPARK-39859: describe extended (formatted) a column without col stats") { withNamespaceAndTable("ns", "tbl") { tbl => sql( s""" From abd0b61bcfb3fc59c13f572a9892a4a53e5c543a Mon Sep 17 00:00:00 2001 From: zouxxyy Date: Thu, 28 Dec 2023 17:16:58 +0800 Subject: [PATCH 3/3] update --- .../spark/sql/execution/command/v2/DescribeTableSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeTableSuite.scala index 951a47099a85..a21baebe24d8 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeTableSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeTableSuite.scala @@ -176,7 +176,7 @@ class DescribeTableSuite extends command.DescribeTableSuiteBase } } - test("SPARK-39859: describe extended (formatted) a column without col stats") { + test("SPARK-46535: describe extended (formatted) a column without col stats") { withNamespaceAndTable("ns", "tbl") { tbl => sql( s"""