Skip to content

[SPARK-20954][SQL] DESCRIBE [EXTENDED] result should be compatible with previous Spark #18203

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
dongjoon-hyun wants to merge 8 commits into from
Closed

[SPARK-20954][SQL] DESCRIBE [EXTENDED] result should be compatible with previous Spark #18203

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
a0b8b1d
[SPARK-20954][SQL] Simple `DESCRIBE` result should be compatible with…
dongjoon-hyun Jun 5, 2017
02913a5
fix
dongjoon-hyun Jun 5, 2017
48329a4
Fix DESC EXTENDED, too.
dongjoon-hyun Jun 8, 2017
744d166
fix typo.
dongjoon-hyun Jun 8, 2017
b8c3b13
Revert "fix typo."
dongjoon-hyun Jun 8, 2017
a589686
Revert "Fix DESC EXTENDED, too."
dongjoon-hyun Jun 8, 2017
1eb712d
Address comments.
dongjoon-hyun Jun 8, 2017
5d27020
fix style.
dongjoon-hyun Jun 8, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 11 additions & 6 deletions sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -522,15 +522,15 @@ case class DescribeTableCommand(
throw new AnalysisException(
s"DESC PARTITION is not allowed on a temporary view: ${table.identifier}")
}
describeSchema(catalog.lookupRelation(table).schema, result)
describeSchema(catalog.lookupRelation(table).schema, result, isExtended)

@gatorsmile gatorsmile Jun 7, 2017

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When specifying Extend, we should also remove it too, right? @cloud-fan

@dongjoon-hyun dongjoon-hyun Jun 7, 2017

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you Right. This PR considered only simple DESC. I'll update 'DESC EXTENDED', too. It is different from DESC FORMATTED as you pointed.

} else {
val metadata = catalog.getTableMetadata(table)
if (metadata.schema.isEmpty) {
// In older version(prior to 2.1) of Spark, the table schema can be empty and should be
// inferred at runtime. We should still support it.
describeSchema(sparkSession.table(metadata.identifier).schema, result)
describeSchema(sparkSession.table(metadata.identifier).schema, result, isExtended)
} else {
describeSchema(metadata.schema, result)
describeSchema(metadata.schema, result, isExtended)
}

describePartitionInfo(metadata, result)
Expand All @@ -550,7 +550,7 @@ case class DescribeTableCommand(
private def describePartitionInfo(table: CatalogTable, buffer: ArrayBuffer[Row]): Unit = {
if (table.partitionColumnNames.nonEmpty) {
append(buffer, "# Partition Information", "", "")
describeSchema(table.partitionSchema, buffer)
describeSchema(table.partitionSchema, buffer, header = true)
}
}

Expand Down Expand Up @@ -601,8 +601,13 @@ case class DescribeTableCommand(
table.storage.toLinkedHashMap.foreach(s => append(buffer, s._1, s._2, ""))
}

private def describeSchema(schema: StructType, buffer: ArrayBuffer[Row]): Unit = {
append(buffer, s"# ${output.head.name}", output(1).name, output(2).name)
private def describeSchema(
schema: StructType,
buffer: ArrayBuffer[Row],
header: Boolean): Unit = {
if (header) {
append(buffer, s"# ${output.head.name}", output(1).name, output(2).name)
}
schema.foreach { column =>
append(buffer, column.name, column.dataType.simpleString, column.getComment().orNull)
}
Expand Down
9 changes: 0 additions & 9 deletions sql/core/src/test/resources/sql-tests/results/change-column.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ DESC test_change
-- !query 1 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 1 output
# col_name data_type comment
a int
b string
c int
Expand All @@ -35,7 +34,6 @@ DESC test_change
-- !query 3 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 3 output
# col_name data_type comment
a int
b string
c int
Expand All @@ -55,7 +53,6 @@ DESC test_change
-- !query 5 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 5 output
# col_name data_type comment
a int
b string
c int
Expand Down Expand Up @@ -94,7 +91,6 @@ DESC test_change
-- !query 8 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 8 output
# col_name data_type comment
a int
b string
c int
Expand Down Expand Up @@ -129,7 +125,6 @@ DESC test_change
-- !query 12 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 12 output
# col_name data_type comment
a int this is column a
b string #*02?`
c int
Expand All @@ -148,7 +143,6 @@ DESC test_change
-- !query 14 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 14 output
# col_name data_type comment
a int this is column a
b string #*02?`
c int
Expand All @@ -168,7 +162,6 @@ DESC test_change
-- !query 16 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 16 output
# col_name data_type comment
a int this is column a
b string #*02?`
c int
Expand All @@ -193,7 +186,6 @@ DESC test_change
-- !query 18 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 18 output
# col_name data_type comment
a int this is column a
b string #*02?`
c int
Expand Down Expand Up @@ -237,7 +229,6 @@ DESC test_change
-- !query 23 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 23 output
# col_name data_type comment
a int this is column A
b string #*02?`
c int
Expand Down
9 changes: 0 additions & 9 deletions sql/core/src/test/resources/sql-tests/results/describe.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@ DESCRIBE t
-- !query 5 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 5 output
# col_name data_type comment
a string
b int
c string
Expand All @@ -70,7 +69,6 @@ DESC default.t
-- !query 6 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 6 output
# col_name data_type comment
a string
b int
c string
Expand All @@ -86,7 +84,6 @@ DESC TABLE t
-- !query 7 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 7 output
# col_name data_type comment
a string
b int
c string
Expand Down Expand Up @@ -162,7 +159,6 @@ DESC t PARTITION (c='Us', d=1)
-- !query 10 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 10 output
# col_name data_type comment
a string
b int
c string
Expand Down Expand Up @@ -268,7 +264,6 @@ DESC temp_v
-- !query 16 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 16 output
# col_name data_type comment
a string
b int
c string
Expand All @@ -280,7 +275,6 @@ DESC TABLE temp_v
-- !query 17 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 17 output
# col_name data_type comment
a string
b int
c string
Expand Down Expand Up @@ -316,7 +310,6 @@ DESC temp_Data_Source_View
-- !query 20 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 20 output
# col_name data_type comment
intType int test comment test1
stringType string
dateType date
Expand Down Expand Up @@ -349,7 +342,6 @@ DESC v
-- !query 22 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 22 output
# col_name data_type comment
a string
b int
c string
Expand All @@ -361,7 +353,6 @@ DESC TABLE v
-- !query 23 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 23 output
# col_name data_type comment
a string
b int
c string
Expand Down
2 changes: 1 addition & 1 deletion sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -806,7 +806,7 @@ class HiveDDLSuite

checkAnswer(
sql(s"DESC $tabName").select("col_name", "data_type", "comment"),
Row("# col_name", "data_type", "comment") :: Row("a", "int", "test") :: Nil
Row("a", "int", "test") :: Nil
)
}
}
Expand Down