[SPARK-14127][SQL] Native "DESC [EXTENDED | FORMATTED] <table>" DDL command #12844
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -19,16 +19,17 @@ package org.apache.spark.sql.execution.command | |
|
|
||
| import java.io.File | ||
| import java.net.URI | ||
| import java.util.Date | ||
|
|
||
| import scala.collection.mutable.ArrayBuffer | ||
|
|
||
| import org.apache.spark.sql.{AnalysisException, Row, SparkSession} | ||
| import org.apache.spark.sql.catalyst.TableIdentifier | ||
| import org.apache.spark.sql.catalyst.catalog.{CatalogColumn, CatalogRelation, CatalogTable, CatalogTableType} | ||
| import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec | ||
| import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference} | ||
| import org.apache.spark.sql.catalyst.plans.logical.{Command, LogicalPlan, UnaryNode} | ||
| import org.apache.spark.sql.types.{BooleanType, MetadataBuilder, StringType, StructType} | ||
| import org.apache.spark.util.Utils | ||
|
|
||
| case class CreateTableAsSelectLogicalPlan( | ||
|
|
@@ -270,10 +271,10 @@ case class LoadData( | |
| /** | ||
| * Command that looks like | ||
| * {{{ | ||
| * DESCRIBE [EXTENDED|FORMATTED] table_name; | ||
| * }}} | ||
| */ | ||
| case class DescribeTableCommand(table: TableIdentifier, isExtended: Boolean, isFormatted: Boolean) | ||
| extends RunnableCommand { | ||
|
|
||
| override val output: Seq[Attribute] = Seq( | ||
|
|
@@ -290,29 +291,92 @@ case class DescribeTableCommand(table: TableIdentifier, isExtended: Boolean) | |
| val result = new ArrayBuffer[Row] | ||
| sparkSession.sessionState.catalog.lookupRelation(table) match { | ||
| case catalogRelation: CatalogRelation => | ||
| if (isExtended) { | ||
| describeExtended(catalogRelation, result) | ||
| } else if (isFormatted) { | ||
| describeFormatted(catalogRelation, result) | ||
| } else { | ||
| describe(catalogRelation, result) | ||
| } | ||
|
|
||
| case relation => | ||
| describeSchema(relation.schema, result) | ||
| } | ||
|
|
||
| result | ||
| } | ||
|
|
||
| // Shows data columns and partitioned columns (if any) | ||
| private def describe(relation: CatalogRelation, buffer: ArrayBuffer[Row]): Unit = { | ||
| describeSchema(relation.catalogTable.schema, buffer) | ||
|
|
||
| if (relation.catalogTable.partitionColumns.nonEmpty) { | ||
| append(buffer, "# Partition Information", "", "") | ||
| append(buffer, s"# ${output(0).name}", output(1).name, output(2).name) | ||
| describeSchema(relation.catalogTable.partitionColumns, buffer) | ||
| } | ||
| } | ||
|
|
||
| private def describeExtended(relation: CatalogRelation, buffer: ArrayBuffer[Row]): Unit = { | ||
| describe(relation, buffer) | ||
|
|
||
| append(buffer, "", "", "") | ||
| append(buffer, "# Detailed Table Information", relation.catalogTable.toString, "") | ||
|
Member
There was a problem hiding this comment. @liancheng To improve the output of I checked what Hive did for the command
Basically, in the implementation of |
||
| } | ||
|
|
||
| private def describeFormatted(relation: CatalogRelation, buffer: ArrayBuffer[Row]): Unit = { | ||
|
Contributor
There was a problem hiding this comment. We could also use some sort of an
Contributor
Author
There was a problem hiding this comment. Makes sense, actually there were several times that I forgot to add the trailing empty string(s) while working on this PR. Thanks! |
||
| describe(relation, buffer) | ||
|
|
||
| val table = relation.catalogTable | ||
|
|
||
| append(buffer, "", "", "") | ||
| append(buffer, "# Detailed Table Information", "", "") | ||
| append(buffer, "Database:", table.database, "") | ||
| append(buffer, "Owner:", table.owner, "") | ||
| append(buffer, "Create Time:", new Date(table.createTime).toString, "") | ||
| append(buffer, "Last Access Time:", new Date(table.lastAccessTime).toString, "") | ||
| append(buffer, "Location:", table.storage.locationUri.getOrElse(""), "") | ||
| append(buffer, "Table Type:", table.tableType.name, "") | ||
|
|
||
| append(buffer, "Table Parameters:", "", "") | ||
| table.properties.foreach { case (key, value) => | ||
| append(buffer, s" $key", value, "") | ||
| } | ||
|
|
||
| append(buffer, "", "", "") | ||
| append(buffer, "# Storage Information", "", "") | ||
| table.storage.serde.foreach(serdeLib => append(buffer, "SerDe Library:", serdeLib, "")) | ||
| table.storage.inputFormat.foreach(format => append(buffer, "InputFormat:", format, "")) | ||
| table.storage.outputFormat.foreach(format => append(buffer, "OutputFormat:", format, "")) | ||
| append(buffer, "Compressed:", if (table.storage.compressed) "Yes" else "No", "") | ||
| append(buffer, "Num Buckets:", table.numBuckets.toString, "") | ||
| append(buffer, "Bucket Columns:", table.bucketColumnNames.mkString("[", ", ", "]"), "") | ||
| append(buffer, "Sort Columns:", table.sortColumnNames.mkString("[", ", ", "]"), "") | ||
|
|
||
| append(buffer, "Storage Desc Parameters:", "", "") | ||
| table.storage.serdeProperties.foreach { case (key, value) => | ||
| append(buffer, s" $key", value, "") | ||
| } | ||
| } | ||
|
|
||
| private def describeSchema(schema: StructType, buffer: ArrayBuffer[Row]): Unit = { | ||
| schema.foreach { column => | ||
| val comment = | ||
| if (column.metadata.contains("comment")) column.metadata.getString("comment") else "" | ||
|
Contributor
There was a problem hiding this comment. Just an idea: So 9/10 times I use
Contributor
Author
There was a problem hiding this comment. I think the metadata is heavily used in ML code. Another thing is that the data type API is already public for a long time. We probably don't want to change it unless there are particular good reasons.
Contributor
There was a problem hiding this comment. I am not saying that we should (re)move it. I am only suggesting that it might be easier if we had such an accessor in StructField. |
||
| append(buffer, column.name, column.dataType.simpleString, comment) | ||
| } | ||
| } | ||
|
|
||
| private def describeSchema(schema: Seq[CatalogColumn], buffer: ArrayBuffer[Row]): Unit = { | ||
| schema.foreach { column => | ||
| append(buffer, column.name, column.dataType.toLowerCase, column.comment.orNull) | ||
| } | ||
| } | ||
|
|
||
| private def append( | ||
| buffer: ArrayBuffer[Row], column: String, dataType: String, comment: String): Unit = { | ||
| buffer += Row(column, dataType, comment) | ||
| } | ||
| } | ||
|
|
||
|
|
||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Is this ever true? If it isn't we could leave it out.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Nvm. Hive can pass compressed tables.