From 1deffcdf66bb2ffdadf7daf758f5eb35d36eccf4 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Tue, 20 Sep 2016 12:57:47 -0700 Subject: [PATCH 01/13] [SPARK-17612][SQL] Support `DESCRIBE table PARTITION` SQL syntax --- .../spark/sql/execution/SparkSqlParser.scala | 10 +++++++-- .../spark/sql/execution/command/tables.scala | 12 ++++++++-- .../sql/hive/execution/SQLQuerySuite.scala | 22 ++++++++++++++++++- 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala index 5359cedc8097..913c0f7694b4 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala @@ -276,13 +276,19 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder { * Create a [[DescribeTableCommand]] logical plan. */ override def visitDescribeTable(ctx: DescribeTableContext): LogicalPlan = withOrigin(ctx) { - // Describe partition and column are not supported yet. Return null and let the parser decide + // Describe column are not supported yet. Return null and let the parser decide // what to do with this (create an exception or pass it on to a different system). - if (ctx.describeColName != null || ctx.partitionSpec != null) { + if (ctx.describeColName != null) { null } else { + val partitionSpec = if (ctx.partitionSpec != null) { + visitPartitionSpec(ctx.partitionSpec).filter(_._2.isDefined).map(x => (x._1, x._2.get)) + } else { + Map.empty[String, String] + } DescribeTableCommand( visitTableIdentifier(ctx.tableIdentifier), + partitionSpec, ctx.EXTENDED != null, ctx.FORMATTED != null) } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index 6a91c997bac6..f15a09e75f63 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -390,10 +390,14 @@ case class TruncateTableCommand( /** * Command that looks like * {{{ - * DESCRIBE [EXTENDED|FORMATTED] table_name; + * DESCRIBE [EXTENDED|FORMATTED] table_name partitionSpec?; * }}} */ -case class DescribeTableCommand(table: TableIdentifier, isExtended: Boolean, isFormatted: Boolean) +case class DescribeTableCommand( + table: TableIdentifier, + partitionSpec: TablePartitionSpec, + isExtended: Boolean, + isFormatted: Boolean) extends RunnableCommand { override val output: Seq[Attribute] = Seq( @@ -410,6 +414,10 @@ case class DescribeTableCommand(table: TableIdentifier, isExtended: Boolean, isF val result = new ArrayBuffer[Row] val catalog = sparkSession.sessionState.catalog + if (partitionSpec.nonEmpty) { + catalog.getPartition(table, partitionSpec) + } + if (catalog.isTemporaryTable(table)) { describeSchema(catalog.lookupRelation(table).schema, result) } else { diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index dc4d099f0f66..59e8c09e6367 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -26,7 +26,8 @@ import org.apache.hadoop.fs.Path import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.catalyst.analysis.{EliminateSubqueryAliases, FunctionRegistry} +import org.apache.spark.sql.catalyst.analysis.{EliminateSubqueryAliases, FunctionRegistry, + NoSuchPartitionException} import org.apache.spark.sql.catalyst.catalog.CatalogTableType import org.apache.spark.sql.catalyst.parser.ParseException import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation} @@ -341,6 +342,25 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { } } + test("describe partition") { + withTable("partitioned_table") { + sql("CREATE TABLE partitioned_table (a STRING, b INT) PARTITIONED BY (c STRING, d STRING)") + sql("ALTER TABLE partitioned_table ADD PARTITION (c='Us', d=1)") + + sql("DESC partitioned_table PARTITION (c='Us', d=1)") + + val m = intercept[NoSuchPartitionException] { + sql("DESC partitioned_table PARTITION (c='Us', d=2)") + }.getMessage() + assert(m.contains("Partition not found in table 'partitioned_table'")) + + val m2 = intercept[AnalysisException] { + sql("DESC partitioned_table PARTITION (c='Us')") + }.getMessage() + assert(m2.contains("Partition spec is invalid.")) + } + } + test("SPARK-5371: union with null and sum") { val df = Seq((1, 1)).toDF("c1", "c2") df.createOrReplaceTempView("table1") From 30077013a4a5026765e7a0c61536243bc8d5f525 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Fri, 23 Sep 2016 12:32:48 -0700 Subject: [PATCH 02/13] Add 'EXTENDED' and 'FORMATTED' feature. --- .../sql/catalyst/catalog/interface.scala | 13 +- .../spark/sql/execution/SparkSqlParser.scala | 9 +- .../spark/sql/execution/command/tables.scala | 61 ++++-- .../resources/sql-tests/inputs/describe.sql | 24 +++ .../sql-tests/results/describe.sql.out | 180 ++++++++++++++++++ .../sql/hive/execution/SQLQuerySuite.scala | 22 +-- 6 files changed, 271 insertions(+), 38 deletions(-) create mode 100644 sql/core/src/test/resources/sql-tests/inputs/describe.sql create mode 100644 sql/core/src/test/resources/sql-tests/results/describe.sql.out diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala index e52251f960ff..51326ca25e9c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala @@ -86,7 +86,18 @@ object CatalogStorageFormat { case class CatalogTablePartition( spec: CatalogTypes.TablePartitionSpec, storage: CatalogStorageFormat, - parameters: Map[String, String] = Map.empty) + parameters: Map[String, String] = Map.empty) { + + override def toString: String = { + val output = + Seq( + s"Partition Values: [${spec.values.mkString(", ")}]", + s"$storage", + s"Partition Parameters:{${parameters.map(p => p._1 + "=" + p._2).mkString(", ")}}") + + output.filter(_.nonEmpty).mkString("CatalogPartition(\n\t", "\n\t", ")") + } +} /** diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala index 913c0f7694b4..144d84945469 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala @@ -282,7 +282,14 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder { null } else { val partitionSpec = if (ctx.partitionSpec != null) { - visitPartitionSpec(ctx.partitionSpec).filter(_._2.isDefined).map(x => (x._1, x._2.get)) + // According to the syntax, visitPartitionSpec returns `Map[String, Option[String]]`. + val (valid, nonValid) = visitPartitionSpec(ctx.partitionSpec).partition(_._2.isDefined) + if (nonValid.nonEmpty) { + // For non-valid specification for `DESC` command, this raises a ParseException. + return null + } else { + valid.map(x => (x._1, x._2.get)) + } } else { Map.empty[String, String] } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index f15a09e75f63..9757e783f8b0 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -29,7 +29,7 @@ import org.apache.hadoop.fs.Path import org.apache.spark.sql.{AnalysisException, Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogStorageFormat, CatalogTable, CatalogTableType} +import org.apache.spark.sql.catalyst.catalog._ import org.apache.spark.sql.catalyst.catalog.CatalogTableType._ import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference} @@ -414,22 +414,28 @@ case class DescribeTableCommand( val result = new ArrayBuffer[Row] val catalog = sparkSession.sessionState.catalog - if (partitionSpec.nonEmpty) { - catalog.getPartition(table, partitionSpec) - } - if (catalog.isTemporaryTable(table)) { describeSchema(catalog.lookupRelation(table).schema, result) } else { val metadata = catalog.getTableMetadata(table) describeSchema(metadata.schema, result) - if (isExtended) { - describeExtended(metadata, result) - } else if (isFormatted) { - describeFormatted(metadata, result) + describePartitionInfo(metadata, result) + + if (partitionSpec.isEmpty) { + if (isExtended) { + describeExtendedTableInfo(metadata, result) + } else if (isFormatted) { + describeFormattedTableInfo(metadata, result) + } } else { - describePartitionInfo(metadata, result) + val partition = catalog.getPartition(table, partitionSpec) + if (isExtended) { + describeExtendedDetailPartitionInfo(table, metadata, partition, result) + } else if (isFormatted) { + describeFormattedDetailPartitionInfo(table, metadata, partition, result) + describeStorageInfo(metadata, result) + } } } @@ -444,16 +450,41 @@ case class DescribeTableCommand( } } - private def describeExtended(table: CatalogTable, buffer: ArrayBuffer[Row]): Unit = { - describePartitionInfo(table, buffer) + private def describeExtendedDetailPartitionInfo( + tableIdentifier: TableIdentifier, + table: CatalogTable, + partition: CatalogTablePartition, + buffer: ArrayBuffer[Row]): Unit = { + append(buffer, "", "", "") + append(buffer, "Detailed Partition Information " + partition.toString, "", "") + } + + private def describeFormattedDetailPartitionInfo( + tableIdentifier: TableIdentifier, + table: CatalogTable, + partition: CatalogTablePartition, + buffer: ArrayBuffer[Row]): Unit = { + append(buffer, "", "", "") + append(buffer, "# Detailed Partition Information", "", "") + append(buffer, "Partition Value:", s"[${partition.spec.values.mkString(", ")}]", "") + append(buffer, "Database:", table.database, "") + append(buffer, "Table:", tableIdentifier.table, "") + append(buffer, "Create Time:", "UNKNOWN", "") + append(buffer, "Last Access Time:", "UNKNOWN", "") + append(buffer, "Protect Mode:", "None", "") + append(buffer, "Location:", partition.storage.locationUri.getOrElse(""), "") + append(buffer, "Partition Parameters:", "", "") + partition.parameters.foreach { case (key, value) => + append(buffer, s" $key", value, "") + } + } + private def describeExtendedTableInfo(table: CatalogTable, buffer: ArrayBuffer[Row]): Unit = { append(buffer, "", "", "") append(buffer, "# Detailed Table Information", table.toString, "") } - private def describeFormatted(table: CatalogTable, buffer: ArrayBuffer[Row]): Unit = { - describePartitionInfo(table, buffer) - + private def describeFormattedTableInfo(table: CatalogTable, buffer: ArrayBuffer[Row]): Unit = { append(buffer, "", "", "") append(buffer, "# Detailed Table Information", "", "") append(buffer, "Database:", table.database, "") diff --git a/sql/core/src/test/resources/sql-tests/inputs/describe.sql b/sql/core/src/test/resources/sql-tests/inputs/describe.sql new file mode 100644 index 000000000000..87e72791b271 --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/inputs/describe.sql @@ -0,0 +1,24 @@ +CREATE TABLE t (a STRING, b INT) PARTITIONED BY (c STRING, d STRING); + +ALTER TABLE t ADD PARTITION (c='Us', d=1); + +DESC t; + +DESC EXTENDED t; + +DESC FORMATTED t; + +DESC t PARTITION (c='Us', d=1); + +DESC EXTENDED t PARTITION (c='Us', d=1); + +DESC FORMATTED t PARTITION (c='Us', d=1); + +-- NoSuchPartitionException: Partition not found in table 'partitioned_table' +DESC t PARTITION (c='Us', d=2); + +-- AnalysisException: Partition spec is invalid +DESC t PARTITION (c='Us'); + +-- ParseException: Unsupported SQL statement +DESC t PARTITION (c='Us', d); diff --git a/sql/core/src/test/resources/sql-tests/results/describe.sql.out b/sql/core/src/test/resources/sql-tests/results/describe.sql.out new file mode 100644 index 000000000000..c16fe16636bf --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/results/describe.sql.out @@ -0,0 +1,180 @@ +-- Automatically generated by SQLQueryTestSuite +-- Number of queries: 11 + + +-- !query 0 +CREATE TABLE t (a STRING, b INT) PARTITIONED BY (c STRING, d STRING) +-- !query 0 schema +struct<> +-- !query 0 output + + + +-- !query 1 +ALTER TABLE t ADD PARTITION (c='Us', d=1) +-- !query 1 schema +struct<> +-- !query 1 output + + + +-- !query 2 +DESC t +-- !query 2 schema +struct +-- !query 2 output +# Partition Information +# col_name data_type comment +a string +b int +c string +c string +d string +d string + + +-- !query 3 +DESC EXTENDED t +-- !query 3 schema +struct +-- !query 3 output +# Detailed Table Information CatalogTable( + Table: `default`.`t` + Created: Fri Sep 23 12:30:40 PDT 2016 + Last Access: Wed Dec 31 15:59:59 PST 1969 + Type: MANAGED + Schema: [StructField(a,StringType,true), StructField(b,IntegerType,true), StructField(c,StringType,true), StructField(d,StringType,true)] + Provider: hive + Partition Columns: [`c`, `d`] + Storage(InputFormat: org.apache.hadoop.mapred.TextInputFormat, OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat)) +# Partition Information +# col_name data_type comment +a string +b int +c string +c string +d string +d string + + +-- !query 4 +DESC FORMATTED t +-- !query 4 schema +struct +-- !query 4 output +# Detailed Table Information +# Partition Information +# Storage Information +# col_name data_type comment +Compressed: No +Create Time: Fri Sep 23 12:30:40 PDT 2016 +Database: default +InputFormat: org.apache.hadoop.mapred.TextInputFormat +Last Access Time: Wed Dec 31 15:59:59 PST 1969 +Location: +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Owner: +Storage Desc Parameters: +Table Parameters: +Table Type: MANAGED +a string +b int +c string +c string +d string +d string + + +-- !query 5 +DESC t PARTITION (c='Us', d=1) +-- !query 5 schema +struct +-- !query 5 output +# Partition Information +# col_name data_type comment +a string +b int +c string +c string +d string +d string + + +-- !query 6 +DESC EXTENDED t PARTITION (c='Us', d=1) +-- !query 6 schema +struct +-- !query 6 output +# Partition Information +# col_name data_type comment +Detailed Partition Information CatalogPartition( + Partition Values: [Us, 1] + Storage(InputFormat: org.apache.hadoop.mapred.TextInputFormat, OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat) + Partition Parameters:{}) +a string +b int +c string +c string +d string +d string + + +-- !query 7 +DESC FORMATTED t PARTITION (c='Us', d=1) +-- !query 7 schema +struct +-- !query 7 output +# Detailed Partition Information +# Partition Information +# Storage Information +# col_name data_type comment +Compressed: No +Create Time: UNKNOWN +Database: default +InputFormat: org.apache.hadoop.mapred.TextInputFormat +Last Access Time: UNKNOWN +Location: +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Partition Parameters: +Partition Value: [Us, 1] +Protect Mode: None +Storage Desc Parameters: +Table: t +a string +b int +c string +c string +d string +d string + + +-- !query 8 +DESC t PARTITION (c='Us', d=2) +-- !query 8 schema +struct<> +-- !query 8 output +org.apache.spark.sql.catalyst.analysis.NoSuchPartitionException +Partition not found in table 't' database 'default': +c -> Us +d -> 2; + + +-- !query 9 +DESC t PARTITION (c='Us') +-- !query 9 schema +struct<> +-- !query 9 output +org.apache.spark.sql.AnalysisException +Partition spec is invalid. The spec (c) must match the partition spec (c, d) defined in table '`default`.`t`'; + + +-- !query 10 +DESC t PARTITION (c='Us', d) +-- !query 10 schema +struct<> +-- !query 10 output +org.apache.spark.sql.catalyst.parser.ParseException + +Unsupported SQL statement +== SQL == +DESC t PARTITION (c='Us', d) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index 59e8c09e6367..dc4d099f0f66 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -26,8 +26,7 @@ import org.apache.hadoop.fs.Path import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.catalyst.analysis.{EliminateSubqueryAliases, FunctionRegistry, - NoSuchPartitionException} +import org.apache.spark.sql.catalyst.analysis.{EliminateSubqueryAliases, FunctionRegistry} import org.apache.spark.sql.catalyst.catalog.CatalogTableType import org.apache.spark.sql.catalyst.parser.ParseException import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation} @@ -342,25 +341,6 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { } } - test("describe partition") { - withTable("partitioned_table") { - sql("CREATE TABLE partitioned_table (a STRING, b INT) PARTITIONED BY (c STRING, d STRING)") - sql("ALTER TABLE partitioned_table ADD PARTITION (c='Us', d=1)") - - sql("DESC partitioned_table PARTITION (c='Us', d=1)") - - val m = intercept[NoSuchPartitionException] { - sql("DESC partitioned_table PARTITION (c='Us', d=2)") - }.getMessage() - assert(m.contains("Partition not found in table 'partitioned_table'")) - - val m2 = intercept[AnalysisException] { - sql("DESC partitioned_table PARTITION (c='Us')") - }.getMessage() - assert(m2.contains("Partition spec is invalid.")) - } - } - test("SPARK-5371: union with null and sum") { val df = Seq((1, 1)).toDF("c1", "c2") df.createOrReplaceTempView("table1") From 24205775fcbfac782588785f4bde9cca295f78fd Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Fri, 23 Sep 2016 13:13:33 -0700 Subject: [PATCH 03/13] Reposition newly added function to make GitHub diff easy for eyes. --- .../spark/sql/execution/command/tables.scala | 58 +++++++++---------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index 9757e783f8b0..95f5a0be56b6 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -450,35 +450,6 @@ case class DescribeTableCommand( } } - private def describeExtendedDetailPartitionInfo( - tableIdentifier: TableIdentifier, - table: CatalogTable, - partition: CatalogTablePartition, - buffer: ArrayBuffer[Row]): Unit = { - append(buffer, "", "", "") - append(buffer, "Detailed Partition Information " + partition.toString, "", "") - } - - private def describeFormattedDetailPartitionInfo( - tableIdentifier: TableIdentifier, - table: CatalogTable, - partition: CatalogTablePartition, - buffer: ArrayBuffer[Row]): Unit = { - append(buffer, "", "", "") - append(buffer, "# Detailed Partition Information", "", "") - append(buffer, "Partition Value:", s"[${partition.spec.values.mkString(", ")}]", "") - append(buffer, "Database:", table.database, "") - append(buffer, "Table:", tableIdentifier.table, "") - append(buffer, "Create Time:", "UNKNOWN", "") - append(buffer, "Last Access Time:", "UNKNOWN", "") - append(buffer, "Protect Mode:", "None", "") - append(buffer, "Location:", partition.storage.locationUri.getOrElse(""), "") - append(buffer, "Partition Parameters:", "", "") - partition.parameters.foreach { case (key, value) => - append(buffer, s" $key", value, "") - } - } - private def describeExtendedTableInfo(table: CatalogTable, buffer: ArrayBuffer[Row]): Unit = { append(buffer, "", "", "") append(buffer, "# Detailed Table Information", table.toString, "") @@ -538,6 +509,35 @@ case class DescribeTableCommand( } } + private def describeExtendedDetailPartitionInfo( + tableIdentifier: TableIdentifier, + table: CatalogTable, + partition: CatalogTablePartition, + buffer: ArrayBuffer[Row]): Unit = { + append(buffer, "", "", "") + append(buffer, "Detailed Partition Information " + partition.toString, "", "") + } + + private def describeFormattedDetailPartitionInfo( + tableIdentifier: TableIdentifier, + table: CatalogTable, + partition: CatalogTablePartition, + buffer: ArrayBuffer[Row]): Unit = { + append(buffer, "", "", "") + append(buffer, "# Detailed Partition Information", "", "") + append(buffer, "Partition Value:", s"[${partition.spec.values.mkString(", ")}]", "") + append(buffer, "Database:", table.database, "") + append(buffer, "Table:", tableIdentifier.table, "") + append(buffer, "Create Time:", "UNKNOWN", "") + append(buffer, "Last Access Time:", "UNKNOWN", "") + append(buffer, "Protect Mode:", "None", "") + append(buffer, "Location:", partition.storage.locationUri.getOrElse(""), "") + append(buffer, "Partition Parameters:", "", "") + partition.parameters.foreach { case (key, value) => + append(buffer, s" $key", value, "") + } + } + private def describeSchema(schema: StructType, buffer: ArrayBuffer[Row]): Unit = { schema.foreach { column => append(buffer, column.name, column.dataType.simpleString, column.getComment().orNull) From f419c4c0af9279b57c8b140868ec31d935e649c0 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Fri, 23 Sep 2016 20:55:47 -0700 Subject: [PATCH 04/13] Move EXTENDED/FORMATTED into SQLQuerySuite. --- .../resources/sql-tests/inputs/describe.sql | 12 +- .../sql-tests/results/describe.sql.out | 120 ++---------------- .../sql/hive/execution/SQLQuerySuite.scala | 48 ++++++- 3 files changed, 63 insertions(+), 117 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/describe.sql b/sql/core/src/test/resources/sql-tests/inputs/describe.sql index 87e72791b271..5ddcadfecc5a 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/describe.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/describe.sql @@ -4,15 +4,15 @@ ALTER TABLE t ADD PARTITION (c='Us', d=1); DESC t; -DESC EXTENDED t; - -DESC FORMATTED t; +-- Ignore these because there exists timestamp results like `create_table` +-- DESC EXTENDED t; +-- DESC FORMATTED t; DESC t PARTITION (c='Us', d=1); -DESC EXTENDED t PARTITION (c='Us', d=1); - -DESC FORMATTED t PARTITION (c='Us', d=1); +-- Ignore these because there exists timestamp results like `create_table` +-- DESC EXTENDED t PARTITION (c='Us', d=1); +-- DESC FORMATTED t PARTITION (c='Us', d=1); -- NoSuchPartitionException: Partition not found in table 'partitioned_table' DESC t PARTITION (c='Us', d=2); diff --git a/sql/core/src/test/resources/sql-tests/results/describe.sql.out b/sql/core/src/test/resources/sql-tests/results/describe.sql.out index c16fe16636bf..3ab0c7c99c01 100644 --- a/sql/core/src/test/resources/sql-tests/results/describe.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/describe.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 11 +-- Number of queries: 7 -- !query 0 @@ -34,19 +34,10 @@ d string -- !query 3 -DESC EXTENDED t +DESC t PARTITION (c='Us', d=1) -- !query 3 schema struct -- !query 3 output -# Detailed Table Information CatalogTable( - Table: `default`.`t` - Created: Fri Sep 23 12:30:40 PDT 2016 - Last Access: Wed Dec 31 15:59:59 PST 1969 - Type: MANAGED - Schema: [StructField(a,StringType,true), StructField(b,IntegerType,true), StructField(c,StringType,true), StructField(d,StringType,true)] - Provider: hive - Partition Columns: [`c`, `d`] - Storage(InputFormat: org.apache.hadoop.mapred.TextInputFormat, OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat)) # Partition Information # col_name data_type comment a string @@ -58,121 +49,30 @@ d string -- !query 4 -DESC FORMATTED t --- !query 4 schema -struct --- !query 4 output -# Detailed Table Information -# Partition Information -# Storage Information -# col_name data_type comment -Compressed: No -Create Time: Fri Sep 23 12:30:40 PDT 2016 -Database: default -InputFormat: org.apache.hadoop.mapred.TextInputFormat -Last Access Time: Wed Dec 31 15:59:59 PST 1969 -Location: -OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat -Owner: -Storage Desc Parameters: -Table Parameters: -Table Type: MANAGED -a string -b int -c string -c string -d string -d string - - --- !query 5 -DESC t PARTITION (c='Us', d=1) --- !query 5 schema -struct --- !query 5 output -# Partition Information -# col_name data_type comment -a string -b int -c string -c string -d string -d string - - --- !query 6 -DESC EXTENDED t PARTITION (c='Us', d=1) --- !query 6 schema -struct --- !query 6 output -# Partition Information -# col_name data_type comment -Detailed Partition Information CatalogPartition( - Partition Values: [Us, 1] - Storage(InputFormat: org.apache.hadoop.mapred.TextInputFormat, OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat) - Partition Parameters:{}) -a string -b int -c string -c string -d string -d string - - --- !query 7 -DESC FORMATTED t PARTITION (c='Us', d=1) --- !query 7 schema -struct --- !query 7 output -# Detailed Partition Information -# Partition Information -# Storage Information -# col_name data_type comment -Compressed: No -Create Time: UNKNOWN -Database: default -InputFormat: org.apache.hadoop.mapred.TextInputFormat -Last Access Time: UNKNOWN -Location: -OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat -Partition Parameters: -Partition Value: [Us, 1] -Protect Mode: None -Storage Desc Parameters: -Table: t -a string -b int -c string -c string -d string -d string - - --- !query 8 DESC t PARTITION (c='Us', d=2) --- !query 8 schema +-- !query 4 schema struct<> --- !query 8 output +-- !query 4 output org.apache.spark.sql.catalyst.analysis.NoSuchPartitionException Partition not found in table 't' database 'default': c -> Us d -> 2; --- !query 9 +-- !query 5 DESC t PARTITION (c='Us') --- !query 9 schema +-- !query 5 schema struct<> --- !query 9 output +-- !query 5 output org.apache.spark.sql.AnalysisException Partition spec is invalid. The spec (c) must match the partition spec (c, d) defined in table '`default`.`t`'; --- !query 10 +-- !query 6 DESC t PARTITION (c='Us', d) --- !query 10 schema +-- !query 6 schema struct<> --- !query 10 output +-- !query 6 output org.apache.spark.sql.catalyst.parser.ParseException Unsupported SQL statement diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index dc4d099f0f66..16b887628df6 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -26,7 +26,7 @@ import org.apache.hadoop.fs.Path import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.catalyst.analysis.{EliminateSubqueryAliases, FunctionRegistry} +import org.apache.spark.sql.catalyst.analysis.{EliminateSubqueryAliases, FunctionRegistry, NoSuchPartitionException} import org.apache.spark.sql.catalyst.catalog.CatalogTableType import org.apache.spark.sql.catalyst.parser.ParseException import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation} @@ -341,6 +341,52 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { } } + test("describe partition") { + withTable("partitioned_table") { + sql("CREATE TABLE partitioned_table (a STRING, b INT) PARTITIONED BY (c STRING, d STRING)") + sql("ALTER TABLE partitioned_table ADD PARTITION (c='Us', d=1)") + + checkKeywordsExist(sql("DESC partitioned_table PARTITION (c='Us', d=1)"), + "# Partition Information", + "# col_name") + + checkKeywordsExist(sql("DESC EXTENDED partitioned_table PARTITION (c='Us', d=1)"), + "# Partition Information", + "# col_name", + "Detailed Partition Information CatalogPartition(", + "Partition Values: [Us, 1]", + "Storage(Location:", + "Partition Parameters") + + checkKeywordsExist(sql("DESC FORMATTED partitioned_table PARTITION (c='Us', d=1)"), + "# Partition Information", + "# col_name", + "# Detailed Partition Information", + "Partition Value:", + "Database:", + "Table:", + "Create Time:", + "Location:", + "Partition Parameters:", + "# Storage Information") + + val m = intercept[NoSuchPartitionException] { + sql("DESC partitioned_table PARTITION (c='Us', d=2)") + }.getMessage() + assert(m.contains("Partition not found in table")) + + val m2 = intercept[AnalysisException] { + sql("DESC partitioned_table PARTITION (c='Us')") + }.getMessage() + assert(m2.contains("Partition spec is invalid")) + + val m3 = intercept[ParseException] { + sql("DESC partitioned_table PARTITION (c='Us', d)") + }.getMessage() + assert(m3.contains("Unsupported SQL statement")) + } + } + test("SPARK-5371: union with null and sum") { val df = Seq((1, 1)).toDF("c1", "c2") df.createOrReplaceTempView("table1") From 7371e485cd056920acfb0ad584333901da935b2a Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Sat, 24 Sep 2016 23:32:55 -0700 Subject: [PATCH 05/13] Add a exception handling for datasource tables. --- .../org/apache/spark/sql/execution/command/tables.scala | 4 ++++ .../apache/spark/sql/hive/execution/SQLQuerySuite.scala | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index 95f5a0be56b6..aca604b6bf71 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -429,6 +429,10 @@ case class DescribeTableCommand( describeFormattedTableInfo(metadata, result) } } else { + if (DDLUtils.isDatasourceTable(metadata)) { + throw new AnalysisException( + s"DESC PARTITION is not allowed on a datasource table: ${table.identifier}") + } val partition = catalog.getPartition(table, partitionSpec) if (isExtended) { describeExtendedDetailPartitionInfo(table, metadata, partition, result) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index 16b887628df6..e3acb4b5b7b3 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -384,6 +384,15 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { sql("DESC partitioned_table PARTITION (c='Us', d)") }.getMessage() assert(m3.contains("Unsupported SQL statement")) + + spark + .range(1).select('id as 'a, 'id as 'b, 'id as 'c, 'id as 'd).write + .partitionBy("d") + .saveAsTable("datasource_table") + val m4 = intercept[AnalysisException] { + sql("DESC datasource_table PARTITION (d=2)") + }.getMessage() + assert(m4.contains("DESC PARTITION is not allowed on a datasource table")) } } From 9b445c268da956cd3cc078655e42ff9e834837cc Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Sun, 25 Sep 2016 20:10:55 -0700 Subject: [PATCH 06/13] Add a exception handling logic for temporary table. --- .../org/apache/spark/sql/execution/command/tables.scala | 4 ++++ .../apache/spark/sql/hive/execution/SQLQuerySuite.scala | 8 +++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index aca604b6bf71..433139f6890c 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -415,6 +415,10 @@ case class DescribeTableCommand( val catalog = sparkSession.sessionState.catalog if (catalog.isTemporaryTable(table)) { + if (partitionSpec.nonEmpty) { + throw new AnalysisException( + s"DESC PARTITION is not allowed on a temporary view: ${table.identifier}") + } describeSchema(catalog.lookupRelation(table).schema, result) } else { val metadata = catalog.getTableMetadata(table) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index e3acb4b5b7b3..b5d0996371b1 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -342,7 +342,7 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { } test("describe partition") { - withTable("partitioned_table") { + withTable("partitioned_table", "datasource_table") { sql("CREATE TABLE partitioned_table (a STRING, b INT) PARTITIONED BY (c STRING, d STRING)") sql("ALTER TABLE partitioned_table ADD PARTITION (c='Us', d=1)") @@ -393,6 +393,12 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { sql("DESC datasource_table PARTITION (d=2)") }.getMessage() assert(m4.contains("DESC PARTITION is not allowed on a datasource table")) + + val m5 = intercept[AnalysisException] { + spark.range(10).select('id as 'a, 'id as 'b).createTempView("view1") + sql("DESC view1 PARTITION (c='Us', d=1)").show() + }.getMessage() + assert(m5.contains("DESC PARTITION is not allowed on a temporary view")) } } From 9e814005d98b5aa93c4cb9658ae9ce629c0cd66c Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Sun, 25 Sep 2016 23:48:41 -0700 Subject: [PATCH 07/13] Remove meaningless output and add a permanent view handling logic. --- .../apache/spark/sql/execution/command/tables.scala | 7 ++++--- .../spark/sql/hive/execution/SQLQuerySuite.scala | 11 +++++++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index 433139f6890c..0798da59b8e5 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -433,6 +433,10 @@ case class DescribeTableCommand( describeFormattedTableInfo(metadata, result) } } else { + if (metadata.tableType == CatalogTableType.VIEW) { + throw new AnalysisException( + s"DESC PARTITION is not allowed on a view: ${table.identifier}") + } if (DDLUtils.isDatasourceTable(metadata)) { throw new AnalysisException( s"DESC PARTITION is not allowed on a datasource table: ${table.identifier}") @@ -536,9 +540,6 @@ case class DescribeTableCommand( append(buffer, "Partition Value:", s"[${partition.spec.values.mkString(", ")}]", "") append(buffer, "Database:", table.database, "") append(buffer, "Table:", tableIdentifier.table, "") - append(buffer, "Create Time:", "UNKNOWN", "") - append(buffer, "Last Access Time:", "UNKNOWN", "") - append(buffer, "Protect Mode:", "None", "") append(buffer, "Location:", partition.storage.locationUri.getOrElse(""), "") append(buffer, "Partition Parameters:", "", "") partition.parameters.foreach { case (key, value) => diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index b5d0996371b1..e9a37b1e4880 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -365,7 +365,6 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { "Partition Value:", "Database:", "Table:", - "Create Time:", "Location:", "Partition Parameters:", "# Storage Information") @@ -396,9 +395,17 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { val m5 = intercept[AnalysisException] { spark.range(10).select('id as 'a, 'id as 'b).createTempView("view1") - sql("DESC view1 PARTITION (c='Us', d=1)").show() + sql("DESC view1 PARTITION (c='Us', d=1)") }.getMessage() assert(m5.contains("DESC PARTITION is not allowed on a temporary view")) + + withView("permanent_view") { + val m = intercept[AnalysisException] { + sql("CREATE VIEW permanent_view AS SELECT * FROM partitioned_table") + sql("DESC permanent_view PARTITION (c='Us', d=1)") + }.getMessage() + assert(m.contains("DESC PARTITION is not allowed on a view")) + } } } From 89d44240faad4a5028c4005842402f7191a00e18 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Mon, 26 Sep 2016 00:39:47 -0700 Subject: [PATCH 08/13] Create 'describeDetailPartitionInfo' function and split test cases. --- .../spark/sql/execution/command/tables.scala | 37 +++++++++++-------- .../sql/hive/execution/SQLQuerySuite.scala | 9 ++++- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index 0798da59b8e5..644a0b560197 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -433,21 +433,7 @@ case class DescribeTableCommand( describeFormattedTableInfo(metadata, result) } } else { - if (metadata.tableType == CatalogTableType.VIEW) { - throw new AnalysisException( - s"DESC PARTITION is not allowed on a view: ${table.identifier}") - } - if (DDLUtils.isDatasourceTable(metadata)) { - throw new AnalysisException( - s"DESC PARTITION is not allowed on a datasource table: ${table.identifier}") - } - val partition = catalog.getPartition(table, partitionSpec) - if (isExtended) { - describeExtendedDetailPartitionInfo(table, metadata, partition, result) - } else if (isFormatted) { - describeFormattedDetailPartitionInfo(table, metadata, partition, result) - describeStorageInfo(metadata, result) - } + describeDetailPartitionInfo(catalog, metadata, result) } } @@ -521,6 +507,27 @@ case class DescribeTableCommand( } } + private def describeDetailPartitionInfo( + catalog: SessionCatalog, + metadata: CatalogTable, + result: ArrayBuffer[Row]): Unit = { + if (metadata.tableType == CatalogTableType.VIEW) { + throw new AnalysisException( + s"DESC PARTITION is not allowed on a view: ${table.identifier}") + } + if (DDLUtils.isDatasourceTable(metadata)) { + throw new AnalysisException( + s"DESC PARTITION is not allowed on a datasource table: ${table.identifier}") + } + val partition = catalog.getPartition(table, partitionSpec) + if (isExtended) { + describeExtendedDetailPartitionInfo(table, metadata, partition, result) + } else if (isFormatted) { + describeFormattedDetailPartitionInfo(table, metadata, partition, result) + describeStorageInfo(metadata, result) + } + } + private def describeExtendedDetailPartitionInfo( tableIdentifier: TableIdentifier, table: CatalogTable, diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index e9a37b1e4880..54684e7a853e 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -342,7 +342,7 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { } test("describe partition") { - withTable("partitioned_table", "datasource_table") { + withTable("partitioned_table") { sql("CREATE TABLE partitioned_table (a STRING, b INT) PARTITIONED BY (c STRING, d STRING)") sql("ALTER TABLE partitioned_table ADD PARTITION (c='Us', d=1)") @@ -368,6 +368,13 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { "Location:", "Partition Parameters:", "# Storage Information") + } + } + + test("describe partition - error handling") { + withTable("partitioned_table", "datasource_table") { + sql("CREATE TABLE partitioned_table (a STRING, b INT) PARTITIONED BY (c STRING, d STRING)") + sql("ALTER TABLE partitioned_table ADD PARTITION (c='Us', d=1)") val m = intercept[NoSuchPartitionException] { sql("DESC partitioned_table PARTITION (c='Us', d=2)") From e3d2faf24c9d4e5241ff07b7263605170dfebffb Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Mon, 26 Sep 2016 01:17:48 -0700 Subject: [PATCH 09/13] Rename functions to use 'detailed' instead of 'detail'. --- .../apache/spark/sql/execution/command/tables.scala | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index 644a0b560197..08de6cd4242c 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -433,7 +433,7 @@ case class DescribeTableCommand( describeFormattedTableInfo(metadata, result) } } else { - describeDetailPartitionInfo(catalog, metadata, result) + describeDetailedPartitionInfo(catalog, metadata, result) } } @@ -507,7 +507,7 @@ case class DescribeTableCommand( } } - private def describeDetailPartitionInfo( + private def describeDetailedPartitionInfo( catalog: SessionCatalog, metadata: CatalogTable, result: ArrayBuffer[Row]): Unit = { @@ -521,14 +521,14 @@ case class DescribeTableCommand( } val partition = catalog.getPartition(table, partitionSpec) if (isExtended) { - describeExtendedDetailPartitionInfo(table, metadata, partition, result) + describeExtendedDetailedPartitionInfo(table, metadata, partition, result) } else if (isFormatted) { - describeFormattedDetailPartitionInfo(table, metadata, partition, result) + describeFormattedDetailedPartitionInfo(table, metadata, partition, result) describeStorageInfo(metadata, result) } } - private def describeExtendedDetailPartitionInfo( + private def describeExtendedDetailedPartitionInfo( tableIdentifier: TableIdentifier, table: CatalogTable, partition: CatalogTablePartition, @@ -537,7 +537,7 @@ case class DescribeTableCommand( append(buffer, "Detailed Partition Information " + partition.toString, "", "") } - private def describeFormattedDetailPartitionInfo( + private def describeFormattedDetailedPartitionInfo( tableIdentifier: TableIdentifier, table: CatalogTable, partition: CatalogTablePartition, From ec574ff73650394e899c602eec9a05ff07083f74 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Tue, 27 Sep 2016 11:14:28 -0700 Subject: [PATCH 10/13] Update `describe.sql`, too. --- .../resources/sql-tests/inputs/describe.sql | 10 +-- .../sql-tests/results/describe.sql.out | 63 ++++++++++++++++--- 2 files changed, 59 insertions(+), 14 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/describe.sql b/sql/core/src/test/resources/sql-tests/inputs/describe.sql index 5ddcadfecc5a..1be5f4c70149 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/describe.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/describe.sql @@ -4,17 +4,17 @@ ALTER TABLE t ADD PARTITION (c='Us', d=1); DESC t; --- Ignore these because there exists timestamp results like `create_table` +-- Ignore these because there exists timestamp results, e.g., `Create Table` -- DESC EXTENDED t; -- DESC FORMATTED t; DESC t PARTITION (c='Us', d=1); --- Ignore these because there exists timestamp results like `create_table` --- DESC EXTENDED t PARTITION (c='Us', d=1); --- DESC FORMATTED t PARTITION (c='Us', d=1); +DESC EXTENDED t PARTITION (c='Us', d=1); --- NoSuchPartitionException: Partition not found in table 'partitioned_table' +DESC FORMATTED t PARTITION (c='Us', d=1); + +-- NoSuchPartitionException: Partition not found in table DESC t PARTITION (c='Us', d=2); -- AnalysisException: Partition spec is invalid diff --git a/sql/core/src/test/resources/sql-tests/results/describe.sql.out b/sql/core/src/test/resources/sql-tests/results/describe.sql.out index 3ab0c7c99c01..5b605f9a18b2 100644 --- a/sql/core/src/test/resources/sql-tests/results/describe.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/describe.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 7 +-- Number of queries: 9 -- !query 0 @@ -49,30 +49,75 @@ d string -- !query 4 -DESC t PARTITION (c='Us', d=2) +DESC EXTENDED t PARTITION (c='Us', d=1) -- !query 4 schema -struct<> +struct -- !query 4 output +# Partition Information +# col_name data_type comment +Detailed Partition Information CatalogPartition( + Partition Values: [Us, 1] + Storage(InputFormat: org.apache.hadoop.mapred.TextInputFormat, OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat) + Partition Parameters:{}) +a string +b int +c string +c string +d string +d string + + +-- !query 5 +DESC FORMATTED t PARTITION (c='Us', d=1) +-- !query 5 schema +struct +-- !query 5 output +# Detailed Partition Information +# Partition Information +# Storage Information +# col_name data_type comment +Compressed: No +Database: default +InputFormat: org.apache.hadoop.mapred.TextInputFormat +Location: +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Partition Parameters: +Partition Value: [Us, 1] +Storage Desc Parameters: +Table: t +a string +b int +c string +c string +d string +d string + + +-- !query 6 +DESC t PARTITION (c='Us', d=2) +-- !query 6 schema +struct<> +-- !query 6 output org.apache.spark.sql.catalyst.analysis.NoSuchPartitionException Partition not found in table 't' database 'default': c -> Us d -> 2; --- !query 5 +-- !query 7 DESC t PARTITION (c='Us') --- !query 5 schema +-- !query 7 schema struct<> --- !query 5 output +-- !query 7 output org.apache.spark.sql.AnalysisException Partition spec is invalid. The spec (c) must match the partition spec (c, d) defined in table '`default`.`t`'; --- !query 6 +-- !query 8 DESC t PARTITION (c='Us', d) --- !query 6 schema +-- !query 8 schema struct<> --- !query 6 output +-- !query 8 output org.apache.spark.sql.catalyst.parser.ParseException Unsupported SQL statement From 2ed1069ea17d4f511a245f59c0479839d483a6c4 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Tue, 27 Sep 2016 11:25:21 -0700 Subject: [PATCH 11/13] Ignore EXTENDED and FORMATTED again for `transient_lastDdlTime`. --- .../resources/sql-tests/inputs/describe.sql | 8 +-- .../sql-tests/results/describe.sql.out | 63 +++---------------- 2 files changed, 13 insertions(+), 58 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/describe.sql b/sql/core/src/test/resources/sql-tests/inputs/describe.sql index 1be5f4c70149..3eb1579a896f 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/describe.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/describe.sql @@ -4,15 +4,15 @@ ALTER TABLE t ADD PARTITION (c='Us', d=1); DESC t; --- Ignore these because there exists timestamp results, e.g., `Create Table` +-- Ignore these because there exist timestamp results, e.g., `Create Table`. -- DESC EXTENDED t; -- DESC FORMATTED t; DESC t PARTITION (c='Us', d=1); -DESC EXTENDED t PARTITION (c='Us', d=1); - -DESC FORMATTED t PARTITION (c='Us', d=1); +-- Ignore these because there exist timestamp results, e.g., transient_lastDdlTime. +-- DESC EXTENDED t PARTITION (c='Us', d=1); +-- DESC FORMATTED t PARTITION (c='Us', d=1); -- NoSuchPartitionException: Partition not found in table DESC t PARTITION (c='Us', d=2); diff --git a/sql/core/src/test/resources/sql-tests/results/describe.sql.out b/sql/core/src/test/resources/sql-tests/results/describe.sql.out index 5b605f9a18b2..3ab0c7c99c01 100644 --- a/sql/core/src/test/resources/sql-tests/results/describe.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/describe.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 9 +-- Number of queries: 7 -- !query 0 @@ -49,75 +49,30 @@ d string -- !query 4 -DESC EXTENDED t PARTITION (c='Us', d=1) --- !query 4 schema -struct --- !query 4 output -# Partition Information -# col_name data_type comment -Detailed Partition Information CatalogPartition( - Partition Values: [Us, 1] - Storage(InputFormat: org.apache.hadoop.mapred.TextInputFormat, OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat) - Partition Parameters:{}) -a string -b int -c string -c string -d string -d string - - --- !query 5 -DESC FORMATTED t PARTITION (c='Us', d=1) --- !query 5 schema -struct --- !query 5 output -# Detailed Partition Information -# Partition Information -# Storage Information -# col_name data_type comment -Compressed: No -Database: default -InputFormat: org.apache.hadoop.mapred.TextInputFormat -Location: -OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat -Partition Parameters: -Partition Value: [Us, 1] -Storage Desc Parameters: -Table: t -a string -b int -c string -c string -d string -d string - - --- !query 6 DESC t PARTITION (c='Us', d=2) --- !query 6 schema +-- !query 4 schema struct<> --- !query 6 output +-- !query 4 output org.apache.spark.sql.catalyst.analysis.NoSuchPartitionException Partition not found in table 't' database 'default': c -> Us d -> 2; --- !query 7 +-- !query 5 DESC t PARTITION (c='Us') --- !query 7 schema +-- !query 5 schema struct<> --- !query 7 output +-- !query 5 output org.apache.spark.sql.AnalysisException Partition spec is invalid. The spec (c) must match the partition spec (c, d) defined in table '`default`.`t`'; --- !query 8 +-- !query 6 DESC t PARTITION (c='Us', d) --- !query 8 schema +-- !query 6 schema struct<> --- !query 8 output +-- !query 6 output org.apache.spark.sql.catalyst.parser.ParseException Unsupported SQL statement From fc132e6061197c2d0c681572ab1cdc923b5ba81c Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Tue, 27 Sep 2016 11:48:02 -0700 Subject: [PATCH 12/13] Simplify the logic and throw meaningful exception message. --- .../apache/spark/sql/execution/SparkSqlParser.scala | 10 ++++------ .../src/test/resources/sql-tests/inputs/describe.sql | 2 +- .../test/resources/sql-tests/results/describe.sql.out | 4 +++- .../spark/sql/hive/execution/SQLQuerySuite.scala | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala index 144d84945469..3f34d0f25393 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala @@ -283,12 +283,10 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder { } else { val partitionSpec = if (ctx.partitionSpec != null) { // According to the syntax, visitPartitionSpec returns `Map[String, Option[String]]`. - val (valid, nonValid) = visitPartitionSpec(ctx.partitionSpec).partition(_._2.isDefined) - if (nonValid.nonEmpty) { - // For non-valid specification for `DESC` command, this raises a ParseException. - return null - } else { - valid.map(x => (x._1, x._2.get)) + visitPartitionSpec(ctx.partitionSpec).map { + case (key, Some(value)) => key -> value + case (key, _) => + throw new ParseException(s"PARTITION specification is incomplete: `$key`", ctx) } } else { Map.empty[String, String] diff --git a/sql/core/src/test/resources/sql-tests/inputs/describe.sql b/sql/core/src/test/resources/sql-tests/inputs/describe.sql index 3eb1579a896f..4f3b3a4cd3f9 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/describe.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/describe.sql @@ -20,5 +20,5 @@ DESC t PARTITION (c='Us', d=2); -- AnalysisException: Partition spec is invalid DESC t PARTITION (c='Us'); --- ParseException: Unsupported SQL statement +-- ParseException: PARTITION specification is incomplete DESC t PARTITION (c='Us', d); diff --git a/sql/core/src/test/resources/sql-tests/results/describe.sql.out b/sql/core/src/test/resources/sql-tests/results/describe.sql.out index 3ab0c7c99c01..6ca06251adbc 100644 --- a/sql/core/src/test/resources/sql-tests/results/describe.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/describe.sql.out @@ -75,6 +75,8 @@ struct<> -- !query 6 output org.apache.spark.sql.catalyst.parser.ParseException -Unsupported SQL statement +PARTITION specification is incomplete: `d`(line 1, pos 0) + == SQL == DESC t PARTITION (c='Us', d) +^^^ diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index 54684e7a853e..6c77a0deb52a 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -389,7 +389,7 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { val m3 = intercept[ParseException] { sql("DESC partitioned_table PARTITION (c='Us', d)") }.getMessage() - assert(m3.contains("Unsupported SQL statement")) + assert(m3.contains("PARTITION specification is incomplete: `d`")) spark .range(1).select('id as 'a, 'id as 'b, 'id as 'c, 'id as 'd).write From 04c75e52a4f4e24dd49aad8148ae75a31c630cc7 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Tue, 27 Sep 2016 14:03:17 -0700 Subject: [PATCH 13/13] Drop test table. --- .../src/test/resources/sql-tests/inputs/describe.sql | 3 +++ .../test/resources/sql-tests/results/describe.sql.out | 10 +++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/describe.sql b/sql/core/src/test/resources/sql-tests/inputs/describe.sql index 4f3b3a4cd3f9..3f0ae902e052 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/describe.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/describe.sql @@ -22,3 +22,6 @@ DESC t PARTITION (c='Us'); -- ParseException: PARTITION specification is incomplete DESC t PARTITION (c='Us', d); + +-- DROP TEST TABLE +DROP TABLE t; diff --git a/sql/core/src/test/resources/sql-tests/results/describe.sql.out b/sql/core/src/test/resources/sql-tests/results/describe.sql.out index 6ca06251adbc..37bf303f1bfe 100644 --- a/sql/core/src/test/resources/sql-tests/results/describe.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/describe.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 7 +-- Number of queries: 8 -- !query 0 @@ -80,3 +80,11 @@ PARTITION specification is incomplete: `d`(line 1, pos 0) == SQL == DESC t PARTITION (c='Us', d) ^^^ + + +-- !query 7 +DROP TABLE t +-- !query 7 schema +struct<> +-- !query 7 output +