diff --git a/docs/sql-ref-ansi-compliance.md b/docs/sql-ref-ansi-compliance.md index ccfc60122d31c..0769089f4dabe 100644 --- a/docs/sql-ref-ansi-compliance.md +++ b/docs/sql-ref-ansi-compliance.md @@ -395,7 +395,6 @@ Below is a list of all the keywords in Spark SQL. |DAY|non-reserved|non-reserved|non-reserved| |DAYOFYEAR|non-reserved|non-reserved|non-reserved| |DBPROPERTIES|non-reserved|non-reserved|non-reserved| -|DEFAULT|non-reserved|non-reserved|non-reserved| |DEFINED|non-reserved|non-reserved|non-reserved| |DELETE|non-reserved|non-reserved|reserved| |DELIMITED|non-reserved|non-reserved|non-reserved| diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4 index e84d4fa45eb99..6c731bb02bc39 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4 @@ -146,7 +146,6 @@ DATABASES: 'DATABASES'; DATEADD: 'DATEADD'; DATEDIFF: 'DATEDIFF'; DBPROPERTIES: 'DBPROPERTIES'; -DEFAULT: 'DEFAULT'; DEFINED: 'DEFINED'; DELETE: 'DELETE'; DELIMITED: 'DELIMITED'; diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 index fb3bccacaf94b..fe81f0ccb8a48 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 @@ -83,7 +83,7 @@ statement (RESTRICT | CASCADE)? #dropNamespace | SHOW namespaces ((FROM | IN) multipartIdentifier)? (LIKE? pattern=STRING)? #showNamespaces - | createTableHeader (LEFT_PAREN createOrReplaceTableColTypeList RIGHT_PAREN)? tableProvider? + | createTableHeader (LEFT_PAREN colTypeList RIGHT_PAREN)? tableProvider? createTableClauses (AS? query)? #createTable | CREATE TABLE (IF NOT EXISTS)? target=tableIdentifier @@ -93,7 +93,7 @@ statement createFileFormat | locationSpec | (TBLPROPERTIES tableProps=propertyList))* #createTableLike - | replaceTableHeader (LEFT_PAREN createOrReplaceTableColTypeList RIGHT_PAREN)? tableProvider? + | replaceTableHeader (LEFT_PAREN colTypeList RIGHT_PAREN)? tableProvider? createTableClauses (AS? query)? #replaceTable | ANALYZE TABLE multipartIdentifier partitionSpec? COMPUTE STATISTICS @@ -917,11 +917,7 @@ qualifiedColTypeWithPositionList ; qualifiedColTypeWithPosition - : name=multipartIdentifier dataType (NOT NULL)? defaultExpression? commentSpec? colPosition? - ; - -defaultExpression - : DEFAULT expression + : name=multipartIdentifier dataType (NOT NULL)? commentSpec? colPosition? ; colTypeList @@ -932,14 +928,6 @@ colType : colName=errorCapturingIdentifier dataType (NOT NULL)? commentSpec? ; -createOrReplaceTableColTypeList - : createOrReplaceTableColType (COMMA createOrReplaceTableColType)* - ; - -createOrReplaceTableColType - : colName=errorCapturingIdentifier dataType (NOT NULL)? defaultExpression? commentSpec? - ; - complexColTypeList : complexColType (COMMA complexColType)* ; @@ -1046,8 +1034,6 @@ alterColumnAction | commentSpec | colPosition | setOrDrop=(SET | DROP) NOT NULL - | SET defaultExpression - | dropDefault=DROP DEFAULT ; @@ -1105,7 +1091,6 @@ ansiNonReserved | DAY | DAYOFYEAR | DBPROPERTIES - | DEFAULT | DEFINED | DELETE | DELIMITED @@ -1361,7 +1346,6 @@ nonReserved | DAY | DAYOFYEAR | DBPROPERTIES - | DEFAULT | DEFINED | DELETE | DELIMITED diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index 5eb72af6b2f09..3c8f0770e19af 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -2728,13 +2728,6 @@ class AstBuilder extends SqlBaseParserBaseVisitor[AnyRef] with SQLConfHelper wit StructType(Option(ctx).toSeq.flatMap(visitColTypeList)) } - /** - * Create top level table schema. - */ - protected def createSchema(ctx: CreateOrReplaceTableColTypeListContext): StructType = { - StructType(Option(ctx).toSeq.flatMap(visitCreateOrReplaceTableColTypeList)) - } - /** * Create a [[StructType]] from a number of column definitions. */ @@ -2761,41 +2754,6 @@ class AstBuilder extends SqlBaseParserBaseVisitor[AnyRef] with SQLConfHelper wit metadata = builder.build()) } - /** - * Create a [[StructType]] from a number of CREATE TABLE column definitions. - */ - override def visitCreateOrReplaceTableColTypeList( - ctx: CreateOrReplaceTableColTypeListContext): Seq[StructField] = withOrigin(ctx) { - ctx.createOrReplaceTableColType().asScala.map(visitCreateOrReplaceTableColType).toSeq - } - - /** - * Create a top level [[StructField]] from a CREATE TABLE column definition. - */ - override def visitCreateOrReplaceTableColType( - ctx: CreateOrReplaceTableColTypeContext): StructField = withOrigin(ctx) { - import ctx._ - - val builder = new MetadataBuilder - // Add comment to metadata - Option(commentSpec()).map(visitCommentSpec).foreach { - builder.putString("comment", _) - } - - // Process the 'DEFAULT expression' clause in the column definition, if any. - val name: String = colName.getText - val defaultExpr = Option(ctx.defaultExpression()).map(visitDefaultExpression) - if (defaultExpr.isDefined) { - throw QueryParsingErrors.defaultColumnNotImplementedYetError(ctx) - } - - StructField( - name = name, - dataType = typedVisit[DataType](ctx.dataType), - nullable = NULL == null, - metadata = builder.build()) - } - /** * Create a [[StructType]] from a sequence of [[StructField]]s. */ @@ -3499,8 +3457,7 @@ class AstBuilder extends SqlBaseParserBaseVisitor[AnyRef] with SQLConfHelper wit override def visitCreateTable(ctx: CreateTableContext): LogicalPlan = withOrigin(ctx) { val (table, temp, ifNotExists, external) = visitCreateTableHeader(ctx.createTableHeader) - val columns = Option(ctx.createOrReplaceTableColTypeList()) - .map(visitCreateOrReplaceTableColTypeList).getOrElse(Nil) + val columns = Option(ctx.colTypeList()).map(visitColTypeList).getOrElse(Nil) val provider = Option(ctx.tableProvider).map(_.multipartIdentifier.getText) val (partTransforms, partCols, bucketSpec, properties, options, location, comment, serdeInfo) = visitCreateTableClauses(ctx.createTableClauses()) @@ -3579,8 +3536,7 @@ class AstBuilder extends SqlBaseParserBaseVisitor[AnyRef] with SQLConfHelper wit val orCreate = ctx.replaceTableHeader().CREATE() != null val (partTransforms, partCols, bucketSpec, properties, options, location, comment, serdeInfo) = visitCreateTableClauses(ctx.createTableClauses()) - val columns = Option(ctx.createOrReplaceTableColTypeList()) - .map(visitCreateOrReplaceTableColTypeList).getOrElse(Nil) + val columns = Option(ctx.colTypeList()).map(visitColTypeList).getOrElse(Nil) val provider = Option(ctx.tableProvider).map(_.multipartIdentifier.getText) if (provider.isDefined && serdeInfo.isDefined) { @@ -3699,10 +3655,6 @@ class AstBuilder extends SqlBaseParserBaseVisitor[AnyRef] with SQLConfHelper wit override def visitQualifiedColTypeWithPosition( ctx: QualifiedColTypeWithPositionContext): QualifiedColType = withOrigin(ctx) { val name = typedVisit[Seq[String]](ctx.name) - val defaultExpr = Option(ctx.defaultExpression()).map(visitDefaultExpression) - if (defaultExpr.isDefined) { - throw QueryParsingErrors.defaultColumnNotImplementedYetError(ctx) - } QualifiedColType( path = if (name.length > 1) Some(UnresolvedFieldName(name.init)) else None, colName = name.last, @@ -3791,12 +3743,6 @@ class AstBuilder extends SqlBaseParserBaseVisitor[AnyRef] with SQLConfHelper wit } else { None } - if (action.defaultExpression != null) { - throw QueryParsingErrors.defaultColumnNotImplementedYetError(ctx) - } - if (action.dropDefault != null) { - throw QueryParsingErrors.defaultColumnNotImplementedYetError(ctx) - } assert(Seq(dataType, nullable, comment, position).count(_.nonEmpty) == 1) @@ -3865,9 +3811,6 @@ class AstBuilder extends SqlBaseParserBaseVisitor[AnyRef] with SQLConfHelper wit throw QueryParsingErrors.operationInHiveStyleCommandUnsupportedError( "Replacing with a nested column", "REPLACE COLUMNS", ctx) } - if (Option(colType.defaultExpression()).map(visitDefaultExpression).isDefined) { - throw QueryParsingErrors.defaultColumnNotImplementedYetError(ctx) - } col }.toSeq ) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryParsingErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryParsingErrors.scala index c03b1b45f644d..c09295884aa24 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryParsingErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryParsingErrors.scala @@ -433,8 +433,4 @@ object QueryParsingErrors { new ParseException( s"DROP TEMPORARY FUNCTION requires a single part name but got: ${name.quoted}", ctx) } - - def defaultColumnNotImplementedYetError(ctx: ParserRuleContext): Throwable = { - new ParseException("Support for DEFAULT column values is not implemented yet", ctx) - } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala index a339e6d33f5f3..507b17bbb5636 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala @@ -2238,57 +2238,4 @@ class DDLParserSuite extends AnalysisTest { comparePlans(parsePlan(timestampTypeSql), insertPartitionPlan(timestamp)) comparePlans(parsePlan(binaryTypeSql), insertPartitionPlan(binaryStr)) } - - test("SPARK-38335: Implement parser support for DEFAULT values for columns in tables") { - // The following commands will support DEFAULT columns, but this has not been implemented yet. - for (sql <- Seq( - "ALTER TABLE t1 ADD COLUMN x int NOT NULL DEFAULT 42", - "ALTER TABLE t1 ALTER COLUMN a.b.c SET DEFAULT 42", - "ALTER TABLE t1 ALTER COLUMN a.b.c DROP DEFAULT", - "ALTER TABLE t1 REPLACE COLUMNS (x STRING DEFAULT 42)", - "CREATE TABLE my_tab(a INT COMMENT 'test', b STRING NOT NULL DEFAULT \"abc\") USING parquet", - "REPLACE TABLE my_tab(a INT COMMENT 'test', b STRING NOT NULL DEFAULT \"xyz\") USING parquet" - )) { - val exc = intercept[ParseException] { - parsePlan(sql); - } - assert(exc.getMessage.contains("Support for DEFAULT column values is not implemented yet")); - } - // In each of the following cases, the DEFAULT reference parses as an unresolved attribute - // reference. We can handle these cases after the parsing stage, at later phases of analysis. - comparePlans(parsePlan("VALUES (1, 2, DEFAULT) AS val"), - SubqueryAlias("val", - UnresolvedInlineTable(Seq("col1", "col2", "col3"), Seq(Seq(Literal(1), Literal(2), - UnresolvedAttribute("DEFAULT")))))) - comparePlans(parsePlan( - "INSERT INTO t PARTITION(part = date'2019-01-02') VALUES ('a', DEFAULT)"), - InsertIntoStatement( - UnresolvedRelation(Seq("t")), - Map("part" -> Some("2019-01-02")), - userSpecifiedCols = Seq.empty[String], - query = UnresolvedInlineTable(Seq("col1", "col2"), Seq(Seq(Literal("a"), - UnresolvedAttribute("DEFAULT")))), - overwrite = false, ifPartitionNotExists = false)) - parseCompare( - """ - |MERGE INTO testcat1.ns1.ns2.tbl AS target - |USING testcat2.ns1.ns2.tbl AS source - |ON target.col1 = source.col1 - |WHEN MATCHED AND (target.col2='delete') THEN DELETE - |WHEN MATCHED AND (target.col2='update') THEN UPDATE SET target.col2 = DEFAULT - |WHEN NOT MATCHED AND (target.col2='insert') - |THEN INSERT (target.col1, target.col2) VALUES (source.col1, DEFAULT) - """.stripMargin, - MergeIntoTable( - SubqueryAlias("target", UnresolvedRelation(Seq("testcat1", "ns1", "ns2", "tbl"))), - SubqueryAlias("source", UnresolvedRelation(Seq("testcat2", "ns1", "ns2", "tbl"))), - EqualTo(UnresolvedAttribute("target.col1"), UnresolvedAttribute("source.col1")), - Seq(DeleteAction(Some(EqualTo(UnresolvedAttribute("target.col2"), Literal("delete")))), - UpdateAction(Some(EqualTo(UnresolvedAttribute("target.col2"), Literal("update"))), - Seq(Assignment(UnresolvedAttribute("target.col2"), - UnresolvedAttribute("DEFAULT"))))), - Seq(InsertAction(Some(EqualTo(UnresolvedAttribute("target.col2"), Literal("insert"))), - Seq(Assignment(UnresolvedAttribute("target.col1"), UnresolvedAttribute("source.col1")), - Assignment(UnresolvedAttribute("target.col2"), UnresolvedAttribute("DEFAULT"))))))) - } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala index a4e72e04507b5..fed02dddecf78 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala @@ -318,7 +318,7 @@ class SparkSqlAstBuilder extends AstBuilder { val (_, _, _, _, options, location, _, _) = visitCreateTableClauses(ctx.createTableClauses()) val provider = Option(ctx.tableProvider).map(_.multipartIdentifier.getText).getOrElse( throw QueryParsingErrors.createTempTableNotSpecifyProviderError(ctx)) - val schema = Option(ctx.createOrReplaceTableColTypeList()).map(createSchema) + val schema = Option(ctx.colTypeList()).map(createSchema) logWarning(s"CREATE TEMPORARY TABLE ... USING ... is deprecated, please use " + "CREATE TEMPORARY VIEW ... USING ... instead")