From 66cd5379a17e05707ae162bb20e9c64812737d78 Mon Sep 17 00:00:00 2001 From: Giambattista Bloisi Date: Sat, 3 Nov 2018 17:04:09 +0100 Subject: [PATCH 1/3] Because of SPARK-17590 support of WITH clause in subqueries requires only grammar support. --- .../org/apache/spark/sql/catalyst/parser/SqlBase.g4 | 4 ++-- .../spark/sql/catalyst/parser/AstBuilder.scala | 4 ++-- .../scala/org/apache/spark/sql/SQLQuerySuite.scala | 13 +++++++++++++ 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 index e2d34d1650dd..7f28b3100866 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 @@ -372,7 +372,7 @@ queryPrimary : querySpecification #queryPrimaryDefault | TABLE tableIdentifier #table | inlineTable #inlineTableDefault1 - | '(' queryNoWith ')' #subquery + | '(' query ')' #subquery ; sortItem @@ -510,7 +510,7 @@ identifierComment relationPrimary : tableIdentifier sample? tableAlias #tableName - | '(' queryNoWith ')' sample? tableAlias #aliasedQuery + | '(' query ')' sample? tableAlias #aliasedQuery | '(' relation ')' sample? tableAlias #aliasedRelation | inlineTable #inlineTableDefault2 | functionTable #tableValuedFunction diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index 672bffcfc0ca..72312a25178e 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -795,7 +795,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging * Create a logical plan for a sub-query. */ override def visitSubquery(ctx: SubqueryContext): LogicalPlan = withOrigin(ctx) { - plan(ctx.queryNoWith) + plan(ctx.query) } /** @@ -883,7 +883,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging * }}} */ override def visitAliasedQuery(ctx: AliasedQueryContext): LogicalPlan = withOrigin(ctx) { - val relation = plan(ctx.queryNoWith).optionalMap(ctx.sample)(withSample) + val relation = plan(ctx.query).optionalMap(ctx.sample)(withSample) if (ctx.tableAlias.strictIdentifier == null) { // For un-aliased subqueries, use a default alias name that is not likely to conflict with // normal subquery names, so that parent operators can only access the columns in subquery by diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index 631ab1b7ece7..a0b00dc0b050 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -535,7 +535,18 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext { |q2 as (select * from testData where key = '4') |select * from q1 union all select * from q2""".stripMargin), Row(5, "5") :: Row(4, "4") :: Nil) + } + test("SPARK-19799: Support WITH clause in subqueries") { + checkAnswer( + sql(""" + |select avg(b) from ( + | with maxearnings as ( + | select course, year, max(earnings) as earnings FROM courseSales GROUP BY course, year + | ) + | select course, sum(earnings) as b FROM maxearnings GROUP BY course + | )""".stripMargin), + Row(54000) :: Nil) } test("Allow only a single WITH clause per query") { @@ -545,6 +556,8 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext { } } + + test("date row") { checkAnswer(sql( """select cast("2015-01-28" as date) from testData limit 1"""), From 5377087612e52eba09961e7591ae0c436c0a4de1 Mon Sep 17 00:00:00 2001 From: gbloisi Date: Wed, 7 Nov 2018 21:09:25 +0100 Subject: [PATCH 2/3] Fix scala check issue FIX: Scalastyle checks failed at following occurrences: [error] /home/jenkins/workspace/SparkPullRequestBuilder/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala:545: File line length exceeds 100 characters --- .../src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index a0b00dc0b050..ee56bba0b0e0 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -542,7 +542,8 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext { sql(""" |select avg(b) from ( | with maxearnings as ( - | select course, year, max(earnings) as earnings FROM courseSales GROUP BY course, year + | select course, year, max(earnings) as earnings FROM courseSales + | GROUP BY course, year | ) | select course, sum(earnings) as b FROM maxearnings GROUP BY course | )""".stripMargin), @@ -556,8 +557,6 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext { } } - - test("date row") { checkAnswer(sql( """select cast("2015-01-28" as date) from testData limit 1"""), From 3080ed4eab23b74bcafd7f933808b116402fe622 Mon Sep 17 00:00:00 2001 From: gbloisi Date: Wed, 7 Nov 2018 21:24:55 +0100 Subject: [PATCH 3/3] Resolve scalastyle check error FIX: [error] /home/jenkins/workspace/SparkPullRequestBuilder/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala:545:79: Whitespace at end of line --- .../src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index ee56bba0b0e0..ab3b5c16ea31 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -542,7 +542,7 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext { sql(""" |select avg(b) from ( | with maxearnings as ( - | select course, year, max(earnings) as earnings FROM courseSales + | select course, year, max(earnings) as earnings FROM courseSales | GROUP BY course, year | ) | select course, sum(earnings) as b FROM maxearnings GROUP BY course