Skip to content

SQL: Add PIVOT support #46489

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
costin merged 10 commits into from
Sep 23, 2019
Merged

SQL: Add PIVOT support #46489

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
84cee40
SQL: Add PIVOT support
costin Aug 20, 2019
a0caa34
Merge branch 'master' into sql/piv
costin Sep 17, 2019
48c6a83
Address feedback
costin Sep 17, 2019
3f356ba
Address feedback
costin Sep 17, 2019
7af8f37
Restrict usage of PIVOT
costin Sep 18, 2019
d28984a
Merge remote-tracking branch 'remotes/upstream/master' into sql/piv
costin Sep 18, 2019
d668751
Prevent usage of InnerAgg for now
costin Sep 18, 2019
854d09c
Remove unneeded file
costin Sep 18, 2019
2606f36
Fix typos
costin Sep 23, 2019
7bbe026
Merge remote-tracking branch 'remotes/upstream/master' into sql/piv
costin Sep 23, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion x-pack/plugin/sql/qa/src/main/resources/debug/debug.csv-spec
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@
//
Copy link
Member Author

@costin costin Sep 9, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This file shouldn't be here - I'll remove it on the next commit once the feedback lands to avoid an extra build.


debug
SELECT first_name f, last_name l, dep.from_date d FROM test_emp WHERE dep.dep_name = 'Production' ORDER BY f LIMIT 5;
SELECT * FROM (SELECT languages, gender, salary FROM test_emp) PIVOT (AVG(salary) FOR gender IN ('M', 'F'));
//SELECT languages FROM (SELECT languages, gender, salary FROM test_emp) PIVOT (AVG(salary) FOR gender IN ('M', 'F'));

f:s | l:s | d:ts

Expand Down
242 changes: 242 additions & 0 deletions x-pack/plugin/sql/qa/src/main/resources/pivot.csv-spec
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,242 @@
averageWithOneValue
schema::languages:bt|'F':d
SELECT * FROM (SELECT languages, gender, salary FROM test_emp) PIVOT (AVG(salary) FOR gender IN ('F'));

languages | 'F'
---------------+------------------
null |62140.666666666664
1 |47073.25
2 |50684.4
3 |53660.0
4 |49291.5
5 |46705.555555555555
;

averageWithAliasAndOneValue
schema::languages:bt|'F':d
SELECT * FROM (SELECT languages, gender, salary FROM test_emp) PIVOT (AVG(salary) AS "AVG" FOR gender IN ('F'));

languages | 'F'
---------------+------------------
null |62140.666666666664
1 |47073.25
2 |50684.4
3 |53660.0
4 |49291.5
5 |46705.555555555555
;

averageWithAliasedValue
schema::languages:bt|XX:d
SELECT * FROM (SELECT languages, gender, salary FROM test_emp) PIVOT (AVG(salary) FOR gender IN ('F' AS "XX"));

languages | XX
---------------+------------------
null |62140.666666666664
1 |47073.25
2 |50684.4
3 |53660.0
4 |49291.5
5 |46705.555555555555
;

averageWithTwoValues
schema::languages:bt|'M':d|'F':d
SELECT * FROM (SELECT languages, gender, salary FROM test_emp) PIVOT (AVG(salary) FOR gender IN ('M', 'F'));

languages | 'M' | 'F'
---------------+-----------------+------------------
null |48396.28571428572|62140.666666666664
1 |49767.22222222222|47073.25
2 |44103.90909090909|50684.4
3 |51741.90909090909|53660.0
4 |47058.90909090909|49291.5
5 |39052.875 |46705.555555555555
;

averageWithTwoValuesAndAlias
schema::languages:bt|XY:d|XX:d
SELECT * FROM (SELECT languages, gender, salary FROM test_emp) PIVOT (AVG(salary) FOR gender IN ('M' AS "XY", 'F' "XX"));

languages | XY | XX
---------------+-----------------+------------------
null |48396.28571428572|62140.666666666664
1 |49767.22222222222|47073.25
2 |44103.90909090909|50684.4
3 |51741.90909090909|53660.0
4 |47058.90909090909|49291.5
5 |39052.875 |46705.555555555555
;

averageWithThreeValuesIncludingNull
schema::languages:bt|'M':d|'F':d
SELECT * FROM (SELECT languages, gender, salary FROM test_emp) PIVOT (AVG(salary) FOR gender IN ('M', 'F'));

languages | 'M' | 'F'
---------------+-----------------+------------------
null |48396.28571428572|62140.666666666664
1 |49767.22222222222|47073.25
2 |44103.90909090909|50684.4
3 |51741.90909090909|53660.0
4 |47058.90909090909|49291.5
5 |39052.875 |46705.555555555555
;


averageWithOneValueAndLimit
schema::languages:bt|'F':d
SELECT * FROM (SELECT languages, gender, salary FROM test_emp) PIVOT (AVG(salary) FOR gender IN ('F')) LIMIT 3;

languages | 'F'
---------------+------------------
null |62140.666666666664
1 |47073.25
2 |50684.4
;

averageWithTwoValuesAndLimit
schema::languages:bt|'M':d|'F':d
SELECT * FROM (SELECT languages, gender, salary FROM test_emp) PIVOT (AVG(salary) FOR gender IN ('M', 'F')) LIMIT 3;

languages | 'M' | 'F'
---------------+-----------------+------------------
null |48396.28571428572|62140.666666666664
1 |49767.22222222222|47073.25
2 |44103.90909090909|50684.4
;


averageWithTwoValuesAndTinyLimit
schema::languages:bt|'M':d|'F':d
SELECT * FROM (SELECT languages, gender, salary FROM test_emp) PIVOT (AVG(salary) FOR gender IN ('M', 'F')) LIMIT 1;

languages | 'M' | 'F'
---------------+-----------------+------------------
null |48396.28571428572|62140.666666666664
;


averageWithTwoValuesAndSmallLimit
schema::languages:bt|'M':d|'F':d
SELECT * FROM (SELECT languages, gender, salary FROM test_emp) PIVOT (AVG(salary) FOR gender IN ('M', 'F')) LIMIT 2;

languages | 'M' | 'F'
---------------+-----------------+------------------
null |48396.28571428572|62140.666666666664
1 |49767.22222222222|47073.25
;

averageWithOneValueAndOrder
schema::languages:bt|'F':d
SELECT * FROM (SELECT languages, gender, salary FROM test_emp) PIVOT (AVG(salary) FOR gender IN ('F')) ORDER BY languages DESC LIMIT 4;

languages | 'F'
---------------+------------------
5 |46705.555555555555
4 |49291.5
3 |53660.0
2 |50684.4
;

averageWithTwoValuesAndOrder
schema::languages:bt|'M':d|'F':d
SELECT * FROM (SELECT languages, gender, salary FROM test_emp) PIVOT (AVG(salary) FOR gender IN ('M', 'F')) ORDER BY languages DESC;

languages | 'M' | 'F'
---------------+-----------------+------------------
5 |39052.875 |46705.555555555555
4 |47058.90909090909|49291.5
3 |51741.90909090909|53660.0
2 |44103.90909090909|50684.4
1 |49767.22222222222|47073.25
null |48396.28571428572|62140.666666666664
;


averageWithOneValueAndAlias
Copy link
Contributor

@matriv matriv Sep 10, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this is the same as: https://github.com/elastic/elasticsearch/pull/46489/files#diff-7cd262e8f087be8a976b37e385749ccdR29

schema::languages:bt|ONE:d
SELECT * FROM (SELECT languages, gender, salary FROM test_emp) PIVOT (AVG(salary) FOR gender IN ('F' AS "ONE"));

languages | ONE
---------------+------------------
null |62140.666666666664
1 |47073.25
2 |50684.4
3 |53660.0
4 |49291.5
5 |46705.555555555555
;

sumWithoutSubquery
schema::birth_date:ts|emp_no:i|first_name:s|gender:s|hire_date:ts|last_name:s|1:i|2:i
SELECT * FROM test_emp PIVOT (SUM(salary) FOR languages IN (1, 2)) LIMIT 5;

birth_date | emp_no | first_name | gender | hire_date | last_name | 1 | 2
---------------------+---------------+---------------+---------------+---------------------+---------------+---------------+---------------
null |10041 |Uri |F |1989-11-12 00:00:00.0|Lenart |56415 |null
null |10043 |Yishay |M |1990-10-20 00:00:00.0|Tzvieli |34341 |null
null |10044 |Mingsen |F |1994-05-21 00:00:00.0|Casley |39728 |null
1952-04-19 00:00:00.0|10009 |Sumant |F |1985-02-18 00:00:00.0|Peac |66174 |null
1953-01-07 00:00:00.0|10067 |Claudi |M |1987-03-04 00:00:00.0|Stavenow |null |52044
1953-01-23 00:00:00.0|10019 |Lillian |null |1999-04-30 00:00:00.0|Haddadi |73717 |null
;

averageWithOneValueAndMath
schema::languages:bt|'F':d
SELECT * FROM (SELECT languages, gender, salary FROM test_emp) PIVOT (ROUND(AVG(salary) / 2) FOR gender IN ('F'));

languages | 'F'
---------------+---------------
null |31070.0
1 |23537.0
2 |25342.0
3 |26830.0
4 |24646.0
5 |23353.0
;

averageWithScalarOverAggregateAndFoldedValue
schema::status:s|client_ip:s
SELECT status, client_ip FROM logs
Copy link
Contributor

@matriv matriv Sep 10, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why don't we include the pivoted value here?

PIVOT (MAX(bytes_out) FOR "@timestamp" IN (CAST('2017-11-10T20:35:55Z' AS DATETIME), CAST('2017-11-10T21:15:40Z' AS DATETIME)));

status | client_ip
---------------+---------------
Error |null
Error |10.0.0.107
Error |10.0.0.130
Error |10.0.0.147
Error |10.0.1.166
Error |10.0.1.177
OK |null
OK |10.0.0.105
OK |10.0.0.109
OK |10.0.0.113
OK |10.0.0.118
OK |10.0.0.128
OK |10.0.0.129
OK |10.0.0.134
OK |10.0.0.144
OK |10.0.0.145
OK |10.0.1.1
OK |10.0.1.7
OK |10.0.1.8
OK |10.0.1.9
OK |10.0.1.10
OK |10.0.1.11
OK |10.0.1.12
OK |10.0.1.13
OK |10.0.1.122
OK |10.0.1.166
OK |10.0.1.199
OK |10.0.1.200
OK |10.0.1.201
OK |10.0.1.202
OK |10.0.1.203
OK |10.0.1.204
OK |10.0.1.205
OK |10.0.1.206
OK |10.0.1.207
OK |10.0.1.208
OK |10.0.1.222
;
25 changes: 22 additions & 3 deletions x-pack/plugin/sql/src/main/antlr/SqlBase.g4
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,15 +90,15 @@ orderBy
;

querySpecification
: SELECT setQuantifier? selectItem (',' selectItem)*
: SELECT setQuantifier? selectItems
fromClause?
(WHERE where=booleanExpression)?
(GROUP BY groupBy)?
(HAVING having=booleanExpression)?
;

fromClause
: FROM relation (',' relation)*
: FROM relation (',' relation)* pivotClause?
;

groupBy
Expand All @@ -123,6 +123,10 @@ setQuantifier
| ALL
;

selectItems
: selectItem (',' selectItem)*
;

selectItem
: expression (AS? identifier)? #selectExpression
;
Expand Down Expand Up @@ -154,6 +158,18 @@ relationPrimary
| '(' relation ')' (AS? qualifiedName)? #aliasedRelation
;

pivotClause
: PIVOT '(' aggs=pivotArgs FOR column=qualifiedName IN '(' vals=pivotArgs ')' ')'
;

pivotArgs
: namedValueExpression (',' namedValueExpression)*
;

namedValueExpression
: valueExpression (AS? identifier)?
;

expression
: booleanExpression
;
Expand Down Expand Up @@ -343,6 +359,7 @@ whenClause
;

// http://developer.mimer.se/validator/sql-reserved-words.tml
// https://developer.mimer.com/wp-content/uploads/standard-sql-reserved-words-summary.pdf
nonReserved
: ANALYZE | ANALYZED
| CATALOGS | COLUMNS | CURRENT_DATE | CURRENT_TIME | CURRENT_TIMESTAMP
Expand All @@ -355,7 +372,7 @@ nonReserved
| LAST | LIMIT
| MAPPED | MINUTE | MONTH
| OPTIMIZED
| PARSED | PHYSICAL | PLAN
| PARSED | PHYSICAL | PIVOT | PLAN
| QUERY
| RLIKE
| SCHEMAS | SECOND | SHOW | SYS
Expand Down Expand Up @@ -397,6 +414,7 @@ EXPLAIN: 'EXPLAIN';
EXTRACT: 'EXTRACT';
FALSE: 'FALSE';
FIRST: 'FIRST';
FOR: 'FOR';
FORMAT: 'FORMAT';
FROM: 'FROM';
FROZEN: 'FROZEN';
Expand Down Expand Up @@ -434,6 +452,7 @@ ORDER: 'ORDER';
OUTER: 'OUTER';
PARSED: 'PARSED';
PHYSICAL: 'PHYSICAL';
PIVOT: 'PIVOT';
PLAN: 'PLAN';
RIGHT: 'RIGHT';
RLIKE: 'RLIKE';
Expand Down
Loading