SQL: Add PIVOT support

x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/FetchSizeTestCase.java

@@ -149,4 +149,59 @@ private void assertNestedDocuments(ResultSet rs, int i) throws SQLException {
             assertTrue("No more entries left after row " + rs.getRow(), (i+j == 23 || rs.next()));
         }
     }
-}
+
+    /**
+     * Explicit pagination test for PIVOT.
+     * Checks that the paging properly consumes the necessary amount of aggregations and the
+     * page size affects the result not the intermediate query.
+     */
+    public void testPivotPaging() throws Exception {
+        Request request = new Request("PUT", "/test_pivot/_bulk");
+        request.addParameter("refresh", "true");
+        StringBuilder bulk = new StringBuilder();
+        String[] continent = new String[] { "AF", "AS", "EU", "NA", "SA", "AQ", "AU" };
+        for (int i = 0; i <= 100; i++) {
+            bulk.append("{\"index\":{}}\n");
+            bulk.append("{\"item\":").append(i % 10)
+                .append(", \"entry\":").append(i)
+                .append(", \"amount\" : ").append(randomInt(999))
+                .append(", \"location\" : \"").append(continent[i % (continent.length)]).append("\"")
+                .append("}\n");
+        }
+        request.setJsonEntity(bulk.toString());
+        assertEquals(200, client().performRequest(request).getStatusLine().getStatusCode());
+
+        try (Connection c = esJdbc();
+             Statement s = c.createStatement()) {
+
+            String query = "SELECT * FROM "
+                    + "(SELECT item, amount, location FROM test_pivot)"
+                    + " PIVOT (AVG(amount) FOR location IN ( 'AF', 'AS', 'EU', 'NA', 'SA', 'AQ', 'AU') )";
+            // set size smaller than an agg page
+            s.setFetchSize(3);
+            try (ResultSet rs = s.executeQuery(query)) {
+                assertEquals(8, rs.getMetaData().getColumnCount());
+                for (int i = 0; i < 10; i++) {
+                    assertTrue(rs.next());
+                    // the page was set to a pivot row (since the initial 3 is lower as a pivot page takes number of pivot entries + 1)
+                    assertEquals(1, rs.getFetchSize());
+                    assertEquals(Long.valueOf(i), rs.getObject("item"));
+                }
+                assertFalse(rs.next());
+            }
+
+            // now try with a larger fetch size (8 * 2 + something) - should be 2
+            s.setFetchSize(20);
+            try (ResultSet rs = s.executeQuery(query)) {
+                for (int i = 0; i < 10; i++) {
+                    assertTrue(rs.next());
+                    //
+                    assertEquals(2, rs.getFetchSize());
+                    assertEquals(Long.valueOf(i), rs.getObject("item"));
+                }
+                assertFalse(rs.next());
+            }
+        }
+        assertNoSearchContexts();
+    }
+}

x-pack/plugin/sql/qa/src/main/resources/debug/debug.csv-spec

@@ -3,7 +3,8 @@
 //
 
 debug
-SELECT first_name f, last_name l, dep.from_date d FROM test_emp WHERE dep.dep_name = 'Production' ORDER BY f LIMIT 5;
+SELECT * FROM (SELECT languages, gender, salary FROM test_emp) PIVOT (AVG(salary) FOR gender IN ('M', 'F'));
+//SELECT languages FROM (SELECT languages, gender, salary FROM test_emp) PIVOT (AVG(salary) FOR gender IN ('M', 'F'));
 
 f:s                  | l:s                  | d:ts
 

x-pack/plugin/sql/qa/src/main/resources/pivot.csv-spec

@@ -0,0 +1,206 @@
+averageWithOneValue
+schema::languages:bt|'F':d
+SELECT * FROM (SELECT languages, gender, salary FROM test_emp) PIVOT (AVG(salary) FOR gender IN ('F'));
+
+   languages   |       'F'        
+---------------+------------------
+null           |62140.666666666664
+1              |47073.25          
+2              |50684.4           
+3              |53660.0           
+4              |49291.5           
+5              |46705.555555555555
+;
+
+averageWithAliasAndOneValue
+schema::languages:bt|'F':d
+SELECT * FROM (SELECT languages, gender, salary FROM test_emp) PIVOT (AVG(salary) AS "AVG" FOR gender IN ('F'));
+
+   languages   |       'F'        
+---------------+------------------
+null           |62140.666666666664
+1              |47073.25          
+2              |50684.4           
+3              |53660.0           
+4              |49291.5           
+5              |46705.555555555555
+;
+
+averageWithAliasedValue
+schema::languages:bt|XX:d
+SELECT * FROM (SELECT languages, gender, salary FROM test_emp) PIVOT (AVG(salary) FOR gender IN ('F' AS "XX"));
+
+   languages   |       XX        
+---------------+------------------
+null           |62140.666666666664
+1              |47073.25          
+2              |50684.4           
+3              |53660.0           
+4              |49291.5           
+5              |46705.555555555555
+;
+
+averageWithTwoValues
+schema::languages:bt|'M':d|'F':d
+SELECT * FROM (SELECT languages, gender, salary FROM test_emp) PIVOT (AVG(salary) FOR gender IN ('M', 'F'));
+
+   languages   |       'M'       |       'F'        
+---------------+-----------------+------------------
+null           |48396.28571428572|62140.666666666664
+1              |49767.22222222222|47073.25          
+2              |44103.90909090909|50684.4           
+3              |51741.90909090909|53660.0           
+4              |47058.90909090909|49291.5           
+5              |39052.875        |46705.555555555555
+;
+
+averageWithTwoValuesAndAlias
+schema::languages:bt|XY:d|XX:d
+SELECT * FROM (SELECT languages, gender, salary FROM test_emp) PIVOT (AVG(salary) FOR gender IN ('M' AS "XY", 'F' "XX"));
+
+   languages   |        XY       |        XX        
+---------------+-----------------+------------------
+null           |48396.28571428572|62140.666666666664
+1              |49767.22222222222|47073.25          
+2              |44103.90909090909|50684.4           
+3              |51741.90909090909|53660.0           
+4              |47058.90909090909|49291.5           
+5              |39052.875        |46705.555555555555
+;
+
+averageWithThreeValuesIncludingNull
+schema::languages:bt|'M':d|'F':d
+SELECT * FROM (SELECT languages, gender, salary FROM test_emp) PIVOT (AVG(salary) FOR gender IN ('M', 'F'));
+
+   languages   |       'M'       |       'F'        
+---------------+-----------------+------------------
+null           |48396.28571428572|62140.666666666664
+1              |49767.22222222222|47073.25          
+2              |44103.90909090909|50684.4           
+3              |51741.90909090909|53660.0           
+4              |47058.90909090909|49291.5           
+5              |39052.875        |46705.555555555555
+;
+
+
+averageWithOneValueAndLimit
+schema::languages:bt|'F':d
+SELECT * FROM (SELECT languages, gender, salary FROM test_emp) PIVOT (AVG(salary) FOR gender IN ('F')) LIMIT 3;
+
+   languages   |       'F'        
+---------------+------------------
+null           |62140.666666666664
+1              |47073.25          
+2              |50684.4           
+;
+
+averageWithTwoValuesAndLimit
+schema::languages:bt|'M':d|'F':d
+SELECT * FROM (SELECT languages, gender, salary FROM test_emp) PIVOT (AVG(salary) FOR gender IN ('M', 'F')) LIMIT 3;
+
+   languages   |       'M'       |       'F'        
+---------------+-----------------+------------------
+null           |48396.28571428572|62140.666666666664
+1              |49767.22222222222|47073.25          
+2              |44103.90909090909|50684.4           
+;
+
+
+averageWithTwoValuesAndTinyLimit
+schema::languages:bt|'M':d|'F':d
+SELECT * FROM (SELECT languages, gender, salary FROM test_emp) PIVOT (AVG(salary) FOR gender IN ('M', 'F')) LIMIT 1;
+
+   languages   |       'M'       |       'F'        
+---------------+-----------------+------------------
+null           |48396.28571428572|62140.666666666664
+;
+
+
+averageWithTwoValuesAndSmallLimit
+schema::languages:bt|'M':d|'F':d
+SELECT * FROM (SELECT languages, gender, salary FROM test_emp) PIVOT (AVG(salary) FOR gender IN ('M', 'F')) LIMIT 2;
+
+   languages   |       'M'       |       'F'        
+---------------+-----------------+------------------
+null           |48396.28571428572|62140.666666666664
+1              |49767.22222222222|47073.25  
+;
+
+averageWithOneValueAndOrder
+schema::languages:bt|'F':d
+SELECT * FROM (SELECT languages, gender, salary FROM test_emp) PIVOT (AVG(salary) FOR gender IN ('F')) ORDER BY languages DESC LIMIT 4;
+
+   languages   |       'F'        
+---------------+------------------
+5              |46705.555555555555
+4              |49291.5           
+3              |53660.0
+2              |50684.4                   
+;
+
+averageWithTwoValuesAndOrderDesc
+schema::languages:bt|'M':d|'F':d
+SELECT * FROM (SELECT languages, gender, salary FROM test_emp) PIVOT (AVG(salary) FOR gender IN ('M', 'F')) ORDER BY languages DESC;
+
+   languages   |       'M'       |       'F'        
+---------------+-----------------+------------------
+5              |39052.875        |46705.555555555555
+4              |47058.90909090909|49291.5           
+3              |51741.90909090909|53660.0           
+2              |44103.90909090909|50684.4           
+1              |49767.22222222222|47073.25          
+null           |48396.28571428572|62140.666666666664
+;
+
+averageWithTwoValuesAndOrderDesc
+schema::languages:bt|'M':d|'F':d
+SELECT * FROM (SELECT languages, gender, salary FROM test_emp) PIVOT (AVG(salary) FOR gender IN ('M', 'F')) ORDER BY languages DESC LIMIT 2;
+
+   languages   |       'M'       |       'F'        
+---------------+-----------------+------------------
+5              |39052.875        |46705.555555555555
+4              |47058.90909090909|49291.5           
+;
+
+averageWithTwoValuesAndOrderAsc
+schema::languages:bt|'M':d|'F':d
+SELECT * FROM (SELECT languages, gender, salary FROM test_emp) PIVOT (AVG(salary) FOR gender IN ('M', 'F')) ORDER BY languages ASC;
+
+   languages   |       'M'       |       'F'        
+---------------+-----------------+------------------
+null           |48396.28571428572|62140.666666666664
+1              |49767.22222222222|47073.25          
+2              |44103.90909090909|50684.4           
+3              |51741.90909090909|53660.0           
+4              |47058.90909090909|49291.5           
+5              |39052.875        |46705.555555555555
+;
+
+
+sumWithoutSubquery
+schema::birth_date:ts|emp_no:i|first_name:s|gender:s|hire_date:ts|last_name:s|1:i|2:i
+SELECT * FROM test_emp PIVOT (SUM(salary) FOR languages IN (1, 2)) LIMIT 5;
+
+     birth_date      |    emp_no     |  first_name   |    gender     |      hire_date      |   last_name   |       1       |       2       
+---------------------+---------------+---------------+---------------+---------------------+---------------+---------------+---------------
+null                 |10041          |Uri            |F              |1989-11-12 00:00:00.0|Lenart         |56415          |null           
+null                 |10043          |Yishay         |M              |1990-10-20 00:00:00.0|Tzvieli        |34341          |null           
+null                 |10044          |Mingsen        |F              |1994-05-21 00:00:00.0|Casley         |39728          |null           
+1952-04-19 00:00:00.0|10009          |Sumant         |F              |1985-02-18 00:00:00.0|Peac           |66174          |null           
+1953-01-07 00:00:00.0|10067          |Claudi         |M              |1987-03-04 00:00:00.0|Stavenow       |null           |52044          
+1953-01-23 00:00:00.0|10019          |Lillian        |null           |1999-04-30 00:00:00.0|Haddadi        |73717          |null    
+;
+
+averageWithOneValueAndMath
+schema::languages:bt|'F':d
+SELECT * FROM (SELECT languages, gender, salary FROM test_emp) PIVOT (ROUND(AVG(salary) / 2) FOR gender IN ('F'));
+
+   languages   |      'F'      
+---------------+---------------
+null           |31070.0        
+1              |23537.0        
+2              |25342.0        
+3              |26830.0        
+4              |24646.0        
+5              |23353.0        
+;

x-pack/plugin/sql/src/main/antlr/SqlBase.g4

@@ -90,15 +90,15 @@ orderBy
     ;
 
 querySpecification
-    : SELECT setQuantifier? selectItem (',' selectItem)*
+    : SELECT setQuantifier? selectItems
       fromClause?
       (WHERE where=booleanExpression)?
       (GROUP BY groupBy)?
       (HAVING having=booleanExpression)?
     ;
 
 fromClause
-    : FROM relation (',' relation)*
+    : FROM relation (',' relation)* pivotClause?
     ;
 
 groupBy
@@ -123,6 +123,10 @@ setQuantifier
     | ALL
     ;
 
+selectItems                                       
+    : selectItem (',' selectItem)*
+    ;
+
 selectItem
     : expression (AS? identifier)?                #selectExpression
     ;
@@ -154,6 +158,18 @@ relationPrimary
     | '(' relation ')' (AS? qualifiedName)?                           #aliasedRelation
     ;
 
+pivotClause
+    : PIVOT '(' aggs=pivotArgs FOR column=qualifiedName IN '(' vals=pivotArgs ')' ')'
+    ;
+
+pivotArgs
+    : namedValueExpression (',' namedValueExpression)* 
+    ;
+
+namedValueExpression
+    : valueExpression  (AS? identifier)?
+    ;
+
 expression
     : booleanExpression
     ;
@@ -343,6 +359,7 @@ whenClause
     ;
 
 // http://developer.mimer.se/validator/sql-reserved-words.tml
+// https://developer.mimer.com/wp-content/uploads/standard-sql-reserved-words-summary.pdf
 nonReserved
     : ANALYZE | ANALYZED 
     | CATALOGS | COLUMNS | CURRENT_DATE | CURRENT_TIME | CURRENT_TIMESTAMP
@@ -355,7 +372,7 @@ nonReserved
     | LAST | LIMIT 
     | MAPPED | MINUTE | MONTH
     | OPTIMIZED 
-    | PARSED | PHYSICAL | PLAN 
+    | PARSED | PHYSICAL | PIVOT | PLAN 
     | QUERY 
     | RLIKE
     | SCHEMAS | SECOND | SHOW | SYS
@@ -397,6 +414,7 @@ EXPLAIN: 'EXPLAIN';
 EXTRACT: 'EXTRACT';
 FALSE: 'FALSE';
 FIRST: 'FIRST';
+FOR: 'FOR';
 FORMAT: 'FORMAT';
 FROM: 'FROM';
 FROZEN: 'FROZEN';
@@ -434,6 +452,7 @@ ORDER: 'ORDER';
 OUTER: 'OUTER';
 PARSED: 'PARSED';
 PHYSICAL: 'PHYSICAL';
+PIVOT: 'PIVOT';
 PLAN: 'PLAN';
 RIGHT: 'RIGHT';
 RLIKE: 'RLIKE';