Skip to content

[SPARK-28343][SQL][TEST] Enabling cartesian product and ansi mode for PostgreSQL testing #25109

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wangyum wants to merge 5 commits into from
Closed

[SPARK-28343][SQL][TEST] Enabling cartesian product and ansi mode for PostgreSQL testing #25109

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
86ca933
PostgreSQL enabled cartesian product by default.
wangyum Jul 11, 2019
45af58f
Add ANSI_SQL_PARSER
wangyum Jul 13, 2019
002c549
Merge remote-tracking branch 'upstream/master' into SPARK-28343
wangyum Jul 13, 2019
4a66a89
update with.sql
wangyum Jul 13, 2019
8ce7af4
Add comment
wangyum Jul 14, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 14 additions & 13 deletions sql/core/src/test/resources/sql-tests/inputs/pgSQL/boolean.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,19 +17,20 @@ SELECT 1 AS one;

SELECT true AS true;

SELECT false AS false;
-- [SPARK-28349] We do not need to follow PostgreSQL to support reserved words in column alias
SELECT false AS `false`;

SELECT boolean('t') AS true;

-- [SPARK-27931] Trim the string when cast string type to boolean type
SELECT boolean(' f ') AS false;
SELECT boolean(' f ') AS `false`;

SELECT boolean('true') AS true;

-- [SPARK-27923] PostgreSQL does not accept 'test' but Spark SQL accepts it and sets it to NULL
SELECT boolean('test') AS error;

SELECT boolean('false') AS false;
SELECT boolean('false') AS `false`;

-- [SPARK-27923] PostgreSQL does not accept 'foo' but Spark SQL accepts it and sets it to NULL
SELECT boolean('foo') AS error;
Expand All @@ -41,20 +42,20 @@ SELECT boolean('yes') AS true;
-- [SPARK-27923] PostgreSQL does not accept 'yeah' but Spark SQL accepts it and sets it to NULL
SELECT boolean('yeah') AS error;

SELECT boolean('n') AS false;
SELECT boolean('n') AS `false`;

SELECT boolean('no') AS false;
SELECT boolean('no') AS `false`;

-- [SPARK-27923] PostgreSQL does not accept 'nay' but Spark SQL accepts it and sets it to NULL
SELECT boolean('nay') AS error;

-- [SPARK-27931] Accept 'on' and 'off' as input for boolean data type
SELECT boolean('on') AS true;

SELECT boolean('off') AS false;
SELECT boolean('off') AS `false`;

-- [SPARK-27931] Accept unique prefixes thereof
SELECT boolean('of') AS false;
SELECT boolean('of') AS `false`;

-- [SPARK-27923] PostgreSQL does not accept 'o' but Spark SQL accepts it and sets it to NULL
SELECT boolean('o') AS error;
Expand All @@ -70,7 +71,7 @@ SELECT boolean('1') AS true;
-- [SPARK-27923] PostgreSQL does not accept '11' but Spark SQL accepts it and sets it to NULL
SELECT boolean('11') AS error;

SELECT boolean('0') AS false;
SELECT boolean('0') AS `false`;

-- [SPARK-27923] PostgreSQL does not accept '000' but Spark SQL accepts it and sets it to NULL
SELECT boolean('000') AS error;
Expand All @@ -82,11 +83,11 @@ SELECT boolean('') AS error;

SELECT boolean('t') or boolean('f') AS true;

SELECT boolean('t') and boolean('f') AS false;
SELECT boolean('t') and boolean('f') AS `false`;

SELECT not boolean('f') AS true;

SELECT boolean('t') = boolean('f') AS false;
SELECT boolean('t') = boolean('f') AS `false`;

SELECT boolean('t') <> boolean('f') AS true;

Expand All @@ -99,11 +100,11 @@ SELECT boolean('f') < boolean('t') AS true;
SELECT boolean('f') <= boolean('t') AS true;

-- explicit casts to/from text
SELECT boolean(string('TrUe')) AS true, boolean(string('fAlse')) AS false;
SELECT boolean(string('TrUe')) AS true, boolean(string('fAlse')) AS `false`;
-- [SPARK-27931] Trim the string when cast to boolean type
SELECT boolean(string(' true ')) AS true,
boolean(string(' FALSE')) AS false;
SELECT string(boolean(true)) AS true, string(boolean(false)) AS false;
boolean(string(' FALSE')) AS `false`;
SELECT string(boolean(true)) AS true, string(boolean(false)) AS `false`;

-- [SPARK-27923] PostgreSQL does not accept ' tru e ' but Spark SQL accepts it and sets it to NULL
SELECT boolean(string(' tru e ')) AS invalid; -- error
Expand Down
4 changes: 0 additions & 4 deletions sql/core/src/test/resources/sql-tests/inputs/pgSQL/case.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,6 @@
-- https://github.com/postgres/postgres/blob/REL_12_BETA2/src/test/regress/sql/case.sql
-- Test the CASE statement
--
-- This test suite contains two Cartesian products without using explicit CROSS JOIN syntax.
-- Thus, we set spark.sql.crossJoin.enabled to true.
set spark.sql.crossJoin.enabled=true;
CREATE TABLE CASE_TBL (
i integer,
f double
Expand Down Expand Up @@ -264,4 +261,3 @@ SELECT CASE

DROP TABLE CASE_TBL;
DROP TABLE CASE2_TBL;
set spark.sql.crossJoin.enabled=false;
3 changes: 2 additions & 1 deletion sql/core/src/test/resources/sql-tests/inputs/pgSQL/int4.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,8 @@ SELECT int('2') * smallint('2') = smallint('16') / int('4') AS true;

SELECT smallint('2') * int('2') = int('16') / smallint('4') AS true;

SELECT int('1000') < int('999') AS false;
-- [SPARK-28349] We do not need to follow PostgreSQL to support reserved words in column alias
SELECT int('1000') < int('999') AS `false`;

-- [SPARK-28027] Our ! and !! has different meanings
-- SELECT 4! AS twenty_four;
Expand Down
5 changes: 3 additions & 2 deletions sql/core/src/test/resources/sql-tests/inputs/pgSQL/int8.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,10 +67,11 @@ SELECT * FROM INT8_TBL WHERE smallint('123') <= q1;
SELECT * FROM INT8_TBL WHERE smallint('123') >= q1;


SELECT '' AS five, q1 AS plus, -q1 AS minus FROM INT8_TBL;
-- [SPARK-28349] We do not need to follow PostgreSQL to support reserved words in column alias
SELECT '' AS five, q1 AS plus, -q1 AS `minus` FROM INT8_TBL;

SELECT '' AS five, q1, q2, q1 + q2 AS plus FROM INT8_TBL;
SELECT '' AS five, q1, q2, q1 - q2 AS minus FROM INT8_TBL;
SELECT '' AS five, q1, q2, q1 - q2 AS `minus` FROM INT8_TBL;
Copy link
Member

@dongjoon-hyun dongjoon-hyun Jul 13, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please add SPARK-28349 before this line.

SELECT '' AS three, q1, q2, q1 * q2 AS multiply FROM INT8_TBL;
SELECT '' AS three, q1, q2, q1 * q2 AS multiply FROM INT8_TBL
WHERE q1 < 1000 or (q2 > 0 and q2 < 1000);
Expand Down
5 changes: 0 additions & 5 deletions sql/core/src/test/resources/sql-tests/inputs/pgSQL/with.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,6 @@
-- WITH
-- https://github.com/postgres/postgres/blob/REL_12_BETA2/src/test/regress/sql/with.sql
--
-- This test suite contains two Cartesian products without using explicit CROSS JOIN syntax.
-- Thus, we set spark.sql.crossJoin.enabled to true.
set spark.sql.crossJoin.enabled=true;
-- This test uses the generate_series(...) function which is rewritten to EXPLODE(SEQUENCE(...)) as
-- it's feature tracking ticket SPARK-27767 is closed as Won't Do.

Expand Down Expand Up @@ -1182,10 +1179,8 @@ SELECT * FROM parent;
--DROP RULE y_rule ON y;

-- check that parser lookahead for WITH doesn't cause any odd behavior
set spark.sql.parser.ansi.enabled=true;
create table foo (with baz); -- fail, WITH is a reserved word
create table foo (with ordinality); -- fail, WITH is a reserved word
set spark.sql.parser.ansi.enabled=false;
with ordinality as (select 1 as x) select * from ordinality;

-- check sane response to attempt to modify CTE relation
Expand Down
26 changes: 13 additions & 13 deletions sql/core/src/test/resources/sql-tests/results/pgSQL/boolean.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ true


-- !query 2
SELECT false AS false
SELECT false AS `false`
-- !query 2 schema
struct<false:boolean>
-- !query 2 output
Expand All @@ -35,7 +35,7 @@ true


-- !query 4
SELECT boolean(' f ') AS false
SELECT boolean(' f ') AS `false`
-- !query 4 schema
struct<false:boolean>
-- !query 4 output
Expand All @@ -59,7 +59,7 @@ NULL


-- !query 7
SELECT boolean('false') AS false
SELECT boolean('false') AS `false`
-- !query 7 schema
struct<false:boolean>
-- !query 7 output
Expand Down Expand Up @@ -99,15 +99,15 @@ NULL


-- !query 12
SELECT boolean('n') AS false
SELECT boolean('n') AS `false`
-- !query 12 schema
struct<false:boolean>
-- !query 12 output
false


-- !query 13
SELECT boolean('no') AS false
SELECT boolean('no') AS `false`
-- !query 13 schema
struct<false:boolean>
-- !query 13 output
Expand All @@ -131,15 +131,15 @@ NULL


-- !query 16
SELECT boolean('off') AS false
SELECT boolean('off') AS `false`
-- !query 16 schema
struct<false:boolean>
-- !query 16 output
NULL


-- !query 17
SELECT boolean('of') AS false
SELECT boolean('of') AS `false`
-- !query 17 schema
struct<false:boolean>
-- !query 17 output
Expand Down Expand Up @@ -187,7 +187,7 @@ NULL


-- !query 23
SELECT boolean('0') AS false
SELECT boolean('0') AS `false`
-- !query 23 schema
struct<false:boolean>
-- !query 23 output
Expand Down Expand Up @@ -219,7 +219,7 @@ true


-- !query 27
SELECT boolean('t') and boolean('f') AS false
SELECT boolean('t') and boolean('f') AS `false`
-- !query 27 schema
struct<false:boolean>
-- !query 27 output
Expand All @@ -235,7 +235,7 @@ true


-- !query 29
SELECT boolean('t') = boolean('f') AS false
SELECT boolean('t') = boolean('f') AS `false`
-- !query 29 schema
struct<false:boolean>
-- !query 29 output
Expand Down Expand Up @@ -283,7 +283,7 @@ true


-- !query 35
SELECT boolean(string('TrUe')) AS true, boolean(string('fAlse')) AS false
SELECT boolean(string('TrUe')) AS true, boolean(string('fAlse')) AS `false`
-- !query 35 schema
struct<true:boolean,false:boolean>
-- !query 35 output
Expand All @@ -292,15 +292,15 @@ true false

-- !query 36
SELECT boolean(string(' true ')) AS true,
boolean(string(' FALSE')) AS false
boolean(string(' FALSE')) AS `false`
-- !query 36 schema
struct<true:boolean,false:boolean>
-- !query 36 output
NULL NULL


-- !query 37
SELECT string(boolean(true)) AS true, string(boolean(false)) AS false
SELECT string(boolean(true)) AS true, string(boolean(false)) AS `false`
-- !query 37 schema
struct<true:string,false:string>
-- !query 37 output
Expand Down
Loading