-
Notifications
You must be signed in to change notification settings - Fork 29k
[SPARK-28276][SQL][PYTHON][TEST] Convert and port 'cross-join.sql' into UDF test base #25168
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Closed
Closed
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit
Hold shift + click to select a range
af80ac7
Add converted sql test file and output.
viirya 65c5eb0
Merge remote-tracking branch 'upstream/master' into SPARK-28276
viirya ac20743
Update test result.
viirya 762ffb6
Merge remote-tracking branch 'upstream/master' into SPARK-28276
viirya 0dbc985
Updated output file.
viirya File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
37 changes: 37 additions & 0 deletions
37
sql/core/src/test/resources/sql-tests/inputs/udf/udf-cross-join.sql
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| -- Cross join detection and error checking is done in JoinSuite since explain output is | ||
| -- used in the error message and the ids are not stable. Only positive cases are checked here. | ||
| -- This test file was converted from cross-join.sql. | ||
|
|
||
| create temporary view nt1 as select * from values | ||
| ("one", 1), | ||
| ("two", 2), | ||
| ("three", 3) | ||
| as nt1(k, v1); | ||
|
|
||
| create temporary view nt2 as select * from values | ||
| ("one", 1), | ||
| ("two", 22), | ||
| ("one", 5) | ||
| as nt2(k, v2); | ||
|
|
||
| -- Cross joins with and without predicates | ||
| SELECT * FROM nt1 cross join nt2; | ||
| SELECT * FROM nt1 cross join nt2 where udf(nt1.k) = udf(nt2.k); | ||
| SELECT * FROM nt1 cross join nt2 on (udf(nt1.k) = udf(nt2.k)); | ||
| SELECT * FROM nt1 cross join nt2 where udf(nt1.v1) = "1" and udf(nt2.v2) = "22"; | ||
|
|
||
| SELECT udf(a.key), udf(b.key) FROM | ||
| (SELECT udf(k) key FROM nt1 WHERE v1 < 2) a | ||
| CROSS JOIN | ||
| (SELECT udf(k) key FROM nt2 WHERE v2 = 22) b; | ||
|
|
||
| -- Join reordering | ||
| create temporary view A(a, va) as select * from nt1; | ||
| create temporary view B(b, vb) as select * from nt1; | ||
| create temporary view C(c, vc) as select * from nt1; | ||
| create temporary view D(d, vd) as select * from nt1; | ||
|
|
||
| -- Allowed since cross join with C is explicit | ||
| select * from ((A join B on (udf(a) = udf(b))) cross join C) join D on (udf(a) = udf(d)); | ||
| -- Cross joins with non-equal predicates | ||
| SELECT * FROM nt1 CROSS JOIN nt2 ON (udf(nt1.k) > udf(nt2.k)); |
146 changes: 146 additions & 0 deletions
146
sql/core/src/test/resources/sql-tests/results/udf/udf-cross-join.sql.out
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,146 @@ | ||
| -- Automatically generated by SQLQueryTestSuite | ||
| -- Number of queries: 13 | ||
|
|
||
|
|
||
| -- !query 0 | ||
| create temporary view nt1 as select * from values | ||
| ("one", 1), | ||
| ("two", 2), | ||
| ("three", 3) | ||
| as nt1(k, v1) | ||
| -- !query 0 schema | ||
| struct<> | ||
| -- !query 0 output | ||
|
|
||
|
|
||
|
|
||
| -- !query 1 | ||
| create temporary view nt2 as select * from values | ||
| ("one", 1), | ||
| ("two", 22), | ||
| ("one", 5) | ||
| as nt2(k, v2) | ||
| -- !query 1 schema | ||
| struct<> | ||
| -- !query 1 output | ||
|
|
||
|
|
||
|
|
||
| -- !query 2 | ||
| SELECT * FROM nt1 cross join nt2 | ||
| -- !query 2 schema | ||
| struct<k:string,v1:int,k:string,v2:int> | ||
| -- !query 2 output | ||
| one 1 one 1 | ||
| one 1 one 5 | ||
| one 1 two 22 | ||
| three 3 one 1 | ||
| three 3 one 5 | ||
| three 3 two 22 | ||
| two 2 one 1 | ||
| two 2 one 5 | ||
| two 2 two 22 | ||
|
|
||
|
|
||
| -- !query 3 | ||
| SELECT * FROM nt1 cross join nt2 where udf(nt1.k) = udf(nt2.k) | ||
| -- !query 3 schema | ||
| struct<k:string,v1:int,k:string,v2:int> | ||
| -- !query 3 output | ||
| one 1 one 1 | ||
| one 1 one 5 | ||
| two 2 two 22 | ||
|
|
||
|
|
||
| -- !query 4 | ||
| SELECT * FROM nt1 cross join nt2 on (udf(nt1.k) = udf(nt2.k)) | ||
| -- !query 4 schema | ||
| struct<k:string,v1:int,k:string,v2:int> | ||
| -- !query 4 output | ||
| one 1 one 1 | ||
| one 1 one 5 | ||
| two 2 two 22 | ||
|
|
||
|
|
||
| -- !query 5 | ||
| SELECT * FROM nt1 cross join nt2 where udf(nt1.v1) = "1" and udf(nt2.v2) = "22" | ||
| -- !query 5 schema | ||
| struct<k:string,v1:int,k:string,v2:int> | ||
| -- !query 5 output | ||
| one 1 two 22 | ||
|
|
||
|
|
||
| -- !query 6 | ||
| SELECT udf(a.key), udf(b.key) FROM | ||
| (SELECT udf(k) key FROM nt1 WHERE v1 < 2) a | ||
| CROSS JOIN | ||
| (SELECT udf(k) key FROM nt2 WHERE v2 = 22) b | ||
| -- !query 6 schema | ||
| struct<udf(key):string,udf(key):string> | ||
| -- !query 6 output | ||
| one two | ||
|
|
||
|
|
||
| -- !query 7 | ||
| create temporary view A(a, va) as select * from nt1 | ||
| -- !query 7 schema | ||
| struct<> | ||
| -- !query 7 output | ||
|
|
||
|
|
||
|
|
||
| -- !query 8 | ||
| create temporary view B(b, vb) as select * from nt1 | ||
| -- !query 8 schema | ||
| struct<> | ||
| -- !query 8 output | ||
|
|
||
|
|
||
|
|
||
| -- !query 9 | ||
| create temporary view C(c, vc) as select * from nt1 | ||
| -- !query 9 schema | ||
| struct<> | ||
| -- !query 9 output | ||
|
|
||
|
|
||
|
|
||
| -- !query 10 | ||
| create temporary view D(d, vd) as select * from nt1 | ||
| -- !query 10 schema | ||
| struct<> | ||
| -- !query 10 output | ||
|
|
||
|
|
||
|
|
||
| -- !query 11 | ||
| select * from ((A join B on (udf(a) = udf(b))) cross join C) join D on (udf(a) = udf(d)) | ||
| -- !query 11 schema | ||
| struct<> | ||
| -- !query 11 output | ||
| org.apache.spark.sql.AnalysisException | ||
| Detected implicit cartesian product for INNER join between logical plans | ||
| Filter (udf(a#x) = udf(b#x)) | ||
| +- Join Inner | ||
| :- Project [k#x AS a#x, v1#x AS va#x] | ||
| : +- LocalRelation [k#x, v1#x] | ||
| +- Project [k#x AS b#x, v1#x AS vb#x] | ||
| +- LocalRelation [k#x, v1#x] | ||
| and | ||
| Project [k#x AS d#x, v1#x AS vd#x] | ||
| +- LocalRelation [k#x, v1#x] | ||
| Join condition is missing or trivial. | ||
| Either: use the CROSS JOIN syntax to allow cartesian products between these | ||
| relations, or: enable implicit cartesian products by setting the configuration | ||
| variable spark.sql.crossJoin.enabled=true; | ||
|
|
||
|
|
||
| -- !query 12 | ||
| SELECT * FROM nt1 CROSS JOIN nt2 ON (udf(nt1.k) > udf(nt2.k)) | ||
| -- !query 12 schema | ||
| struct<k:string,v1:int,k:string,v2:int> | ||
| -- !query 12 output | ||
| three 3 one 1 | ||
| three 3 one 5 | ||
| two 2 one 1 | ||
| two 2 one 5 | ||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.