From 89212b73627a42ff6e0725ccc3c16bdd839d0805 Mon Sep 17 00:00:00 2001 From: Terry Kim Date: Thu, 11 Jul 2019 15:55:30 -0700 Subject: [PATCH 1/4] initial commit --- .../sql-tests/inputs/udf/udf-inline-table.sql | 54 +++++++ .../results/udf/udf-inline-table.sql.out | 153 ++++++++++++++++++ 2 files changed, 207 insertions(+) create mode 100644 sql/core/src/test/resources/sql-tests/inputs/udf/udf-inline-table.sql create mode 100644 sql/core/src/test/resources/sql-tests/results/udf/udf-inline-table.sql.out diff --git a/sql/core/src/test/resources/sql-tests/inputs/udf/udf-inline-table.sql b/sql/core/src/test/resources/sql-tests/inputs/udf/udf-inline-table.sql new file mode 100644 index 0000000000000..25c335ac516f7 --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/inputs/udf/udf-inline-table.sql @@ -0,0 +1,54 @@ +-- This test file was converted from intersect-all.sql. +-- Note that currently registered UDF returns a string. So there are some differences, for instance +-- in string cast within UDF in Scala and Python. + +-- single row, without table and column alias +select * from values ("one", 1); + +-- single row, without column alias +select * from values ("one", 1) as data; + +-- single row +select udf(a), b from values ("one", 1) as data(a, b); + +-- single column multiple rows +select udf(a) from values 1, 2, 3 as data(a); + +-- three rows +select udf(a), b from values ("one", 1), ("two", 2), ("three", null) as data(a, b); + +-- null type +select udf(a), b from values ("one", null), ("two", null) as data(a, b); + +-- int and long coercion +select udf(a), b from values ("one", 1), ("two", 2L) as data(a, b); + +-- foldable expressions +select udf(a), udf(b) from values ("one", 1 + 0), ("two", 1 + 3L) as data(a, b); + +-- complex types +select udf(a), b from values ("one", array(0, 1)), ("two", array(2, 3)) as data(a, b); + +-- decimal and double coercion +select udf(a), b from values ("one", 2.0), ("two", 3.0D) as data(a, b); + +-- error reporting: nondeterministic function rand +select udf(a), b from values ("one", rand(5)), ("two", 3.0D) as data(a, b); + +-- error reporting: different number of columns +select udf(a), udf(b) from values ("one", 2.0), ("two") as data(a, b); + +-- error reporting: types that are incompatible +select udf(a), udf(b) from values ("one", array(0, 1)), ("two", struct(1, 2)) as data(a, b); + +-- error reporting: number aliases different from number data values +select udf(a), udf(b) from values ("one"), ("two") as data(a, b); + +-- error reporting: unresolved expression +select udf(a), udf(b) from values ("one", random_not_exist_func(1)), ("two", 2) as data(a, b); + +-- error reporting: aggregate expression +select udf(a), udf(b) from values ("one", count(1)), ("two", 2) as data(a, b); + +-- string to timestamp +select * from values (timestamp('1991-12-06 00:00:00.0'), array(timestamp('1991-12-06 01:00:00.0'), timestamp('1991-12-06 12:00:00.0'))) as data(a, b); diff --git a/sql/core/src/test/resources/sql-tests/results/udf/udf-inline-table.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/udf-inline-table.sql.out new file mode 100644 index 0000000000000..f2e2cd7c84b07 --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/results/udf/udf-inline-table.sql.out @@ -0,0 +1,153 @@ +-- Automatically generated by SQLQueryTestSuite +-- Number of queries: 17 + + +-- !query 0 +select * from values ("one", 1) +-- !query 0 schema +struct +-- !query 0 output +one 1 + + +-- !query 1 +select * from values ("one", 1) as data +-- !query 1 schema +struct +-- !query 1 output +one 1 + + +-- !query 2 +select udf(a), b from values ("one", 1) as data(a, b) +-- !query 2 schema +struct +-- !query 2 output +one 1 + + +-- !query 3 +select udf(a) from values 1, 2, 3 as data(a) +-- !query 3 schema +struct +-- !query 3 output +1 +2 +3 + + +-- !query 4 +select udf(a), b from values ("one", 1), ("two", 2), ("three", null) as data(a, b) +-- !query 4 schema +struct +-- !query 4 output +one 1 +three NULL +two 2 + + +-- !query 5 +select udf(a), b from values ("one", null), ("two", null) as data(a, b) +-- !query 5 schema +struct +-- !query 5 output +one NULL +two NULL + + +-- !query 6 +select udf(a), b from values ("one", 1), ("two", 2L) as data(a, b) +-- !query 6 schema +struct +-- !query 6 output +one 1 +two 2 + + +-- !query 7 +select udf(a), udf(b) from values ("one", 1 + 0), ("two", 1 + 3L) as data(a, b) +-- !query 7 schema +struct +-- !query 7 output +one 1 +two 4 + + +-- !query 8 +select udf(a), b from values ("one", array(0, 1)), ("two", array(2, 3)) as data(a, b) +-- !query 8 schema +struct> +-- !query 8 output +one [0,1] +two [2,3] + + +-- !query 9 +select udf(a), b from values ("one", 2.0), ("two", 3.0D) as data(a, b) +-- !query 9 schema +struct +-- !query 9 output +one 2.0 +two 3.0 + + +-- !query 10 +select udf(a), b from values ("one", rand(5)), ("two", 3.0D) as data(a, b) +-- !query 10 schema +struct<> +-- !query 10 output +org.apache.spark.sql.AnalysisException +cannot evaluate expression rand(5) in inline table definition; line 1 pos 37 + + +-- !query 11 +select udf(a), udf(b) from values ("one", 2.0), ("two") as data(a, b) +-- !query 11 schema +struct<> +-- !query 11 output +org.apache.spark.sql.AnalysisException +expected 2 columns but found 1 columns in row 1; line 1 pos 27 + + +-- !query 12 +select udf(a), udf(b) from values ("one", array(0, 1)), ("two", struct(1, 2)) as data(a, b) +-- !query 12 schema +struct<> +-- !query 12 output +org.apache.spark.sql.AnalysisException +incompatible types found in column b for inline table; line 1 pos 27 + + +-- !query 13 +select udf(a), udf(b) from values ("one"), ("two") as data(a, b) +-- !query 13 schema +struct<> +-- !query 13 output +org.apache.spark.sql.AnalysisException +expected 2 columns but found 1 columns in row 0; line 1 pos 27 + + +-- !query 14 +select udf(a), udf(b) from values ("one", random_not_exist_func(1)), ("two", 2) as data(a, b) +-- !query 14 schema +struct<> +-- !query 14 output +org.apache.spark.sql.AnalysisException +Undefined function: 'random_not_exist_func'. This function is neither a registered temporary function nor a permanent function registered in the database 'default'.; line 1 pos 42 + + +-- !query 15 +select udf(a), udf(b) from values ("one", count(1)), ("two", 2) as data(a, b) +-- !query 15 schema +struct<> +-- !query 15 output +org.apache.spark.sql.AnalysisException +cannot evaluate expression count(1) in inline table definition; line 1 pos 42 + + +-- !query 16 +select * from values (timestamp('1991-12-06 00:00:00.0'), array(timestamp('1991-12-06 01:00:00.0'), timestamp('1991-12-06 12:00:00.0'))) as data(a, b) +-- !query 16 schema +struct> +-- !query 16 output +1991-12-06 00:00:00 [1991-12-06 01:00:00.0,1991-12-06 12:00:00.0] From bd7aea85611a5e24298d54b0f2f82d41acf75bc6 Mon Sep 17 00:00:00 2001 From: Terry Kim Date: Thu, 18 Jul 2019 14:12:59 -0700 Subject: [PATCH 2/4] Address PR Comments. --- .../sql-tests/inputs/udf/udf-inline-table.sql | 15 ++++---- .../results/udf/udf-inline-table.sql.out | 36 +++++++++---------- 2 files changed, 25 insertions(+), 26 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/udf/udf-inline-table.sql b/sql/core/src/test/resources/sql-tests/inputs/udf/udf-inline-table.sql index 25c335ac516f7..0dfac44cb645b 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/udf/udf-inline-table.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/udf/udf-inline-table.sql @@ -1,12 +1,11 @@ --- This test file was converted from intersect-all.sql. --- Note that currently registered UDF returns a string. So there are some differences, for instance --- in string cast within UDF in Scala and Python. +-- This test file was converted from inline-table.sql. +-- [SPARK-28291] UDFs cannot be evaluated within inline table definition -- single row, without table and column alias -select * from values ("one", 1); +select udf(col1), udf(col2) from values ("one", 1); -- single row, without column alias -select * from values ("one", 1) as data; +select udf(col1), udf(udf(col2)) from values ("one", 1) as data; -- single row select udf(a), b from values ("one", 1) as data(a, b); @@ -18,13 +17,13 @@ select udf(a) from values 1, 2, 3 as data(a); select udf(a), b from values ("one", 1), ("two", 2), ("three", null) as data(a, b); -- null type -select udf(a), b from values ("one", null), ("two", null) as data(a, b); +select a, udf(b) from values ("one", null), ("two", null) as data(a, b); -- int and long coercion select udf(a), b from values ("one", 1), ("two", 2L) as data(a, b); -- foldable expressions -select udf(a), udf(b) from values ("one", 1 + 0), ("two", 1 + 3L) as data(a, b); +select udf(udf(a)), udf(b) from values ("one", 1 + 0), ("two", 1 + 3L) as data(a, b); -- complex types select udf(a), b from values ("one", array(0, 1)), ("two", array(2, 3)) as data(a, b); @@ -51,4 +50,4 @@ select udf(a), udf(b) from values ("one", random_not_exist_func(1)), ("two", 2) select udf(a), udf(b) from values ("one", count(1)), ("two", 2) as data(a, b); -- string to timestamp -select * from values (timestamp('1991-12-06 00:00:00.0'), array(timestamp('1991-12-06 01:00:00.0'), timestamp('1991-12-06 12:00:00.0'))) as data(a, b); +select udf(a), b from values (timestamp('1991-12-06 00:00:00.0'), array(timestamp('1991-12-06 01:00:00.0'), timestamp('1991-12-06 12:00:00.0'))) as data(a, b); diff --git a/sql/core/src/test/resources/sql-tests/results/udf/udf-inline-table.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/udf-inline-table.sql.out index f2e2cd7c84b07..e92b701d0e0ed 100644 --- a/sql/core/src/test/resources/sql-tests/results/udf/udf-inline-table.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/udf/udf-inline-table.sql.out @@ -3,17 +3,17 @@ -- !query 0 -select * from values ("one", 1) +select udf(col1), udf(col2) from values ("one", 1) -- !query 0 schema -struct +struct -- !query 0 output one 1 -- !query 1 -select * from values ("one", 1) as data +select udf(col1), udf(udf(col2)) from values ("one", 1) as data -- !query 1 schema -struct +struct -- !query 1 output one 1 @@ -21,7 +21,7 @@ one 1 -- !query 2 select udf(a), b from values ("one", 1) as data(a, b) -- !query 2 schema -struct +struct -- !query 2 output one 1 @@ -29,7 +29,7 @@ one 1 -- !query 3 select udf(a) from values 1, 2, 3 as data(a) -- !query 3 schema -struct +struct -- !query 3 output 1 2 @@ -39,7 +39,7 @@ struct -- !query 4 select udf(a), b from values ("one", 1), ("two", 2), ("three", null) as data(a, b) -- !query 4 schema -struct +struct -- !query 4 output one 1 three NULL @@ -47,27 +47,27 @@ two 2 -- !query 5 -select udf(a), b from values ("one", null), ("two", null) as data(a, b) +select a, udf(b) from values ("one", null), ("two", null) as data(a, b) -- !query 5 schema -struct +struct<> -- !query 5 output -one NULL -two NULL +org.apache.spark.sql.AnalysisException +cannot resolve 'CAST(udf(cast(b as string)) AS NULL)' due to data type mismatch: cannot cast string to null; line 1 pos 10 -- !query 6 select udf(a), b from values ("one", 1), ("two", 2L) as data(a, b) -- !query 6 schema -struct +struct -- !query 6 output one 1 two 2 -- !query 7 -select udf(a), udf(b) from values ("one", 1 + 0), ("two", 1 + 3L) as data(a, b) +select udf(udf(a)), udf(b) from values ("one", 1 + 0), ("two", 1 + 3L) as data(a, b) -- !query 7 schema -struct +struct -- !query 7 output one 1 two 4 @@ -76,7 +76,7 @@ two 4 -- !query 8 select udf(a), b from values ("one", array(0, 1)), ("two", array(2, 3)) as data(a, b) -- !query 8 schema -struct> +struct> -- !query 8 output one [0,1] two [2,3] @@ -85,7 +85,7 @@ two [2,3] -- !query 9 select udf(a), b from values ("one", 2.0), ("two", 3.0D) as data(a, b) -- !query 9 schema -struct +struct -- !query 9 output one 2.0 two 3.0 @@ -146,8 +146,8 @@ cannot evaluate expression count(1) in inline table definition; line 1 pos 42 -- !query 16 -select * from values (timestamp('1991-12-06 00:00:00.0'), array(timestamp('1991-12-06 01:00:00.0'), timestamp('1991-12-06 12:00:00.0'))) as data(a, b) +select udf(a), b from values (timestamp('1991-12-06 00:00:00.0'), array(timestamp('1991-12-06 01:00:00.0'), timestamp('1991-12-06 12:00:00.0'))) as data(a, b) -- !query 16 schema -struct> +struct> -- !query 16 output 1991-12-06 00:00:00 [1991-12-06 01:00:00.0,1991-12-06 12:00:00.0] From 0562cc9deb3d2fe7b6b91dfd7f39d318656f9e55 Mon Sep 17 00:00:00 2001 From: Terry Kim Date: Thu, 18 Jul 2019 20:34:24 -0700 Subject: [PATCH 3/4] Address PR comments --- .../resources/sql-tests/inputs/udf/udf-inline-table.sql | 2 +- .../sql-tests/results/udf/udf-inline-table.sql.out | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/udf/udf-inline-table.sql b/sql/core/src/test/resources/sql-tests/inputs/udf/udf-inline-table.sql index 0dfac44cb645b..6fb1323c780eb 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/udf/udf-inline-table.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/udf/udf-inline-table.sql @@ -17,7 +17,7 @@ select udf(a) from values 1, 2, 3 as data(a); select udf(a), b from values ("one", 1), ("two", 2), ("three", null) as data(a, b); -- null type -select a, udf(b) from values ("one", null), ("two", null) as data(a, b); +select udf(a), b from values ("one", null), ("two", null) as data(a, b); -- int and long coercion select udf(a), b from values ("one", 1), ("two", 2L) as data(a, b); diff --git a/sql/core/src/test/resources/sql-tests/results/udf/udf-inline-table.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/udf-inline-table.sql.out index e92b701d0e0ed..2cf24e50c80a5 100644 --- a/sql/core/src/test/resources/sql-tests/results/udf/udf-inline-table.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/udf/udf-inline-table.sql.out @@ -47,12 +47,12 @@ two 2 -- !query 5 -select a, udf(b) from values ("one", null), ("two", null) as data(a, b) +select udf(a), b from values ("one", null), ("two", null) as data(a, b) -- !query 5 schema -struct<> +struct -- !query 5 output -org.apache.spark.sql.AnalysisException -cannot resolve 'CAST(udf(cast(b as string)) AS NULL)' due to data type mismatch: cannot cast string to null; line 1 pos 10 +one NULL +two NULL -- !query 6 From 1ba0886b8d2928252efba4c09da816c72c1e8657 Mon Sep 17 00:00:00 2001 From: Terry Kim Date: Fri, 19 Jul 2019 14:59:15 -0700 Subject: [PATCH 4/4] Add TODO as suggested. --- .../src/test/resources/sql-tests/inputs/udf/udf-inline-table.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/core/src/test/resources/sql-tests/inputs/udf/udf-inline-table.sql b/sql/core/src/test/resources/sql-tests/inputs/udf/udf-inline-table.sql index 6fb1323c780eb..179e528d74ca7 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/udf/udf-inline-table.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/udf/udf-inline-table.sql @@ -1,5 +1,6 @@ -- This test file was converted from inline-table.sql. -- [SPARK-28291] UDFs cannot be evaluated within inline table definition +-- TODO: We should add UDFs in VALUES clause when [SPARK-28291] is resolved. -- single row, without table and column alias select udf(col1), udf(col2) from values ("one", 1);