From d3dc77ca63546df08784f69c999afa34d571c4e7 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Thu, 10 Sep 2020 11:20:05 +0000 Subject: [PATCH 1/2] [SPARK-32840][SQL] nvalid interval value can happen to be just adhesive with the unit --- .../sql/catalyst/parser/AstBuilder.scala | 11 +++++- .../resources/sql-tests/inputs/interval.sql | 4 ++ .../sql-tests/results/ansi/interval.sql.out | 38 ++++++++++++++++++- .../sql-tests/results/interval.sql.out | 38 ++++++++++++++++++- 4 files changed, 88 insertions(+), 3 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index b29fe211454f6..307a72eee0376 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -2107,7 +2107,16 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging val kvs = units.indices.map { i => val u = units(i).getText val v = if (values(i).STRING() != null) { - string(values(i).STRING()) + val value = string(values(i).STRING()) + // SPARK-32840: For invalid cases, e.g. INTERVAL '1 day 2' hour, + // INTERVAL 'interval 1' day, we need to check ahead before they are concatenated with + // units and become valid ones, e.g. '1 day 2 hour'. + // Ideally, we only ensure the value parts don't contain any units here. + if (value.exists(Character.isLetter)) { + throw new ParseException("Can only use numbers in the interval value part for" + + s" multiple unit value pairs interval form, but got invalid value: $value", ctx) + } + value } else { values(i).getText } diff --git a/sql/core/src/test/resources/sql-tests/inputs/interval.sql b/sql/core/src/test/resources/sql-tests/inputs/interval.sql index ebc39f5bc44a6..c16e2d9261640 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/interval.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/interval.sql @@ -180,3 +180,7 @@ SELECT to_json(from_json('{"a":"1 days"}', 'a interval')), to_json(map('a', interval 25 month 100 day 130 minute)), from_json(to_json(map('a', interval 25 month 100 day 130 minute)), 'a interval'); + +select interval '1 day 2' day; +select interval 'interval 1' day; +select interval '-\t 1' day; diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out index 6a6a1df7eeb0e..1e1ecc21aa806 100644 --- a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 93 +-- Number of queries: 96 -- !query @@ -956,3 +956,39 @@ SELECT struct,to_json(from_json({"a":"1 days"})):string,to_json(map(a, INTERVAL '2 years 1 months 100 days 2 hours 10 minutes')):string,from_json(to_json(map(a, INTERVAL '2 years 1 months 100 days 2 hours 10 minutes'))):struct> -- !query output {"a":1 days} {"a":"1 days"} {"a":"2 years 1 months 100 days 2 hours 10 minutes"} {"a":2 years 1 months 100 days 2 hours 10 minutes} + + +-- !query +select interval '1 day 2' day +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.catalyst.parser.ParseException + +Can only use numbers in the interval value part for multiple unit value pairs interval form, but got invalid value: 1 day 2(line 1, pos 16) + +== SQL == +select interval '1 day 2' day +----------------^^^ + + +-- !query +select interval 'interval 1' day +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.catalyst.parser.ParseException + +Can only use numbers in the interval value part for multiple unit value pairs interval form, but got invalid value: interval 1(line 1, pos 16) + +== SQL == +select interval 'interval 1' day +----------------^^^ + + +-- !query +select interval '-\t 1' day +-- !query schema +struct +-- !query output +-1 days diff --git a/sql/core/src/test/resources/sql-tests/results/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/interval.sql.out index 63ed9b567c058..3c9bb13ac81b5 100644 --- a/sql/core/src/test/resources/sql-tests/results/interval.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/interval.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 93 +-- Number of queries: 96 -- !query @@ -928,3 +928,39 @@ SELECT struct,to_json(from_json({"a":"1 days"})):string,to_json(map(a, INTERVAL '2 years 1 months 100 days 2 hours 10 minutes')):string,from_json(to_json(map(a, INTERVAL '2 years 1 months 100 days 2 hours 10 minutes'))):struct> -- !query output {"a":1 days} {"a":"1 days"} {"a":"2 years 1 months 100 days 2 hours 10 minutes"} {"a":2 years 1 months 100 days 2 hours 10 minutes} + + +-- !query +select interval '1 day 2' day +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.catalyst.parser.ParseException + +Can only use numbers in the interval value part for multiple unit value pairs interval form, but got invalid value: 1 day 2(line 1, pos 16) + +== SQL == +select interval '1 day 2' day +----------------^^^ + + +-- !query +select interval 'interval 1' day +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.catalyst.parser.ParseException + +Can only use numbers in the interval value part for multiple unit value pairs interval form, but got invalid value: interval 1(line 1, pos 16) + +== SQL == +select interval 'interval 1' day +----------------^^^ + + +-- !query +select interval '-\t 1' day +-- !query schema +struct +-- !query output +-1 days From 3dd80b6478ed36843f85c6115e4d44f79c036f76 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Thu, 10 Sep 2020 21:13:39 +0800 Subject: [PATCH 2/2] conflicts --- .../resources/sql-tests/inputs/interval.sql | 8 +++ .../sql-tests/results/ansi/interval.sql.out | 58 ++++++++++++++++++- .../sql-tests/results/interval.sql.out | 58 ++++++++++++++++++- 3 files changed, 122 insertions(+), 2 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/interval.sql b/sql/core/src/test/resources/sql-tests/inputs/interval.sql index c16e2d9261640..9ad968ecda936 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/interval.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/interval.sql @@ -181,6 +181,14 @@ SELECT to_json(map('a', interval 25 month 100 day 130 minute)), from_json(to_json(map('a', interval 25 month 100 day 130 minute)), 'a interval'); +select interval '+'; +select interval '+.'; +select interval '1'; +select interval '1.2'; +select interval '- 2'; +select interval '1 day -'; +select interval '1 day 1'; + select interval '1 day 2' day; select interval 'interval 1' day; select interval '-\t 1' day; diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out index 1e1ecc21aa806..0185fb0206499 100644 --- a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 96 +-- Number of queries: 103 -- !query @@ -958,6 +958,62 @@ struct,to_json(from_json({"a":"1 da {"a":1 days} {"a":"1 days"} {"a":"2 years 1 months 100 days 2 hours 10 minutes"} {"a":2 years 1 months 100 days 2 hours 10 minutes} +-- !query +select interval '+' +-- !query schema +struct +-- !query output +NULL + + +-- !query +select interval '+.' +-- !query schema +struct +-- !query output +NULL + + +-- !query +select interval '1' +-- !query schema +struct +-- !query output +NULL + + +-- !query +select interval '1.2' +-- !query schema +struct +-- !query output +NULL + + +-- !query +select interval '- 2' +-- !query schema +struct +-- !query output +NULL + + +-- !query +select interval '1 day -' +-- !query schema +struct +-- !query output +NULL + + +-- !query +select interval '1 day 1' +-- !query schema +struct +-- !query output +NULL + + -- !query select interval '1 day 2' day -- !query schema diff --git a/sql/core/src/test/resources/sql-tests/results/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/interval.sql.out index 3c9bb13ac81b5..ff3bdc7ddb012 100644 --- a/sql/core/src/test/resources/sql-tests/results/interval.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/interval.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 96 +-- Number of queries: 103 -- !query @@ -930,6 +930,62 @@ struct,to_json(from_json({"a":"1 da {"a":1 days} {"a":"1 days"} {"a":"2 years 1 months 100 days 2 hours 10 minutes"} {"a":2 years 1 months 100 days 2 hours 10 minutes} +-- !query +select interval '+' +-- !query schema +struct +-- !query output +NULL + + +-- !query +select interval '+.' +-- !query schema +struct +-- !query output +NULL + + +-- !query +select interval '1' +-- !query schema +struct +-- !query output +NULL + + +-- !query +select interval '1.2' +-- !query schema +struct +-- !query output +NULL + + +-- !query +select interval '- 2' +-- !query schema +struct +-- !query output +NULL + + +-- !query +select interval '1 day -' +-- !query schema +struct +-- !query output +NULL + + +-- !query +select interval '1 day 1' +-- !query schema +struct +-- !query output +NULL + + -- !query select interval '1 day 2' day -- !query schema