Skip to content

[SPARK-32559][SQL]Fix the trim logic in UTF8String.toInt/toLong did't handle non-ASCII characters correctly #29375

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
WangGuangxin wants to merge 1 commit into from
Closed

[SPARK-32559][SQL]Fix the trim logic in UTF8String.toInt/toLong did't handle non-ASCII characters correctly #29375

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -575,14 +575,14 @@ public UTF8String trim() {
public UTF8String trimAll() {
int s = 0;
// skip all of the whitespaces (<=0x20) in the left side
while (s < this.numBytes && getByte(s) <= ' ') s++;
while (s < this.numBytes && Character.isWhitespace(getByte(s))) s++;
if (s == this.numBytes) {
// Everything trimmed
return EMPTY_UTF8;
}
// skip all of the whitespaces (<=0x20) in the right side
int e = this.numBytes - 1;
while (e > s && getByte(e) <= ' ') e--;
while (e > s && Character.isWhitespace(getByte(e))) e--;
if (s == 0 && e == numBytes - 1) {
// Nothing trimmed
return this;
Expand Down Expand Up @@ -1119,11 +1119,11 @@ public boolean toLong(LongWrapper toLongResult) {

private boolean toLong(LongWrapper toLongResult, boolean allowDecimal) {
int offset = 0;
while (offset < this.numBytes && getByte(offset) <= ' ') offset++;
while (offset < this.numBytes && Character.isWhitespace(getByte(offset))) offset++;
if (offset == this.numBytes) return false;

int end = this.numBytes - 1;
while (end > offset && getByte(end) <= ' ') end--;
while (end > offset && Character.isWhitespace(getByte(end))) end--;

byte b = getByte(offset);
final boolean negative = b == '-';
Expand Down Expand Up @@ -1216,11 +1216,11 @@ public boolean toInt(IntWrapper intWrapper) {

private boolean toInt(IntWrapper intWrapper, boolean allowDecimal) {
int offset = 0;
while (offset < this.numBytes && getByte(offset) <= ' ') offset++;
while (offset < this.numBytes && Character.isWhitespace(getByte(offset))) offset++;
if (offset == this.numBytes) return false;

int end = this.numBytes - 1;
while (end > offset && getByte(end) <= ' ') end--;
while (end > offset && Character.isWhitespace(getByte(end))) end--;

byte b = getByte(offset);
final boolean negative = b == '-';
Expand Down
5 changes: 5 additions & 0 deletions sql/core/src/test/resources/sql-tests/inputs/cast.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,11 @@ select cast(' 1' as bigint);
select cast(' 1' as float);
select cast(' 1 ' as DOUBLE);
select cast('1.0 ' as DEC);
select cast('1中文' as tinyint);
Copy link
Member

@srowen srowen Aug 9, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is purely to educate me, but those characters are considered whitespace?

Copy link
Contributor

@cloud-fan cloud-fan Aug 10, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This kind of character needs multiple bytes, so getByte(s) <= ' ' may not work.

select cast('1中文' as smallint);
select cast('1中文' as INT);
select cast('中文1' as bigint);
select cast('1中文' as bigint);

-- trim string before cast to boolean
select cast('\t\t true \n\r ' as boolean);
Expand Down
2 changes: 2 additions & 0 deletions sql/core/src/test/resources/sql-tests/inputs/datetime.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ select year('1500-01-01'), month('1500-01-01'), dayOfYear('1500-01-01');

select date '2019-01-01\t';
select timestamp '2019-01-01\t';
select date '2020-01-01中文';
select timestamp '2019-01-01中文';

-- time add/sub
select timestamp'2011-11-11 11:11:11' + interval '2' day;
Expand Down
1 change: 1 addition & 0 deletions sql/core/src/test/resources/sql-tests/inputs/interval.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,7 @@ select interval '2-2\t' year to month;
select interval '-\t2-2\t' year to month;
select interval '\n0 12:34:46.789\t' day to second;
select interval '\n-\t10\t 12:34:46.789\t' day to second;
select interval '中文 interval 1 day';

-- interval overflow if (ansi) exception else NULL
select -(a) from values (interval '-2147483648 months', interval '2147483647 months') t(a, b);
Expand Down
30 changes: 29 additions & 1 deletion sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
-- Number of queries: 106
-- Number of queries: 108


-- !query
Expand Down Expand Up @@ -226,6 +226,34 @@ struct<TIMESTAMP '2019-01-01 00:00:00':timestamp>
2019-01-01 00:00:00


-- !query
select date '2020-01-01中文'
-- !query schema
struct<>
-- !query output
org.apache.spark.sql.catalyst.parser.ParseException

Cannot parse the DATE value: 2020-01-01中文(line 1, pos 7)

== SQL ==
select date '2020-01-01中文'
-------^^^


-- !query
select timestamp '2019-01-01中文'
-- !query schema
struct<>
-- !query output
org.apache.spark.sql.catalyst.parser.ParseException

Cannot parse the TIMESTAMP value: 2019-01-01中文(line 1, pos 7)

== SQL ==
select timestamp '2019-01-01中文'
-------^^^


-- !query
select timestamp'2011-11-11 11:11:11' + interval '2' day
-- !query schema
Expand Down
16 changes: 15 additions & 1 deletion sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
-- Number of queries: 93
-- Number of queries: 94


-- !query
Expand Down Expand Up @@ -891,6 +891,20 @@ select interval '\n-\t10\t 12:34:46.789\t' day to second
----------------^^^


-- !query
select interval '中文 interval 1 day'
-- !query schema
struct<>
-- !query output
org.apache.spark.sql.catalyst.parser.ParseException

Cannot parse the INTERVAL value: 中文 interval 1 day(line 1, pos 7)

== SQL ==
select interval '中文 interval 1 day'
-------^^^


-- !query
select -(a) from values (interval '-2147483648 months', interval '2147483647 months') t(a, b)
-- !query schema
Expand Down
42 changes: 41 additions & 1 deletion sql/core/src/test/resources/sql-tests/results/cast.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
-- Number of queries: 46
-- Number of queries: 51


-- !query
Expand Down Expand Up @@ -353,6 +353,46 @@ struct<CAST(1.0 AS DECIMAL(10,0)):decimal(10,0)>
1


-- !query
select cast('1中文' as tinyint)
-- !query schema
struct<CAST(1中文 AS TINYINT):tinyint>
-- !query output
NULL


-- !query
select cast('1中文' as smallint)
-- !query schema
struct<CAST(1中文 AS SMALLINT):smallint>
-- !query output
NULL


-- !query
select cast('1中文' as INT)
-- !query schema
struct<CAST(1中文 AS INT):int>
-- !query output
NULL


-- !query
select cast('中文1' as bigint)
-- !query schema
struct<CAST(中文1 AS BIGINT):bigint>
-- !query output
NULL


-- !query
select cast('1中文' as bigint)
-- !query schema
struct<CAST(1中文 AS BIGINT):bigint>
-- !query output
NULL


-- !query
select cast('\t\t true \n\r ' as boolean)
-- !query schema
Expand Down
30 changes: 29 additions & 1 deletion sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
-- Number of queries: 106
-- Number of queries: 108


-- !query
Expand Down Expand Up @@ -200,6 +200,34 @@ struct<TIMESTAMP '2019-01-01 00:00:00':timestamp>
2019-01-01 00:00:00


-- !query
select date '2020-01-01中文'
-- !query schema
struct<>
-- !query output
org.apache.spark.sql.catalyst.parser.ParseException

Cannot parse the DATE value: 2020-01-01中文(line 1, pos 7)

== SQL ==
select date '2020-01-01中文'
-------^^^


-- !query
select timestamp '2019-01-01中文'
-- !query schema
struct<>
-- !query output
org.apache.spark.sql.catalyst.parser.ParseException

Cannot parse the TIMESTAMP value: 2019-01-01中文(line 1, pos 7)

== SQL ==
select timestamp '2019-01-01中文'
-------^^^


-- !query
select timestamp'2011-11-11 11:11:11' + interval '2' day
-- !query schema
Expand Down
30 changes: 29 additions & 1 deletion sql/core/src/test/resources/sql-tests/results/datetime.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
-- Number of queries: 106
-- Number of queries: 108


-- !query
Expand Down Expand Up @@ -200,6 +200,34 @@ struct<TIMESTAMP '2019-01-01 00:00:00':timestamp>
2019-01-01 00:00:00


-- !query
select date '2020-01-01中文'
-- !query schema
struct<>
-- !query output
org.apache.spark.sql.catalyst.parser.ParseException

Cannot parse the DATE value: 2020-01-01中文(line 1, pos 7)

== SQL ==
select date '2020-01-01中文'
-------^^^


-- !query
select timestamp '2019-01-01中文'
-- !query schema
struct<>
-- !query output
org.apache.spark.sql.catalyst.parser.ParseException

Cannot parse the TIMESTAMP value: 2019-01-01中文(line 1, pos 7)

== SQL ==
select timestamp '2019-01-01中文'
-------^^^


-- !query
select timestamp'2011-11-11 11:11:11' + interval '2' day
-- !query schema
Expand Down
16 changes: 15 additions & 1 deletion sql/core/src/test/resources/sql-tests/results/interval.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
-- Number of queries: 93
-- Number of queries: 94


-- !query
Expand Down Expand Up @@ -868,6 +868,20 @@ select interval '\n-\t10\t 12:34:46.789\t' day to second
----------------^^^


-- !query
select interval '中文 interval 1 day'
-- !query schema
struct<>
-- !query output
org.apache.spark.sql.catalyst.parser.ParseException

Cannot parse the INTERVAL value: 中文 interval 1 day(line 1, pos 7)

== SQL ==
select interval '中文 interval 1 day'
-------^^^


-- !query
select -(a) from values (interval '-2147483648 months', interval '2147483647 months') t(a, b)
-- !query schema
Expand Down