From 831056fd47580591d2e6c26466cb915859c575fa Mon Sep 17 00:00:00 2001 From: Anton Yanchenko Date: Thu, 11 Jul 2019 00:54:27 +0200 Subject: [PATCH 1/5] Validate LongType in _make_type_verifier --- python/pyspark/sql/types.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py index da84fc1e0066c..6056d1200a017 100644 --- a/python/pyspark/sql/types.py +++ b/python/pyspark/sql/types.py @@ -1211,6 +1211,10 @@ def _make_type_verifier(dataType, nullable=True, name=None): >>> _make_type_verifier(StructType([]))(None) >>> _make_type_verifier(StringType())("") >>> _make_type_verifier(LongType())(0) + >>> _make_type_verifier(LongType())(1 << 64) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + ValueError:... >>> _make_type_verifier(ArrayType(ShortType()))(list(range(3))) >>> _make_type_verifier(ArrayType(StringType()))(set()) # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): @@ -1319,6 +1323,16 @@ def verify_integer(obj): verify_value = verify_integer + elif isinstance(dataType, LongType): + def verify_integer(obj): + assert_acceptable_types(obj) + verify_acceptable_types(obj) + if not (-9223372036854775808 <= obj <= 9223372036854775807): + raise ValueError( + new_msg("object of LongType out of range, got: %s" % obj)) + + verify_value = verify_integer + elif isinstance(dataType, ArrayType): element_verifier = _make_type_verifier( dataType.elementType, dataType.containsNull, name="element in array %s" % name) From 71b7b36a00bdc44cb4c4a837083c25a780e46ea8 Mon Sep 17 00:00:00 2001 From: Anton Yanchenko Date: Sat, 20 Jul 2019 00:07:56 +0200 Subject: [PATCH 2/5] Fix tests --- python/pyspark/sql/tests/test_types.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/pyspark/sql/tests/test_types.py b/python/pyspark/sql/tests/test_types.py index 5132eec61d6f5..1cd84e0cd24e8 100644 --- a/python/pyspark/sql/tests/test_types.py +++ b/python/pyspark/sql/tests/test_types.py @@ -830,7 +830,8 @@ def __init__(self, **kwargs): (2**31 - 1, IntegerType()), # Long - (2**64, LongType()), + (-(2**63), LongType()), + (2**63 - 1, LongType()), # Float & Double (1.0, FloatType()), From ffa9b119136ef38cc742379b9e306454ac4c5074 Mon Sep 17 00:00:00 2001 From: Anton Yanchenko Date: Sat, 20 Jul 2019 09:52:35 +0200 Subject: [PATCH 3/5] More consistent code style --- python/pyspark/sql/types.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py index 6056d1200a017..0c7f4ce3ddc67 100644 --- a/python/pyspark/sql/types.py +++ b/python/pyspark/sql/types.py @@ -1324,14 +1324,14 @@ def verify_integer(obj): verify_value = verify_integer elif isinstance(dataType, LongType): - def verify_integer(obj): + def verify_long(obj): assert_acceptable_types(obj) verify_acceptable_types(obj) - if not (-9223372036854775808 <= obj <= 9223372036854775807): + if obj < -9223372036854775808 or obj > 9223372036854775807: raise ValueError( new_msg("object of LongType out of range, got: %s" % obj)) - verify_value = verify_integer + verify_value = verify_long elif isinstance(dataType, ArrayType): element_verifier = _make_type_verifier( From fde0a906e813042d301f94a699ced931f084beb6 Mon Sep 17 00:00:00 2001 From: Anton Yanchenko Date: Wed, 31 Jul 2019 20:24:18 +0200 Subject: [PATCH 4/5] Update docs --- docs/sql-migration-guide-upgrade.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/sql-migration-guide-upgrade.md b/docs/sql-migration-guide-upgrade.md index e9d99b66353e2..2e740bb72ad5f 100644 --- a/docs/sql-migration-guide-upgrade.md +++ b/docs/sql-migration-guide-upgrade.md @@ -149,6 +149,8 @@ license: | - Since Spark 3.0, if files or subdirectories disappear during recursive directory listing (i.e. they appear in an intermediate listing but then cannot be read or listed during later phases of the recursive directory listing, due to either concurrent file deletions or object store consistency issues) then the listing will fail with an exception unless `spark.sql.files.ignoreMissingFiles` is `true` (default `false`). In previous versions, these missing files or subdirectories would be ignored. Note that this change of behavior only applies during initial table file listing (or during `REFRESH TABLE`), not during query execution: the net change is that `spark.sql.files.ignoreMissingFiles` is now obeyed during table file listing / query planning, not only at query execution time. + - Since Spark 3.0, missing validation of `LongType` value in pyspark's `createDataframe` added. Previously `LongType` was not verified and resulted in `None` in case of wrong value. To have `None` instead of any validation error set `verifySchema=False`. + - Since Spark 3.0, substitution order of nested WITH clauses is changed and an inner CTE definition takes precedence over an outer. In version 2.4 and earlier, `WITH t AS (SELECT 1), t2 AS (WITH t AS (SELECT 2) SELECT * FROM t) SELECT * FROM t2` returns `1` while in version 3.0 it returns `2`. The previous behaviour can be restored by setting `spark.sql.legacy.ctePrecedence.enabled` to `true`. - Since Spark 3.0, the `add_months` function does not adjust the resulting date to a last day of month if the original date is a last day of months. For example, `select add_months(DATE'2019-02-28', 1)` results `2019-03-28`. In Spark version 2.4 and earlier, the resulting date is adjusted when the original date is a last day of months. For example, adding a month to `2019-02-28` results in `2019-03-31`. From 1f02d0c8c20654668fbbd3a59387efed3f101b88 Mon Sep 17 00:00:00 2001 From: Anton Yanchenko Date: Wed, 7 Aug 2019 11:51:19 +0200 Subject: [PATCH 5/5] Review fix --- docs/sql-migration-guide-upgrade.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sql-migration-guide-upgrade.md b/docs/sql-migration-guide-upgrade.md index 2e740bb72ad5f..88a9d8b9aa10c 100644 --- a/docs/sql-migration-guide-upgrade.md +++ b/docs/sql-migration-guide-upgrade.md @@ -149,7 +149,7 @@ license: | - Since Spark 3.0, if files or subdirectories disappear during recursive directory listing (i.e. they appear in an intermediate listing but then cannot be read or listed during later phases of the recursive directory listing, due to either concurrent file deletions or object store consistency issues) then the listing will fail with an exception unless `spark.sql.files.ignoreMissingFiles` is `true` (default `false`). In previous versions, these missing files or subdirectories would be ignored. Note that this change of behavior only applies during initial table file listing (or during `REFRESH TABLE`), not during query execution: the net change is that `spark.sql.files.ignoreMissingFiles` is now obeyed during table file listing / query planning, not only at query execution time. - - Since Spark 3.0, missing validation of `LongType` value in pyspark's `createDataframe` added. Previously `LongType` was not verified and resulted in `None` in case of wrong value. To have `None` instead of any validation error set `verifySchema=False`. + - Since Spark 3.0, `createDataFrame(..., verifySchema=True)` validates `LongType` as well in PySpark. Previously, `LongType` was not verified and resulted in `None` in case the value overflows. To restore this behavior, `verifySchema` can be set to `False` to disable the validation. - Since Spark 3.0, substitution order of nested WITH clauses is changed and an inner CTE definition takes precedence over an outer. In version 2.4 and earlier, `WITH t AS (SELECT 1), t2 AS (WITH t AS (SELECT 2) SELECT * FROM t) SELECT * FROM t2` returns `1` while in version 3.0 it returns `2`. The previous behaviour can be restored by setting `spark.sql.legacy.ctePrecedence.enabled` to `true`.