From f127d8eded1cb9986a975d11fb7411678f65f245 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Thu, 29 Jul 2021 19:22:08 -0700 Subject: [PATCH 1/5] upgrade arrow --- ci/cpu/build.sh | 2 ++ ci/gpu/build.sh | 2 ++ ci/gpu/java.sh | 3 ++- conda/environments/cudf_dev_cuda11.0.yml | 4 ++-- conda/environments/cudf_dev_cuda11.2.yml | 4 ++-- conda/recipes/cudf/meta.yaml | 2 +- conda/recipes/libcudf/meta.yaml | 2 +- cpp/cmake/thirdparty/CUDF_GetArrow.cmake | 2 +- 8 files changed, 13 insertions(+), 8 deletions(-) diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh index 00dffa576834..e776093a4576 100755 --- a/ci/cpu/build.sh +++ b/ci/cpu/build.sh @@ -42,6 +42,8 @@ gpuci_logger "Activate conda env" . /opt/conda/etc/profile.d/conda.sh conda activate rapids +conda install -y "arrow-cpp=5.0.0" "pyarrow=5.0.0" 'arrow-cpp-proc=*=cuda' + # Remove rapidsai-nightly channel if we are building main branch if [ "$SOURCE_BRANCH" = "main" ]; then conda config --system --remove channels rapidsai-nightly diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index 326fc2f1119b..e9fa203b1b49 100755 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -85,6 +85,8 @@ gpuci_mamba_retry install -y \ # https://docs.rapids.ai/maintainers/depmgmt/ # gpuci_mamba_retry remove --force rapids-build-env rapids-notebook-env # gpuci_mamba_retry install -y "your-pkg=1.0.0" +gpuci_conda_retry remove --force rapids-build-env rapids-notebook-env +gpuci_conda_retry install -y "arrow-cpp=5.0.0" "pyarrow=5.0.0" 'arrow-cpp-proc=*=cuda' "pyorc" "python-confluent-kafka>=1.3.0" gpuci_logger "Check compiler versions" diff --git a/ci/gpu/java.sh b/ci/gpu/java.sh index 8c4b597d12dc..ef4677d799fe 100755 --- a/ci/gpu/java.sh +++ b/ci/gpu/java.sh @@ -87,7 +87,8 @@ gpuci_conda_retry install -y \ # https://docs.rapids.ai/maintainers/depmgmt/ # gpuci_conda_retry remove --force rapids-build-env rapids-notebook-env # gpuci_conda_retry install -y "your-pkg=1.0.0" - +gpuci_conda_retry remove --force rapids-build-env rapids-notebook-env +gpuci_conda_retry install -y "arrow-cpp=5.0.0" "pyarrow=5.0.0" 'arrow-cpp-proc=*=cuda' "pyorc" "python-confluent-kafka>=1.3.0" gpuci_logger "Check compiler versions" python --version diff --git a/conda/environments/cudf_dev_cuda11.0.yml b/conda/environments/cudf_dev_cuda11.0.yml index 70bbe88a00c9..c509e2953180 100644 --- a/conda/environments/cudf_dev_cuda11.0.yml +++ b/conda/environments/cudf_dev_cuda11.0.yml @@ -17,7 +17,7 @@ dependencies: - numba>=0.53.1 - numpy - pandas>=1.0,<1.3.0dev0 - - pyarrow=4.0.1=*cuda + - pyarrow=5.0.0=*cuda - fastavro>=0.22.9 - notebook>=0.5.0 - cython>=0.29,<0.30 @@ -43,7 +43,7 @@ dependencies: - dask>=2021.6.0 - distributed>=2021.6.0 - streamz - - arrow-cpp=4.0.1 + - arrow-cpp=5.0.0 - dlpack>=0.5,<0.6.0a0 - arrow-cpp-proc * cuda - double-conversion diff --git a/conda/environments/cudf_dev_cuda11.2.yml b/conda/environments/cudf_dev_cuda11.2.yml index 6d2abdda449c..b623f4537d2d 100644 --- a/conda/environments/cudf_dev_cuda11.2.yml +++ b/conda/environments/cudf_dev_cuda11.2.yml @@ -17,7 +17,7 @@ dependencies: - numba>=0.53.1 - numpy - pandas>=1.0,<1.3.0dev0 - - pyarrow=4.0.1=*cuda + - pyarrow=5.0.0=*cuda - fastavro>=0.22.9 - notebook>=0.5.0 - cython>=0.29,<0.30 @@ -43,7 +43,7 @@ dependencies: - dask>=2021.6.0 - distributed>=2021.6.0 - streamz - - arrow-cpp=4.0.1 + - arrow-cpp=5.0.0 - dlpack>=0.5,<0.6.0a0 - arrow-cpp-proc * cuda - double-conversion diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index 9023e89c2f59..ca36acccfbbd 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -30,7 +30,7 @@ requirements: - setuptools - numba >=0.53.1 - dlpack>=0.5,<0.6.0a0 - - pyarrow 4.0.1 *cuda + - pyarrow 5.0.0 *cuda - libcudf {{ version }} - rmm {{ minor_version }} - cudatoolkit {{ cuda_version }} diff --git a/conda/recipes/libcudf/meta.yaml b/conda/recipes/libcudf/meta.yaml index 6c4175a25391..e42bbeaf9bf5 100644 --- a/conda/recipes/libcudf/meta.yaml +++ b/conda/recipes/libcudf/meta.yaml @@ -37,7 +37,7 @@ requirements: host: - librmm {{ minor_version }}.* - cudatoolkit {{ cuda_version }}.* - - arrow-cpp 4.0.1 *cuda + - arrow-cpp 5.0.0 *cuda - arrow-cpp-proc * cuda - dlpack>=0.5,<0.6.0a0 run: diff --git a/cpp/cmake/thirdparty/CUDF_GetArrow.cmake b/cpp/cmake/thirdparty/CUDF_GetArrow.cmake index 8cef3e8b9d04..90917763ac88 100644 --- a/cpp/cmake/thirdparty/CUDF_GetArrow.cmake +++ b/cpp/cmake/thirdparty/CUDF_GetArrow.cmake @@ -146,7 +146,7 @@ function(find_and_configure_arrow VERSION BUILD_STATIC ENABLE_S3 ENABLE_PYTHON E endfunction() -set(CUDF_VERSION_Arrow 4.0.1) +set(CUDF_VERSION_Arrow 5.0.0) find_and_configure_arrow( ${CUDF_VERSION_Arrow} From 4f40b2c04a5a73bc3fc3c999119b4b56a707ba7f Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Thu, 29 Jul 2021 23:51:32 -0700 Subject: [PATCH 2/5] fix decimal creation --- python/cudf/cudf/core/column/column.py | 45 +++++++++++++++----------- python/cudf/cudf/tests/test_binops.py | 40 +++++++++++------------ python/cudf/cudf/tests/test_decimal.py | 2 +- 3 files changed, 48 insertions(+), 39 deletions(-) diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index a5e49b026f35..cbc6ace50783 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -2044,24 +2044,8 @@ def as_column( memoryview(arbitrary), dtype=dtype, nan_as_null=nan_as_null ) except TypeError: - pa_type = None - np_type = None - try: - if dtype is not None: - if is_categorical_dtype(dtype) or is_interval_dtype(dtype): - raise TypeError - if is_list_dtype(dtype): - data = pa.array(arbitrary) - if type(data) not in (pa.ListArray, pa.NullArray): - raise ValueError( - "Cannot create list column from given data" - ) - return as_column(data, nan_as_null=nan_as_null) - elif isinstance( - dtype, cudf.StructDtype - ) and not isinstance(dtype, cudf.IntervalDtype): - data = pa.array(arbitrary, type=dtype.to_arrow()) - return as_column(data, nan_as_null=nan_as_null) + if dtype is not None: + try: if isinstance(dtype, cudf.core.dtypes.Decimal64Dtype): data = pa.array( arbitrary, @@ -2082,6 +2066,31 @@ def as_column( return cudf.core.column.Decimal32Column.from_arrow( data ) + except pa.ArrowTypeError as e: + # Arrow throws a type error if the input is of + # mixed-precision and cannot fit into the provided + # decimal type properly, see: + # https://github.com/apache/arrow/pull/9948 + raise e + pa_type = None + np_type = None + try: + if dtype is not None: + if is_categorical_dtype(dtype) or is_interval_dtype(dtype): + raise TypeError + if is_list_dtype(dtype): + data = pa.array(arbitrary) + if type(data) not in (pa.ListArray, pa.NullArray): + raise ValueError( + "Cannot create list column from given data" + ) + return as_column(data, nan_as_null=nan_as_null) + elif isinstance( + dtype, cudf.StructDtype + ) and not isinstance(dtype, cudf.IntervalDtype): + data = pa.array(arbitrary, type=dtype.to_arrow()) + return as_column(data, nan_as_null=nan_as_null) + dtype = pd.api.types.pandas_dtype(dtype) np_type = np.dtype(dtype).type if np_type == np.bool_: diff --git a/python/cudf/cudf/tests/test_binops.py b/python/cudf/cudf/tests/test_binops.py index 8277b8e7b32b..f8fd2502a7d5 100644 --- a/python/cudf/cudf/tests/test_binops.py +++ b/python/cudf/cudf/tests/test_binops.py @@ -1758,16 +1758,16 @@ def test_binops_with_NA_consistent(dtype, op): ( operator.add, ["1.5", "2.0"], - cudf.Decimal64Dtype(scale=2, precision=2), + cudf.Decimal64Dtype(scale=2, precision=3), ["1.5", "2.0"], - cudf.Decimal64Dtype(scale=2, precision=2), - ["3.0", "4.0"], cudf.Decimal64Dtype(scale=2, precision=3), + ["3.0", "4.0"], + cudf.Decimal64Dtype(scale=2, precision=4), ), ( operator.add, ["1.5", "2.0"], - cudf.Decimal64Dtype(scale=2, precision=2), + cudf.Decimal64Dtype(scale=2, precision=3), ["2.25", "1.005"], cudf.Decimal64Dtype(scale=3, precision=4), ["3.75", "3.005"], @@ -1785,7 +1785,7 @@ def test_binops_with_NA_consistent(dtype, op): ( operator.sub, ["1.5", "2.0"], - cudf.Decimal64Dtype(scale=2, precision=2), + cudf.Decimal64Dtype(scale=1, precision=2), ["2.25", "1.005"], cudf.Decimal64Dtype(scale=3, precision=4), ["-0.75", "0.995"], @@ -1794,7 +1794,7 @@ def test_binops_with_NA_consistent(dtype, op): ( operator.sub, ["1.5", "2.0"], - cudf.Decimal64Dtype(scale=2, precision=2), + cudf.Decimal64Dtype(scale=1, precision=2), ["2.25", "1.005"], cudf.Decimal64Dtype(scale=3, precision=4), ["-0.75", "0.995"], @@ -1812,11 +1812,11 @@ def test_binops_with_NA_consistent(dtype, op): ( operator.mul, ["1.5", "2.0"], - cudf.Decimal64Dtype(scale=2, precision=2), + cudf.Decimal64Dtype(scale=2, precision=3), ["1.5", "3.0"], cudf.Decimal64Dtype(scale=3, precision=4), ["2.25", "6.0"], - cudf.Decimal64Dtype(scale=5, precision=7), + cudf.Decimal64Dtype(scale=5, precision=8), ), ( operator.mul, @@ -1866,16 +1866,16 @@ def test_binops_with_NA_consistent(dtype, op): ( operator.add, ["1.5", None, "2.0"], - cudf.Decimal64Dtype(scale=2, precision=2), + cudf.Decimal64Dtype(scale=1, precision=2), ["1.5", None, "2.0"], - cudf.Decimal64Dtype(scale=2, precision=2), + cudf.Decimal64Dtype(scale=1, precision=2), ["3.0", None, "4.0"], - cudf.Decimal64Dtype(scale=2, precision=3), + cudf.Decimal64Dtype(scale=1, precision=3), ), ( operator.add, ["1.5", None], - cudf.Decimal64Dtype(scale=2, precision=2), + cudf.Decimal64Dtype(scale=2, precision=3), ["2.25", "1.005"], cudf.Decimal64Dtype(scale=3, precision=4), ["3.75", None], @@ -1884,7 +1884,7 @@ def test_binops_with_NA_consistent(dtype, op): ( operator.sub, ["1.5", None], - cudf.Decimal64Dtype(scale=2, precision=2), + cudf.Decimal64Dtype(scale=2, precision=3), ["2.25", None], cudf.Decimal64Dtype(scale=3, precision=4), ["-0.75", None], @@ -1893,7 +1893,7 @@ def test_binops_with_NA_consistent(dtype, op): ( operator.sub, ["1.5", "2.0"], - cudf.Decimal64Dtype(scale=2, precision=2), + cudf.Decimal64Dtype(scale=2, precision=3), ["2.25", None], cudf.Decimal64Dtype(scale=3, precision=4), ["-0.75", None], @@ -1902,11 +1902,11 @@ def test_binops_with_NA_consistent(dtype, op): ( operator.mul, ["1.5", None], - cudf.Decimal64Dtype(scale=2, precision=2), + cudf.Decimal64Dtype(scale=2, precision=3), ["1.5", None], cudf.Decimal64Dtype(scale=3, precision=4), ["2.25", None], - cudf.Decimal64Dtype(scale=5, precision=7), + cudf.Decimal64Dtype(scale=5, precision=8), ), ( operator.mul, @@ -2432,10 +2432,10 @@ def test_binops_decimal_comp_mixed_integer(args, integer_dtype, reflected): ( operator.truediv, ["100", "200"], - cudf.Decimal64Dtype(scale=2, precision=4), + cudf.Decimal64Dtype(scale=2, precision=5), decimal.Decimal(2), ["50", "100"], - cudf.Decimal64Dtype(scale=2, precision=6), + cudf.Decimal64Dtype(scale=2, precision=7), False, ), ( @@ -2459,10 +2459,10 @@ def test_binops_decimal_comp_mixed_integer(args, integer_dtype, reflected): ( operator.truediv, ["100", "200"], - cudf.Decimal64Dtype(scale=2, precision=3), + cudf.Decimal64Dtype(scale=2, precision=5), 1, ["0", "0"], - cudf.Decimal64Dtype(scale=-2, precision=5), + cudf.Decimal64Dtype(scale=-2, precision=7), True, ), ( diff --git a/python/cudf/cudf/tests/test_decimal.py b/python/cudf/cudf/tests/test_decimal.py index d2de44b0c8f7..72fa3bf0fc3f 100644 --- a/python/cudf/cudf/tests/test_decimal.py +++ b/python/cudf/cudf/tests/test_decimal.py @@ -24,7 +24,7 @@ [1], [-1], [1, 2, 3, 4], - [42, 1729, 4104], + [42, 17, 41], [1, 2, None, 4], [None, None, None], [], From 9356ae19006ad34695219d4725ead29a05124c34 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Thu, 29 Jul 2021 23:58:40 -0700 Subject: [PATCH 3/5] add error tests --- python/cudf/cudf/tests/test_decimal.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/python/cudf/cudf/tests/test_decimal.py b/python/cudf/cudf/tests/test_decimal.py index 72fa3bf0fc3f..9d93898dcd97 100644 --- a/python/cudf/cudf/tests/test_decimal.py +++ b/python/cudf/cudf/tests/test_decimal.py @@ -347,3 +347,11 @@ def test_serialize_decimal_columns(data): df = cudf.DataFrame(data) recreated = df.__class__.deserialize(*df.serialize()) assert_eq(recreated, df) + + +def test_decimal_invalid_precision(): + with pytest.raises(pa.ArrowInvalid): + _ = cudf.Series([10, 20, 30], dtype=cudf.Decimal64Dtype(2, 2)) + + with pytest.raises(pa.ArrowInvalid): + _ = cudf.Series([Decimal("300")], dtype=cudf.Decimal64Dtype(2, 1)) From d77683acec9f596c2c64641d31128ada794dc89a Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Fri, 30 Jul 2021 08:39:35 -0700 Subject: [PATCH 4/5] remove try/except --- python/cudf/cudf/core/column/column.py | 49 ++++++++++++-------------- 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index cbc6ace50783..f0530ec2a9a5 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -2045,33 +2045,28 @@ def as_column( ) except TypeError: if dtype is not None: - try: - if isinstance(dtype, cudf.core.dtypes.Decimal64Dtype): - data = pa.array( - arbitrary, - type=pa.decimal128( - precision=dtype.precision, scale=dtype.scale - ), - ) - return cudf.core.column.Decimal64Column.from_arrow( - data - ) - if isinstance(dtype, cudf.core.dtypes.Decimal32Dtype): - data = pa.array( - arbitrary, - type=pa.decimal128( - precision=dtype.precision, scale=dtype.scale - ), - ) - return cudf.core.column.Decimal32Column.from_arrow( - data - ) - except pa.ArrowTypeError as e: - # Arrow throws a type error if the input is of - # mixed-precision and cannot fit into the provided - # decimal type properly, see: - # https://github.com/apache/arrow/pull/9948 - raise e + # Arrow throws a type error if the input is of + # mixed-precision and cannot fit into the provided + # decimal type properly, see: + # https://github.com/apache/arrow/pull/9948 + # Hence we should let the exception propagate to + # the user. + if isinstance(dtype, cudf.core.dtypes.Decimal64Dtype): + data = pa.array( + arbitrary, + type=pa.decimal128( + precision=dtype.precision, scale=dtype.scale + ), + ) + return cudf.core.column.Decimal64Column.from_arrow(data) + if isinstance(dtype, cudf.core.dtypes.Decimal32Dtype): + data = pa.array( + arbitrary, + type=pa.decimal128( + precision=dtype.precision, scale=dtype.scale + ), + ) + return cudf.core.column.Decimal32Column.from_arrow(data) pa_type = None np_type = None try: From 69ea9038d6d5b6664a4f4e65164eb8d473a0500f Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 3 Aug 2021 13:33:19 -0700 Subject: [PATCH 5/5] remove temporary build commands --- ci/cpu/build.sh | 2 -- ci/gpu/build.sh | 2 -- ci/gpu/java.sh | 3 +-- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh index e776093a4576..00dffa576834 100755 --- a/ci/cpu/build.sh +++ b/ci/cpu/build.sh @@ -42,8 +42,6 @@ gpuci_logger "Activate conda env" . /opt/conda/etc/profile.d/conda.sh conda activate rapids -conda install -y "arrow-cpp=5.0.0" "pyarrow=5.0.0" 'arrow-cpp-proc=*=cuda' - # Remove rapidsai-nightly channel if we are building main branch if [ "$SOURCE_BRANCH" = "main" ]; then conda config --system --remove channels rapidsai-nightly diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index e9fa203b1b49..326fc2f1119b 100755 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -85,8 +85,6 @@ gpuci_mamba_retry install -y \ # https://docs.rapids.ai/maintainers/depmgmt/ # gpuci_mamba_retry remove --force rapids-build-env rapids-notebook-env # gpuci_mamba_retry install -y "your-pkg=1.0.0" -gpuci_conda_retry remove --force rapids-build-env rapids-notebook-env -gpuci_conda_retry install -y "arrow-cpp=5.0.0" "pyarrow=5.0.0" 'arrow-cpp-proc=*=cuda' "pyorc" "python-confluent-kafka>=1.3.0" gpuci_logger "Check compiler versions" diff --git a/ci/gpu/java.sh b/ci/gpu/java.sh index ef4677d799fe..8c4b597d12dc 100755 --- a/ci/gpu/java.sh +++ b/ci/gpu/java.sh @@ -87,8 +87,7 @@ gpuci_conda_retry install -y \ # https://docs.rapids.ai/maintainers/depmgmt/ # gpuci_conda_retry remove --force rapids-build-env rapids-notebook-env # gpuci_conda_retry install -y "your-pkg=1.0.0" -gpuci_conda_retry remove --force rapids-build-env rapids-notebook-env -gpuci_conda_retry install -y "arrow-cpp=5.0.0" "pyarrow=5.0.0" 'arrow-cpp-proc=*=cuda' "pyorc" "python-confluent-kafka>=1.3.0" + gpuci_logger "Check compiler versions" python --version