diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 570a37ab8746..4ecf44256b25 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -346,8 +346,6 @@ jobs: with: build_type: pull-request script: ci/test_cpp.sh - # Skip tests on RTX PRO 6000 (Blackwell). xref: https://github.com/rapidsai/cudf/issues/21953 - matrix_filter: map(select(.GPU != "rtxpro6000")) conda-python-build: needs: conda-cpp-build secrets: inherit @@ -373,8 +371,6 @@ jobs: with: build_type: pull-request script: "ci/test_python_cudf.sh" - # Skip tests on RTX PRO 6000 (Blackwell). xref: https://github.com/rapidsai/cudf/issues/21953 - matrix_filter: map(select(.GPU != "rtxpro6000")) conda-python-other-tests: # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism needs: [conda-python-build-noarch, changed-files] @@ -384,8 +380,6 @@ jobs: with: build_type: pull-request script: "ci/test_python_other.sh" - # Skip tests on RTX PRO 6000 (Blackwell). xref: https://github.com/rapidsai/cudf/issues/21953 - matrix_filter: map(select(.GPU != "rtxpro6000")) conda-java-tests: needs: [conda-cpp-build, changed-files] secrets: inherit @@ -463,8 +457,6 @@ jobs: with: build_type: pull-request script: ci/test_wheel_cudf.sh - # Skip tests on RTX PRO 6000 (Blackwell). xref: https://github.com/rapidsai/cudf/issues/21953 - matrix_filter: map(select(.GPU != "rtxpro6000")) wheel-build-cudf-polars: needs: wheel-build-pylibcudf secrets: inherit @@ -496,8 +488,7 @@ jobs: with: # This selects "ARCH=amd64 + the latest supported Python + CUDA" to minimize CI usage. # (rapidsmpf compatibility already validated in rapidsmpf CI) - # Skip tests on RTX PRO 6000 (Blackwell). xref: https://github.com/rapidsai/cudf/issues/21953 - matrix_filter: map(select(.GPU != "rtxpro6000")) | map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) + matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) build_type: pull-request script: "ci/test_cudf_polars_experimental.sh" cudf-polars-polars-tests: @@ -507,8 +498,7 @@ jobs: if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_wheels && fromJSON(needs.changed-files.outputs.changed_file_groups).neither_cudf_nor_dask_cudf with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". - # Skip tests on RTX PRO 6000 (Blackwell). xref: https://github.com/rapidsai/cudf/issues/21953 - matrix_filter: map(select(.GPU != "rtxpro6000")) | map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) + matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) build_type: pull-request script: "ci/test_cudf_polars_polars_tests.sh" wheel-build-dask-cudf: @@ -531,8 +521,7 @@ jobs: if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_wheels && fromJSON(needs.changed-files.outputs.changed_file_groups).not_cudf_polars with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". - # Skip tests on RTX PRO 6000 (Blackwell). xref: https://github.com/rapidsai/cudf/issues/21953 - matrix_filter: map(select(.GPU != "rtxpro6000")) | map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) + matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) build_type: pull-request script: ci/test_wheel_dask_cudf.sh devcontainer: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 0bd9fb4ad95b..95439e65744b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -39,8 +39,6 @@ jobs: date: ${{ inputs.date }} script: ci/test_cpp.sh sha: ${{ inputs.sha }} - # Skip tests on RTX PRO 6000 (Blackwell). xref: https://github.com/rapidsai/cudf/issues/21953 - matrix_filter: map(select(.GPU != "rtxpro6000")) conda-cpp-memcheck-tests: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main @@ -72,8 +70,6 @@ jobs: date: ${{ inputs.date }} sha: ${{ inputs.sha }} script: "ci/test_python_cudf.sh" - # Skip tests on RTX PRO 6000 (Blackwell). xref: https://github.com/rapidsai/cudf/issues/21953 - matrix_filter: map(select(.GPU != "rtxpro6000")) conda-python-other-tests: # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism secrets: inherit @@ -84,8 +80,6 @@ jobs: date: ${{ inputs.date }} sha: ${{ inputs.sha }} script: "ci/test_python_other.sh" - # Skip tests on RTX PRO 6000 (Blackwell). xref: https://github.com/rapidsai/cudf/issues/21953 - matrix_filter: map(select(.GPU != "rtxpro6000")) conda-java-tests: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main @@ -119,8 +113,6 @@ jobs: date: ${{ inputs.date }} sha: ${{ inputs.sha }} script: ci/test_wheel_cudf.sh - # Skip tests on RTX PRO 6000 (Blackwell). xref: https://github.com/rapidsai/cudf/issues/21953 - matrix_filter: map(select(.GPU != "rtxpro6000")) wheel-tests-dask-cudf: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@main @@ -130,8 +122,6 @@ jobs: date: ${{ inputs.date }} sha: ${{ inputs.sha }} script: ci/test_wheel_dask_cudf.sh - # Skip tests on RTX PRO 6000 (Blackwell). xref: https://github.com/rapidsai/cudf/issues/21953 - matrix_filter: map(select(.GPU != "rtxpro6000")) unit-tests-cudf-pandas: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@main @@ -169,8 +159,7 @@ jobs: with: # This selects "ARCH=amd64 + the latest supported Python + CUDA" to minimize CI usage. # (rapidsmpf compatibility already validated in rapidsmpf CI) - # Skip tests on RTX PRO 6000 (Blackwell). xref: https://github.com/rapidsai/cudf/issues/21953 - matrix_filter: map(select(.GPU != "rtxpro6000")) | map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) + matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) build_type: ${{ inputs.build_type }} branch: ${{ inputs.branch }} date: ${{ inputs.date }} @@ -186,8 +175,6 @@ jobs: date: ${{ inputs.date }} sha: ${{ inputs.sha }} script: "ci/test_cudf_polars_polars_tests.sh" - # Skip tests on RTX PRO 6000 (Blackwell). xref: https://github.com/rapidsai/cudf/issues/21953 - matrix_filter: map(select(.GPU != "rtxpro6000")) narwhals-tests: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main diff --git a/ci/run_cudf_examples.sh b/ci/run_cudf_examples.sh index 2a58c1cfeefa..617a02048d8c 100755 --- a/ci/run_cudf_examples.sh +++ b/ci/run_cudf_examples.sh @@ -10,50 +10,68 @@ trap "EXITCODE=1" ERR # Support customizing the examples' install location cd "${INSTALL_PREFIX:-${CONDA_PREFIX:-/usr}}/bin/examples/libcudf" || exit +# TODO: Temporary workaround for compute-sanitizer bug 5824899 that occurs only on the examples +GPU_COMPUTE_CAP=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -n1 | tr -d '[:space:]') +USE_COMPUTE_SANITIZER=true +if [[ "${GPU_COMPUTE_CAP}" == "12.0" ]]; then + USE_COMPUTE_SANITIZER=false + echo "Disabling compute-sanitizer for examples for sm_120 device" +fi + +run_example() { + local parent + parent=$(basename "${PWD}") + local cmd=("$@") + if ${USE_COMPUTE_SANITIZER}; then + cmd=(compute-sanitizer --tool memcheck "${cmd[@]}") + fi + echo "Running ${parent} example: ${cmd[*]}" + "${cmd[@]}" +} + pushd basic || exit -compute-sanitizer --tool memcheck basic_example +run_example ./basic_example popd || exit pushd hybrid_scan_io || exit -compute-sanitizer --tool memcheck hybrid_scan_io example.parquet string_col 0000001 PINNED_BUFFER -compute-sanitizer --tool memcheck hybrid_scan_pipeline example.parquet 2 HOST_BUFFER ROW_GROUPS 2 -compute-sanitizer --tool memcheck hybrid_scan_pipeline example.parquet 2 FILEPATH BYTE_RANGES 2 -compute-sanitizer --tool memcheck hybrid_scan_multifile_single_step example.parquet 10 2 YES DEVICE_BUFFER 2 -compute-sanitizer --tool memcheck hybrid_scan_multifile_single_step example.parquet 10 2 NO FILEPATH 1 -compute-sanitizer --tool memcheck hybrid_scan_multifile_two_step example.parquet 10 2 string_col 0000001 PINNED_BUFFER 2 -compute-sanitizer --tool memcheck hybrid_scan_multifile_two_step example.parquet 10 2 string_col 0000001 HOST_BUFFER 1 +run_example ./hybrid_scan_io example.parquet string_col 0000001 PINNED_BUFFER +run_example ./hybrid_scan_pipeline example.parquet 2 HOST_BUFFER ROW_GROUPS 2 +run_example ./hybrid_scan_pipeline example.parquet 2 FILEPATH BYTE_RANGES 2 +run_example ./hybrid_scan_multifile_single_step example.parquet 10 2 YES DEVICE_BUFFER 2 +run_example ./hybrid_scan_multifile_single_step example.parquet 10 2 NO FILEPATH 1 +run_example ./hybrid_scan_multifile_two_step example.parquet 10 2 string_col 0000001 PINNED_BUFFER 2 +run_example ./hybrid_scan_multifile_two_step example.parquet 10 2 string_col 0000001 HOST_BUFFER 1 popd || exit pushd nested_types || exit -compute-sanitizer --tool memcheck deduplication +run_example ./deduplication popd || exit pushd parquet_io || exit -compute-sanitizer --tool memcheck parquet_io example.parquet -compute-sanitizer --tool memcheck parquet_io example.parquet output.parquet DELTA_BINARY_PACKED ZSTD TRUE - -compute-sanitizer --tool memcheck parquet_io_multithreaded example.parquet -compute-sanitizer --tool memcheck parquet_io_multithreaded example.parquet 4 DEVICE_BUFFER 2 2 +run_example ./parquet_io example.parquet +run_example ./parquet_io example.parquet output.parquet DELTA_BINARY_PACKED ZSTD TRUE +run_example ./parquet_io_multithreaded example.parquet +run_example ./parquet_io_multithreaded example.parquet 4 DEVICE_BUFFER 2 2 popd || exit pushd parquet_inspect || exit -compute-sanitizer --tool memcheck parquet_inspect example.parquet +run_example ./parquet_inspect example.parquet popd || exit pushd strings || exit -compute-sanitizer --tool memcheck custom_optimized names.csv -compute-sanitizer --tool memcheck custom_prealloc names.csv -compute-sanitizer --tool memcheck custom_with_malloc names.csv +run_example ./custom_optimized names.csv +run_example ./custom_prealloc names.csv +run_example ./custom_with_malloc names.csv popd || exit pushd string_transformers || exit -compute-sanitizer --tool memcheck compute_checksum_jit info.csv output.csv -compute-sanitizer --tool memcheck extract_email_jit info.csv output.csv -compute-sanitizer --tool memcheck extract_email_precompiled info.csv output.csv -compute-sanitizer --tool memcheck format_phone_jit info.csv output.csv -compute-sanitizer --tool memcheck format_phone_precompiled info.csv output.csv -compute-sanitizer --tool memcheck localize_phone_jit info.csv output.csv -compute-sanitizer --tool memcheck localize_phone_precompiled info.csv output.csv +run_example ./compute_checksum_jit info.csv output.csv +run_example ./extract_email_jit info.csv output.csv +run_example ./extract_email_precompiled info.csv output.csv +run_example ./format_phone_jit info.csv output.csv +run_example ./format_phone_precompiled info.csv output.csv +run_example ./localize_phone_jit info.csv output.csv +run_example ./localize_phone_precompiled info.csv output.csv popd || exit exit ${EXITCODE}