Skip to content

Fix RTX PRO 6000 Blackwell CI #21999

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rapids-bot merged 6 commits into from
Apr 7, 2026
Merged

Fix RTX PRO 6000 Blackwell CI #21999

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
c7b801a
Revert "Dont allow rtxpro6000 runners to pick up CI jobs (#21954)"
bdice Apr 2, 2026
24788a4
disable compute-sanitizer for hybrid_scan_io examples
davidwendt Apr 6, 2026
46bb54a
fix examples script
davidwendt Apr 6, 2026
dba9051
add comment about nvbug workaround
davidwendt Apr 6, 2026
bb9c988
Merge branch 'main' into enable-blackwell
davidwendt Apr 6, 2026
4254f63
Refactor call site to print example name/info
bdice Apr 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 3 additions & 14 deletions .github/workflows/pr.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -346,8 +346,6 @@ jobs:
with:
build_type: pull-request
script: ci/test_cpp.sh
# Skip tests on RTX PRO 6000 (Blackwell). xref: https://github.com/rapidsai/cudf/issues/21953
matrix_filter: map(select(.GPU != "rtxpro6000"))
conda-python-build:
needs: conda-cpp-build
secrets: inherit
Expand All @@ -373,8 +371,6 @@ jobs:
with:
build_type: pull-request
script: "ci/test_python_cudf.sh"
# Skip tests on RTX PRO 6000 (Blackwell). xref: https://github.com/rapidsai/cudf/issues/21953
matrix_filter: map(select(.GPU != "rtxpro6000"))
conda-python-other-tests:
# Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism
needs: [conda-python-build-noarch, changed-files]
Expand All @@ -384,8 +380,6 @@ jobs:
with:
build_type: pull-request
script: "ci/test_python_other.sh"
# Skip tests on RTX PRO 6000 (Blackwell). xref: https://github.com/rapidsai/cudf/issues/21953
matrix_filter: map(select(.GPU != "rtxpro6000"))
conda-java-tests:
needs: [conda-cpp-build, changed-files]
secrets: inherit
Expand Down Expand Up @@ -463,8 +457,6 @@ jobs:
with:
build_type: pull-request
script: ci/test_wheel_cudf.sh
# Skip tests on RTX PRO 6000 (Blackwell). xref: https://github.com/rapidsai/cudf/issues/21953
matrix_filter: map(select(.GPU != "rtxpro6000"))
wheel-build-cudf-polars:
needs: wheel-build-pylibcudf
secrets: inherit
Expand Down Expand Up @@ -496,8 +488,7 @@ jobs:
with:
# This selects "ARCH=amd64 + the latest supported Python + CUDA" to minimize CI usage.
# (rapidsmpf compatibility already validated in rapidsmpf CI)
# Skip tests on RTX PRO 6000 (Blackwell). xref: https://github.com/rapidsai/cudf/issues/21953
matrix_filter: map(select(.GPU != "rtxpro6000")) | map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]))
matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]))
build_type: pull-request
script: "ci/test_cudf_polars_experimental.sh"
cudf-polars-polars-tests:
Expand All @@ -507,8 +498,7 @@ jobs:
if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_wheels && fromJSON(needs.changed-files.outputs.changed_file_groups).neither_cudf_nor_dask_cudf
with:
# This selects "ARCH=amd64 + the latest supported Python + CUDA".
# Skip tests on RTX PRO 6000 (Blackwell). xref: https://github.com/rapidsai/cudf/issues/21953
matrix_filter: map(select(.GPU != "rtxpro6000")) | map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]))
matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]))
build_type: pull-request
script: "ci/test_cudf_polars_polars_tests.sh"
wheel-build-dask-cudf:
Expand All @@ -531,8 +521,7 @@ jobs:
if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_wheels && fromJSON(needs.changed-files.outputs.changed_file_groups).not_cudf_polars
with:
# This selects "ARCH=amd64 + the latest supported Python + CUDA".
# Skip tests on RTX PRO 6000 (Blackwell). xref: https://github.com/rapidsai/cudf/issues/21953
matrix_filter: map(select(.GPU != "rtxpro6000")) | map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]))
matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]))
build_type: pull-request
script: ci/test_wheel_dask_cudf.sh
devcontainer:
Expand Down
15 changes: 1 addition & 14 deletions .github/workflows/test.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,6 @@ jobs:
date: ${{ inputs.date }}
script: ci/test_cpp.sh
sha: ${{ inputs.sha }}
# Skip tests on RTX PRO 6000 (Blackwell). xref: https://github.com/rapidsai/cudf/issues/21953
matrix_filter: map(select(.GPU != "rtxpro6000"))
conda-cpp-memcheck-tests:
secrets: inherit
uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main
Expand Down Expand Up @@ -72,8 +70,6 @@ jobs:
date: ${{ inputs.date }}
sha: ${{ inputs.sha }}
script: "ci/test_python_cudf.sh"
# Skip tests on RTX PRO 6000 (Blackwell). xref: https://github.com/rapidsai/cudf/issues/21953
matrix_filter: map(select(.GPU != "rtxpro6000"))
conda-python-other-tests:
# Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism
secrets: inherit
Expand All @@ -84,8 +80,6 @@ jobs:
date: ${{ inputs.date }}
sha: ${{ inputs.sha }}
script: "ci/test_python_other.sh"
# Skip tests on RTX PRO 6000 (Blackwell). xref: https://github.com/rapidsai/cudf/issues/21953
matrix_filter: map(select(.GPU != "rtxpro6000"))
conda-java-tests:
secrets: inherit
uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main
Expand Down Expand Up @@ -119,8 +113,6 @@ jobs:
date: ${{ inputs.date }}
sha: ${{ inputs.sha }}
script: ci/test_wheel_cudf.sh
# Skip tests on RTX PRO 6000 (Blackwell). xref: https://github.com/rapidsai/cudf/issues/21953
matrix_filter: map(select(.GPU != "rtxpro6000"))
wheel-tests-dask-cudf:
secrets: inherit
uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@main
Expand All @@ -130,8 +122,6 @@ jobs:
date: ${{ inputs.date }}
sha: ${{ inputs.sha }}
script: ci/test_wheel_dask_cudf.sh
# Skip tests on RTX PRO 6000 (Blackwell). xref: https://github.com/rapidsai/cudf/issues/21953
matrix_filter: map(select(.GPU != "rtxpro6000"))
unit-tests-cudf-pandas:
secrets: inherit
uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@main
Expand Down Expand Up @@ -169,8 +159,7 @@ jobs:
with:
# This selects "ARCH=amd64 + the latest supported Python + CUDA" to minimize CI usage.
# (rapidsmpf compatibility already validated in rapidsmpf CI)
# Skip tests on RTX PRO 6000 (Blackwell). xref: https://github.com/rapidsai/cudf/issues/21953
matrix_filter: map(select(.GPU != "rtxpro6000")) | map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]))
matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]))
build_type: ${{ inputs.build_type }}
branch: ${{ inputs.branch }}
date: ${{ inputs.date }}
Expand All @@ -186,8 +175,6 @@ jobs:
date: ${{ inputs.date }}
sha: ${{ inputs.sha }}
script: "ci/test_cudf_polars_polars_tests.sh"
# Skip tests on RTX PRO 6000 (Blackwell). xref: https://github.com/rapidsai/cudf/issues/21953
matrix_filter: map(select(.GPU != "rtxpro6000"))
narwhals-tests:
secrets: inherit
uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main
Expand Down
68 changes: 43 additions & 25 deletions ci/run_cudf_examples.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,50 +10,68 @@ trap "EXITCODE=1" ERR
# Support customizing the examples' install location
cd "${INSTALL_PREFIX:-${CONDA_PREFIX:-/usr}}/bin/examples/libcudf" || exit

# TODO: Temporary workaround for compute-sanitizer bug 5824899 that occurs only on the examples
GPU_COMPUTE_CAP=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -n1 | tr -d '[:space:]')
USE_COMPUTE_SANITIZER=true
if [[ "${GPU_COMPUTE_CAP}" == "12.0" ]]; then
USE_COMPUTE_SANITIZER=false
echo "Disabling compute-sanitizer for examples for sm_120 device"
fi

run_example() {
local parent
parent=$(basename "${PWD}")
local cmd=("$@")
if ${USE_COMPUTE_SANITIZER}; then
cmd=(compute-sanitizer --tool memcheck "${cmd[@]}")
fi
echo "Running ${parent} example: ${cmd[*]}"
"${cmd[@]}"
}

pushd basic || exit
compute-sanitizer --tool memcheck basic_example
run_example ./basic_example
popd || exit

pushd hybrid_scan_io || exit
compute-sanitizer --tool memcheck hybrid_scan_io example.parquet string_col 0000001 PINNED_BUFFER
compute-sanitizer --tool memcheck hybrid_scan_pipeline example.parquet 2 HOST_BUFFER ROW_GROUPS 2
compute-sanitizer --tool memcheck hybrid_scan_pipeline example.parquet 2 FILEPATH BYTE_RANGES 2
compute-sanitizer --tool memcheck hybrid_scan_multifile_single_step example.parquet 10 2 YES DEVICE_BUFFER 2
compute-sanitizer --tool memcheck hybrid_scan_multifile_single_step example.parquet 10 2 NO FILEPATH 1
compute-sanitizer --tool memcheck hybrid_scan_multifile_two_step example.parquet 10 2 string_col 0000001 PINNED_BUFFER 2
compute-sanitizer --tool memcheck hybrid_scan_multifile_two_step example.parquet 10 2 string_col 0000001 HOST_BUFFER 1
run_example ./hybrid_scan_io example.parquet string_col 0000001 PINNED_BUFFER
run_example ./hybrid_scan_pipeline example.parquet 2 HOST_BUFFER ROW_GROUPS 2
run_example ./hybrid_scan_pipeline example.parquet 2 FILEPATH BYTE_RANGES 2
run_example ./hybrid_scan_multifile_single_step example.parquet 10 2 YES DEVICE_BUFFER 2
run_example ./hybrid_scan_multifile_single_step example.parquet 10 2 NO FILEPATH 1
run_example ./hybrid_scan_multifile_two_step example.parquet 10 2 string_col 0000001 PINNED_BUFFER 2
run_example ./hybrid_scan_multifile_two_step example.parquet 10 2 string_col 0000001 HOST_BUFFER 1
popd || exit

pushd nested_types || exit
compute-sanitizer --tool memcheck deduplication
run_example ./deduplication
popd || exit

pushd parquet_io || exit
compute-sanitizer --tool memcheck parquet_io example.parquet
compute-sanitizer --tool memcheck parquet_io example.parquet output.parquet DELTA_BINARY_PACKED ZSTD TRUE

compute-sanitizer --tool memcheck parquet_io_multithreaded example.parquet
compute-sanitizer --tool memcheck parquet_io_multithreaded example.parquet 4 DEVICE_BUFFER 2 2
run_example ./parquet_io example.parquet
run_example ./parquet_io example.parquet output.parquet DELTA_BINARY_PACKED ZSTD TRUE
run_example ./parquet_io_multithreaded example.parquet
run_example ./parquet_io_multithreaded example.parquet 4 DEVICE_BUFFER 2 2
popd || exit

pushd parquet_inspect || exit
compute-sanitizer --tool memcheck parquet_inspect example.parquet
run_example ./parquet_inspect example.parquet
popd || exit

pushd strings || exit
compute-sanitizer --tool memcheck custom_optimized names.csv
compute-sanitizer --tool memcheck custom_prealloc names.csv
compute-sanitizer --tool memcheck custom_with_malloc names.csv
run_example ./custom_optimized names.csv
run_example ./custom_prealloc names.csv
run_example ./custom_with_malloc names.csv
popd || exit

pushd string_transformers || exit
compute-sanitizer --tool memcheck compute_checksum_jit info.csv output.csv
compute-sanitizer --tool memcheck extract_email_jit info.csv output.csv
compute-sanitizer --tool memcheck extract_email_precompiled info.csv output.csv
compute-sanitizer --tool memcheck format_phone_jit info.csv output.csv
compute-sanitizer --tool memcheck format_phone_precompiled info.csv output.csv
compute-sanitizer --tool memcheck localize_phone_jit info.csv output.csv
compute-sanitizer --tool memcheck localize_phone_precompiled info.csv output.csv
run_example ./compute_checksum_jit info.csv output.csv
run_example ./extract_email_jit info.csv output.csv
run_example ./extract_email_precompiled info.csv output.csv
run_example ./format_phone_jit info.csv output.csv
run_example ./format_phone_precompiled info.csv output.csv
run_example ./localize_phone_jit info.csv output.csv
run_example ./localize_phone_precompiled info.csv output.csv
popd || exit

exit ${EXITCODE}
Loading