Skip to content

[ci] updating data ci tests to py310 #58378

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Nov 4, 2025
Merged

[ci] updating data ci tests to py310 #58378

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1574e40
updating data ci tests to py310
elliot-barn Nov 3, 2025
d5b7297
enabling ml-mltipy for 3.10
elliot-barn Nov 3, 2025
c1c3d1c
Merge branch 'master' into elliot-barn/upgrading-data-ci-tests-py310
aslonnie Nov 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .buildkite/base.rayci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ steps:
label: "wanda: oss-ci-base_ml-py{{matrix}}"
wanda: ci/docker/base.ml.wanda.yaml
matrix:
- "3.10"
- "3.11"
- "3.12"
env:
Expand Down
77 changes: 52 additions & 25 deletions .buildkite/data.rayci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,38 +1,56 @@
group: data tests
depends_on:
- forge
- oss-ci-base_ml
- oss-ci-base_ml-multipy
- ray-core-build
- ray-dashboard-build
steps:
# builds
- name: data9build
- name: data9build-multipy
label: "wanda: data9build-py{{matrix}}"
wanda: ci/docker/data9.build.wanda.yaml
matrix:
- "3.10"
env:
PYTHON: "{{matrix}}"
tags: cibase

- name: datalbuild
- name: datalbuild-multipy
label: "wanda: datalbuild-py{{matrix}}"
wanda: ci/docker/datal.build.wanda.yaml
tags: cibase
matrix:
- "3.10"
env:
PYTHON: "{{matrix}}"

- name: databuild-multipy
label: "wanda: databuild-py{{matrix}}"
wanda: ci/docker/data.build.wanda.yaml
matrix: ["3.12"]
matrix: ["3.10", "3.12"]
env:
PYTHON: "{{matrix}}"
depends_on: oss-ci-base_ml-multipy
tags: cibase

- name: datanbuild
wanda: ci/docker/datan.build.wanda.yaml
tags: cibase

- name: datamongobuild
- name: datamongobuild-multipy
label: "wanda: datamongobuild-py{{matrix}}"
wanda: ci/docker/datamongo.build.wanda.yaml
matrix:
- "3.10"
env:
PYTHON: "{{matrix}}"
tags: cibase

- name: datatfxbslbuild
- name: datatfxbslbuild-multipy
label: "wanda: datatfxbslbuild-py{{matrix}}"
wanda: ci/docker/datatfxbsl.build.wanda.yaml
matrix:
- "3.10"
env:
PYTHON: "{{matrix}}"
tags: cibase

# tests
Expand Down Expand Up @@ -73,9 +91,10 @@ steps:
- bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... //python/ray/air/... data
--workers "$${BUILDKITE_PARALLEL_JOB_COUNT}"
--worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
--build-name datalbuild
--build-name databuild-py3.10
--python-version 3.10
--except-tags data_integration,doctest,data_non_parallel,dask,needs_credentials
depends_on: datalbuild
depends_on: datalbuild-multipy

- label: ":database: data: arrow v21 tests (data_non_parallel)"
tags:
Expand All @@ -88,9 +107,10 @@ steps:
- bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... //python/ray/air/... data
--workers "$${BUILDKITE_PARALLEL_JOB_COUNT}"
--worker-id "$${BUILDKITE_PARALLEL_JOB}"
--build-name datalbuild
--build-name databuild-py3.10
--python-version 3.10
--only-tags data_non_parallel
depends_on: datalbuild
depends_on: datalbuild-multipy

- label: ":database: data: arrow v21 {{matrix.python}} tests ({{matrix.worker_id}})"
key: datal_python_tests
Expand Down Expand Up @@ -171,9 +191,10 @@ steps:
instance_type: medium
commands:
- bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... data
--build-name datatfxbslbuild
--build-name datatfxbslbuild-py3.10
--python-version 3.10
--only-tags tfxbsl
depends_on: datatfxbslbuild
depends_on: datatfxbslbuild-multipy

- label: ":database: data: doc tests"
tags:
Expand All @@ -183,17 +204,19 @@ steps:
commands:
# doc tests
- bazel run //ci/ray_ci:test_in_docker -- python/ray/... //doc/... data
--build-name datalbuild
--build-name databuild-py3.10
--python-version 3.10
--except-tags gpu
--only-tags doctest
--parallelism-per-worker 2
# doc examples
- bazel run //ci/ray_ci:test_in_docker -- //doc/... data
--build-name datalbuild
--build-name databuild-py3.10
--python-version 3.10
--except-tags gpu,post_wheel_build,doctest,dask
--parallelism-per-worker 2
--skip-ray-installation
depends_on: datalbuild
depends_on: datalbuild-multipy

- label: ":database: data: dask doc tests"
tags:
Expand Down Expand Up @@ -237,11 +260,12 @@ steps:
instance_type: medium
commands:
- bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... data
--build-name datamongobuild
--build-name datamongobuild-py3.10
--python-version 3.10
--build-type java
--only-tags data_integration
--except-tags doctest
depends_on: datamongobuild
depends_on: datamongobuild-multipy

- label: ":database: data: dashboard tests"
tags:
Expand All @@ -251,9 +275,10 @@ steps:
instance_type: small
commands:
- bazel run //ci/ray_ci:test_in_docker -- python/ray/dashboard/... data
--build-name datalbuild
--build-name databuild-py3.10
--python-version 3.10
--parallelism-per-worker 3
depends_on: datalbuild
depends_on: datalbuild-multipy

- label: ":database: data: flaky tests"
key: data_flaky_tests
Expand All @@ -267,9 +292,10 @@ steps:
commands:
- bazel run //ci/ray_ci:test_in_docker -- //... data --run-flaky-tests
--parallelism-per-worker 3
--build-name datalbuild
--build-name databuild-py3.10
--python-version 3.10
--except-tags gpu_only,gpu
depends_on: datalbuild
depends_on: datalbuild-multipy

- label: ":database: data: flaky gpu tests"
key: data_flaky_gpu_tests
Expand All @@ -296,7 +322,8 @@ steps:
commands:
- $(python ci/env/setup_credentials.py)
- bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... data
--build-name datalbuild
--build-name databuild-py3.10
--python-version 3.10
--only-tags needs_credentials
--test-env=SNOWFLAKE_USER --test-env=SNOWFLAKE_ACCOUNT --test-env=SNOWFLAKE_DATABASE --test-env=SNOWFLAKE_SCHEMA --test-env=SNOWFLAKE_WAREHOUSE --test-env=SNOWFLAKE_PRIVATE_KEY
depends_on: datalbuild
depends_on: datalbuild-multipy
2 changes: 1 addition & 1 deletion ci/docker/data-tfxbsl.build.Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# syntax=docker/dockerfile:1.3-labs

ARG DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_ml
ARG DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_ml-py3.10
FROM $DOCKER_IMAGE_BASE_BUILD

ARG ARROW_VERSION=14.*
Expand Down
2 changes: 1 addition & 1 deletion ci/docker/data.build.Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# syntax=docker/dockerfile:1.3-labs

ARG DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_ml
ARG DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_ml-py3.10
Copy link

@cursor cursor bot Nov 3, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Bug: Hardcoded Python base image causes version mismatch

The data.build.Dockerfile hardcodes its default base image to Python 3.10. The data9.build.wanda.yaml uses a parameterized base image (-py$PYTHON) but doesn't pass DOCKER_IMAGE_BASE_BUILD as a build argument to override this default. Consequently, databuild-multipy jobs for Python versions like 3.12 will incorrectly use the Python 3.10 base image.

Additional Locations (1)

Fix in Cursor Fix in Web

FROM $DOCKER_IMAGE_BASE_BUILD

ARG ARROW_VERSION=20.*
Expand Down
2 changes: 1 addition & 1 deletion ci/docker/data9.build.wanda.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
name: "data9build"
froms: ["cr.ray.io/rayproject/oss-ci-base_ml"]
froms: ["cr.ray.io/rayproject/oss-ci-base_ml-py$PYTHON"]
dockerfile: ci/docker/data.build.Dockerfile
srcs:
- ci/env/install-dependencies.sh
Expand Down
6 changes: 3 additions & 3 deletions ci/docker/datal.build.wanda.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
name: "datalbuild"
froms: ["cr.ray.io/rayproject/oss-ci-base_ml"]
name: "databuild-py$PYTHON"
froms: ["cr.ray.io/rayproject/oss-ci-base_ml-py$PYTHON"]
dockerfile: ci/docker/data.build.Dockerfile
srcs:
- ci/env/install-dependencies.sh
Expand All @@ -12,4 +12,4 @@ srcs:
build_args:
- ARROW_VERSION=21.*
tags:
- cr.ray.io/rayproject/datalbuild
- cr.ray.io/rayproject/databuild-py$PYTHON
7 changes: 4 additions & 3 deletions ci/docker/datamongo.build.wanda.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
name: "datamongobuild"
froms: ["cr.ray.io/rayproject/oss-ci-base_ml"]
name: "datamongobuild-py$PYTHON"
froms: ["cr.ray.io/rayproject/oss-ci-base_ml-py$PYTHON"]
dockerfile: ci/docker/data.build.Dockerfile
srcs:
- ci/env/install-dependencies.sh
Expand All @@ -13,5 +13,6 @@ build_args:
- ARROW_VERSION=9.*
- ARROW_MONGO_VERSION=0.5.*
- RAY_CI_JAVA_BUILD=1
- DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_ml-py$PYTHON
tags:
- cr.ray.io/rayproject/datamongobuild
- cr.ray.io/rayproject/datamongobuild-py$PYTHON
6 changes: 3 additions & 3 deletions ci/docker/datatfxbsl.build.wanda.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
name: "datatfxbslbuild"
froms: ["cr.ray.io/rayproject/oss-ci-base_ml"]
name: "datatfxbslbuild-py$PYTHON"
froms: ["cr.ray.io/rayproject/oss-ci-base_ml-py$PYTHON"]
dockerfile: ci/docker/data-tfxbsl.build.Dockerfile
srcs:
- ci/env/install-dependencies.sh
Expand All @@ -11,4 +11,4 @@ srcs:
build_args:
- ARROW_VERSION=14.*
tags:
- cr.ray.io/rayproject/datatfxbslbuild
- cr.ray.io/rayproject/datatfxbslbuild-py$PYTHON