[images][deps] raydepsets base extra depset (#58461)

generating depsets for base extra python requirements
Installing requirements in base extra image

---------

Signed-off-by: elliot-barn <[email protected]>

Author: elliot-barn

.buildkite/_images.rayci.yml

@@ -28,6 +28,7 @@ steps:
       PYTHON_VERSION: "{{matrix}}"
       IMAGE_TYPE: "ray"
       ARCH_SUFFIX: ""
+      REQUIREMENTS_FILE: "ray_base_extra_py{{matrix}}.lock"
     depends_on: raycpubase
 
   - name: raycudabase
@@ -81,6 +82,7 @@ steps:
       CUDA_VERSION: "{{matrix.cuda}}"
       IMAGE_TYPE: "ray"
       ARCH_SUFFIX: ""
+      REQUIREMENTS_FILE: "ray_base_extra_py{{matrix.python}}.lock"
     depends_on: raycudabase
 
   - name: ray-llmbase
@@ -114,6 +116,7 @@ steps:
       CUDA_VERSION: "{{matrix.cuda}}"
       IMAGE_TYPE: "ray-llm"
       ARCH_SUFFIX: ""
+      REQUIREMENTS_FILE: "ray_base_extra_py{{matrix.python}}.lock"
     depends_on: ray-llmbase
 
   - name: ray-mlcpubase
@@ -141,6 +144,7 @@ steps:
       PYTHON_VERSION: "{{matrix}}"
       IMAGE_TYPE: "ray-ml"
       ARCH_SUFFIX: ""
+      REQUIREMENTS_FILE: "ray_base_extra_py{{matrix}}.lock"
     depends_on: ray-mlcpubase
 
   - name: ray-mlcudabase
@@ -178,6 +182,7 @@ steps:
       CUDA_VERSION: "{{matrix.cuda}}"
       IMAGE_TYPE: "ray-ml"
       ARCH_SUFFIX: ""
+      REQUIREMENTS_FILE: "ray_base_extra_py{{matrix.python}}.lock"
     depends_on: ray-mlcudabase
 
   - name: ray-slimcpubase

.buildkite/linux_aarch64.rayci.yml

@@ -54,6 +54,7 @@ steps:
       PYTHON_VERSION: "{{matrix}}"
       IMAGE_TYPE: "ray"
       ARCH_SUFFIX: "-aarch64"
+      REQUIREMENTS_FILE: "ray_base_extra_py{{matrix}}.lock"
     depends_on: raycpubase-aarch64
 
   - name: raycudabase-aarch64
@@ -109,6 +110,7 @@ steps:
       CUDA_VERSION: "{{matrix.cuda}}"
       IMAGE_TYPE: "ray"
       ARCH_SUFFIX: "-aarch64"
+      REQUIREMENTS_FILE: "ray_base_extra_py{{matrix.python}}.lock"
     depends_on: raycudabase-aarch64
 
   - name: ray-core-build-aarch64

ci/raydepsets/configs/rayimg.depsets.yaml

@@ -134,3 +134,22 @@ depsets:
     build_arg_sets:
       - py39
       - py310
+
+    # Second layer of the ray release images. Contains anyscale cli deps (and others)
+    # and is a subset of the base extra test deps defined above (release image depset)
+  - name: ray_base_extra_${PYTHON_SHORT}
+    operation: subset
+    source_depset: ray_base_extra_testdeps_${PYTHON_SHORT}
+    requirements:
+      - docker/base-deps/requirements.in
+      - docker/base-extra/requirements.in
+    output: python/deplocks/base_extra/ray_base_extra_py${PYTHON_VERSION}.lock
+    append_flags:
+      - --unsafe-package ray
+      - --python-version=${PYTHON_VERSION}
+      - --python-platform=linux
+    build_arg_sets:
+      - py39
+      - py310
+      - py311
+      - py312

docker/base-extra/Dockerfile

@@ -8,6 +8,10 @@ ENV TERM=xterm
 
 ARG SSH_PORT=5020
 
+ARG PYTHON_DEPSET
+
+COPY "$PYTHON_DEPSET" /home/ray/python_depset.lock
+
 RUN <<EOF
 #!/bin/bash
 
@@ -88,29 +92,10 @@ if [[ "$ARCH" == "x86_64" ]]; then
     rm -rf "${DYNOLOG_TMP}"
 fi
 
-# Python dependencies to install. To specify a version, please make the change
-# in OSS ray repository, but not here.
-PYTHON_REQUIREMENTS=(
-    azure-identity
-    jupyterlab
-    ipywidgets
-    grpcio
-    grpcio-tools
-
-    # Pinning jupyter_server_terminals==0.4.4 , the higher version will break the
-    # webterminal when using an older version of terminado.
-    jupyter_server_terminals==0.4.4
-
-    # [backend] is for installing anyscale CLI for use in the anyscale cloud.
-    "anyscale[backend]"
-)
-
-
 PYTHON_VERSION="$(python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')"
 
-uv pip install --system --no-cache-dir --index-strategy unsafe-best-match \
-    -c /home/ray/requirements_compiled.txt \
-    "${PYTHON_REQUIREMENTS[@]}"
+uv pip install --system --no-cache-dir --no-deps --index-strategy unsafe-best-match \
+    -r $HOME/python_depset.lock
 
 # Install awscli v2
 AWSCLI_TMP="$(mktemp -d)"

docker/base-extra/cpu.wanda.yaml

@@ -1,7 +1,10 @@
 name: "$IMAGE_TYPE-py$PYTHON_VERSION-cpu-base-extra$ARCH_SUFFIX"
 froms: ["cr.ray.io/rayproject/$IMAGE_TYPE-py$PYTHON_VERSION-cpu-base$ARCH_SUFFIX"]
 dockerfile: docker/base-extra/Dockerfile
+srcs:
+  - python/deplocks/base_extra/$REQUIREMENTS_FILE
 build_args:
   - BASE_IMAGE=cr.ray.io/rayproject/$IMAGE_TYPE-py$PYTHON_VERSION-cpu-base$ARCH_SUFFIX
+  - PYTHON_DEPSET=python/deplocks/base_extra/$REQUIREMENTS_FILE
 tags:
   - cr.ray.io/rayproject/$IMAGE_TYPE-py$PYTHON_VERSION-cpu-base-extra$ARCH_SUFFIX

docker/base-extra/cuda.wanda.yaml

@@ -1,7 +1,10 @@
 name: "$IMAGE_TYPE-py$PYTHON_VERSION-cu$CUDA_VERSION-base-extra$ARCH_SUFFIX"
 froms: ["cr.ray.io/rayproject/$IMAGE_TYPE-py$PYTHON_VERSION-cu$CUDA_VERSION-base$ARCH_SUFFIX"]
 dockerfile: docker/base-extra/Dockerfile
+srcs:
+  - python/deplocks/base_extra/$REQUIREMENTS_FILE
 build_args:
   - BASE_IMAGE=cr.ray.io/rayproject/$IMAGE_TYPE-py$PYTHON_VERSION-cu$CUDA_VERSION-base$ARCH_SUFFIX
+  - PYTHON_DEPSET=python/deplocks/base_extra/$REQUIREMENTS_FILE
 tags:
   - cr.ray.io/rayproject/$IMAGE_TYPE-py$PYTHON_VERSION-cu$CUDA_VERSION-base-extra$ARCH_SUFFIX