[deps][ci] Creating depset for docs & updating pydantic doc ver (#57947)

creating a depset for doc builds
Pinning pydantic==2.50 for doc requirements due to api_policy_check
failing due to dependencies (failure below)
Installing ray without deps

Failure due to dependencies:
https://buildkite.com/ray-project/premerge/builds/52231#019a04a9-e9f8-4151-b5e5-d4bceb48a3cc

Successful api_policy_check run on this PR:
https://buildkite.com/ray-project/microcheck/builds/29312#019a05b8-ef40-4449-9ca2-6dd9d8e790a7

---------

Signed-off-by: elliot-barn <[email protected]>
Co-authored-by: Lonnie Liu <[email protected]>

Author: elliot-barn

.buildkite/base.rayci.yml

@@ -86,6 +86,7 @@ steps:
       - "3.12"
     env:
       PYTHON: "{{matrix}}"
+      REQUIREMENTS_FILE: "python/deplocks/docs/docbuild_depset_py{{matrix}}.lock"
     tags: cibase
 
   - name: docgpubuild

.buildkite/dependencies.rayci.yml

@@ -47,3 +47,12 @@ steps:
       - bazel run //ci/raydepsets:raydepsets -- build ci/raydepsets/configs/release_multimodal_inference_benchmarks_tests.depsets.yaml --check
     job_env: manylinux
     depends_on: manylinux
+
+  - label: ":tapioca: build: raydepsets: compile doc dependencies"
+    key: raydepsets_compile_doc_dependencies
+    tags: always
+    instance_type: medium
+    commands:
+      - bazel run //ci/raydepsets:raydepsets -- build ci/raydepsets/configs/docs.depsets.yaml --check
+    job_env: manylinux
+    depends_on: manylinux

.readthedocs.yaml

@@ -20,4 +20,4 @@ sphinx:
 # https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
 python:
   install:
-    - requirements: doc/requirements-doc.txt
+    - requirements: python/deplocks/docs/docbuild_depset_py3.12.lock

ci/docker/doc.build.Dockerfile

@@ -7,10 +7,10 @@ FROM $DOCKER_IMAGE_RAY_DASHBOARD AS ray_dashboard
 
 FROM $DOCKER_IMAGE_BASE_BUILD
 
-SHELL ["/bin/bash", "-ice"]
-
 COPY . .
 
+SHELL ["/bin/bash", "-ice"]
+
 RUN --mount=type=bind,from=ray_core,target=/mnt/ray-core \
     --mount=type=bind,from=ray_dashboard,target=/mnt/ray-dashboard \
     <<EOF
@@ -24,6 +24,6 @@ cp /mnt/ray-core/ray_pkg.zip /opt/ray-build/ray_pkg.zip
 cp /mnt/ray-core/ray_py_proto.zip /opt/ray-build/ray_py_proto.zip
 cp /mnt/ray-dashboard/dashboard.tar.gz /opt/ray-build/dashboard.tar.gz
 
-pip install -r doc/requirements-doc.txt
+pip install -r python/deplocks/docs/docbuild_depset_py${PYTHON}.lock
 
 EOF

ci/docker/doc.build.wanda.yaml

@@ -5,7 +5,7 @@ froms:
   - cr.ray.io/rayproject/ray-dashboard
 dockerfile: ci/docker/doc.build.Dockerfile
 srcs:
-  - doc/requirements-doc.txt
+  - $REQUIREMENTS_FILE
 build_args:
   - DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_build-py$PYTHON
   - DOCKER_IMAGE_RAY_CORE=cr.ray.io/rayproject/ray-core-py$PYTHON

ci/lint/lint.sh

@@ -104,21 +104,21 @@ test_coverage() {
   python ci/pipeline/check-test-run.py
 }
 
-_install_ray() {
+_install_ray_no_deps() {
   if [[ -d /opt/ray-build ]]; then
     unzip -o -q /opt/ray-build/ray_pkg.zip -d python
     unzip -o -q /opt/ray-build/ray_py_proto.zip -d python
     mkdir -p python/ray/dashboard/client/build
     tar -xzf /opt/ray-build/dashboard.tar.gz -C python/ray/dashboard/client/build
-    SKIP_BAZEL_BUILD=1 pip install -e "python[all]"
+    SKIP_BAZEL_BUILD=1 pip install -e "python[all]" --no-deps
   else
-    RAY_DISABLE_EXTRA_CPP=1 pip install -e "python[all]"
+    RAY_DISABLE_EXTRA_CPP=1 pip install -e "python[all]" --no-deps
   fi
 }
 
 api_annotations() {
   echo "--- Install Ray"
-  _install_ray
+  _install_ray_no_deps
 
   echo "--- Check API annotations"
   ./ci/lint/check_api_annotations.py
@@ -130,7 +130,7 @@ api_policy_check() {
   make -C doc/ html
 
   echo "--- Install Ray"
-  _install_ray
+  _install_ray_no_deps
 
   echo "--- Check API/doc consistency"
   bazel run //ci/ray_ci/doc:cmd_check_api_discrepancy -- /ray "$@"

ci/raydepsets/configs/docs.depsets.yaml

@@ -0,0 +1,28 @@
+build_arg_sets:
+  py39:
+    PYTHON_VERSION: "3.9"
+    PYTHON_SHORT: "39"
+  py310:
+    PYTHON_VERSION: "3.10"
+    PYTHON_SHORT: "310"
+  py312:
+    PYTHON_VERSION: "3.12"
+    PYTHON_SHORT: "312"
+
+
+depsets:
+  - name: docbuild_depset_${PYTHON_SHORT}
+    operation: compile
+    depsets:
+      - ray_img_depset_${PYTHON_SHORT}
+    requirements:
+      - doc/requirements-doc.txt
+    output: python/deplocks/docs/docbuild_depset_py${PYTHON_VERSION}.lock
+    append_flags:
+      - --python-version=${PYTHON_VERSION}
+      - --python-platform=linux
+      - --unsafe-package ray
+    build_arg_sets:
+      - py39
+      - py310
+      - py312

doc/requirements-doc.txt

@@ -22,12 +22,11 @@ autodoc_pydantic==2.2.0
 appnope
 sphinx-docsearch==0.0.7
 
-pydantic!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,<3
+pydantic==2.5.0
 
 # MyST
 myst-parser==2.0.0 # Needed to parse markdown
 myst-nb==1.0.0rc0 # Most recent version of myst-nb; pin when new release is made
-
 # Jupyter conversion
 jupytext==1.15.2