[data] Cleanup: Use SortKey instead of mixed typing for Aggregates

BUILD.bazel

@@ -620,6 +620,7 @@ ray_cc_library(
     deps = [
         ":reporter_rpc",
         ":stats_metric",
+        "//src/ray/util:size_literals",
         "@com_github_grpc_grpc//:grpc_opencensus_plugin",
     ],
 )

bazel/BUILD.jemalloc

@@ -15,7 +15,7 @@ configure_make(
     out_shared_libs = ["libjemalloc.so"],
     # See https://salsa.debian.org/debian/jemalloc/-/blob/c0a88c37a551be7d12e4863435365c9a6a51525f/debian/rules#L8-23
     # for why we are setting "--with-lg-page" on non x86 hardware here.
-    configure_options = ["--disable-static", "--enable-prof"] +
+    configure_options = ["--disable-static", "--enable-prof", "--enable-prof-libunwind"] +
         select({
             "@platforms//cpu:x86_64": [],
             "//conditions:default": ["--with-lg-page=16"],

bazel/ray_deps_setup.bzl

@@ -212,12 +212,14 @@ def ray_deps_setup():
 
     # OpenCensus depends on Abseil so we have to explicitly pull it in.
     # This is how diamond dependencies are prevented.
+    #
+    # TODO(owner): Upgrade abseil to latest version after protobuf updated, which requires to upgrade `rules_cc` first.
     auto_http_archive(
         name = "com_google_absl",
-        sha256 = "5366d7e7fa7ba0d915014d387b66d0d002c03236448e1ba9ef98122c13b35c36",
-        strip_prefix = "abseil-cpp-20230125.3",
+        sha256 = "987ce98f02eefbaf930d6e38ab16aa05737234d7afbab2d5c4ea7adbe50c28ed",
+        strip_prefix = "abseil-cpp-20230802.1",
         urls = [
-            "https://github.com/abseil/abseil-cpp/archive/20230125.3.tar.gz",
+            "https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.1.tar.gz",
         ],
     )
 

doc/source/data/api/dataset.rst

@@ -23,3 +23,15 @@ Developer API
   block.BlockExecStats
   block.BlockMetadata
   block.BlockAccessor
+
+Deprecated API
+--------------
+
+.. currentmodule:: ray.data
+
+.. autosummary::
+  :nosignatures:
+  :toctree: doc/
+
+  Dataset.iter_tf_batches
+  Dataset.to_torch

doc/source/ray-core/handling-dependencies.rst

@@ -405,6 +405,23 @@ The ``runtime_env`` is a Python dictionary or a Python class :class:`ray.runtime
   When specifying a path to a ``requirements.txt`` file, the file must be present on your local machine and it must be a valid absolute path or relative filepath relative to your local current working directory, *not* relative to the ``working_dir`` specified in the ``runtime_env``.
   Furthermore, referencing local files *within* a ``requirements.txt`` file isn't directly supported (e.g., ``-r ./my-laptop/more-requirements.txt``, ``./my-pkg.whl``). Instead, use the ``${RAY_RUNTIME_ENV_CREATE_WORKING_DIR}`` environment variable in the creation process. For example, use ``-r ${RAY_RUNTIME_ENV_CREATE_WORKING_DIR}/my-laptop/more-requirements.txt`` or ``${RAY_RUNTIME_ENV_CREATE_WORKING_DIR}/my-pkg.whl`` to reference local files, while ensuring they're in the ``working_dir``.
 
+- ``uv`` (dict | List[str] | str): Alpha version feature. Either (1) a list of uv `requirements specifiers <https://pip.pypa.io/en/stable/cli/pip_install/#requirement-specifiers>`_, (2) a string containing
+  the path to a local uv `“requirements.txt” <https://pip.pypa.io/en/stable/user_guide/#requirements-files>`_ file, or (3) a python dictionary that has three fields: (a) ``packages`` (required, List[str]): a list of uv packages,
+  (b) ``uv_version`` (optional, str): the version of uv; Ray will spell the package name "uv" in front of the ``uv_version`` to form the final requirement string.
+  The syntax of a requirement specifier is the same as ``pip`` requirements.
+  This will be installed in the Ray workers at runtime.  Packages in the preinstalled cluster environment will still be available.
+  To use a library like Ray Serve or Ray Tune, you will need to include ``"ray[serve]"`` or ``"ray[tune]"`` here.
+  The Ray version must match that of the cluster.
+
+  - Example: ``["requests==1.0.0", "aiohttp", "ray[serve]"]``
+
+  - Example: ``"./requirements.txt"``
+
+  - Example: ``{"packages":["tensorflow", "requests"], "uv_version": "==0.4.0;python_version=='3.8.11'"}``
+
+  When specifying a path to a ``requirements.txt`` file, the file must be present on your local machine and it must be a valid absolute path or relative filepath relative to your local current working directory, *not* relative to the ``working_dir`` specified in the ``runtime_env``.
+  Furthermore, referencing local files *within* a ``requirements.txt`` file isn't directly supported (e.g., ``-r ./my-laptop/more-requirements.txt``, ``./my-pkg.whl``). Instead, use the ``${RAY_RUNTIME_ENV_CREATE_WORKING_DIR}`` environment variable in the creation process. For example, use ``-r ${RAY_RUNTIME_ENV_CREATE_WORKING_DIR}/my-laptop/more-requirements.txt`` or ``${RAY_RUNTIME_ENV_CREATE_WORKING_DIR}/my-pkg.whl`` to reference local files, while ensuring they're in the ``working_dir``.
+
 - ``conda`` (dict | str): Either (1) a dict representing the conda environment YAML, (2) a string containing the path to a local
   `conda “environment.yml” <https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#create-env-file-manually>`_ file,
   or (3) the name of a local conda environment already installed on each node in your cluster (e.g., ``"pytorch_p36"``) or its absolute path (e.g. ``"/home/youruser/anaconda3/envs/pytorch_p36"``) .

doc/source/ray-overview/installation.rst

@@ -431,7 +431,7 @@ We publish the dependencies that are installed in our ``ray`` Docker images for
     .. tab-item:: ray (Python 3.9)
         :sync: ray (Python 3.9)
 
-        Ray version: nightly (`09cf5bb <https://github.com/ray-project/ray/commit/09cf5bba9e8f2e3af4336f0b2b76fb4727b09214>`_)
+        Ray version: nightly (`e393a71 <https://github.com/ray-project/ray/commit/e393a716d8742a987a36df555defb2ca90bb94d4>`_)
 
         .. literalinclude:: ./pip_freeze_ray-py39-cpu.txt
 

doc/source/ray-overview/pip_freeze_ray-ml-py39-cpu.txt

@@ -16,7 +16,7 @@ aioitertools==0.11.0
 aiorwlock==1.4.0
 aiosignal==1.3.1
 aiosqlite==0.19.0
-ale-py==0.8.1
+ale-py==0.10.1
 alembic==1.12.1
 anaconda-anon-usage @ file:///croot/anaconda-anon-usage_1710965072196/work
 annotated-types==0.6.0
@@ -34,16 +34,14 @@ asttokens==2.4.1
 astunparse==1.6.3
 async-timeout==4.0.3
 attrs==21.4.0
-AutoROM==0.6.1
-AutoROM.accept-rom-license==0.6.1
 ax-platform==0.3.2
 azure-cli-core==2.40.0
 azure-cli-telemetry==1.0.8
 azure-common==1.1.28
-azure-core==1.31.0
+azure-core==1.32.0
 azure-identity==1.10.0
 azure-mgmt-compute==23.1.0
-azure-mgmt-core==1.4.0
+azure-mgmt-core==1.5.0
 azure-mgmt-network==19.0.0
 azure-mgmt-resource==20.0.0
 Babel==2.13.1
@@ -62,13 +60,10 @@ botocore==1.29.76
 botorch==0.8.5
 Brotli @ file:///croot/brotli-split_1714483155106/work
 build==1.2.2.post1
-cached-property==1.5.2
 cachetools==5.3.2
 certifi==2023.11.17
 cffi==1.16.0
 charset-normalizer==3.3.2
-chess==1.7.0
-chex==0.1.7
 click==8.1.7
 cloudpickle==2.2.0
 cma==3.2.2
@@ -80,7 +75,7 @@ colorful==0.5.5
 colorlog==6.7.0
 comet-ml==3.44.1
 comm==0.2.0
-conda @ file:///home/conda/feedstock_root/build_artifacts/conda_1727884725164/work
+conda @ file:///home/conda/feedstock_root/build_artifacts/conda_1729155160301/work
 conda-content-trust @ file:///croot/conda-content-trust_1714483159009/work
 conda-libmamba-solver @ file:///croot/conda-libmamba-solver_1706733287605/work/src
 conda-package-handling @ file:///croot/conda-package-handling_1718138267740/work
@@ -113,7 +108,6 @@ dm-tree==0.1.8
 docker==6.1.3
 docker-pycreds==0.4.0
 docstring-parser==0.15
-dopamine-rl==4.0.5
 dulwich==0.21.6
 entrypoints==0.4
 etils==1.5.2
@@ -131,7 +125,6 @@ filelock==3.13.1
 Flask==2.1.3
 Flask-Cors==4.0.0
 flatbuffers==23.5.26
-flax==0.7.2
 fonttools==4.45.1
 fqdn==1.5.1
 frozendict @ file:///croot/frozendict_1713194832637/work
@@ -143,7 +136,6 @@ fugue-sql-antlr==0.2.0
 future==1.0.0
 gast==0.4.0
 gcs-oauth2-boto-plugin==3.0
-gin-config==0.5.0
 gitdb==4.0.11
 GitPython==3.1.40
 glfw==2.6.3
@@ -163,9 +155,7 @@ greenlet==3.0.1
 grpcio==1.66.2
 gsutil==5.27
 gunicorn==20.1.0
-gym==0.26.2
-gym-notices==0.0.8
-gymnasium==0.28.1
+gymnasium==1.0.0
 h11==0.12.0
 h5py==3.10.0
 higher==0.2.1
@@ -190,9 +180,6 @@ ipywidgets==8.1.3
 isodate==0.7.2
 isoduration==20.11.0
 itsdangerous==2.1.2
-jax==0.4.13
-jax-jumpy==1.0.0
-jaxlib==0.4.13
 jedi==0.19.1
 Jinja2==3.1.2
 jmespath==1.0.1
@@ -213,7 +200,6 @@ jupyterlab==3.6.1
 jupyterlab_pygments==0.3.0
 jupyterlab_server==2.24.0
 jupyterlab_widgets==3.0.11
-kaggle-environments==1.7.11
 keras==2.15.0
 kiwisolver==1.4.5
 knack==0.10.1
@@ -259,7 +245,7 @@ msgpack==1.0.7
 msgpack-numpy==0.4.8
 msrest==0.7.1
 msrestazure==0.6.4
-mujoco==2.3.6
+mujoco==3.2.4
 multidict==6.0.5
 multipledispatch==1.0.0
 multiprocess==0.70.15
@@ -299,16 +285,14 @@ opencensus-context==0.1.3
 opencv-python==4.8.1.78
 opentelemetry-api==1.1.0
 opentelemetry-exporter-otlp==1.1.0
-opentelemetry-exporter-otlp-proto-common==1.27.0
+opentelemetry-exporter-otlp-proto-common==1.28.1
 opentelemetry-exporter-otlp-proto-grpc==1.1.0
-opentelemetry-exporter-otlp-proto-http==1.27.0
+opentelemetry-exporter-otlp-proto-http==1.28.1
 opentelemetry-proto==1.1.0
 opentelemetry-sdk==1.1.0
 opentelemetry-semantic-conventions==0.20b0
 opt-einsum==3.3.0
-optax==0.1.7
 optuna==3.2.0
-orbax-checkpoint==0.2.3
 packaging==23.0
 pandas==1.5.3
 pandocfilters==1.5.0
@@ -317,7 +301,7 @@ paramz==0.9.6
 parso==0.8.3
 partd==1.4.1
 patsy==0.5.3
-pettingzoo==1.23.1
+pettingzoo==1.24.3
 pexpect==4.8.0
 pickleshare==0.7.5
 pillow==10.3.0
@@ -332,7 +316,7 @@ promise==2.3
 prompt-toolkit==3.0.41
 propcache==0.2.0
 prophet==1.1.5
-proto-plus==1.24.0
+proto-plus==1.25.0
 protobuf==3.20.3
 psutil==5.9.6
 ptyprocess==0.7.0
@@ -377,8 +361,7 @@ PyYAML==6.0.1
 pyzmq==26.0.3
 qpd==0.4.4
 querystring-parser==1.2.4
-ray @ file:///home/ray/ray-3.0.0.dev0-cp39-cp39-manylinux2014_x86_64.whl#sha256=48accb26a1a6dfaed7fc2cf3f97025523deace08c2e6975e7803d759caf879f7
-recsim==0.2.4
+ray @ file:///home/ray/ray-3.0.0.dev0-cp39-cp39-manylinux2014_x86_64.whl#sha256=94d614fd3aa3279dd580da1e3006a60200d5152f41e488556b74efd727d40de1
 redis==3.5.3
 referencing==0.35.1
 regex==2024.5.15
@@ -391,7 +374,7 @@ retry_decorator==1.1.1
 rfc3339-validator==0.1.4
 rfc3986-validator==0.1.1
 rich==13.3.2
-rpds-py==0.20.0
+rpds-py==0.21.0
 rsa==4.7.2
 ruamel.yaml==0.17.40
 ruamel.yaml.clib @ file:///croot/ruamel.yaml.clib_1727769819935/work
@@ -408,7 +391,7 @@ sentry-sdk==2.10.0
 serpent==1.41
 setproctitle==1.3.3
 shellingham==1.5.4
-Shimmy==1.3.0
+Shimmy==2.0.0
 shortuuid==1.0.1
 SimpleITK==2.3.1
 simplejson==3.19.2
@@ -426,7 +409,7 @@ stanio==0.3.0
 starlette==0.36.3
 statsforecast==1.7.0
 statsmodels==0.14.0
-SuperSuit==3.8.0
+SuperSuit==3.9.3
 sympy==1.13.1
 tabulate==0.9.0
 tblib==3.0.0
@@ -440,17 +423,15 @@ tensorflow-estimator==2.15.0
 tensorflow-io-gcs-filesystem==0.31.0
 tensorflow-metadata==1.14.0
 tensorflow-probability==0.23.0
-tensorstore==0.1.63
 termcolor==2.4.0
 terminado==0.18.1
-textual==0.83.0
-tf-slim==1.1.0
+textual==0.85.2
 tf2onnx==1.15.1
 threadpoolctl==3.1.0
 tifffile==2024.7.21
 timm==0.9.2
 tinycss2==1.3.0
-tinyscaler==1.2.6
+tinyscaler==1.2.8
 tokenizers==0.15.2
 toml==0.10.2
 tomli==2.0.1

doc/source/ray-overview/pip_freeze_ray-py39-cpu.txt

@@ -13,10 +13,10 @@ attrs==21.4.0
 azure-cli-core==2.40.0
 azure-cli-telemetry==1.0.8
 azure-common==1.1.28
-azure-core==1.31.0
+azure-core==1.32.0
 azure-identity==1.10.0
 azure-mgmt-compute==23.1.0
-azure-mgmt-core==1.4.0
+azure-mgmt-core==1.5.0
 azure-mgmt-network==19.0.0
 azure-mgmt-resource==20.0.0
 backoff==1.10.0
@@ -32,7 +32,7 @@ charset-normalizer==3.3.2
 click==8.1.7
 cloudpickle==2.2.0
 colorful==0.5.5
-conda @ file:///home/conda/feedstock_root/build_artifacts/conda_1727884725164/work
+conda @ file:///home/conda/feedstock_root/build_artifacts/conda_1729155160301/work
 conda-content-trust @ file:///croot/conda-content-trust_1714483159009/work
 conda-libmamba-solver @ file:///croot/conda-libmamba-solver_1706733287605/work/src
 conda-package-handling @ file:///croot/conda-package-handling_1718138267740/work
@@ -59,7 +59,7 @@ google-auth-httplib2==0.2.0
 google-oauth==1.0.1
 googleapis-common-protos==1.61.0
 grpcio==1.66.2
-gymnasium==0.28.1
+gymnasium==1.0.0
 h11==0.12.0
 httplib2==0.22.0
 httptools==0.6.4
@@ -68,7 +68,6 @@ idna @ file:///croot/idna_1714398848350/work
 imageio==2.34.2
 importlib-metadata==6.11.0
 isodate==0.7.2
-jax-jumpy==1.0.0
 Jinja2==3.1.2
 jmespath==1.0.1
 jsonpatch @ file:///croot/jsonpatch_1714483231291/work
@@ -130,7 +129,7 @@ python-dateutil==2.8.2
 python-dotenv==1.0.1
 pytz==2022.7.1
 PyYAML==6.0.1
-ray @ file:///home/ray/ray-3.0.0.dev0-cp39-cp39-manylinux2014_x86_64.whl#sha256=0ad223b423c13a8d8dc7e73a3592e5e9bcff73492bf15e2b13e29b68eae2bfc7
+ray @ file:///home/ray/ray-3.0.0.dev0-cp39-cp39-manylinux2014_x86_64.whl#sha256=51dff03e70d4cb811226c482302b7c73c6b9a9b73177f4d49e3c85d05b555803
 redis==3.5.3
 requests==2.31.0
 requests-oauthlib==2.0.0