Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions python/ray/_private/runtime_env/uv.py
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,20 @@ async def _check_uv_existence(
except Exception:
return False

async def _uv_check(sef, python: str, cwd: str, logger: logging.Logger) -> None:
"""Check virtual env dependency compatibility.
If any incompatibility detected, exception will be thrown.

param:
python: the path for python executable within virtual environment.
"""
cmd = [python, "-m", "uv", "pip", "check"]
await check_output_cmd(
cmd,
logger=logger,
cwd=cwd,
)

async def _install_uv_packages(
self,
path: str,
Expand Down Expand Up @@ -172,6 +186,10 @@ async def _install_uv_packages(
logger.info("Installing python requirements to %s", virtualenv_path)
await check_output_cmd(pip_install_cmd, logger=logger, cwd=cwd, env=pip_env)

# Check python environment for conflicts.
if self._uv_config.get("uv_check", False):
await self._uv_check(python, cwd, logger)

async def _run(self):
path = self._target_dir
logger = self._logger
Expand Down
20 changes: 12 additions & 8 deletions python/ray/_private/runtime_env/validation.py
Original file line number Diff line number Diff line change
Expand Up @@ -115,10 +115,6 @@ def parse_and_validate_conda(conda: Union[str, dict]) -> Union[str, dict]:
return result


# TODO(hjiang): More package installation options to implement:
# 1. `pip_check` has different semantics for `uv` and `pip`, see
# https://github.com/astral-sh/uv/pull/2544/files, consider whether we need to support
# it; or simply ignore the field when people come from `pip`.
def parse_and_validate_uv(uv: Union[str, List[str], Dict]) -> Optional[Dict]:
"""Parses and validates a user-provided 'uv' option.

Expand All @@ -128,6 +124,8 @@ def parse_and_validate_uv(uv: Union[str, List[str], Dict]) -> Optional[Dict]:
2) a string containing the path to a local pip “requirements.txt” file.
3) A python dictionary that has one field:
a) packages (required, List[str]): a list of uv packages, it same as 1).
b) uv_check (optional, bool): whether to enable pip check at the end of uv
install, default to False.

The returned parsed value will be a list of packages. If a Ray library
(e.g. "ray[serve]") is specified, it will be deleted and replaced by its
Expand All @@ -144,27 +142,33 @@ def parse_and_validate_uv(uv: Union[str, List[str], Dict]) -> Optional[Dict]:
result: str = ""
if isinstance(uv, str):
uv_list = _handle_local_deps_requirement_file(uv)
result = dict(packages=uv_list)
result = dict(packages=uv_list, uv_check=False)
elif isinstance(uv, list) and all(isinstance(dep, str) for dep in uv):
result = dict(packages=uv)
result = dict(packages=uv, uv_check=False)
elif isinstance(uv, dict):
if set(uv.keys()) - {"packages", "uv_version"}:
if set(uv.keys()) - {"packages", "uv_check", "uv_version"}:
raise ValueError(
"runtime_env['uv'] can only have these fields: "
"packages and uv_version, but got: "
"packages, uv_check and uv_version, but got: "
f"{list(uv.keys())}"
)
if "packages" not in uv:
raise ValueError(
f"runtime_env['uv'] must include field 'packages', but got {uv}"
)
if "uv_check" in uv and not isinstance(uv["uv_check"], bool):
raise TypeError(
"runtime_env['uv']['uv_check'] must be of type bool, "
f"got {type(uv['uv_check'])}"
)
if "uv_version" in uv and not isinstance(uv["uv_version"], str):
raise TypeError(
"runtime_env['uv']['uv_version'] must be of type str, "
f"got {type(uv['uv_version'])}"
)

result = uv.copy()
result["uv_check"] = uv.get("uv_check", False)
if not isinstance(uv["packages"], list):
raise ValueError(
"runtime_env['uv']['packages'] must be of type list, "
Expand Down
11 changes: 11 additions & 0 deletions python/ray/tests/test_runtime_env_uv.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,17 @@ def f():
assert ray.get(f.remote()) == "2.3.0"


# Package installation succeeds, with compatibility enabled.
def test_package_install_with_uv_and_validation(shutdown_only):
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you add a test where uv check fails?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it's hard.

  • Package installation already checks conflicts, checkout
    def test_runtime_env_cache_with_pip_check(start_cluster):
    # moto require requests>=2.5
    conflict_packages = ["moto==3.0.5", "requests==2.4.0"]
    runtime_env = {
    "pip": {
    "packages": conflict_packages,
    "pip_version": "==20.2.3",
    "pip_check": False,
    }
    }
    @ray.remote
    def f():
    return True
    assert ray.get(f.options(runtime_env=runtime_env).remote())
    runtime_env["pip"]["pip_version"] = "==21.3.1"
    # Just modify filed pip_version, but this time,
    # not hit cache and raise an exception
    with pytest.raises(ray.exceptions.RuntimeEnvSetupError) as error:
    ray.get(f.options(runtime_env=runtime_env).remote())
    assert "The conflict is caused by:" in str(error.value)
    assert "The user requested requests==2.4.0" in str(error.value)
    assert "moto 3.0.5 depends on requests>=2.5" in str(error.value)
  • The only way I could think of faking and testing pip_check failure, is you download a series of compatible packages, enter into the virtual environment and modify the packages manually
  • I don't think it's viable based on our current test setup, since everything's constructed and destructed when ray.remote finishes, which you don't have a way to hack in

@ray.remote(runtime_env={"uv": {"packages": ["requests==2.3.0"], "uv_check": True}})
def f():
import requests

return requests.__version__

assert ray.get(f.remote()) == "2.3.0"


# Package installation fails due to conflict versions.
def test_package_install_has_conflict_with_uv(shutdown_only):
# moto require requests>=2.5
Expand Down
20 changes: 15 additions & 5 deletions python/ray/tests/unit/test_runtime_env_validation.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,19 +32,22 @@ class TestVaidationUv:
def test_parse_and_validate_uv(self, test_directory):
# Valid case w/o duplication.
result = validation.parse_and_validate_uv({"packages": ["tensorflow"]})
assert result == {"packages": ["tensorflow"]}
assert result == {"packages": ["tensorflow"], "uv_check": False}

# Valid case w/ duplication.
result = validation.parse_and_validate_uv(
{"packages": ["tensorflow", "tensorflow"]}
)
assert result == {"packages": ["tensorflow"]}
assert result == {"packages": ["tensorflow"], "uv_check": False}

# Valid case, use `list` to represent necessary packages.
result = validation.parse_and_validate_uv(
["requests==1.0.0", "aiohttp", "ray[serve]"]
)
assert result == {"packages": ["requests==1.0.0", "aiohttp", "ray[serve]"]}
assert result == {
"packages": ["requests==1.0.0", "aiohttp", "ray[serve]"],
"uv_check": False,
}

# Invalid case, unsupport keys.
with pytest.raises(ValueError):
Expand All @@ -54,13 +57,20 @@ def test_parse_and_validate_uv(self, test_directory):
result = validation.parse_and_validate_uv(
{"packages": ["tensorflow"], "uv_version": "==0.4.30"}
)
assert result == {"packages": ["tensorflow"], "uv_version": "==0.4.30"}
assert result == {
"packages": ["tensorflow"],
"uv_version": "==0.4.30",
"uv_check": False,
}

# Valid requirement files.
_, requirements_file = test_directory
requirements_file = requirements_file.resolve()
result = validation.parse_and_validate_uv(str(requirements_file))
assert result == {"packages": ["requests==1.0.0", "pip-install-test"]}
assert result == {
"packages": ["requests==1.0.0", "pip-install-test"],
"uv_check": False,
}

# Invalid requiremnt files.
with pytest.raises(ValueError):
Expand Down