Repository navigation
[Feat] Per Stage Runtime Env #1623
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 2 commits
5271c7c
9d1fd4e
b3624fc
e2ae9a2
b25df9f
eaaf1d4
e243755
98470ca
b7b2351
fc8f510
2666b24
f4ef209
e971625
fab1b90
e89badb
8062427
a60e271
297b8e4
d88d678
dcf7143
7164dd0
beb11ac
952e77f
a882ad4
6792826
00dd3b9
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,104 @@ | ||
| # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| """Resolve per-stage pip_specs into virtualenvs and set _resolved_site_packages_path on stages. | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'm intrigued about this design choice. Is there a reason to not go with
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I have refactored it so we now use runtime_env instead |
||
|
|
||
| Uses the `uv` CLI to create venvs (no Python dependency on uv). When a stage defines | ||
| pip_specs (e.g. ["vllm==0.6.0"]), the resolver creates a venv with those packages | ||
| and sets the stage's _resolved_site_packages_path so executors can inject PYTHONPATH. | ||
| """ | ||
|
|
||
| from pathlib import Path | ||
| from typing import TYPE_CHECKING | ||
|
|
||
| from loguru import logger | ||
|
|
||
| if TYPE_CHECKING: | ||
| from nemo_curator.stages.base import ProcessingStage | ||
|
|
||
|
|
||
| def _site_packages_for_venv(venv_dir: Path) -> Path: | ||
|
suiyoubi marked this conversation as resolved.
Resolved
|
||
| lib = venv_dir / "lib" | ||
| if not lib.exists(): | ||
| msg = f"venv has no lib/: {venv_dir}" | ||
| raise RuntimeError(msg) | ||
| for p in lib.iterdir(): | ||
| if p.is_dir() and p.name.startswith("python"): | ||
| site = p / "site-packages" | ||
| if site.exists(): | ||
| return site.resolve() | ||
| msg = f"no site-packages found under {venv_dir}" | ||
| raise RuntimeError(msg) | ||
|
suiyoubi marked this conversation as resolved.
Resolved
|
||
|
|
||
|
|
||
| def _create_venv_for_specs(specs: list[str], base_dir: Path, subdir_name: str) -> Path: | ||
| import subprocess | ||
|
|
||
| venv_root = base_dir / subdir_name | ||
| venv_root.mkdir(parents=True, exist_ok=True) | ||
| venv_path = venv_root / ".venv" | ||
| python_path = venv_path / "bin" / "python" | ||
|
suiyoubi marked this conversation as resolved.
Resolved
|
||
| subprocess.run( | ||
|
Check failure on line 52 in nemo_curator/utils/stage_pip_env.py
|
||
| ["uv", "venv", str(venv_path)], | ||
|
Check failure on line 53 in nemo_curator/utils/stage_pip_env.py
|
||
| check=True, | ||
| capture_output=True, | ||
| ) | ||
| subprocess.run( | ||
|
Check failure on line 57 in nemo_curator/utils/stage_pip_env.py
|
||
| ["uv", "pip", "install", "--python", str(python_path), *specs], | ||
|
Check failure on line 58 in nemo_curator/utils/stage_pip_env.py
|
||
| check=True, | ||
| capture_output=True, | ||
|
suiyoubi marked this conversation as resolved.
Resolved
|
||
| ) | ||
| return _site_packages_for_venv(venv_path) | ||
|
|
||
|
|
||
| def resolve_stage_pip_envs( | ||
| stages: list["ProcessingStage"], | ||
| base_dir: Path | None = None, | ||
| ) -> None: | ||
| """Create venvs for stages that have pip_specs and set _resolved_site_packages_path. | ||
|
|
||
| Stages with the same pip_specs (order-independent) share one venv. Uses the `uv` | ||
| CLI; must be on PATH. Modifies each stage instance in place. | ||
|
|
||
| Args: | ||
| stages: Flat list of execution stages (e.g. after pipeline build). | ||
| base_dir: Directory for venv roots. If None, a temp directory is used. | ||
| """ | ||
| import subprocess | ||
| import tempfile | ||
|
|
||
| try: | ||
| subprocess.run(["uv", "--version"], check=True, capture_output=True) | ||
|
Check failure on line 82 in nemo_curator/utils/stage_pip_env.py
|
||
| except (subprocess.CalledProcessError, FileNotFoundError) as e: | ||
| logger.warning( | ||
| "resolve_stage_pip_envs requires `uv` on PATH; skipping. Error: %s", | ||
| e, | ||
| ) | ||
| return | ||
|
suiyoubi marked this conversation as resolved.
Resolved
|
||
|
|
||
| base = Path(base_dir) if base_dir else Path(tempfile.mkdtemp(prefix="curator_pip_envs_")) | ||
|
suiyoubi marked this conversation as resolved.
Resolved
|
||
| base.mkdir(parents=True, exist_ok=True) | ||
|
|
||
| # Dedupe: same sorted specs -> one venv | ||
| unique_specs: dict[tuple[str, ...], Path] = {} | ||
| for i, stage in enumerate(stages): | ||
|
Check failure on line 95 in nemo_curator/utils/stage_pip_env.py
|
||
|
suiyoubi marked this conversation as resolved.
Resolved
|
||
| specs = getattr(stage, "pip_specs", None) | ||
| if not specs or not isinstance(specs, list): | ||
| continue | ||
| key = tuple(sorted(specs)) | ||
|
suiyoubi marked this conversation as resolved.
Resolved
|
||
| if key not in unique_specs: | ||
| subdir = f"env_{len(unique_specs)}" | ||
| unique_specs[key] = _create_venv_for_specs(list(specs), base, subdir) | ||
| logger.info("Created venv for pip_specs %s at %s", list(key), unique_specs[key]) | ||
| setattr(stage, "_resolved_site_packages_path", unique_specs[key]) | ||
|
Check failure on line 104 in nemo_curator/utils/stage_pip_env.py
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,162 @@ | ||
| # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| """Tests for per-stage runtime environment: different Python package versions per stage. | ||
|
|
||
| Uses pip_specs + resolve_stage_pip_envs (uv CLI) so each stage runs with its own | ||
| packaging version. Requires `uv` on PATH; test is skipped if uv is not available. | ||
| See tutorials/per_stage_runtime_env_example.py and docs/design/per-stage-runtime-environment.md. | ||
| """ | ||
|
|
||
|
suiyoubi marked this conversation as resolved.
Resolved
|
||
| import subprocess | ||
|
|
||
| import pandas as pd | ||
| import pytest | ||
|
|
||
| from nemo_curator.backends.experimental.ray_data import RayDataExecutor | ||
| from nemo_curator.backends.xenna import XennaExecutor | ||
|
suiyoubi marked this conversation as resolved.
Resolved
|
||
| from nemo_curator.pipeline.pipeline import Pipeline | ||
| from nemo_curator.stages.base import ProcessingStage | ||
| from nemo_curator.stages.resources import Resources | ||
| from nemo_curator.tasks import DocumentBatch | ||
|
|
||
|
|
||
| def _uv_available() -> bool: | ||
| try: | ||
| subprocess.run(["uv", "--version"], check=True, capture_output=True) | ||
|
Check failure on line 37 in tests/pipelines/test_per_stage_runtime_env.py
|
||
| return True | ||
|
Check failure on line 38 in tests/pipelines/test_per_stage_runtime_env.py
|
||
| except (subprocess.CalledProcessError, FileNotFoundError): | ||
| return False | ||
|
|
||
|
|
||
| @pytest.fixture | ||
| def require_uv(): | ||
| if not _uv_available(): | ||
| pytest.skip("uv not on PATH; per-stage pip_specs tests require uv") | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Should tests ever be skipped? Is there a case where uv doesn't exist? |
||
|
|
||
|
|
||
| class VersionStage1(ProcessingStage[DocumentBatch, DocumentBatch]): | ||
| """Stage 1: packaging==23.2 via pip_specs (resolver creates venv, PYTHONPATH).""" | ||
|
|
||
| name = "version_stage_1" | ||
| resources = Resources(cpus=0.5) | ||
| batch_size = 1 | ||
| pip_specs = ["packaging==23.2"] | ||
|
Check failure on line 55 in tests/pipelines/test_per_stage_runtime_env.py
|
||
|
|
||
| def inputs(self) -> tuple[list[str], list[str]]: | ||
| return ["data"], [] | ||
|
|
||
|
suiyoubi marked this conversation as resolved.
Resolved
|
||
| def outputs(self) -> tuple[list[str], list[str]]: | ||
| return ["data"], ["stage1_packaging_version"] | ||
|
|
||
| def process(self, task: DocumentBatch) -> DocumentBatch: | ||
| import packaging # noqa: PLC0415 | ||
|
|
||
| df = task.to_pandas().copy() | ||
| df["stage1_packaging_version"] = packaging.__version__ | ||
| return DocumentBatch( | ||
| task_id=task.task_id, | ||
| dataset_name=task.dataset_name, | ||
| data=df, | ||
| _metadata=task._metadata, | ||
| _stage_perf=task._stage_perf, | ||
| ) | ||
|
|
||
|
|
||
| class VersionStage2(ProcessingStage[DocumentBatch, DocumentBatch]): | ||
| """Stage 2: packaging==24.0 via pip_specs (resolver creates venv, PYTHONPATH).""" | ||
|
|
||
| name = "version_stage_2" | ||
| resources = Resources(cpus=0.5) | ||
| batch_size = 1 | ||
| pip_specs = ["packaging==24.0"] | ||
|
|
||
| def inputs(self) -> tuple[list[str], list[str]]: | ||
| return ["data"], ["stage1_packaging_version"] | ||
|
suiyoubi marked this conversation as resolved.
Resolved
|
||
|
|
||
| def outputs(self) -> tuple[list[str], list[str]]: | ||
| return ["data"], ["stage2_packaging_version"] | ||
|
|
||
| def process(self, task: DocumentBatch) -> DocumentBatch: | ||
| import packaging # noqa: PLC0415 | ||
|
|
||
| df = task.to_pandas().copy() | ||
| df["stage2_packaging_version"] = packaging.__version__ | ||
| return DocumentBatch( | ||
| task_id=task.task_id, | ||
| dataset_name=task.dataset_name, | ||
| data=df, | ||
| _metadata=task._metadata, | ||
| _stage_perf=task._stage_perf, | ||
| ) | ||
|
|
||
|
|
||
|
suiyoubi marked this conversation as resolved.
Resolved
|
||
| @pytest.mark.usefixtures("shared_ray_client", "require_uv") | ||
| def test_per_stage_different_package_versions_ray_data() -> None: | ||
| """Run two stages with different packaging versions via pip_specs; assert each sees its own version. | ||
|
|
||
| Pipeline.run() calls resolve_stage_pip_envs() to create venvs with uv; Ray Data adapter | ||
| injects PYTHONPATH so workers load the correct site-packages per stage. | ||
| """ | ||
| initial = DocumentBatch( | ||
| task_id="per_stage_version_test", | ||
| dataset_name="test", | ||
| data=pd.DataFrame({"text": ["hello"]}), | ||
| ) | ||
| pipeline = Pipeline( | ||
| name="per_stage_version_test", | ||
| stages=[VersionStage1(), VersionStage2()], | ||
| ) | ||
| results = pipeline.run( | ||
| executor=RayDataExecutor(), | ||
| initial_tasks=[initial], | ||
| ) | ||
| assert results is not None | ||
| assert len(results) == 1 | ||
| out = results[0] | ||
| df = out.to_pandas() | ||
| assert "stage1_packaging_version" in df.columns | ||
| assert "stage2_packaging_version" in df.columns | ||
| assert df["stage1_packaging_version"].iloc[0] == "23.2", "Stage 1 should see packaging 23.2" | ||
| assert df["stage2_packaging_version"].iloc[0] == "24.0", "Stage 2 should see packaging 24.0" | ||
|
|
||
|
|
||
| @pytest.mark.usefixtures("shared_ray_client", "require_uv") | ||
| def test_per_stage_different_package_versions_xenna() -> None: | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Take a look at |
||
| """Run two stages with different packaging versions via pip_specs using XennaExecutor. | ||
|
|
||
| Pipeline.run() calls resolve_stage_pip_envs() to create venvs with uv; Xenna adapter | ||
| uses env_info() to set PYTHONPATH so workers load the correct site-packages per stage. | ||
| """ | ||
| initial = DocumentBatch( | ||
| task_id="per_stage_version_test_xenna", | ||
| dataset_name="test", | ||
| data=pd.DataFrame({"text": ["hello"]}), | ||
| ) | ||
| pipeline = Pipeline( | ||
| name="per_stage_version_test_xenna", | ||
| stages=[VersionStage1(), VersionStage2()], | ||
| ) | ||
| results = pipeline.run( | ||
| executor=XennaExecutor(config={"execution_mode": "streaming"}), | ||
| initial_tasks=[initial], | ||
| ) | ||
| assert results is not None | ||
| assert len(results) == 1 | ||
| out = results[0] | ||
| df = out.to_pandas() | ||
| assert "stage1_packaging_version" in df.columns | ||
| assert "stage2_packaging_version" in df.columns | ||
| assert df["stage1_packaging_version"].iloc[0] == "23.2", "Stage 1 should see packaging 23.2" | ||
| assert df["stage2_packaging_version"].iloc[0] == "24.0", "Stage 2 should see packaging 24.0" | ||
Uh oh!
There was an error while loading. Please reload this page.