diff --git a/nemo_gym/global_config.py b/nemo_gym/global_config.py index 33acd69f1f4..1a957cb4471 100644 --- a/nemo_gym/global_config.py +++ b/nemo_gym/global_config.py @@ -78,6 +78,7 @@ DRY_RUN_KEY_NAME = "dry_run" UVICORN_TIMEOUT_WORKER_HEALTHCHECK = "uvicorn_timeout_worker_healthcheck" MODEL_ENDPOINT_READINESS_TIMEOUT_KEY_NAME = "model_endpoint_readiness_timeout_seconds" +ALLOW_OPENAI_VERSION_SKEW_KEY_NAME = "allow_openai_version_skew" UV_CACHE_DIR_KEY_NAME = "uv_cache_dir" UV_VENV_DIR_KEY_NAME = "uv_venv_dir" INHERIT_FROM_KEY_NAME = "_inherit_from" @@ -113,6 +114,7 @@ PORT_RANGE_HIGH_KEY_NAME, DRY_RUN_KEY_NAME, MODEL_ENDPOINT_READINESS_TIMEOUT_KEY_NAME, + ALLOW_OPENAI_VERSION_SKEW_KEY_NAME, UV_CACHE_DIR_KEY_NAME, UV_VENV_DIR_KEY_NAME, INHERIT_FROM_KEY_NAME, @@ -206,6 +208,55 @@ def _load_config_yaml(config_path): raise ConfigError(f"Malformed YAML in '{config_path}'{location}: {problem}") from e +def _nemo_gym_openai_requirement() -> Optional[str]: + """Return nemo-gym's own openai requirement string (name plus specifier, as declared in its metadata). + + Returns ``None`` when the requirement cannot be determined (nemo-gym not + installed as a distribution, ``packaging`` unavailable, only marker'd + requirements found). + """ + try: + # Lazy imports: `packaging` is not a declared nemo-gym dependency; any + # import or lookup failure means "constraint unknown". + from importlib.metadata import requires + + from packaging.requirements import Requirement + from packaging.utils import canonicalize_name + + for req_str in requires("nemo-gym") or []: + req = Requirement(req_str) + if canonicalize_name(req.name) == "openai" and req.marker is None: + return req_str + return None + except Exception: + return None + + +def _openai_version_matches_nemo_gym_constraint(version: str) -> bool: + """True when `version` satisfies nemo-gym's own openai requirement. + + head_server_deps normally pins the parent process's openai version into every + sub-venv for consistency. When the parent environment ships an openai release + outside nemo-gym's own constraint (e.g. the base image preinstalls a newer + openai than nemo-gym's cap allows), that pin makes every + sub-venv resolution unsatisfiable — and the dry-run prefetch then bakes + venvs that contain nothing but pip. The parser uses this to fail fast (or, + with the explicit opt-in, to fall back to nemo-gym's own resolution) instead + of emitting an impossible pin. Returns True (preserving the original + pin-the-parent behavior) when the constraint cannot be determined. + """ + req_str = _nemo_gym_openai_requirement() + if req_str is None: + return True + try: + from packaging.requirements import Requirement + from packaging.version import Version + + return Requirement(req_str).specifier.contains(Version(version), prereleases=True) + except Exception: + return True + + class GlobalConfigDictParser(BaseModel): def parse_global_config_dict_from_cli(self) -> DictConfig: # We need to monkeypatch hydra here so that it doesn't use Hydra help so that we can use our own help down the line @@ -707,13 +758,42 @@ def parse(self, parse_config: Optional[GlobalConfigDictParserConfig] = None) -> global_config_dict[DISALLOWED_PORTS_KEY_NAME] = disallowed_ports # Constrain sensitive package versions - global_config_dict[HEAD_SERVER_DEPS_KEY_NAME] = [ + head_server_deps = [ # The ray version is very sensitive. The children ray versions must exactly match those of the parent ray. # The ray extra [default] should also exactly match the extra in the top-level Gym pyproject.toml. f"ray[default]=={ray_version}", - # OpenAI version is also sensitive since it changes so often and may introduce subtle incompatibilities. - f"openai=={openai_version}", ] + # OpenAI version is also sensitive since it changes so often and may introduce subtle + # incompatibilities — but only pin the parent's version when nemo-gym's own constraint + # accepts it; otherwise the sub-venv resolutions are unsatisfiable and the venvs come + # out empty (see _openai_version_matches_nemo_gym_constraint). + allow_openai_skew = global_config_dict.setdefault(ALLOW_OPENAI_VERSION_SKEW_KEY_NAME, False) + if not isinstance(allow_openai_skew, bool): + raise ConfigError( + f"{ALLOW_OPENAI_VERSION_SKEW_KEY_NAME} must be a boolean (true/false), got {allow_openai_skew!r}." + ) + if _openai_version_matches_nemo_gym_constraint(openai_version): + head_server_deps.append(f"openai=={openai_version}") + elif allow_openai_skew: + logging.warning( + f"Not pinning the parent process's openai=={openai_version} into server venvs: " + f"it does not satisfy nemo-gym's own constraint ({_nemo_gym_openai_requirement()}). " + "Server venvs resolve openai from nemo-gym's constraint instead, so the parent and " + "the servers exchange requests across the HTTP boundary with different openai " + f"versions ({ALLOW_OPENAI_VERSION_SKEW_KEY_NAME}=true)." + ) + else: + raise ConfigError( + f"The parent process runs openai=={openai_version}, which does not satisfy " + f"nemo-gym's own openai constraint ({_nemo_gym_openai_requirement()}). Pinning it " + "into the server venvs would make every server venv resolution unsatisfiable " + "(and the dry-run prefetch would bake venvs containing nothing but pip). " + "Install an openai version compatible with nemo-gym in the parent environment, " + f"or set {ALLOW_OPENAI_VERSION_SKEW_KEY_NAME}=true to let server venvs resolve " + "openai from nemo-gym's own constraint (the parent and the servers then run " + "different openai versions across the HTTP boundary)." + ) + global_config_dict[HEAD_SERVER_DEPS_KEY_NAME] = head_server_deps # Constrain python version since ray is sensitive to this. global_config_dict[PYTHON_VERSION_KEY_NAME] = python_version() diff --git a/tests/unit_tests/test_global_config.py b/tests/unit_tests/test_global_config.py index 5f287069ffc..1559ec83690 100644 --- a/tests/unit_tests/test_global_config.py +++ b/tests/unit_tests/test_global_config.py @@ -12,13 +12,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import importlib.metadata import sys from contextlib import nullcontext as does_not_raise from pathlib import Path from unittest.mock import MagicMock, PropertyMock from omegaconf import OmegaConf -from pytest import MonkeyPatch, mark, raises +from pytest import LogCaptureFixture, MonkeyPatch, mark, raises import nemo_gym.global_config import nemo_gym.server_utils @@ -39,6 +40,7 @@ USE_ABSOLUTE_IP, GlobalConfigDictParser, GlobalConfigDictParserConfig, + _openai_version_matches_nemo_gym_constraint, find_open_port, get_first_server_config_dict, get_global_config_dict, @@ -70,6 +72,7 @@ def _default_global_config_dict_values(self) -> dict: "skip_venv_if_present": False, "dry_run": False, "model_endpoint_readiness_timeout_seconds": 600, + "allow_openai_version_skew": False, "uv_cache_dir": str(CACHE_DIR / "uv"), "uv_venv_dir": str(WORKING_DIR), } @@ -132,6 +135,66 @@ def test_offline_resolution_uses_invalid_port_without_probing(self, monkeypatch: wandb_init.assert_not_called() assert "UV_CACHE_DIR" not in nemo_gym.global_config.environ + def _mock_parse_environment(self, monkeypatch: MonkeyPatch, config_dict: "DictConfig") -> None: + """Standard parser mocks (no env var, no .env.yaml, fixed hydra config).""" + monkeypatch.delenv(NEMO_GYM_CONFIG_DICT_ENV_VAR_NAME, raising=False) + monkeypatch.setattr(nemo_gym.global_config, "_GLOBAL_CONFIG_DICT", None) + + exists_mock = MagicMock() + exists_mock.return_value = False + monkeypatch.setattr(nemo_gym.global_config.Path, "exists", exists_mock) + + hydra_main_mock = MagicMock() + hydra_main_mock.return_value = lambda fn: (lambda: fn(config_dict)) + monkeypatch.setattr(nemo_gym.global_config.hydra, "main", hydra_main_mock) + + def _mock_openai_topology(self, monkeypatch: MonkeyPatch, parent_version: str) -> None: + """Parent process on `parent_version`; nemo-gym's own constraint is openai<=2.7.2.""" + monkeypatch.setattr(nemo_gym.global_config, "openai_version", parent_version) + monkeypatch.setattr(nemo_gym.global_config, "ray_version", "test ray version") + monkeypatch.setattr(nemo_gym.global_config, "python_version", MagicMock(return_value="test python version")) + monkeypatch.setattr(importlib.metadata, "requires", lambda name: ["openai<=2.7.2"]) + + def test_head_server_deps_pins_compatible_parent_openai(self, monkeypatch: MonkeyPatch) -> None: + self._mock_openai_topology(monkeypatch, parent_version="2.6.0") + self._mock_parse_environment(monkeypatch, DictConfig({})) + + global_config_dict = get_global_config_dict() + assert "openai==2.6.0" in global_config_dict["head_server_deps"] + + def test_head_server_deps_raises_on_incompatible_parent_openai(self, monkeypatch: MonkeyPatch) -> None: + self._mock_openai_topology(monkeypatch, parent_version="2.52.0") + self._mock_parse_environment(monkeypatch, DictConfig({})) + + with raises(ConfigError, match="allow_openai_version_skew") as excinfo: + get_global_config_dict() + # The error must name both sides of the conflict. + assert "openai==2.52.0" in str(excinfo.value) + assert "openai<=2.7.2" in str(excinfo.value) + + def test_head_server_deps_omits_pin_with_explicit_skew_opt_in( + self, monkeypatch: MonkeyPatch, caplog: LogCaptureFixture + ) -> None: + self._mock_openai_topology(monkeypatch, parent_version="2.52.0") + self._mock_parse_environment(monkeypatch, DictConfig({"allow_openai_version_skew": True})) + + with caplog.at_level("WARNING"): + global_config_dict = get_global_config_dict() + assert not any(dep.startswith("openai") for dep in global_config_dict["head_server_deps"]) + assert global_config_dict["allow_openai_version_skew"] is True + # The warning must name both sides of the skew. + assert "openai==2.52.0" in caplog.text + assert "openai<=2.7.2" in caplog.text + + def test_head_server_deps_rejects_non_boolean_skew_opt_in(self, monkeypatch: MonkeyPatch) -> None: + self._mock_openai_topology(monkeypatch, parent_version="2.52.0") + # YAML/CLI plumbing can hand the key through as a string; "false" is + # truthy and must not silently enable the skew. + self._mock_parse_environment(monkeypatch, DictConfig({"allow_openai_version_skew": "false"})) + + with raises(ConfigError, match="must be a boolean"): + get_global_config_dict() + def test_get_global_config_dict_global_exists(self, monkeypatch: MonkeyPatch) -> None: # Clear any lingering env vars. monkeypatch.delenv(NEMO_GYM_CONFIG_DICT_ENV_VAR_NAME, raising=False) @@ -1499,3 +1562,37 @@ def test_raise_on_no_server_instances_passes_with_a_server(self) -> None: parser = GlobalConfigDictParser() config = DictConfig({"my_server": {"resources_servers": {"x": {"entrypoint": "app.py", "domain": "other"}}}}) parser.raise_on_no_server_instances(config) + + +class TestOpenAIVersionMatchesNemoGymConstraint: + @mark.parametrize( + "requirements, version, expected", + [ + # Parent openai satisfies nemo-gym's constraint -> keep pinning it. + (["openai<=2.7.2"], "2.7.2", True), + (["openai<=2.7.2"], "2.6.0", True), + # Parent openai violates the constraint -> the pin would be unsatisfiable. + (["openai<=2.7.2"], "2.52.0", False), + (["openai>=2.0,<3"], "2.52.0", True), + # No openai requirement at all -> preserve the pin-the-parent behavior. + (["ray[default]==2.49.0"], "2.52.0", True), + # Package names are case-insensitive (PEP 503): still recognized. + (["OpenAI<=2.7.2"], "2.52.0", False), + # Marker'd requirements are skipped (conservative fallback). + (['openai<=2.7.2; extra == "adapters"'], "2.52.0", True), + # importlib.metadata.requires may return None. + (None, "2.52.0", True), + ], + ) + def test_constraint_matching( + self, monkeypatch: MonkeyPatch, requirements: list, version: str, expected: bool + ) -> None: + monkeypatch.setattr(importlib.metadata, "requires", lambda name: requirements) + assert _openai_version_matches_nemo_gym_constraint(version) is expected + + def test_falls_back_to_pinning_when_constraint_lookup_fails(self, monkeypatch: MonkeyPatch) -> None: + def raise_not_found(name: str) -> None: + raise importlib.metadata.PackageNotFoundError(name) + + monkeypatch.setattr(importlib.metadata, "requires", raise_not_found) + assert _openai_version_matches_nemo_gym_constraint("2.52.0") is True