fix(global_config): fail fast when the parent openai version violates nemo-gym's constraint; explicit opt-in for version skew

nemo_gym/global_config.py

@@ -78,6 +78,7 @@
 DRY_RUN_KEY_NAME = "dry_run"
 UVICORN_TIMEOUT_WORKER_HEALTHCHECK = "uvicorn_timeout_worker_healthcheck"
 MODEL_ENDPOINT_READINESS_TIMEOUT_KEY_NAME = "model_endpoint_readiness_timeout_seconds"
+ALLOW_OPENAI_VERSION_SKEW_KEY_NAME = "allow_openai_version_skew"
 UV_CACHE_DIR_KEY_NAME = "uv_cache_dir"
 UV_VENV_DIR_KEY_NAME = "uv_venv_dir"
 INHERIT_FROM_KEY_NAME = "_inherit_from"
@@ -113,6 +114,7 @@
     PORT_RANGE_HIGH_KEY_NAME,
     DRY_RUN_KEY_NAME,
     MODEL_ENDPOINT_READINESS_TIMEOUT_KEY_NAME,
+    ALLOW_OPENAI_VERSION_SKEW_KEY_NAME,
     UV_CACHE_DIR_KEY_NAME,
     UV_VENV_DIR_KEY_NAME,
     INHERIT_FROM_KEY_NAME,
@@ -206,6 +208,55 @@ def _load_config_yaml(config_path):
         raise ConfigError(f"Malformed YAML in '{config_path}'{location}: {problem}") from e
 
 
+def _nemo_gym_openai_requirement() -> Optional[str]:
+    """Return nemo-gym's own openai requirement string (name plus specifier, as declared in its metadata).
+
+    Returns ``None`` when the requirement cannot be determined (nemo-gym not
+    installed as a distribution, ``packaging`` unavailable, only marker'd
+    requirements found).
+    """
+    try:
+        # Lazy imports: `packaging` is not a declared nemo-gym dependency; any
+        # import or lookup failure means "constraint unknown".
+        from importlib.metadata import requires
+
+        from packaging.requirements import Requirement
+        from packaging.utils import canonicalize_name
+
+        for req_str in requires("nemo-gym") or []:
+            req = Requirement(req_str)
+            if canonicalize_name(req.name) == "openai" and req.marker is None:
+                return req_str
+        return None
+    except Exception:
+        return None
+
+
+def _openai_version_matches_nemo_gym_constraint(version: str) -> bool:
+    """True when `version` satisfies nemo-gym's own openai requirement.
+
+    head_server_deps normally pins the parent process's openai version into every
+    sub-venv for consistency. When the parent environment ships an openai release
+    outside nemo-gym's own constraint (e.g. the base image preinstalls a newer
+    openai than nemo-gym's cap allows), that pin makes every
+    sub-venv resolution unsatisfiable — and the dry-run prefetch then bakes
+    venvs that contain nothing but pip. The parser uses this to fail fast (or,
+    with the explicit opt-in, to fall back to nemo-gym's own resolution) instead
+    of emitting an impossible pin. Returns True (preserving the original
+    pin-the-parent behavior) when the constraint cannot be determined.
+    """
+    req_str = _nemo_gym_openai_requirement()
+    if req_str is None:
+        return True
+    try:
+        from packaging.requirements import Requirement
+        from packaging.version import Version
+
+        return Requirement(req_str).specifier.contains(Version(version), prereleases=True)
+    except Exception:
+        return True
+
+
 class GlobalConfigDictParser(BaseModel):
     def parse_global_config_dict_from_cli(self) -> DictConfig:
         # We need to monkeypatch hydra here so that it doesn't use Hydra help so that we can use our own help down the line
@@ -707,13 +758,42 @@ def parse(self, parse_config: Optional[GlobalConfigDictParserConfig] = None) ->
             global_config_dict[DISALLOWED_PORTS_KEY_NAME] = disallowed_ports
 
             # Constrain sensitive package versions
-            global_config_dict[HEAD_SERVER_DEPS_KEY_NAME] = [
+            head_server_deps = [
                 # The ray version is very sensitive. The children ray versions must exactly match those of the parent ray.
                 # The ray extra [default] should also exactly match the extra in the top-level Gym pyproject.toml.
                 f"ray[default]=={ray_version}",
-                # OpenAI version is also sensitive since it changes so often and may introduce subtle incompatibilities.
-                f"openai=={openai_version}",
             ]
+            # OpenAI version is also sensitive since it changes so often and may introduce subtle
+            # incompatibilities — but only pin the parent's version when nemo-gym's own constraint
+            # accepts it; otherwise the sub-venv resolutions are unsatisfiable and the venvs come
+            # out empty (see _openai_version_matches_nemo_gym_constraint).
+            allow_openai_skew = global_config_dict.setdefault(ALLOW_OPENAI_VERSION_SKEW_KEY_NAME, False)
+            if not isinstance(allow_openai_skew, bool):
+                raise ConfigError(
+                    f"{ALLOW_OPENAI_VERSION_SKEW_KEY_NAME} must be a boolean (true/false), got {allow_openai_skew!r}."
+                )
+            if _openai_version_matches_nemo_gym_constraint(openai_version):
+                head_server_deps.append(f"openai=={openai_version}")
+            elif allow_openai_skew:
+                logging.warning(
+                    f"Not pinning the parent process's openai=={openai_version} into server venvs: "
+                    f"it does not satisfy nemo-gym's own constraint ({_nemo_gym_openai_requirement()}). "
+                    "Server venvs resolve openai from nemo-gym's constraint instead, so the parent and "
+                    "the servers exchange requests across the HTTP boundary with different openai "
+                    f"versions ({ALLOW_OPENAI_VERSION_SKEW_KEY_NAME}=true)."
+                )
+            else:
+                raise ConfigError(
+                    f"The parent process runs openai=={openai_version}, which does not satisfy "
+                    f"nemo-gym's own openai constraint ({_nemo_gym_openai_requirement()}). Pinning it "
+                    "into the server venvs would make every server venv resolution unsatisfiable "
+                    "(and the dry-run prefetch would bake venvs containing nothing but pip). "
+                    "Install an openai version compatible with nemo-gym in the parent environment, "
+                    f"or set {ALLOW_OPENAI_VERSION_SKEW_KEY_NAME}=true to let server venvs resolve "
+                    "openai from nemo-gym's own constraint (the parent and the servers then run "
+                    "different openai versions across the HTTP boundary)."
+                )
+            global_config_dict[HEAD_SERVER_DEPS_KEY_NAME] = head_server_deps
 
             # Constrain python version since ray is sensitive to this.
             global_config_dict[PYTHON_VERSION_KEY_NAME] = python_version()

tests/unit_tests/test_global_config.py

@@ -12,13 +12,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import importlib.metadata
 import sys
 from contextlib import nullcontext as does_not_raise
 from pathlib import Path
 from unittest.mock import MagicMock, PropertyMock
 
 from omegaconf import OmegaConf
-from pytest import MonkeyPatch, mark, raises
+from pytest import LogCaptureFixture, MonkeyPatch, mark, raises
 
 import nemo_gym.global_config
 import nemo_gym.server_utils
@@ -39,6 +40,7 @@
     USE_ABSOLUTE_IP,
     GlobalConfigDictParser,
     GlobalConfigDictParserConfig,
+    _openai_version_matches_nemo_gym_constraint,
     find_open_port,
     get_first_server_config_dict,
     get_global_config_dict,
@@ -70,6 +72,7 @@ def _default_global_config_dict_values(self) -> dict:
             "skip_venv_if_present": False,
             "dry_run": False,
             "model_endpoint_readiness_timeout_seconds": 600,
+            "allow_openai_version_skew": False,
             "uv_cache_dir": str(CACHE_DIR / "uv"),
             "uv_venv_dir": str(WORKING_DIR),
         }
@@ -132,6 +135,66 @@ def test_offline_resolution_uses_invalid_port_without_probing(self, monkeypatch:
         wandb_init.assert_not_called()
         assert "UV_CACHE_DIR" not in nemo_gym.global_config.environ
 
+    def _mock_parse_environment(self, monkeypatch: MonkeyPatch, config_dict: "DictConfig") -> None:
+        """Standard parser mocks (no env var, no .env.yaml, fixed hydra config)."""
+        monkeypatch.delenv(NEMO_GYM_CONFIG_DICT_ENV_VAR_NAME, raising=False)
+        monkeypatch.setattr(nemo_gym.global_config, "_GLOBAL_CONFIG_DICT", None)
+
+        exists_mock = MagicMock()
+        exists_mock.return_value = False
+        monkeypatch.setattr(nemo_gym.global_config.Path, "exists", exists_mock)
+
+        hydra_main_mock = MagicMock()
+        hydra_main_mock.return_value = lambda fn: (lambda: fn(config_dict))
+        monkeypatch.setattr(nemo_gym.global_config.hydra, "main", hydra_main_mock)
+
+    def _mock_openai_topology(self, monkeypatch: MonkeyPatch, parent_version: str) -> None:
+        """Parent process on `parent_version`; nemo-gym's own constraint is openai<=2.7.2."""
+        monkeypatch.setattr(nemo_gym.global_config, "openai_version", parent_version)
+        monkeypatch.setattr(nemo_gym.global_config, "ray_version", "test ray version")
+        monkeypatch.setattr(nemo_gym.global_config, "python_version", MagicMock(return_value="test python version"))
+        monkeypatch.setattr(importlib.metadata, "requires", lambda name: ["openai<=2.7.2"])
+
+    def test_head_server_deps_pins_compatible_parent_openai(self, monkeypatch: MonkeyPatch) -> None:
+        self._mock_openai_topology(monkeypatch, parent_version="2.6.0")
+        self._mock_parse_environment(monkeypatch, DictConfig({}))
+
+        global_config_dict = get_global_config_dict()
+        assert "openai==2.6.0" in global_config_dict["head_server_deps"]
+
+    def test_head_server_deps_raises_on_incompatible_parent_openai(self, monkeypatch: MonkeyPatch) -> None:
+        self._mock_openai_topology(monkeypatch, parent_version="2.52.0")
+        self._mock_parse_environment(monkeypatch, DictConfig({}))
+
+        with raises(ConfigError, match="allow_openai_version_skew") as excinfo:
+            get_global_config_dict()
+        # The error must name both sides of the conflict.
+        assert "openai==2.52.0" in str(excinfo.value)
+        assert "openai<=2.7.2" in str(excinfo.value)
+
+    def test_head_server_deps_omits_pin_with_explicit_skew_opt_in(
+        self, monkeypatch: MonkeyPatch, caplog: LogCaptureFixture
+    ) -> None:
+        self._mock_openai_topology(monkeypatch, parent_version="2.52.0")
+        self._mock_parse_environment(monkeypatch, DictConfig({"allow_openai_version_skew": True}))
+
+        with caplog.at_level("WARNING"):
+            global_config_dict = get_global_config_dict()
+        assert not any(dep.startswith("openai") for dep in global_config_dict["head_server_deps"])
+        assert global_config_dict["allow_openai_version_skew"] is True
+        # The warning must name both sides of the skew.
+        assert "openai==2.52.0" in caplog.text
+        assert "openai<=2.7.2" in caplog.text
+
+    def test_head_server_deps_rejects_non_boolean_skew_opt_in(self, monkeypatch: MonkeyPatch) -> None:
+        self._mock_openai_topology(monkeypatch, parent_version="2.52.0")
+        # YAML/CLI plumbing can hand the key through as a string; "false" is
+        # truthy and must not silently enable the skew.
+        self._mock_parse_environment(monkeypatch, DictConfig({"allow_openai_version_skew": "false"}))
+
+        with raises(ConfigError, match="must be a boolean"):
+            get_global_config_dict()
+
     def test_get_global_config_dict_global_exists(self, monkeypatch: MonkeyPatch) -> None:
         # Clear any lingering env vars.
         monkeypatch.delenv(NEMO_GYM_CONFIG_DICT_ENV_VAR_NAME, raising=False)
@@ -1499,3 +1562,37 @@ def test_raise_on_no_server_instances_passes_with_a_server(self) -> None:
         parser = GlobalConfigDictParser()
         config = DictConfig({"my_server": {"resources_servers": {"x": {"entrypoint": "app.py", "domain": "other"}}}})
         parser.raise_on_no_server_instances(config)
+
+
+class TestOpenAIVersionMatchesNemoGymConstraint:
+    @mark.parametrize(
+        "requirements, version, expected",
+        [
+            # Parent openai satisfies nemo-gym's constraint -> keep pinning it.
+            (["openai<=2.7.2"], "2.7.2", True),
+            (["openai<=2.7.2"], "2.6.0", True),
+            # Parent openai violates the constraint -> the pin would be unsatisfiable.
+            (["openai<=2.7.2"], "2.52.0", False),
+            (["openai>=2.0,<3"], "2.52.0", True),
+            # No openai requirement at all -> preserve the pin-the-parent behavior.
+            (["ray[default]==2.49.0"], "2.52.0", True),
+            # Package names are case-insensitive (PEP 503): still recognized.
+            (["OpenAI<=2.7.2"], "2.52.0", False),
+            # Marker'd requirements are skipped (conservative fallback).
+            (['openai<=2.7.2; extra == "adapters"'], "2.52.0", True),
+            # importlib.metadata.requires may return None.
+            (None, "2.52.0", True),
+        ],
+    )
+    def test_constraint_matching(
+        self, monkeypatch: MonkeyPatch, requirements: list, version: str, expected: bool
+    ) -> None:
+        monkeypatch.setattr(importlib.metadata, "requires", lambda name: requirements)
+        assert _openai_version_matches_nemo_gym_constraint(version) is expected
+
+    def test_falls_back_to_pinning_when_constraint_lookup_fails(self, monkeypatch: MonkeyPatch) -> None:
+        def raise_not_found(name: str) -> None:
+            raise importlib.metadata.PackageNotFoundError(name)
+
+        monkeypatch.setattr(importlib.metadata, "requires", raise_not_found)
+        assert _openai_version_matches_nemo_gym_constraint("2.52.0") is True