fix: handle symlinked Transformers module cache

examples/nemo_gym/nemotron-3-super-omni/super_omni_launch.sh

@@ -146,7 +146,7 @@ export SANDBOX_ENV_VARS="NEMO_SKILLS_SANDBOX_PORT=${NEMO_SKILLS_SANDBOX_PORT}"
 
 export COMMAND="export HF_MODULES_CACHE=${HF_MODULES_CACHE_DIR} ; \
     export PYTHONPATH=${HF_MODULES_CACHE_DIR}:${SNAPSHOT_DIR}:${MEGATRON_BRIDGE_SRC}:${MEGATRON_LM_SRC}:\${PYTHONPATH:-} ; \
-    python -c \"from transformers import AutoConfig, AutoProcessor, AutoTokenizer; p='${MODEL_PATH}'; AutoConfig.from_pretrained(p, trust_remote_code=True); AutoProcessor.from_pretrained(p, trust_remote_code=True, use_fast=True); AutoTokenizer.from_pretrained(p, trust_remote_code=True, use_fast=True); print('Prewarmed HF dynamic modules cache')\" ; \
+    python -c \"import nemo_rl; from transformers import AutoConfig, AutoProcessor, AutoTokenizer; p='${MODEL_PATH}'; AutoConfig.from_pretrained(p, trust_remote_code=True); AutoProcessor.from_pretrained(p, trust_remote_code=True, use_fast=True); AutoTokenizer.from_pretrained(p, trust_remote_code=True, use_fast=True); print('Prewarmed HF dynamic modules cache')\" ; \
     date ; \
     NRL_WG_USE_RAY_REF=1 \
     NRL_MEGATRON_CHECKPOINT_DIR=${NRL_MEGATRON_CHECKPOINT_DIR} \

examples/nemo_gym/nemotron-3-super/super_launch.sh

@@ -139,7 +139,7 @@ export SANDBOX_ENV_VARS="NEMO_SKILLS_SANDBOX_PORT=${NEMO_SKILLS_SANDBOX_PORT}"
 
 # ---- Build the run command ----
 export COMMAND="export HF_MODULES_CACHE=${HF_MODULES_CACHE_DIR} ; \
-    python -c \"from transformers import AutoConfig, AutoTokenizer; p='${MODEL_PATH}'; AutoConfig.from_pretrained(p, trust_remote_code=True); AutoTokenizer.from_pretrained(p, trust_remote_code=True, use_fast=True); print('Prewarmed HF dynamic modules cache')\" ; \
+    python -c \"import nemo_rl; from transformers import AutoConfig, AutoTokenizer; p='${MODEL_PATH}'; AutoConfig.from_pretrained(p, trust_remote_code=True); AutoTokenizer.from_pretrained(p, trust_remote_code=True, use_fast=True); print('Prewarmed HF dynamic modules cache')\" ; \
     date ; \
     NRL_WG_USE_RAY_REF=1 \
     NRL_MEGATRON_CHECKPOINT_DIR=${NRL_MEGATRON_CHECKPOINT_DIR} \

nemo_rl/__init__.py

@@ -331,3 +331,15 @@ def patch_transformers_module_dir(
 
 
 patch_transformers_module_dir(os.environ, apply_to_current_interpreter=True)
+
+
+# Transformers 5.11 and 5.12 follow cached snapshot symlinks into blobs/ while
+# hashing trust_remote_code modules. Install the version-gated upstream fix
+# before model configurations are loaded in this interpreter. Remove this
+# bootstrap after the minimum Transformers version is upgraded to 5.13.0.
+from nemo_rl.transformers_compat import (  # noqa: E402
+    _patch_transformers_dynamic_module_symlink_cache,
+)
+
+_patch_transformers_dynamic_module_symlink_cache()
+del _patch_transformers_dynamic_module_symlink_cache

nemo_rl/models/generation/sglang/utils/patches.py

@@ -83,6 +83,32 @@ def _patch_sglang_safe_unpickler() -> None:
     logger.info("Patched SafeUnpickler allowlist in %s.", file_to_patch)
 
 
+def _patch_sglang_transformers_compat_bootstrap() -> None:
+    """Load NeMo RL's temporary Transformers patch in every SGLang process."""
+    file_to_patch = _get_sglang_file("srt/utils/hf_transformers_patches.py")
+
+    with open(file_to_patch, "r") as f:
+        content = f.read()
+
+    sentinel = (
+        "    import nemo_rl  # noqa: F401  # Install temporary Transformers patch.\n"
+    )
+    if sentinel in content:
+        return
+
+    anchor = "\n    _applied = True\n"
+    if anchor not in content:
+        raise RuntimeError(
+            f"Transformers patch bootstrap anchor '{anchor.strip()}' not found in "
+            f"{file_to_patch}."
+        )
+
+    patched_content = content.replace(anchor, f"{anchor}\n{sentinel}", 1)
+    compile(patched_content, file_to_patch, "exec")
+    _write_and_verify(file_to_patch, patched_content, sentinel)
+    logger.info("Patched Transformers compatibility bootstrap in %s.", file_to_patch)
+
+
 def _override_sglang_imbalance_check_env() -> None:
     """Force-disable sglang's per-GPU memory imbalance check.
 
@@ -172,6 +198,8 @@ def _patch_megatron_training_hook_mode() -> None:
 
 
 def _apply_sglang_compat_patches() -> None:
+    # Remove with nemo_rl.transformers_compat once Transformers >= 5.13 is required.
+    _patch_sglang_transformers_compat_bootstrap()
     _patch_sglang_safe_unpickler()
     _override_sglang_imbalance_check_env()
     _patch_megatron_dynamic_context_hook_mode()

nemo_rl/transformers_compat.py

@@ -0,0 +1,148 @@
+# Copyright (c) 2026, NVIDIA CORPORATION.  All rights reserved.
+# Copyright 2021 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Temporary compatibility patches for supported Transformers releases.
+
+Remove this module and its package bootstrap when NeMo RL requires Transformers
+5.13.0 or newer.
+"""
+
+import hashlib
+import importlib
+import inspect
+import logging
+import os
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version as distribution_version
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+_MIN_AFFECTED_TRANSFORMERS_VERSION = "5.11.0"
+_FIXED_TRANSFORMERS_VERSION = "5.13.0"
+_EXPECTED_HASH_FUNCTION_PARAMETERS = (
+    "pretrained_model_name_or_path",
+    "resolved_module_file",
+)
+
+
+def _compute_local_source_files_hash_with_symlink_fix(
+    pretrained_model_name_or_path: str | os.PathLike,
+    resolved_module_file: str | os.PathLike,
+) -> str:
+    """Hash dynamic-module sources without losing symlinked snapshot filenames.
+
+    Adapted from the implementation shipped upstream in Transformers 5.13.0 by
+    https://github.com/huggingface/transformers/pull/46618. Earlier affected
+    releases resolve snapshot file symlinks into ``blobs/`` before locating
+    relative imports, where blob names no longer have the Python module
+    filenames that those imports reference.
+    """
+    # Keep Transformers optional for lightweight NeMo RL import paths.
+    from transformers.dynamic_module_utils import get_relative_import_files
+
+    model_path = Path(pretrained_model_name_or_path).resolve()
+    resolved_module_file = Path(resolved_module_file)
+
+    def _resolve_relative_source_path(source_file_path: Path) -> str:
+        # Resolve only the parent directory. Calling resolve() on the whole file
+        # follows Hugging Face snapshot symlinks into blobs/ and loses the
+        # source filename needed to locate sibling relative imports.
+        canonical_path = source_file_path.parent.resolve() / source_file_path.name
+        try:
+            return canonical_path.relative_to(model_path).as_posix()
+        except ValueError:
+            return canonical_path.as_posix()
+
+    files_to_hash = [
+        (
+            _resolve_relative_source_path(resolved_module_file),
+            resolved_module_file,
+        )
+    ]
+    for source_file in get_relative_import_files(resolved_module_file):
+        source_file_path = Path(source_file)
+        files_to_hash.append(
+            (
+                _resolve_relative_source_path(source_file_path),
+                source_file_path,
+            )
+        )
+
+    source_files_hash = hashlib.sha256()
+    for relative_path, file_path in sorted(files_to_hash, key=lambda entry: entry[0]):
+        source_files_hash.update(relative_path.encode("utf-8"))
+        source_files_hash.update(file_path.read_bytes())
+
+    return source_files_hash.hexdigest()[:16]
+
+
+def _patch_transformers_dynamic_module_symlink_cache() -> bool:
+    """Backport the Transformers 5.13 dynamic-module symlink fix.
+
+    Returns ``True`` only when this call installs the patch. Missing
+    Transformers and unaffected versions are intentional no-ops. Remove this
+    patch after upgrading the minimum supported Transformers version to 5.13.0.
+    """
+    # Keep Transformers optional for lightweight NeMo RL import paths.
+    try:
+        installed_version = distribution_version("transformers")
+    except PackageNotFoundError:
+        return False
+
+    # packaging is a Transformers dependency, but importing it only after the
+    # distribution check keeps Transformers optional for lightweight imports.
+    from packaging.version import Version
+
+    if not (
+        Version(_MIN_AFFECTED_TRANSFORMERS_VERSION)
+        <= Version(installed_version)
+        < Version(_FIXED_TRANSFORMERS_VERSION)
+    ):
+        return False
+
+    # Do not mask an ImportError from a present but broken Transformers install.
+    dynamic_module_utils = importlib.import_module("transformers.dynamic_module_utils")
+    current_function: Any = getattr(
+        dynamic_module_utils, "_compute_local_source_files_hash", None
+    )
+    if current_function is None:
+        raise RuntimeError(
+            f"Transformers {installed_version} does not expose "
+            "dynamic_module_utils._compute_local_source_files_hash; cannot apply "
+            "the NeMo RL symlink-cache compatibility patch."
+        )
+
+    if current_function is _compute_local_source_files_hash_with_symlink_fix:
+        return False
+
+    actual_parameters = tuple(inspect.signature(current_function).parameters)
+    if actual_parameters != _EXPECTED_HASH_FUNCTION_PARAMETERS:
+        raise RuntimeError(
+            f"Transformers {installed_version} exposes an unexpected "
+            "dynamic_module_utils._compute_local_source_files_hash signature "
+            f"{actual_parameters}; cannot apply the NeMo RL symlink-cache "
+            "compatibility patch."
+        )
+
+    dynamic_module_utils._compute_local_source_files_hash = (  # type: ignore[attr-defined]
+        _compute_local_source_files_hash_with_symlink_fix
+    )
+    logger.info(
+        "Applied the Transformers %s dynamic-module symlink-cache patch "
+        "(https://github.com/huggingface/transformers/pull/46618)",
+        installed_version,
+    )
+    return True

pyrefly.toml

@@ -243,6 +243,7 @@ project-includes = [
   "nemo_rl/telemetry/metrics.py",
   "nemo_rl/telemetry/setup.py",
   "nemo_rl/telemetry/span_groups.py",
+  "nemo_rl/transformers_compat.py",
   "nemo_rl/utils/__init__.py",
   "nemo_rl/utils/checkpoint.py",
   "nemo_rl/utils/checkpoint_engines/__init__.py",

tests/unit/test_transformers_compat.py

@@ -0,0 +1,164 @@
+# Copyright (c) 2026, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for Transformers runtime compatibility patches."""
+
+from importlib.metadata import PackageNotFoundError
+from pathlib import Path
+
+import pytest
+import transformers.dynamic_module_utils as dynamic_module_utils
+
+import nemo_rl.transformers_compat as transformers_compat
+from nemo_rl.transformers_compat import (
+    _compute_local_source_files_hash_with_symlink_fix,
+    _patch_transformers_dynamic_module_symlink_cache,
+)
+
+_SOURCE_FILES = {
+    "configuration_test.py": "from .dependency import VALUE\n",
+    "dependency.py": "from .leaf import LEAF\nVALUE = LEAF\n",
+    "leaf.py": "LEAF = 1\n",
+}
+
+
+def _create_symlinked_source_tree(root: Path) -> Path:
+    blobs = root / "blobs"
+    snapshot = root / "snapshots" / "revision"
+    blobs.mkdir(parents=True)
+    snapshot.mkdir(parents=True)
+    for index, (filename, content) in enumerate(_SOURCE_FILES.items()):
+        blob = blobs / f"hash-{index}"
+        blob.write_text(content, encoding="utf-8")
+        (snapshot / filename).symlink_to(Path("../../blobs") / blob.name)
+    return snapshot
+
+
+def test_get_cached_module_file_handles_symlinked_hub_cache(monkeypatch, tmp_path):
+    modules_cache = tmp_path / "hf_modules_cache"
+    monkeypatch.setattr(dynamic_module_utils, "HF_MODULES_CACHE", str(modules_cache))
+    snapshot = _create_symlinked_source_tree(tmp_path / "models--org--repo")
+
+    cached_module = dynamic_module_utils.get_cached_module_file(
+        str(snapshot), "configuration_test.py"
+    )
+    cached_source_dir = modules_cache / Path(cached_module).parent
+
+    assert dynamic_module_utils._compute_local_source_files_hash is (
+        _compute_local_source_files_hash_with_symlink_fix
+    ), (
+        "The symlink-cache patch is not installed. If Transformers is now "
+        "5.13.0 or newer the upstream fix is already in place: delete "
+        "nemo_rl/transformers_compat.py, its bootstrap at the bottom of "
+        "nemo_rl/__init__.py, and this test file. Do not widen the version gate."
+    )
+    for filename, content in _SOURCE_FILES.items():
+        assert (cached_source_dir / filename).read_text(encoding="utf-8") == content
+
+
+@pytest.mark.parametrize(
+    "version",
+    ("5.11.0", "5.12.0", "5.12.1", "5.12.1+vendor.1"),
+)
+def test_patch_applies_to_affected_transformers_versions(monkeypatch, version):
+    def original_function(pretrained_model_name_or_path, resolved_module_file):
+        return None
+
+    monkeypatch.setattr(transformers_compat, "distribution_version", lambda _: version)
+    monkeypatch.setattr(
+        dynamic_module_utils,
+        "_compute_local_source_files_hash",
+        original_function,
+    )
+
+    assert _patch_transformers_dynamic_module_symlink_cache() is True
+    assert (
+        dynamic_module_utils._compute_local_source_files_hash
+        is _compute_local_source_files_hash_with_symlink_fix
+    )
+
+
+@pytest.mark.parametrize("version", ("5.10.0", "5.13.0", "6.0.0"))
+def test_patch_skips_unaffected_transformers_versions(monkeypatch, version):
+    original_function = object()
+    monkeypatch.setattr(transformers_compat, "distribution_version", lambda _: version)
+    monkeypatch.setattr(
+        dynamic_module_utils,
+        "_compute_local_source_files_hash",
+        original_function,
+    )
+
+    assert _patch_transformers_dynamic_module_symlink_cache() is False
+    assert dynamic_module_utils._compute_local_source_files_hash is original_function
+
+
+def test_patch_is_idempotent_without_function_attribute(monkeypatch):
+    monkeypatch.setattr(transformers_compat, "distribution_version", lambda _: "5.12.1")
+    monkeypatch.setattr(
+        dynamic_module_utils,
+        "_compute_local_source_files_hash",
+        _compute_local_source_files_hash_with_symlink_fix,
+    )
+
+    assert _patch_transformers_dynamic_module_symlink_cache() is False
+    assert not vars(_compute_local_source_files_hash_with_symlink_fix)
+
+
+def test_patch_skips_when_transformers_is_not_installed(monkeypatch):
+    def missing_distribution(_):
+        raise PackageNotFoundError
+
+    monkeypatch.setattr(
+        transformers_compat, "distribution_version", missing_distribution
+    )
+
+    assert _patch_transformers_dynamic_module_symlink_cache() is False
+
+
+def test_patch_propagates_import_error_from_transformers(monkeypatch):
+    monkeypatch.setattr(transformers_compat, "distribution_version", lambda _: "5.12.1")
+
+    def broken_import(_):
+        raise ImportError("broken Transformers installation")
+
+    monkeypatch.setattr(transformers_compat.importlib, "import_module", broken_import)
+
+    with pytest.raises(ImportError, match="broken Transformers installation"):
+        _patch_transformers_dynamic_module_symlink_cache()
+
+
+def test_patch_fails_loudly_if_transformers_target_is_missing(monkeypatch):
+    monkeypatch.setattr(transformers_compat, "distribution_version", lambda _: "5.12.1")
+    monkeypatch.delattr(
+        dynamic_module_utils,
+        "_compute_local_source_files_hash",
+        raising=False,
+    )
+
+    with pytest.raises(RuntimeError, match="cannot apply"):
+        _patch_transformers_dynamic_module_symlink_cache()
+
+
+def test_patch_fails_loudly_if_transformers_target_signature_changed(monkeypatch):
+    def incompatible_function(unexpected_parameter):
+        return None
+
+    monkeypatch.setattr(transformers_compat, "distribution_version", lambda _: "5.12.1")
+    monkeypatch.setattr(
+        dynamic_module_utils,
+        "_compute_local_source_files_hash",
+        incompatible_function,
+    )
+
+    with pytest.raises(RuntimeError, match="unexpected.*signature"):
+        _patch_transformers_dynamic_module_symlink_cache()