From b62e0254c5abfb6b5410b0b88b149ded10a4dcd9 Mon Sep 17 00:00:00 2001 From: Zhiyu Li Date: Thu, 9 Jul 2026 01:47:17 -0700 Subject: [PATCH 1/9] fix: un-blocklist deepseek_v3 in transformers tokenizer path Fixes #2764. transformers 5.4-5.11 lists "deepseek_v3" in two internal registries -- MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASS (a set) and TOKENIZER_MAPPING_NAMES (a dict pinning it to "TokenizersBackend"). Together they force the fast tokenizer backend and suppress trust_remote_code, so AutoTokenizer can only load via a local tokenizer.json. Models like Moonlight-16B-A3B ship no tokenizer.json (only tiktoken.model + a remote-code TikTokenTokenizer), so offline loading fails with "Couldn't instantiate the backend tokenizer". Two coordinated patches: 1. nemo_rl/__init__.py: discard "deepseek_v3" from both registries at nemo_rl import time. Adds a version assertion that fires if transformers is bumped past 5.12 (upstream fix), noting Megatron-Bridge is the current blocker for that bump. 2. nemo_rl/models/megatron/setup.py: set megatron_cfg.tokenizer.trust_remote_code attribute directly. The pre-existing hf_tokenizer_kwargs["trust_remote_code"] dict mutation was dead code because Megatron-Bridge's TokenizerConfig snapshots hf_tokenizer_kwargs into plain attributes at __post_init__ and never re-reads the dict afterwards. Both patches are held together by the same MBridge-caps-transformers-below-5.12 dependency chain and can be dropped together once that pin is relaxed. Verified end-to-end: llm_grpo_moonlight_16b_automodel_1n8g_ep8 trains 10+ steps offline; llm_grpo_moonlight_16ba3b_4n8g_megatron trains 17+ steps offline with KL 0.0003 stable, no NaN, zero backend-tokenizer errors on driver or 32 workers. Signed-off-by: Zhiyu Li --- nemo_rl/__init__.py | 39 ++++++++++++++++++++++++++++++++ nemo_rl/models/megatron/setup.py | 9 ++++---- 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/nemo_rl/__init__.py b/nemo_rl/__init__.py index 31f186c4c23..f9e00e8dcbb 100644 --- a/nemo_rl/__init__.py +++ b/nemo_rl/__init__.py @@ -274,6 +274,45 @@ def _patch_nsight_file(): _patch_nsight_file() +def _patch_transformers_tokenizer_class_set(): + """Undo the transformers block on deepseek_v3 tokenizers. + + Root cause: transformers 5.4-5.11 lists "deepseek_v3" in two internal + registries -- MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASS (a set) and + TOKENIZER_MAPPING_NAMES (a dict pinning it to "TokenizersBackend"). Together + they force the fast tokenizer backend and suppress trust_remote_code, so + AutoTokenizer can only load via a local tokenizer.json. Models like + Moonlight-16B-A3B ship no tokenizer.json (only tiktoken.model + a remote-code + TikTokenTokenizer), so offline loading fails. + + Removing both entries restores the trust_remote_code / auto_map path. + discard/pop-with-default are no-ops when the entries are absent, so this is + safe on any transformers version in the currently-supported range. + """ + import transformers + + # This whole patch exists only because Megatron-Bridge caps the transformers + # upper bound below 5.9 today, which forces us onto a transformers version + # that still has the deepseek_v3 tokenizer-blocklist bug. Once MBridge relaxes + # its transformers upper bound to >=5.12, we can drop this workaround. + assert transformers.__version__ < "5.12.0", ( + f"transformers {transformers.__version__} detected. " + "The deepseek_v3 tokenizer-blocklist patch was written for <5.12. " + "Check if the upstream fix now applies and remove this patch if so." + ) + + from transformers.models.auto.tokenization_auto import ( + MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASS, + TOKENIZER_MAPPING_NAMES, + ) + + MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASS.discard("deepseek_v3") + TOKENIZER_MAPPING_NAMES.pop("deepseek_v3", None) + + +_patch_transformers_tokenizer_class_set() + + # Need to set PYTHONPATH to include transformers downloaded modules. # Assuming the cache directory is the same cross venvs. def patch_transformers_module_dir(env_vars: dict[str, str]): diff --git a/nemo_rl/models/megatron/setup.py b/nemo_rl/models/megatron/setup.py index e8d7f81450c..eea649c2355 100644 --- a/nemo_rl/models/megatron/setup.py +++ b/nemo_rl/models/megatron/setup.py @@ -1206,11 +1206,10 @@ def setup_model_and_optimizer( # Context used for persisting some state between checkpoint saves. checkpointing_context = init_checkpointing_context(megatron_cfg.checkpoint) - # Tokenizer - if megatron_cfg.tokenizer.hf_tokenizer_kwargs is None: - megatron_cfg.tokenizer.hf_tokenizer_kwargs = {} - megatron_cfg.tokenizer.hf_tokenizer_kwargs["trust_remote_code"] = True - megatron_cfg.tokenizer.hf_tokenizer_kwargs["use_fast"] = True + # Set the attribute directly instead of updating hf_tokenizer_kwargs, because + # Megatron-Bridge's TokenizerConfig snapshots hf_tokenizer_kwargs into plain + # attributes at __post_init__ and never re-reads the dict afterwards. + megatron_cfg.tokenizer.trust_remote_code = True build_tokenizer( megatron_cfg.tokenizer, make_vocab_size_divisible_by=megatron_cfg.model.make_vocab_size_divisible_by From 41c8e08724f8cf0fcc1a35e4a637e4674d4d1148 Mon Sep 17 00:00:00 2001 From: Zhiyu Li Date: Thu, 9 Jul 2026 01:54:48 -0700 Subject: [PATCH 2/9] docs: add explicit TODO(#2764) marker above transformers version assert Signed-off-by: Zhiyu Li --- nemo_rl/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nemo_rl/__init__.py b/nemo_rl/__init__.py index f9e00e8dcbb..c0965de7675 100644 --- a/nemo_rl/__init__.py +++ b/nemo_rl/__init__.py @@ -295,6 +295,8 @@ def _patch_transformers_tokenizer_class_set(): # upper bound below 5.9 today, which forces us onto a transformers version # that still has the deepseek_v3 tokenizer-blocklist bug. Once MBridge relaxes # its transformers upper bound to >=5.12, we can drop this workaround. + # TODO(#2764): remove this patch (and the assert below) once MBridge relaxes + # its transformers upper bound past the deepseek_v3 fix (~transformers 5.12). assert transformers.__version__ < "5.12.0", ( f"transformers {transformers.__version__} detected. " "The deepseek_v3 tokenizer-blocklist patch was written for <5.12. " From a74e96617a1eea0f86d5428b12257f350ca64dc5 Mon Sep 17 00:00:00 2001 From: Zhiyu Li Date: Thu, 9 Jul 2026 01:55:45 -0700 Subject: [PATCH 3/9] docs: switch TODO format to canonical style (TODO: + issue URL) Signed-off-by: Zhiyu Li --- nemo_rl/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nemo_rl/__init__.py b/nemo_rl/__init__.py index c0965de7675..46212109ed9 100644 --- a/nemo_rl/__init__.py +++ b/nemo_rl/__init__.py @@ -295,8 +295,9 @@ def _patch_transformers_tokenizer_class_set(): # upper bound below 5.9 today, which forces us onto a transformers version # that still has the deepseek_v3 tokenizer-blocklist bug. Once MBridge relaxes # its transformers upper bound to >=5.12, we can drop this workaround. - # TODO(#2764): remove this patch (and the assert below) once MBridge relaxes - # its transformers upper bound past the deepseek_v3 fix (~transformers 5.12). + # TODO: remove this patch (and the assert below) once MBridge relaxes its + # transformers upper bound past the deepseek_v3 fix (~transformers 5.12). + # https://github.com/NVIDIA-NeMo/RL/issues/2764 assert transformers.__version__ < "5.12.0", ( f"transformers {transformers.__version__} detected. " "The deepseek_v3 tokenizer-blocklist patch was written for <5.12. " From 796546b5b32d3e3485ca3d18f1a2f9ed72357ac6 Mon Sep 17 00:00:00 2001 From: Zhiyu Li Date: Thu, 9 Jul 2026 01:57:41 -0700 Subject: [PATCH 4/9] fix: use packaging.version.Version for semver-correct transformers assert String comparison "5.5.0" < "5.12.0" is False in Python (lexicographic '5' > '1' at the minor position), so the assertion would fire and raise AssertionError on every import for every currently-pinned transformers version (>=5.5.0,<5.9.0). Switch to packaging.version.Version. Signed-off-by: Zhiyu Li --- nemo_rl/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nemo_rl/__init__.py b/nemo_rl/__init__.py index 46212109ed9..df8c7db1d72 100644 --- a/nemo_rl/__init__.py +++ b/nemo_rl/__init__.py @@ -290,6 +290,7 @@ def _patch_transformers_tokenizer_class_set(): safe on any transformers version in the currently-supported range. """ import transformers + from packaging.version import Version # This whole patch exists only because Megatron-Bridge caps the transformers # upper bound below 5.9 today, which forces us onto a transformers version @@ -298,7 +299,7 @@ def _patch_transformers_tokenizer_class_set(): # TODO: remove this patch (and the assert below) once MBridge relaxes its # transformers upper bound past the deepseek_v3 fix (~transformers 5.12). # https://github.com/NVIDIA-NeMo/RL/issues/2764 - assert transformers.__version__ < "5.12.0", ( + assert Version(transformers.__version__) < Version("5.12.0"), ( f"transformers {transformers.__version__} detected. " "The deepseek_v3 tokenizer-blocklist patch was written for <5.12. " "Check if the upstream fix now applies and remove this patch if so." From a1d1a68ca1905126541b0738b411b4411686ee6d Mon Sep 17 00:00:00 2001 From: Zhiyu Li Date: Thu, 9 Jul 2026 01:59:45 -0700 Subject: [PATCH 5/9] refactor: use stdlib tuple compare for transformers version check (drop packaging dep) Signed-off-by: Zhiyu Li --- nemo_rl/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nemo_rl/__init__.py b/nemo_rl/__init__.py index df8c7db1d72..c42bc4c30ed 100644 --- a/nemo_rl/__init__.py +++ b/nemo_rl/__init__.py @@ -290,7 +290,6 @@ def _patch_transformers_tokenizer_class_set(): safe on any transformers version in the currently-supported range. """ import transformers - from packaging.version import Version # This whole patch exists only because Megatron-Bridge caps the transformers # upper bound below 5.9 today, which forces us onto a transformers version @@ -299,7 +298,9 @@ def _patch_transformers_tokenizer_class_set(): # TODO: remove this patch (and the assert below) once MBridge relaxes its # transformers upper bound past the deepseek_v3 fix (~transformers 5.12). # https://github.com/NVIDIA-NeMo/RL/issues/2764 - assert Version(transformers.__version__) < Version("5.12.0"), ( + _version_parts = transformers.__version__.split(".") + _version_tuple = (int(_version_parts[0]), int(_version_parts[1])) + assert _version_tuple < (5, 12), ( f"transformers {transformers.__version__} detected. " "The deepseek_v3 tokenizer-blocklist patch was written for <5.12. " "Check if the upstream fix now applies and remove this patch if so." From ca6a042071383f80449f2051554fadf81fe5ea7f Mon Sep 17 00:00:00 2001 From: Zhiyu Li Date: Thu, 9 Jul 2026 02:00:51 -0700 Subject: [PATCH 6/9] refactor: use PkgVersion alias matching nemo_rl/utils/flops_tracker.py canonical pattern Signed-off-by: Zhiyu Li --- nemo_rl/__init__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nemo_rl/__init__.py b/nemo_rl/__init__.py index c42bc4c30ed..9860d58a808 100644 --- a/nemo_rl/__init__.py +++ b/nemo_rl/__init__.py @@ -290,6 +290,7 @@ def _patch_transformers_tokenizer_class_set(): safe on any transformers version in the currently-supported range. """ import transformers + from packaging.version import Version as PkgVersion # This whole patch exists only because Megatron-Bridge caps the transformers # upper bound below 5.9 today, which forces us onto a transformers version @@ -298,9 +299,7 @@ def _patch_transformers_tokenizer_class_set(): # TODO: remove this patch (and the assert below) once MBridge relaxes its # transformers upper bound past the deepseek_v3 fix (~transformers 5.12). # https://github.com/NVIDIA-NeMo/RL/issues/2764 - _version_parts = transformers.__version__.split(".") - _version_tuple = (int(_version_parts[0]), int(_version_parts[1])) - assert _version_tuple < (5, 12), ( + assert PkgVersion(transformers.__version__) < PkgVersion("5.12.0"), ( f"transformers {transformers.__version__} detected. " "The deepseek_v3 tokenizer-blocklist patch was written for <5.12. " "Check if the upstream fix now applies and remove this patch if so." From 420bd42c58983ef525a639e2de674cf7f1da98f8 Mon Sep 17 00:00:00 2001 From: Zhiyu Li Date: Thu, 9 Jul 2026 02:42:16 -0700 Subject: [PATCH 7/9] fix(lint): guard transformers import against setuptools build isolation The lint CI's editable install pipeline (uv sync + setuptools) imports nemo_rl to statically read __version__ before deps are guaranteed to be present. Without a guard, 'import transformers' at nemo_rl load time crashes the build with 'nemo_rl has no attribute __version__'. Wrap the top-level 'import transformers' in try/except so the patch becomes a no-op under build isolation. Runtime venv always has the package, so the assertion + discards still fire in real use. Signed-off-by: Zhiyu Li --- nemo_rl/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/nemo_rl/__init__.py b/nemo_rl/__init__.py index 9860d58a808..fb3c3556b39 100644 --- a/nemo_rl/__init__.py +++ b/nemo_rl/__init__.py @@ -289,7 +289,12 @@ def _patch_transformers_tokenizer_class_set(): discard/pop-with-default are no-ops when the entries are absent, so this is safe on any transformers version in the currently-supported range. """ - import transformers + try: + import transformers + except ImportError: + # setuptools build isolation may import nemo_rl before transformers is + # installed; skip the patch there. The runtime venv always has it. + return from packaging.version import Version as PkgVersion # This whole patch exists only because Megatron-Bridge caps the transformers From 6b980c4243bcc6e5075cb705d59207f90b3f0d5c Mon Sep 17 00:00:00 2001 From: Zhiyu Li Date: Thu, 9 Jul 2026 02:48:36 -0700 Subject: [PATCH 8/9] refactor(lint): guard tokenizer patch with _is_build_isolation() instead of try/except Same rationale as the existing _check_container_fingerprint guard on line 99: setuptools' uv-based build isolation imports nemo_rl to read __version__, but transformers isn't in the isolated build env. Match the existing pattern -- skip the patch at the call site under build isolation, keep the function body clean (no exception handling for control flow). Runtime venv is never a build isolation, so the patch still fires in real use. Signed-off-by: Zhiyu Li --- nemo_rl/__init__.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/nemo_rl/__init__.py b/nemo_rl/__init__.py index fb3c3556b39..7398e764a38 100644 --- a/nemo_rl/__init__.py +++ b/nemo_rl/__init__.py @@ -289,12 +289,7 @@ def _patch_transformers_tokenizer_class_set(): discard/pop-with-default are no-ops when the entries are absent, so this is safe on any transformers version in the currently-supported range. """ - try: - import transformers - except ImportError: - # setuptools build isolation may import nemo_rl before transformers is - # installed; skip the patch there. The runtime venv always has it. - return + import transformers from packaging.version import Version as PkgVersion # This whole patch exists only because Megatron-Bridge caps the transformers @@ -319,7 +314,8 @@ def _patch_transformers_tokenizer_class_set(): TOKENIZER_MAPPING_NAMES.pop("deepseek_v3", None) -_patch_transformers_tokenizer_class_set() +if not _is_build_isolation(): + _patch_transformers_tokenizer_class_set() # Need to set PYTHONPATH to include transformers downloaded modules. From 04051093ec701b1ee27f2113d6d63d0b91fef9e5 Mon Sep 17 00:00:00 2001 From: Zhiyu Li Date: Thu, 9 Jul 2026 02:59:00 -0700 Subject: [PATCH 9/9] refactor: move deepseek_v3 tokenizer patch from nemo_rl.__init__ to nemo_rl.models.policy.__init__ The patch is only needed by policy code paths that eventually call AutoTokenizer.from_pretrained -- driver's get_tokenizer(), Megatron worker's build_tokenizer, DTensor worker's tokenizer reconstruction. All three share nemo_rl.models.policy as an import ancestor. Placing it in nemo_rl.models.policy.__init__.py fires exactly once per process when policy code is first imported. Non-policy consumers (tests, utilities, environments) no longer mutate the global transformers registries as a side effect of importing nemo_rl. Bonus: setuptools build isolation reads nemo_rl.__version__ from nemo_rl.__init__.py but never cascades into models/policy, so the _is_build_isolation() guard is no longer needed at the call site. Verified locally with 'from nemo_rl.models.policy import PolicyConfig': deepseek_v3 removed from MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASS and TOKENIZER_MAPPING_NAMES as expected. Signed-off-by: Zhiyu Li --- nemo_rl/__init__.py | 44 --------------------------- nemo_rl/models/policy/__init__.py | 49 +++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 44 deletions(-) diff --git a/nemo_rl/__init__.py b/nemo_rl/__init__.py index 7398e764a38..31f186c4c23 100644 --- a/nemo_rl/__init__.py +++ b/nemo_rl/__init__.py @@ -274,50 +274,6 @@ def _patch_nsight_file(): _patch_nsight_file() -def _patch_transformers_tokenizer_class_set(): - """Undo the transformers block on deepseek_v3 tokenizers. - - Root cause: transformers 5.4-5.11 lists "deepseek_v3" in two internal - registries -- MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASS (a set) and - TOKENIZER_MAPPING_NAMES (a dict pinning it to "TokenizersBackend"). Together - they force the fast tokenizer backend and suppress trust_remote_code, so - AutoTokenizer can only load via a local tokenizer.json. Models like - Moonlight-16B-A3B ship no tokenizer.json (only tiktoken.model + a remote-code - TikTokenTokenizer), so offline loading fails. - - Removing both entries restores the trust_remote_code / auto_map path. - discard/pop-with-default are no-ops when the entries are absent, so this is - safe on any transformers version in the currently-supported range. - """ - import transformers - from packaging.version import Version as PkgVersion - - # This whole patch exists only because Megatron-Bridge caps the transformers - # upper bound below 5.9 today, which forces us onto a transformers version - # that still has the deepseek_v3 tokenizer-blocklist bug. Once MBridge relaxes - # its transformers upper bound to >=5.12, we can drop this workaround. - # TODO: remove this patch (and the assert below) once MBridge relaxes its - # transformers upper bound past the deepseek_v3 fix (~transformers 5.12). - # https://github.com/NVIDIA-NeMo/RL/issues/2764 - assert PkgVersion(transformers.__version__) < PkgVersion("5.12.0"), ( - f"transformers {transformers.__version__} detected. " - "The deepseek_v3 tokenizer-blocklist patch was written for <5.12. " - "Check if the upstream fix now applies and remove this patch if so." - ) - - from transformers.models.auto.tokenization_auto import ( - MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASS, - TOKENIZER_MAPPING_NAMES, - ) - - MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASS.discard("deepseek_v3") - TOKENIZER_MAPPING_NAMES.pop("deepseek_v3", None) - - -if not _is_build_isolation(): - _patch_transformers_tokenizer_class_set() - - # Need to set PYTHONPATH to include transformers downloaded modules. # Assuming the cache directory is the same cross venvs. def patch_transformers_module_dir(env_vars: dict[str, str]): diff --git a/nemo_rl/models/policy/__init__.py b/nemo_rl/models/policy/__init__.py index 55f511bb569..8f2448e4e76 100644 --- a/nemo_rl/models/policy/__init__.py +++ b/nemo_rl/models/policy/__init__.py @@ -18,6 +18,55 @@ from nemo_rl.utils.checkpoint import PretrainedCheckpointConfig +def _patch_transformers_tokenizer_class_set(): + """Undo the transformers block on deepseek_v3 tokenizers. + + Root cause: transformers 5.4-5.11 lists "deepseek_v3" in two internal + registries -- MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASS (a set) and + TOKENIZER_MAPPING_NAMES (a dict pinning it to "TokenizersBackend"). Together + they force the fast tokenizer backend and suppress trust_remote_code, so + AutoTokenizer can only load via a local tokenizer.json. Models like + Moonlight-16B-A3B ship no tokenizer.json (only tiktoken.model + a remote-code + TikTokenTokenizer), so offline loading fails. + + Removing both entries restores the trust_remote_code / auto_map path. + discard/pop-with-default are no-ops when the entries are absent, so this is + safe on any transformers version in the currently-supported range. + + Placed here (nemo_rl/models/policy/__init__.py) so it fires exactly once + per process the first time any policy code is imported -- covers the driver + (via nemo_rl.algorithms.grpo) and every policy worker (Megatron / DTensor / + DTensor v2 all import from nemo_rl.models.policy) without polluting nemo_rl + consumers that don't touch tokenizers. + """ + import transformers + from packaging.version import Version as PkgVersion + + # This whole patch exists only because Megatron-Bridge caps the transformers + # upper bound below 5.9 today, which forces us onto a transformers version + # that still has the deepseek_v3 tokenizer-blocklist bug. Once MBridge relaxes + # its transformers upper bound to >=5.12, we can drop this workaround. + # TODO: remove this patch (and the assert below) once MBridge relaxes its + # transformers upper bound past the deepseek_v3 fix (~transformers 5.12). + # https://github.com/NVIDIA-NeMo/RL/issues/2764 + assert PkgVersion(transformers.__version__) < PkgVersion("5.12.0"), ( + f"transformers {transformers.__version__} detected. " + "The deepseek_v3 tokenizer-blocklist patch was written for <5.12. " + "Check if the upstream fix now applies and remove this patch if so." + ) + + from transformers.models.auto.tokenization_auto import ( + MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASS, + TOKENIZER_MAPPING_NAMES, + ) + + MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASS.discard("deepseek_v3") + TOKENIZER_MAPPING_NAMES.pop("deepseek_v3", None) + + +_patch_transformers_tokenizer_class_set() + + class LoRAConfigDisabled(TypedDict): enabled: Literal[False]