diff --git a/megatron/core/optimizer/emerging_optimizers.py b/megatron/core/optimizer/emerging_optimizers.py index 25294beabdf..f10c79dc4be 100644 --- a/megatron/core/optimizer/emerging_optimizers.py +++ b/megatron/core/optimizer/emerging_optimizers.py @@ -232,7 +232,8 @@ def orthogonalize(self, p: torch.Tensor, grad: torch.Tensor, **kwargs: Any) -> t def _eopt_init_state_fn(opt, config=None): """Initialize emerging optimizer state for torch_dist checkpoint format.""" for group in opt.param_groups: - opt._init_group(group) + # Checkpoint init needs state for all parameters, including those without grads yet. + opt._init_group(group, skip_non_grad_params=False) def _kwargs_from_config(optimizer_cls: type, prefix: str, config) -> Dict[str, Any]: diff --git a/pyproject.toml b/pyproject.toml index 52a168aaa3a..3a9d27b6a81 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -190,7 +190,7 @@ flash_mla = [ ] transformer-engine = { git = "https://github.com/NVIDIA/TransformerEngine.git", rev = "5671fd3675906cda1ade26c24a65d3dedd88eb89" } nemo-run = { git = "https://github.com/NVIDIA-NeMo/Run.git", rev = "01a9a8ba360f7b2908728ad0516e0ad9d936966d" } -emerging_optimizers = { git = "https://github.com/NVIDIA-NeMo/Emerging-Optimizers.git", rev = "a8faf641d5fca6a0515cfc010b6cedbf488cc33a" } +emerging_optimizers = { git = "https://github.com/NVIDIA-NeMo/Emerging-Optimizers.git", rev = "bc634ff8c0cf4fb5dbae0a531081281b499be3a0" } fast-hadamard-transform = { git = "https://github.com/Dao-AILab/fast-hadamard-transform.git", rev = "f134af63deb2df17e1171a9ec1ea4a7d8604d5ca" } [tool.isort] diff --git a/uv.lock b/uv.lock index 08482b4b7b8..d1e17d67196 100644 --- a/uv.lock +++ b/uv.lock @@ -1411,7 +1411,7 @@ wheels = [ [[package]] name = "emerging-optimizers" version = "0.2.0" -source = { git = "https://github.com/NVIDIA-NeMo/Emerging-Optimizers.git?rev=a8faf641d5fca6a0515cfc010b6cedbf488cc33a#a8faf641d5fca6a0515cfc010b6cedbf488cc33a" } +source = { git = "https://github.com/NVIDIA-NeMo/Emerging-Optimizers.git?rev=bc634ff8c0cf4fb5dbae0a531081281b499be3a0#bc634ff8c0cf4fb5dbae0a531081281b499be3a0" } dependencies = [ { name = "absl-py", marker = "python_full_version >= '3.12' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "torch", marker = "(python_full_version >= '3.12' and sys_platform == 'never') or (python_full_version < '3.12' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform != 'never' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, @@ -2748,8 +2748,8 @@ requires-dist = [ { name = "datasets", marker = "extra == 'lts'" }, { name = "einops", marker = "extra == 'dev'", specifier = "~=0.8" }, { name = "einops", marker = "extra == 'lts'", specifier = "~=0.8" }, - { name = "emerging-optimizers", marker = "python_full_version >= '3.12' and extra == 'dev'", git = "https://github.com/NVIDIA-NeMo/Emerging-Optimizers.git?rev=a8faf641d5fca6a0515cfc010b6cedbf488cc33a" }, - { name = "emerging-optimizers", marker = "python_full_version >= '3.12' and extra == 'lts'", git = "https://github.com/NVIDIA-NeMo/Emerging-Optimizers.git?rev=a8faf641d5fca6a0515cfc010b6cedbf488cc33a" }, + { name = "emerging-optimizers", marker = "python_full_version >= '3.12' and extra == 'dev'", git = "https://github.com/NVIDIA-NeMo/Emerging-Optimizers.git?rev=bc634ff8c0cf4fb5dbae0a531081281b499be3a0" }, + { name = "emerging-optimizers", marker = "python_full_version >= '3.12' and extra == 'lts'", git = "https://github.com/NVIDIA-NeMo/Emerging-Optimizers.git?rev=bc634ff8c0cf4fb5dbae0a531081281b499be3a0" }, { name = "fastapi", marker = "extra == 'dev'", specifier = "~=0.50" }, { name = "fastapi", marker = "extra == 'lts'", specifier = "~=0.50" }, { name = "flash-linear-attention", marker = "extra == 'dev'", specifier = "~=0.4.0" }, @@ -2823,7 +2823,7 @@ linting = [ { name = "ruff", specifier = "~=0.9.0" }, ] no-pypi-wheels = [ - { name = "emerging-optimizers", marker = "python_full_version >= '3.12'", git = "https://github.com/NVIDIA-NeMo/Emerging-Optimizers.git?rev=a8faf641d5fca6a0515cfc010b6cedbf488cc33a" }, + { name = "emerging-optimizers", marker = "python_full_version >= '3.12'", git = "https://github.com/NVIDIA-NeMo/Emerging-Optimizers.git?rev=bc634ff8c0cf4fb5dbae0a531081281b499be3a0" }, { name = "fast-hadamard-transform", git = "https://github.com/Dao-AILab/fast-hadamard-transform.git?rev=f134af63deb2df17e1171a9ec1ea4a7d8604d5ca" }, ] test = [