chore: Enable cuda-13 build

.github/workflows/cicd-main.yml

@@ -291,7 +291,7 @@ jobs:
       build-ref: ${{ needs.pre-flight.outputs.test_sha }}
       image-name: ${{ vars.CI_CONTAINER_NAME }}
       dockerfile: docker/Dockerfile
-      runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}-gpu-x2
+      runner: ${{ contains(needs.org-member-pre-flight.outputs.runner_prefix, 'azure') && format('{0}-gpu-x2', needs.org-member-pre-flight.outputs.runner_prefix) || contains(needs.org-member-pre-flight.outputs.runner_prefix, 'gcp') && format('{0}-gpu-x4', needs.org-member-pre-flight.outputs.runner_prefix) }}
       image-label: ${{ vars.CI_CONTAINER_NAME }}
       target: release
       registry: ${{ needs.org-member-pre-flight.outputs.registry }}

3rdparty/Megatron-Bridge-workspace/setup.py

@@ -49,7 +49,7 @@
     "megatron-core[dev,mlm]",
     "qwen-vl-utils",
     # TODO(https://github.com/NVIDIA-NeMo/RL/issues/2111): upgrade to core_cu13 when we move to CUDA 13 base container
-    "transformer-engine[pytorch,core_cu12]",
+    "transformer-engine[pytorch,core_cu13]",
     "mamba-ssm",
     "nvidia-resiliency-ext",
     "causal-conv1d",
@@ -74,20 +74,9 @@
     project = data["project"]
     deps_list = project["dependencies"]
     submodule_deps = set(str(d).strip() for d in deps_list)
-
-    # Normalize the transformer-engine CUDA variant extra (core_cu12 vs core_cu13)
-    # so our CUDA 12 override doesn't trip the consistency check against the
-    # submodule's CUDA 13 default.
-    # TODO(https://github.com/NVIDIA-NeMo/RL/issues/2111): remove this when we upgrade to CUDA 13
-    def _normalize_te_cuda(dep):
-        if dep.startswith("transformer-engine") or dep.startswith("transformer_engine"):
-            return dep.replace("core_cu13", "core_cu12")
-        return dep
-
-    normalized_submodule = set(_normalize_te_cuda(d) for d in submodule_deps)
-    normalized_cached = set(_normalize_te_cuda(d) for d in CACHED_DEPENDENCIES)
-    missing_in_cached = normalized_submodule - normalized_cached
-    extra_in_cached = normalized_cached - normalized_submodule
+    cached_deps = set(CACHED_DEPENDENCIES)
+    missing_in_cached = submodule_deps - cached_deps
+    extra_in_cached = cached_deps - submodule_deps
 
     if missing_in_cached or extra_in_cached:
         print(

3rdparty/Megatron-LM-workspace/setup.py

@@ -49,7 +49,7 @@
     # Dev dependencies from pyproject.toml
     "nvidia-modelopt[torch]; sys_platform != 'darwin'",
     # TODO(https://github.com/NVIDIA-NeMo/RL/issues/2111): upgrade to core_cu13 when we move to CUDA 13 base container
-    "transformer-engine[pytorch,core_cu12]",
+    "transformer-engine[pytorch,core_cu13]",
     # VCS dependency - must match pyproject.toml [tool.uv.sources]
     "nvidia-resiliency-ext @ git+https://github.com/NVIDIA/nvidia-resiliency-ext.git@15a851565a4ce846c04431ecb0cf09903ab4837e",
     "tqdm",
@@ -122,20 +122,8 @@ def build_vcs_dependency(pkg_name: str, source_info: dict) -> str:
             submodule_deps_with_vcs.add(dep)
 
     cached_deps_set = set(CACHED_DEPENDENCIES)
-
-    # Normalize the transformer-engine CUDA variant extra (core_cu12 vs core_cu13)
-    # so our CUDA 12 override doesn't trip the consistency check against the
-    # submodule's CUDA 13 default.
-    # TODO(https://github.com/NVIDIA-NeMo/RL/issues/2111): remove this when we upgrade to CUDA 13
-    def _normalize_te_cuda(dep):
-        if dep.startswith("transformer-engine") or dep.startswith("transformer_engine"):
-            return dep.replace("core_cu13", "core_cu12")
-        return dep
-
-    normalized_submodule = set(_normalize_te_cuda(d) for d in submodule_deps_with_vcs)
-    normalized_cached = set(_normalize_te_cuda(d) for d in cached_deps_set)
-    missing_in_cached = normalized_submodule - normalized_cached
-    extra_in_cached = normalized_cached - normalized_submodule
+    missing_in_cached = submodule_deps_with_vcs - cached_deps_set
+    extra_in_cached = cached_deps_set - submodule_deps_with_vcs
 
     if missing_in_cached or extra_in_cached:
         print(

docker/Dockerfile

@@ -16,7 +16,7 @@
 #   --build-arg SKIP_VLLM_BUILD=1    # Skip vLLM dependencies
 #   --build-arg SKIP_SGLANG_BUILD=1  # Skip SGLang dependencies
 
-ARG BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:25.05-cuda12.9-devel-ubuntu24.04
+ARG BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:26.03-cuda13.2-devel-ubuntu24.04
 FROM scratch AS nemo-rl
 ARG NRL_GIT_REF=main
 ADD --keep-git-dir=true https://github.com/NVIDIA-NeMo/RL.git#${NRL_GIT_REF} /
@@ -112,6 +112,10 @@ ENV UV_LINK_MODE=copy
 # Ensure DeepEP is built for H100 and B200 (also mcore inference unified memory API now invokes a torch API that requires these to be set)
 ENV TORCH_CUDA_ARCH_LIST="9.0 10.0"
 
+## CUDA 13 moved standard library headers under include/cccl; expose that path so
+## builds that include <cuda/std/...> (e.g., deep_gemm) can find them.
+ENV CPLUS_INCLUDE_PATH=/usr/local/cuda/include/cccl
+
 # First copy only the dependency files
 COPY --from=nemo-rl pyproject.toml uv.lock ./
 # Copy in the top level __init__.py/package_info.py since build-custom-vllm.sh needs the nemo_rl package to exist.

pyproject.toml

@@ -46,30 +46,40 @@ dependencies = [
   "transformers==5.3.0",
   "num2words>=0.5.14",                                                                                                # for SmolVLM
   "mlflow>=3.11.1",
-  "nvidia-nvshmem-cu12; sys_platform == 'linux' and (platform_machine == 'x86_64' or platform_machine == 'aarch64')", # for deep_ep build
+  "nvidia-nvshmem-cu13; sys_platform == 'linux' and (platform_machine == 'x86_64' or platform_machine == 'aarch64')", # for deep_ep build
   "swanlab",
   "pyzmq",
   "decord2",
   "soundfile>=0.13.1",
   "nccl4py; sys_platform != 'darwin'",                                                                                # for non-colocated refit
   "cuda-bindings; sys_platform != 'darwin'",                                                                          # for non-colocated refit
   "pybase64",                                                                                                         # for sglang refit
-  "nvidia-cudnn-cu12==9.19.0.56; sys_platform != 'darwin'",                                                           # for transformer-engine no build isolation
+  "nvidia-cudnn-cu13==9.20.0.48; sys_platform != 'darwin'",                                                           # for transformer-engine no build isolation
 ]
 
 [project.optional-dependencies]
-fsdp = ["flash-attn==2.8.1", "mamba-ssm", "causal-conv1d"]
+fsdp = [
+  # +cu13 wheels from GitHub match torch cu130; PyPI often resolves to +cu12 (libcudart.so.12).
+  # https://github.com/Dao-AILab/flash-attention/releases/tag/v2.8.1
+  "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.8.1/flash_attn-2.8.1+cu13torch2.10cxx11abiTRUE-cp313-cp313-linux_aarch64.whl ; sys_platform == 'linux' and platform_machine == 'aarch64'",
+  "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.8.1/flash_attn-2.8.1+cu13torch2.10cxx11abiTRUE-cp313-cp313-linux_x86_64.whl ; sys_platform == 'linux' and platform_machine == 'x86_64'",
+  "flash-attn==2.8.1 ; sys_platform != 'linux' or (platform_machine != 'aarch64' and platform_machine != 'x86_64')",
+  "mamba-ssm",
+  "causal-conv1d",
+]
 automodel = [
   "nemo-automodel[moe]",
   # Flash-attn version should be selected to satisfy both TE + vLLM requirements (xformers in particular)
   # https://github.com/NVIDIA/TransformerEngine/blob/v2.3/transformer_engine/pytorch/attention/dot_product_attention/utils.py#L108
   # https://github.com/facebookresearch/xformers/blob/8354497deb2c04c67fbb2e2ad911e86530da0e90/xformers/ops/fmha/flash.py#L76
-  "flash-attn==2.8.1",
+  "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.8.1/flash_attn-2.8.1+cu13torch2.10cxx11abiTRUE-cp313-cp313-linux_aarch64.whl ; sys_platform == 'linux' and platform_machine == 'aarch64'",
+  "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.8.1/flash_attn-2.8.1+cu13torch2.10cxx11abiTRUE-cp313-cp313-linux_x86_64.whl ; sys_platform == 'linux' and platform_machine == 'x86_64'",
+  "flash-attn==2.8.1 ; sys_platform != 'linux' or (platform_machine != 'aarch64' and platform_machine != 'x86_64')",
   "transformers>=5.3.0",
   "mamba-ssm",
   "causal-conv1d",
   "nv-grouped-gemm",
-  "transformer-engine[pytorch,core_cu12] @ git+https://github.com/NVIDIA/TransformerEngine.git@v2.14.1",
+  "transformer-engine[pytorch,core_cu13] @ git+https://github.com/NVIDIA/TransformerEngine.git@v2.14.1",
   "deep_ep @ git+https://github.com/deepseek-ai/DeepEP.git@bfded34800dfec415b71503f8205181de90b2480",
 ]
 vllm = [
@@ -79,7 +89,12 @@ vllm = [
   # sudo apt-get update
   # sudo apt-get install libibverbs-dev
   "deep_ep @ git+https://github.com/deepseek-ai/DeepEP.git@bfded34800dfec415b71503f8205181de90b2480",
-  "vllm==0.17.1",
+  # +cu130 wheels from GitHub match torch (pytorch-cu130). PyPI can resolve to +cu12
+  # binaries on Linux (e.g. libcudart.so.12 vs torch cu13). See v0.17.1 release assets:
+  # https://github.com/vllm-project/vllm/releases/tag/v0.17.1
+  "vllm @ https://github.com/vllm-project/vllm/releases/download/v0.17.1/vllm-0.17.1+cu130-cp38-abi3-manylinux_2_35_aarch64.whl ; sys_platform == 'linux' and platform_machine == 'aarch64'",
+  "vllm @ https://github.com/vllm-project/vllm/releases/download/v0.17.1/vllm-0.17.1+cu130-cp38-abi3-manylinux_2_35_x86_64.whl ; sys_platform == 'linux' and platform_machine == 'x86_64'",
+  "vllm==0.17.1 ; sys_platform != 'linux' or (platform_machine != 'aarch64' and platform_machine != 'x86_64')",
   "num2words>=0.5.14",
   "flashinfer-python==0.6.4",
   "flashinfer-cubin==0.6.4",
@@ -99,13 +114,15 @@ mcore = [
   # This dependency also needs to be compatible with the spec in Megatron-Bridge/pyproject.toml.
   # It is specified here since we don't directly use Megatron-Bridge/pyproject.toml, but a proxy setup.py+pyproject.toml combo
   # outside to allow "optionally" installing the megatron path. It's simpler to deal with transformer-engine here in the NeMo RL pyproject.toml
-  "transformer-engine[pytorch,core_cu12] @ git+https://github.com/NVIDIA/TransformerEngine.git@v2.14.1",
+  "transformer-engine[pytorch,core_cu13] @ git+https://github.com/NVIDIA/TransformerEngine.git@v2.14.1",
   "megatron-core",
   "megatron-bridge",
   # Flash-attn version should be selected to satisfy both TE + vLLM requirements (xformers in particular)
   # https://github.com/NVIDIA/TransformerEngine/blob/v2.3/transformer_engine/pytorch/attention/dot_product_attention/utils.py#L108
   # https://github.com/facebookresearch/xformers/blob/8354497deb2c04c67fbb2e2ad911e86530da0e90/xformers/ops/fmha/flash.py#L76
-  "flash-attn==2.8.1",
+  "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.8.1/flash_attn-2.8.1+cu13torch2.10cxx11abiTRUE-cp313-cp313-linux_aarch64.whl ; sys_platform == 'linux' and platform_machine == 'aarch64'",
+  "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.8.1/flash_attn-2.8.1+cu13torch2.10cxx11abiTRUE-cp313-cp313-linux_x86_64.whl ; sys_platform == 'linux' and platform_machine == 'x86_64'",
+  "flash-attn==2.8.1 ; sys_platform != 'linux' or (platform_machine != 'aarch64' and platform_machine != 'x86_64')",
   "emerging-optimizers==0.2.0",
   "deep_ep @ git+https://github.com/deepseek-ai/DeepEP.git@bfded34800dfec415b71503f8205181de90b2480",
 ]
@@ -168,19 +185,19 @@ nemo_gym = { workspace = true }
 nemo_run = { git = "https://github.com/NVIDIA-NeMo/Run", rev = "414f0077c648fde2c71bb1186e97ccbf96d6844c" }
 # torch/torchaudio/torchvision/triton all come from the torch index in order to pick up aarch64 wheels
 torch = [
-  { index = "pytorch-cu129", marker = "sys_platform != 'darwin'" },
+  { index = "pytorch-cu130", marker = "sys_platform != 'darwin'" },
   { index = "pypi", marker = "sys_platform == 'darwin'" },
 ]
 torchvision = [
-  { index = "pytorch-cu129", marker = "sys_platform != 'darwin'" },
+  { index = "pytorch-cu130", marker = "sys_platform != 'darwin'" },
   { index = "pypi", marker = "sys_platform == 'darwin'" },
 ]
 torchaudio = [
-  { index = "pytorch-cu129", marker = "sys_platform != 'darwin'" },
+  { index = "pytorch-cu130", marker = "sys_platform != 'darwin'" },
   { index = "pypi", marker = "sys_platform == 'darwin'" },
 ]
 triton = [
-  { index = "pytorch-cu129", marker = "sys_platform != 'darwin'" },
+  { index = "pytorch-cu130", marker = "sys_platform != 'darwin'" },
   { index = "pypi", marker = "sys_platform == 'darwin'" },
 ]
 causal-conv1d = { git = "https://github.com/Dao-AILab/causal-conv1d", rev = "67e0a9dfe1518fc0036444e9ab5fe06ab78299e0" }
@@ -210,13 +227,13 @@ url = "https://pypi.org/simple"
 explicit = true
 
 [[tool.uv.index]]
-name = "pytorch-cu129"
-url = "https://download.pytorch.org/whl/cu129"
+name = "pytorch-cu130"
+url = "https://download.pytorch.org/whl/cu130"
 explicit = true
 
 [tool.uv]
 preview = true # Enable preview features like extra-build-dependencies
-extra-build-variables = { sglang-kernel = { CMAKE_BUILD_PARALLEL_LEVEL = "24", FLASHINFER_CUDA_ARCH_LIST = "9.0a 10.0a", CMAKE_ARGS = "-DCMAKE_POLICY_VERSION_MINIMUM=3.5" } }
+extra-build-variables = { sglang-kernel = { CMAKE_BUILD_PARALLEL_LEVEL = "8", FLASHINFER_CUDA_ARCH_LIST = "9.0a 10.0a", CMAKE_ARGS = "-DCMAKE_POLICY_VERSION_MINIMUM=3.5" } }
 no-build-isolation-package = [
   "transformer-engine-torch",
   "transformer-engine",
@@ -242,8 +259,9 @@ link-mode = "copy"
 # The timm override is needed because current automodel pins to 1.0.16. This can be removed once we move ToT automodel
 # The nvidia-modelopt override is needed because mcore is still on 0.33
 override-dependencies = [
-  "transformer-engine[pytorch,core_cu12] @ git+https://github.com/NVIDIA/TransformerEngine.git@v2.14.1",
-  "nvidia-cudnn-cu12==9.19.0.56; sys_platform != 'darwin'",
+  "transformer-engine[pytorch,core_cu13] @ git+https://github.com/NVIDIA/TransformerEngine.git@v2.14.1",
+  "nvidia-cublas==13.3.0.5; sys_platform != 'darwin'",
+  "nvidia-cudnn-cu13==9.20.0.48; sys_platform != 'darwin'",
   "opencv-python-headless>=4.11.0",
   "timm<=1.0.22",
   "nvidia-modelopt[torch]>=0.39.0",
@@ -410,7 +428,7 @@ requires-dist = [
   "blobfile==3.0.0",
   "build",
   "compressed-tensors",
-  "cuda-python==12.9",
+  "cuda-python==13.0",
   "decord2",
   "datasets",
   "einops",
@@ -496,7 +514,7 @@ requires-dist = [
   # Non-workspace path deps cannot depend on workspace members (uv name-shadowing restriction).
   "qwen-vl-utils",
   # TODO(https://github.com/NVIDIA-NeMo/RL/issues/2111): upgrade to core_cu13 when we move to CUDA 13
-  "transformer-engine[pytorch,core_cu12]",
+  "transformer-engine[pytorch,core_cu13]",
   "mamba-ssm",
   "nvidia-resiliency-ext",
   "causal-conv1d",