Skip to content
Merged
Show file tree
Hide file tree
Changes from 46 commits
Commits
Show all changes
48 commits
Select commit Hold shift + click to select a range
e7fb146
[hardware] feat: Add platform abstraction layer and FlagOS multi-chip…
physics31415926 Apr 21, 2026
900d02d
[hardware] fix: address review feedback on FlagCX validation and Flag…
physics31415926 Apr 21, 2026
ad8cde2
[doc] feat: add FL multi-chip support architecture documentation
physics31415926 Apr 21, 2026
5c53652
[doc] fix: use ASCII box drawing for better alignment
physics31415926 Apr 21, 2026
c7e10bc
[doc] fix: correct ASCII diagram alignment
physics31415926 Apr 21, 2026
d0358ea
Document common configurations for multi-chip support
heavyrain-lzy Apr 21, 2026
ebdc901
[hardware] refactor: replace in-tree FL engine imports with dynamic l…
physics31415926 Apr 22, 2026
87e9d6a
[hardware] fix: make VERL_ENGINE_DEVICE generic and revert unrelated …
physics31415926 Apr 22, 2026
e674b2d
feat: add custom_engine_module config for FL plugin dynamic loading
physics31415926 Apr 22, 2026
7d331e8
chore: anonymize personal paths in run_qwen3-0.6b_fl.sh
physics31415926 Apr 22, 2026
626da57
fix: add custom_engine_module field to fsdp.yaml for Hydra override s…
physics31415926 Apr 22, 2026
1da626e
fix: add + prefix for use_legacy_worker_impl Hydra override
physics31415926 Apr 22, 2026
801e826
fix: remove deprecated use_legacy_worker_impl parameter
physics31415926 Apr 22, 2026
60f7226
test: replace flagos with generic custom_device in registry tests
physics31415926 Apr 22, 2026
2789a2b
refactor: VERL_ENGINE_DEVICE auto-set by plugin, update docs and exam…
physics31415926 Apr 22, 2026
bfa7f71
test: remove redundant test files, keep test_platform_abstraction.py
physics31415926 Apr 22, 2026
a463232
[hardware] refactor: remove VERL_ENGINE_DEVICE, use last-writer-wins …
physics31415926 Apr 22, 2026
ee548d9
[misc] chore: fix trailing newline (ruff format)
physics31415926 Apr 23, 2026
eee6913
[hardware] refactor: use VERL_USE_EXTERNAL_MODULES instead of custom_…
physics31415926 Apr 23, 2026
399ca52
[hardware] fix: remove CPU platform backend
physics31415926 Apr 28, 2026
ce79737
[hardware] test/docs: update platform tests and docs to remove CPU ba…
physics31415926 Apr 28, 2026
fe1905e
[hardware] docs/test: document external plugin platform injection via…
physics31415926 Apr 28, 2026
ca40cea
[hardware] refactor: update plugin entry point and clean up platform …
physics31415926 Apr 28, 2026
7277fd9
[hardware] fix: remove redundant cpu check in _context_switch
physics31415926 Apr 28, 2026
42466d5
[hardware] feat: add PlatformRegistry with @register decorator for dy…
physics31415926 Apr 28, 2026
d5de081
[hardware] fix: align base.py with upstream and enable last-writer-wi…
physics31415926 Apr 28, 2026
bca60f8
[hardware] fix: warn instead of raise on CPU-only env, fix doc toctre…
physics31415926 Apr 30, 2026
b0a7db6
[hardware] fix: rename example script to follow naming convention, up…
physics31415926 May 12, 2026
fddd688
fix: patch groupwise module directly in monkeypatch test
physics31415926 May 12, 2026
7818b00
fix: restore NsightSystemsProfiler.__init__ attributes and update tes…
physics31415926 May 12, 2026
e5ac39f
refactor: remove redundant logic from NsightSystemsProfiler, align wi…
physics31415926 May 13, 2026
90166a5
style: fix ruff unused import in test_platform_abstraction.py
physics31415926 May 13, 2026
ff0fd7d
docs: clarify cross-platform profiling contract in PlatformBase
physics31415926 May 13, 2026
62c5d86
fix: guard get_device_capability against CUDA unavailable
physics31415926 May 13, 2026
af3d054
fix: handle torch.npu.get_device_capability returning None
physics31415926 May 13, 2026
89a5258
fix: remove sys.modules stub injection in test_platform_abstraction
physics31415926 May 14, 2026
a07b0fe
add extra configs for ray/vllm in platform
heavyrain-lzy May 22, 2026
8340bc2
update pre-commit
heavyrain-lzy May 22, 2026
2968f9b
add 'vendor' attribute for engine and platform to distinguish cuda-co…
heavyrain-lzy May 25, 2026
8b9a8b3
update platform name
heavyrain-lzy May 26, 2026
c9af119
change apply_model_patches attribute
heavyrain-lzy May 28, 2026
b524b8a
fix conflict
heavyrain-lzy Jun 3, 2026
dd98329
fix platform test
heavyrain-lzy Jun 3, 2026
aed996e
add log when autodetect platform
heavyrain-lzy Jun 3, 2026
4c72ac2
fix the bug when check is_available in num_gpu=0 ray actor
heavyrain-lzy Jun 3, 2026
53984ae
add new api for detect platform
heavyrain-lzy Jun 3, 2026
4391020
fix confict
heavyrain-lzy Jun 4, 2026
ee0649d
polish code according to the review
heavyrain-lzy Jun 4, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
175 changes: 175 additions & 0 deletions docs/hardware/multi_chip_support.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1,175 @@
Multi-Chip Support
==================

Last updated: 06/03/2026.

Overview
--------

verl supports RL training across multiple hardware platforms through a unified
plugin system. The architecture consists of two main subsystems:

1. **Platform Plugin System** (``verl.plugin.platform``) — A hardware
abstraction layer with auto-detection and a unified device API.
2. **Engine Plugin System** (``verl.workers.engine.base``) — Training engine
extensions that add chip-specific optimizations on top of existing
FSDP/Megatron engines.

Hardware Support
----------------

**Built-in (verl core):**

- NVIDIA GPU (CUDA)
- Huawei Ascend NPU

**Via verl-hardware-plugin (reference implementations):**

Other hardware platforms are supported through the external
`verl-hardware-plugin <https://github.com/verl-project/verl-hardware-plugin>`_
package, which provides reference implementations for vendors to adapt:

- Intel XPU (Data Center GPU Max / Arc)
- Cambricon MLU (MLU370 / MLU590)
- MetaX (CUDA-compatible)

.. note::

The implementations in verl-hardware-plugin are **examples only**. Full
production support requires collaboration with the respective hardware
vendors. Vendors can use these as templates to build and maintain their own
plugins.

Design Principles
-----------------

1. **Plugin Architecture**: Platform backends and engine extensions register
via decorator-based registries (``PlatformRegistry``, ``EngineRegistry``),
requiring no modifications to verl core code.

2. **Auto-Detection + Manual Override**: The platform auto-detects hardware
type by probing ``is_available(use_smi_check=True)`` on each registered
platform. Can be explicitly overridden via the ``VERL_PLATFORM`` environment
variable.

3. **Two-Dimensional Engine Lookup**: Engines register with both ``device``
(torch device type) and ``vendor`` (hardware vendor). Lookup priority:

- Exact match ``(device, vendor)`` — vendor-specific engine
- Fallback to device-only key — base engine for that device type
- For CUDA-compatible devices, fallback to base CUDA engine

4. **Backward Compatibility**: The legacy ``verl.utils.device`` API is
preserved as a thin wrapper over the platform plugin system. Existing code
continues to work without modification.

Architecture Overview
---------------------

::

+-------------------------------------------------------------------+
| verl Multi-Chip Architecture |
+-------------------------------------------------------------------+
| |
| +---------------------------------------------------------+ |
| | Platform Plugin System | |
| | (verl.plugin.platform) | |
| | | |
| | PlatformRegistry | |
| | ├─ "nvidia" → PlatformCUDA (built-in) | |
| | ├─ "huawei" → PlatformNPU (built-in) | |
| | ├─ "intel" → PlatformXPU (plugin) | |
| | ├─ "cambricon" → PlatformMLU (plugin) | |
| | └─ "metax" → PlatformMetaX (plugin) | |
| | | |
| +---------------------------------------------------------+ |
| |
| +---------------------------------------------------------+ |
| | Engine Plugin System | |
| | (verl.workers.engine.base) | |
| | | |
| | EngineRegistry (device, vendor) → Engine class | |
| | | | |
| | +-- ("cuda", None) → FSDPEngineWithLMHead | |
| | +-- ("npu", None) → FSDPNPUEngineWithLMHead | |
| | +-- ("cuda", "metax") → FSDPMetaXEngineWithLMHead | |
| | +-- ("xpu", "intel") → FSDPXPUEngineWithLMHead | |
| | +-- ("mlu","cambricon")→ FSDPMLUEngineWithLMHead | |
| | | |
| +---------------------------------------------------------+ |
| |
+-------------------------------------------------------------------+

Plugin Loading
--------------

verl discovers plugins through two mechanisms:

1. **setuptools entry_points** (``verl.plugins`` group) — standard Python
packaging mechanism. After ``pip install``, the plugin is auto-discovered.

2. **``VERL_USE_EXTERNAL_MODULES``** environment variable — for development
or non-packaged plugins:

.. code-block:: bash

export VERL_USE_EXTERNAL_MODULES=verl_hardware_plugin

Platform Registration
---------------------

Each platform class registers via decorator:

.. code-block:: python

@PlatformRegistry.register(platform="my_vendor")
class PlatformMyDevice(PlatformBase):
@property
def device_name(self) -> str:
return "my_device" # torch device type

@property
def vendor_name(self) -> str:
return "my_vendor" # used for engine lookup

Platform selection priority:

1. ``VERL_PLATFORM`` environment variable (explicit override)
2. Auto-detection via ``is_available(use_smi_check=True)``
3. Fallback to ``"nvidia"``

Engine Registration
-------------------

Engine classes register with device and vendor:

.. code-block:: python

@EngineRegistry.register(
model_type="language_model",
backend=["fsdp", "fsdp2"],
device="cuda", # torch device type
vendor="my_vendor", # vendor name
)
class FSDPMyVendorEngineWithLMHead(FSDPEngineWithLMHead):
def initialize(self):
super().initialize()
# vendor-specific initialization

Engine lookup calls ``get_device_name()`` and ``get_vendor()`` from the active
platform, then resolves the engine by ``(device_name, vendor_name)`` key.

Environment variable overrides for engine selection:

- ``VERL_ENGINE_DEVICE`` — override detected device name
- ``VERL_ENGINE_VENDOR`` — override detected vendor name

Adding New Hardware
-------------------

For a step-by-step guide on adding support for a new hardware platform, see
the `verl-hardware-plugin Development Guide <https://github.com/verl-project/verl-hardware-plugin/blob/main/docs/development.md>`_.

The core platform and engine registry mechanism is implemented in
`PR #6086 <https://github.com/verl-project/verl/pull/6086>`_.
1 change: 1 addition & 0 deletions docs/index.rst
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,7 @@ verl is fast with:
:maxdepth: 2
:caption: Hardware Support

hardware/multi_chip_support
amd_tutorial/amd_build_dockerfile_page.rst
amd_tutorial/amd_vllm_page.rst
amd_tutorial/amd_quick_start.rst
Expand Down
1 change: 1 addition & 0 deletions tests/plugin/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
# Copyright (c) 2026 BAAI. All rights reserved.
207 changes: 207 additions & 0 deletions tests/plugin/test_platform_abstraction.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,207 @@
# Copyright (c) 2026 BAAI. All rights reserved.
"""Unit tests for the platform abstraction layer."""

import os
from contextlib import contextmanager
from unittest import mock

import pytest

from verl.plugin.platform import get_platform, set_platform
from verl.plugin.platform.platform_base import PlatformBase
from verl.plugin.platform.platform_manager import (
PlatformRegistry,
_create_platform,
_detect_platform_name,
)


def _make_mock_platform(name="mock_xpu"):
"""Return a minimal concrete PlatformBase subclass for testing."""

class _Mock(PlatformBase):
@property
def device_name(self):
return name

@property
def vendor_name(self):
return f"mock_{name}"

@property
def device_module(self):
import torch

return torch

def is_available(self):
return True

def is_platform_available(self, use_smi_check=False):
return True

def current_device(self):
return 0

def device_count(self):
return 1

def set_device(self, device_index):
pass

def synchronize(self, device_index=None):
pass

def manual_seed(self, seed):
pass

def manual_seed_all(self, seed):
pass

def set_allocator_settings(self, settings):
pass

def empty_cache(self):
pass

def get_device_capability(self, device_index=0):
return (None, None)

def communication_backend_name(self):
return "mock_ccl"

def visible_devices_envvar(self):
return "MOCK_VISIBLE_DEVICES"

def ray_resource_name(self):
return "MOCK"

def ray_noset_envvars(self):
return ["RAY_EXPERIMENTAL_NOSET_MOCK_VISIBLE_DEVICES"]

def is_ipc_supported(self):
return False

@contextmanager
def nvtx_range(self, msg):
yield

def profiler_start(self):
pass

def profiler_stop(self):
pass

def cudart(self):
return None

_Mock.__name__ = f"Mock_{name}"
return _Mock


class TestPlatformDetection:
"""Test platform auto-detection logic."""

def setup_method(self):
import verl.plugin.platform.platform_manager as pm

pm._current_platform = None

def test_env_override_nvidia(self):
with mock.patch.dict(os.environ, {"VERL_PLATFORM": "nvidia"}):
assert _detect_platform_name() == "nvidia"

def test_env_override_huawei(self):
with mock.patch.dict(os.environ, {"VERL_PLATFORM": "huawei"}):
assert _detect_platform_name() == "huawei"

def test_invalid_value_passes_through(self):
# When an explicit platform name is set, _detect_platform_name returns
# it as-is (validation happens later in _create_platform).
with mock.patch.dict(os.environ, {"VERL_PLATFORM": "invalid"}):
assert _detect_platform_name() == "invalid"

def test_case_insensitive(self):
with mock.patch.dict(os.environ, {"VERL_PLATFORM": "NVIDIA"}):
assert _detect_platform_name() == "nvidia"

def test_empty_triggers_auto_detection(self):
with mock.patch.dict(os.environ, {"VERL_PLATFORM": ""}):
assert _detect_platform_name() in ("nvidia", "huawei")


class TestPlatformCreation:
"""Test platform creation."""

def test_cuda_warns_if_unavailable(self):
with mock.patch("torch.cuda.is_available", return_value=False):
with mock.patch("verl.plugin.platform.platform_manager.logger") as mock_logger:
platform = _create_platform("nvidia")
mock_logger.warning.assert_called_once()
assert platform is not None

def test_invalid_platform_raises(self):
with pytest.raises(ValueError):
_create_platform("invalid_platform")


class TestPlatformSingleton:
"""Test singleton and external injection."""

def setup_method(self):
import verl.plugin.platform.platform_manager as pm

pm._current_platform = None

def test_get_platform_returns_singleton(self):
p1 = get_platform()
p2 = get_platform()
assert p1 is p2

def test_set_platform_external_injection(self):
"""Simulates an entry_points plugin calling set_platform()."""
custom = _make_mock_platform("mock_xpu")()
set_platform(custom)
assert get_platform() is custom
assert get_platform().device_name == "mock_xpu"


class TestPlatformRegistry:
"""Test PlatformRegistry dynamic registration."""

def setup_method(self):
import verl.plugin.platform.platform_manager as pm

pm._current_platform = None

def test_builtin_platforms_registered(self):
names = PlatformRegistry.registered_names()
assert "nvidia" in names
assert "huawei" in names

def test_register_custom_platform(self):
"""External plugin registers a new platform via @PlatformRegistry.register()."""
MockCls = _make_mock_platform("mock_test")
PlatformRegistry.register(platform="mock_test")(MockCls)

assert "mock_test" in PlatformRegistry.registered_names()
assert PlatformRegistry.get("mock_test") is MockCls
assert _create_platform("mock_test").device_name == "mock_test"

del PlatformRegistry._platforms["mock_test"]

def test_register_override(self):
"""Re-registering the same name overrides the previous class (last writer wins)."""
original = PlatformRegistry.get("nvidia")
FakeCls = _make_mock_platform("nvidia")
PlatformRegistry.register(platform="nvidia")(FakeCls)
assert PlatformRegistry.get("nvidia") is FakeCls
PlatformRegistry._platforms["nvidia"] = original

def test_unregistered_platform_raises(self):
with pytest.raises(ValueError):
_create_platform("nonexistent_platform")


if __name__ == "__main__":
pytest.main([__file__, "-v"])
Loading
Loading