From 6bdf09be50438bacccbca8484112e958ecbac679 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Wed, 30 Sep 2026 12:23:31 -0300 Subject: [PATCH 1/3] Studio: avoid slow cold MIOpen searches on gfx1151 --- .../backend/tests/test_rocm_miopen_cutoff.py | 78 +++++++++++++++++++ studio/backend/utils/hardware/hardware.py | 20 +++++ 2 files changed, 98 insertions(+) create mode 100644 studio/backend/tests/test_rocm_miopen_cutoff.py diff --git a/studio/backend/tests/test_rocm_miopen_cutoff.py b/studio/backend/tests/test_rocm_miopen_cutoff.py new file mode 100644 index 000000000000..014ba9175e10 --- /dev/null +++ b/studio/backend/tests/test_rocm_miopen_cutoff.py @@ -0,0 +1,78 @@ +# SPDX-License-Identifier: AGPL-3.0-only +import os +import sys +from types import FunctionType, SimpleNamespace +from unittest.mock import Mock + +import pytest + +from utils.hardware import hardware as hw + + +@pytest.fixture(autouse = True) +def isolated_environment(monkeypatch): + monkeypatch.setattr(os, "environ", os.environ.copy()) + + +@pytest.mark.parametrize( + "arches, expected", + [ + (["gfx1151"], "1"), + (["gfx1151:xnack-", "gfx1151"], "1"), + (["gfx1100"], None), + (["gfx1151", "gfx1100"], None), + (["gfx1151", ""], None), + ([], None), + ], +) +def test_visible_device_scope(monkeypatch, arches, expected): + monkeypatch.delenv("MIOPEN_SEARCH_CUTOFF", raising = False) + cuda = SimpleNamespace( + device_count = lambda: len(arches), + get_device_properties = lambda i: SimpleNamespace(gcnArchName = arches[i]), + ) + hw._configure_rocm_miopen(SimpleNamespace(cuda = cuda)) + assert os.environ.get("MIOPEN_SEARCH_CUTOFF") == expected + + +@pytest.mark.parametrize("value", ["0", "1", "", "custom"]) +def test_explicit_override_is_preserved_without_probing(monkeypatch, value): + monkeypatch.setenv("MIOPEN_SEARCH_CUTOFF", value) + torch = Mock() + hw._configure_rocm_miopen(torch) + torch.cuda.device_count.assert_not_called() + assert os.environ["MIOPEN_SEARCH_CUTOFF"] == value + + +def test_probe_failure_does_not_break_startup(monkeypatch): + monkeypatch.delenv("MIOPEN_SEARCH_CUTOFF", raising = False) + torch = Mock() + torch.cuda.device_count.side_effect = RuntimeError("driver unavailable") + hw._configure_rocm_miopen(torch) + assert "MIOPEN_SEARCH_CUTOFF" not in os.environ + + +@pytest.mark.parametrize( + "hip, version, enabled", + [("7.13", "2.11", True), (None, "2.11+rocm7.13", True), (None, "2.11+cu130", False)], +) +def test_detection_configures_only_rocm(monkeypatch, hip, version, enabled): + torch = SimpleNamespace( + version = SimpleNamespace(hip = hip), + __version__ = version, + cuda = SimpleNamespace( + is_available = lambda: True, get_device_properties = lambda i: SimpleNamespace(name = "GPU") + ), + ) + monkeypatch.setitem(sys.modules, "torch", torch) + monkeypatch.setattr(hw, "_has_torch", lambda: True) + monkeypatch.setattr(hw, "_print_cuda_device_list", lambda _: None) + configure = Mock() + monkeypatch.setattr(hw, "_configure_rocm_miopen", configure) + monkeypatch.delenv("UNSLOTH_FORCE_XPU", raising = False) + monkeypatch.delenv("ZE_AFFINITY_MASK", raising = False) + monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False) + # Run the real body with isolated globals, including any future state fields. + detect = FunctionType(hw._detect_hardware_locked.__code__, vars(hw).copy()) + assert detect() == hw.DeviceType.CUDA + assert configure.call_count == int(enabled) diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py index 377afebc308a..69e218de8927 100644 --- a/studio/backend/utils/hardware/hardware.py +++ b/studio/backend/utils/hardware/hardware.py @@ -1407,6 +1407,25 @@ def _xpu_device_name_or_placeholder(torch) -> str: return "" +def _configure_rocm_miopen(torch) -> None: + """Bound cold solver search on measured GPUs without removing fallback solvers.""" + if "MIOPEN_SEARCH_CUTOFF" in os.environ: + return + try: + count = torch.cuda.device_count() + # MIOpen caches this process-wide setting at first use. Do not apply a + # single GPU's workaround to an untested GPU in a mixed-architecture host. + if not count or any( + _props_gfx_arch(torch.cuda.get_device_properties(i)) != "gfx1151" for i in range(count) + ): + return + except Exception as exc: + logger.debug("MIOpen search cutoff device probe failed: %s", exc) + return + os.environ.setdefault("MIOPEN_SEARCH_CUTOFF", "1") + logger.info("ROCm gfx1151: enabled MIOpen search cutoff (MIOPEN_SEARCH_CUTOFF=1)") + + def _detect_hardware_locked() -> DeviceType: """detect_hardware() body. Call only with _DETECT_LOCK held.""" global DEVICE, CHAT_ONLY, CHAT_ONLY_REASON, CHAT_ONLY_DETAIL, IS_ROCM @@ -1471,6 +1490,7 @@ def _detect_hardware_locked() -> DeviceType: _hip_ver = getattr(torch.version, "hip", None) if _hip_ver is not None or "rocm" in torch.__version__.lower(): IS_ROCM = True + _configure_rocm_miopen(torch) _hip_label = _hip_ver or torch.__version__ print(f"Hardware detected: ROCm (HIP {_hip_label}) -- {device_name}") else: From d1039ba9e68bd846f37d90d5cde8cab8c8df23eb Mon Sep 17 00:00:00 2001 From: Daniel Han <23090290+danielhanchen@users.noreply.github.com> Date: Thu, 1 Oct 2026 01:17:04 -0700 Subject: [PATCH 2/3] Import MIOPEN_SEARCH_CUTOFF from the login shell on Desktop launches --- studio/backend/tests/test_rocm_miopen_cutoff.py | 5 +++++ studio/backend/utils/desktop_shell_env.py | 2 ++ 2 files changed, 7 insertions(+) diff --git a/studio/backend/tests/test_rocm_miopen_cutoff.py b/studio/backend/tests/test_rocm_miopen_cutoff.py index 014ba9175e10..539b3e30ff06 100644 --- a/studio/backend/tests/test_rocm_miopen_cutoff.py +++ b/studio/backend/tests/test_rocm_miopen_cutoff.py @@ -76,3 +76,8 @@ def test_detection_configures_only_rocm(monkeypatch, hip, version, enabled): detect = FunctionType(hw._detect_hardware_locked.__code__, vars(hw).copy()) assert detect() == hw.DeviceType.CUDA assert configure.call_count == int(enabled) + + +def test_desktop_launch_imports_a_shell_override(): + from utils import desktop_shell_env as dse + assert "MIOPEN_SEARCH_CUTOFF" in dse.ROCM_SHELL_ENV_ALLOWLIST diff --git a/studio/backend/utils/desktop_shell_env.py b/studio/backend/utils/desktop_shell_env.py index 35f084d0b9e3..f956061cb7f2 100644 --- a/studio/backend/utils/desktop_shell_env.py +++ b/studio/backend/utils/desktop_shell_env.py @@ -48,6 +48,8 @@ "MIOPEN_USER_DB_PATH", "MIOPEN_CUSTOM_CACHE_DIR", "MIOPEN_FIND_MODE", + # A shell opt-out (0) or opt-in must beat hardware.py's gfx1151 default. + "MIOPEN_SEARCH_CUTOFF", "HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "GPU_DEVICE_ORDINAL", From fbe2344422e19665a483620911ae44b848e34ce4 Mon Sep 17 00:00:00 2001 From: Daniel Han <23090290+danielhanchen@users.noreply.github.com> Date: Thu, 1 Oct 2026 02:12:10 -0700 Subject: [PATCH 3/3] Trim comments in the MIOpen cutoff helper --- studio/backend/utils/hardware/hardware.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py index 69e218de8927..fd5c7f549652 100644 --- a/studio/backend/utils/hardware/hardware.py +++ b/studio/backend/utils/hardware/hardware.py @@ -1408,13 +1408,11 @@ def _xpu_device_name_or_placeholder(torch) -> str: def _configure_rocm_miopen(torch) -> None: - """Bound cold solver search on measured GPUs without removing fallback solvers.""" if "MIOPEN_SEARCH_CUTOFF" in os.environ: return try: count = torch.cuda.device_count() - # MIOpen caches this process-wide setting at first use. Do not apply a - # single GPU's workaround to an untested GPU in a mixed-architecture host. + # Process-wide in MIOpen: any untested arch on a mixed host keeps the default. if not count or any( _props_gfx_arch(torch.cuda.get_device_properties(i)) != "gfx1151" for i in range(count) ):