From caac78acd07dbb6f2eaf1eee2e36ae3e9a74ed1d Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Sat, 27 Jun 2026 15:12:44 -0300 Subject: [PATCH 01/14] fix: make get_rocm_arch() prefer the best AMD GPU (dGPU > iGPU, more VRAM > less) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prior behavior: get_rocm_arch() returned the first available AMD GPU, which is always the iGPU (added first in the devices array). On systems with a gfx1036 iGPU + gfx1201 dGPU, this returned 'gfx1036' — causing all ROCm backends to download binaries for the wrong architecture, install TheRock for the wrong GPU, and silently fall back to CPU. New behavior: iterates ALL AMD GPUs and returns the highest-scoring architecture using a VRAM-based ranking: - Primary factor: VRAM capacity (more VRAM = better for LLM inference) - Tiebreaker: dGPU bonus (GPUs with >= 4 GB VRAM are likely discrete) This mirrors get_cuda_arch() which also iterates all NVIDIA GPUs and picks the highest compute capability. Fixes: #2296, #2319 (GPU selection), helps #1787, #1991, #2002, #2371 --- src/cpp/server/system_info.cpp | 48 +++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/src/cpp/server/system_info.cpp b/src/cpp/server/system_info.cpp index 056297bad7..3b54603586 100644 --- a/src/cpp/server/system_info.cpp +++ b/src/cpp/server/system_info.cpp @@ -2106,11 +2106,23 @@ std::string identify_npu_arch() { return ""; } +// Score a ROCm architecture for ranking: higher = better for LLM inference. +// Primary factor is VRAM (more VRAM = larger models fit). Tiebreaker is +// discrete vs integrated (dGPU dedicated VRAM is faster than shared RAM). +static int rocm_arch_score(const std::string& arch, double vram_gb) { + // Primary: VRAM capacity in GB × 1000 (so 16 GB → 16000, 0.5 GB → 500) + int score = static_cast(vram_gb * 1000.0); + // Tiebreaker: dGPU bonus (GPUs with ≥ 4 GB VRAM are likely discrete) + if (vram_gb >= 4.0) score += 100000; + return score; +} + std::string SystemInfo::get_rocm_arch() { - // Returns the ROCm architecture for the best available AMD GPU on this system - // Checks iGPU first, then dGPUs. Returns empty string if no compatible GPU found. + // Returns the ROCm architecture for the BEST available AMD GPU on this + // system. Iterates ALL AMD GPUs and returns the highest-scoring supported + // architecture, preferring dGPUs over iGPUs and newer archs over older. + // This mirrors get_cuda_arch() which also picks the best NVIDIA GPU. try { - // Use cached system info to avoid re-detecting GPUs json system_info = SystemInfoCache::get_system_info_with_cache(); if (!system_info.contains("devices")) { @@ -2119,20 +2131,32 @@ std::string SystemInfo::get_rocm_arch() { const auto& devices = system_info["devices"]; - // Check AMD GPUs + std::string best_arch; + int best_score = -1; + if (devices.contains("amd_gpu") && devices["amd_gpu"].is_array()) { for (const auto& gpu : devices["amd_gpu"]) { - if (gpu.value("available", false)) { - std::string name = gpu.value("name", ""); - if (!name.empty()) { - std::string arch = identify_rocm_arch_from_name(name); - if (!arch.empty()) { - return arch; - } - } + if (!gpu.value("available", false)) continue; + + std::string name = gpu.value("name", ""); + if (name.empty()) continue; + + std::string arch = identify_rocm_arch_from_name(name); + if (arch.empty()) continue; + + double vram = gpu.value("vram_gb", 0.0); + + int score = rocm_arch_score(arch, vram); + if (score > best_score) { + best_score = score; + best_arch = arch; } } } + + if (!best_arch.empty()) { + return best_arch; + } } catch (...) { // Detection failed } From 654840f946aacfb60e5a25520d66e12710436e95 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Sat, 27 Jun 2026 19:52:19 -0300 Subject: [PATCH 02/14] fix: remove unused rocm_arch_score param, align comment, add selection logging --- src/cpp/server/system_info.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/cpp/server/system_info.cpp b/src/cpp/server/system_info.cpp index 3b54603586..f02d4c6199 100644 --- a/src/cpp/server/system_info.cpp +++ b/src/cpp/server/system_info.cpp @@ -2109,7 +2109,7 @@ std::string identify_npu_arch() { // Score a ROCm architecture for ranking: higher = better for LLM inference. // Primary factor is VRAM (more VRAM = larger models fit). Tiebreaker is // discrete vs integrated (dGPU dedicated VRAM is faster than shared RAM). -static int rocm_arch_score(const std::string& arch, double vram_gb) { +static int rocm_arch_score(double vram_gb) { // Primary: VRAM capacity in GB × 1000 (so 16 GB → 16000, 0.5 GB → 500) int score = static_cast(vram_gb * 1000.0); // Tiebreaker: dGPU bonus (GPUs with ≥ 4 GB VRAM are likely discrete) @@ -2120,7 +2120,7 @@ static int rocm_arch_score(const std::string& arch, double vram_gb) { std::string SystemInfo::get_rocm_arch() { // Returns the ROCm architecture for the BEST available AMD GPU on this // system. Iterates ALL AMD GPUs and returns the highest-scoring supported - // architecture, preferring dGPUs over iGPUs and newer archs over older. + // architecture, preferring dGPUs over iGPUs (by VRAM). // This mirrors get_cuda_arch() which also picks the best NVIDIA GPU. try { json system_info = SystemInfoCache::get_system_info_with_cache(); @@ -2146,7 +2146,7 @@ std::string SystemInfo::get_rocm_arch() { double vram = gpu.value("vram_gb", 0.0); - int score = rocm_arch_score(arch, vram); + int score = rocm_arch_score(vram); if (score > best_score) { best_score = score; best_arch = arch; @@ -2155,6 +2155,8 @@ std::string SystemInfo::get_rocm_arch() { } if (!best_arch.empty()) { + LOG(INFO, "SystemInfo") << "Selected ROCm arch " << best_arch + << " (score=" << best_score << ")" << std::endl; return best_arch; } } catch (...) { From 17a0306683c83866d94f7a0a7af6169629055baf Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Sat, 27 Jun 2026 20:06:21 -0300 Subject: [PATCH 03/14] fix(vllm): set VLLM_TARGET_DEVICE=rocm to prevent dual-platform crash on hybrid AMD+NVIDIA laptops On systems with both AMD iGPU and NVIDIA dGPU, vLLM's platform detection finds both 'cuda' and 'rocm' plugins and refuses to start with: RuntimeError: Only one platform plugin can be activated, but got: ['cuda', 'rocm'] Set VLLM_TARGET_DEVICE=rocm in the vLLM subprocess environment so the ROCm launcher activates only the ROCm platform plugin and skips the CUDA/NVML probe. Closes #1863 Co-Authored-By: Claude --- src/cpp/server/backends/vllm_server.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/cpp/server/backends/vllm_server.cpp b/src/cpp/server/backends/vllm_server.cpp index 7584d56d98..4b998ac80b 100644 --- a/src/cpp/server/backends/vllm_server.cpp +++ b/src/cpp/server/backends/vllm_server.cpp @@ -204,6 +204,10 @@ void VLLMServer::load(const std::string& model_name, env_vars.push_back({"FLASH_ATTENTION_TRITON_AMD_ENABLE", "TRUE"}); // Prevent system/user Python packages from leaking into the bundled vLLM environment env_vars.push_back({"PYTHONNOUSERSITE", "1"}); + // vLLM platform detection sees both CUDA and ROCm on hybrid AMD+NVIDIA + // laptops and refuses to start. Tell the vLLM ROCm launcher to activate + // only the ROCm platform plugin, skipping the CUDA/NVML probe entirely. + env_vars.push_back({"VLLM_TARGET_DEVICE", "rocm"}); // Start process bool inherit_output = (log_level_ == "info") || is_debug(); From 4d5fc846c35fbed02c1204bed577f7d4fa1d3062 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Sat, 27 Jun 2026 20:22:48 -0300 Subject: [PATCH 04/14] fix(llamacpp): hide CUDA devices from ROCm backend to fix hybrid-GPU memory crash MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On hybrid AMD+NVIDIA systems, llama.cpp's ggml memory budgeter counts both GPUs' VRAM pools, then tries to allocate the full model on the ROCm device — exceeding available memory and failing with: ggml_backend_cuda_buffer_type_alloc_buffer: cudaMalloc failed: out of memory Setting CUDA_VISIBLE_DEVICES="" hides the NVIDIA GPU from ggml's memory calculation while the ROCm backend discovers AMD GPUs through ROCr directly. User-confirmed workaround on Strix Halo + RTX 3090 Ti. Closes #2074 Co-Authored-By: Claude --- src/cpp/server/backends/llamacpp_server.cpp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/cpp/server/backends/llamacpp_server.cpp b/src/cpp/server/backends/llamacpp_server.cpp index a8b731f63e..9737a6580f 100644 --- a/src/cpp/server/backends/llamacpp_server.cpp +++ b/src/cpp/server/backends/llamacpp_server.cpp @@ -434,6 +434,24 @@ void LlamaCppServer::load(const std::string& model_name, env_vars.push_back({"LD_LIBRARY_PATH", lib_path}); LOG(DEBUG, "LlamaCpp") << "Setting LD_LIBRARY_PATH=" << lib_path << std::endl; + + // On hybrid AMD+NVIDIA systems, llama.cpp's ggml memory budgeting + // counts all GPU memory pools including the NVIDIA dGPU, then tries + // to allocate the full model on the ROCm device. Hiding CUDA devices + // keeps the NVIDIA GPU out of ggml's memory calculation while the + // ROCm backend discovers AMD GPUs through ROCr directly. + const char* existing_cuda_visible = std::getenv("CUDA_VISIBLE_DEVICES"); + if (!existing_cuda_visible || existing_cuda_visible[0] == '\0') { + env_vars.push_back({"CUDA_VISIBLE_DEVICES", ""}); + LOG(INFO, "LlamaCpp") + << "Hiding CUDA devices from llama.cpp ROCm backend" + << " to prevent hybrid-GPU memory budgeting skew" + << std::endl; + } else { + LOG(INFO, "LlamaCpp") + << "Respecting existing CUDA_VISIBLE_DEVICES=" + << existing_cuda_visible << std::endl; + } } else if (is_llamacpp_cuda_backend(llamacpp_backend)) { // The llama.cpp-builds Linux tarballs ship the bundled CUDA runtime // (libcudart.so, libcublas.so, etc.) alongside llama-server, so add the From 55d5133123c3f1c290c9c58bd6953214bae7c987 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Sat, 27 Jun 2026 20:40:55 -0300 Subject: [PATCH 05/14] =?UTF-8?q?fix:=20prevent=20NPU=20backend=20hangs=20?= =?UTF-8?q?=E2=80=94=20CURL=20timeout,=20watchdog=20for=20all=20backends,?= =?UTF-8?q?=20memory=20guard?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three-layer defense against RyzenAI/NPU inference hangs on memory-constrained systems (e.g. Qwen3-14B-Hybrid on 24 GB RAM): B1+B2: Apply default_timeout_seconds_ (300s) fallback to HttpClient::post() and post_stream(), matching the existing safety net in get(). Previously post() passed timeout_seconds=0 directly to CURLOPT_TIMEOUT, meaning 0=infinite. A hung backend would block the HTTP worker thread forever, freezing the UI. C2: Start the backend watchdog in every backend's load() method (llama.cpp, vLLM, RyzenAI, Whisper, Moonshine, SD, Kokoro). Only FastFlowLM had it before. All backends already call stop_backend_watchdog() in unload(). The watchdog probes /health and resets a hung backend after the grace period (default 90s). A2: Pre-reject NPU model loads when auto-tune can only reserve the fallback ctx_size (AUTO_CTX_FALLBACK=4096) and the model exceeds 10 GB. On memory-constrained systems, the NPU driver's runtime overhead pushes past the limit during first inference, causing the backend to hang. Rejecting early gives a clear error instead of a silent hang. Closes #1151 Co-Authored-By: Claude --- src/cpp/server/backends/kokoro_server.cpp | 2 ++ src/cpp/server/backends/llamacpp_server.cpp | 2 ++ src/cpp/server/backends/moonshine_server.cpp | 2 ++ src/cpp/server/backends/ryzenaiserver.cpp | 2 ++ src/cpp/server/backends/sd_server.cpp | 2 ++ src/cpp/server/backends/vllm_server.cpp | 2 ++ src/cpp/server/backends/whisper_server.cpp | 2 ++ src/cpp/server/router.cpp | 14 ++++++++++++++ src/cpp/server/utils/http_client.cpp | 6 ++++-- 9 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/cpp/server/backends/kokoro_server.cpp b/src/cpp/server/backends/kokoro_server.cpp index 7a707cd7ef..4548dad532 100644 --- a/src/cpp/server/backends/kokoro_server.cpp +++ b/src/cpp/server/backends/kokoro_server.cpp @@ -146,6 +146,8 @@ void KokoroServer::load(const std::string& model_name, const ModelInfo& model_in unload(); throw std::runtime_error("koko failed to start or become ready"); } + + start_backend_watchdog("/"); } void KokoroServer::unload() { diff --git a/src/cpp/server/backends/llamacpp_server.cpp b/src/cpp/server/backends/llamacpp_server.cpp index 9737a6580f..9c631fd658 100644 --- a/src/cpp/server/backends/llamacpp_server.cpp +++ b/src/cpp/server/backends/llamacpp_server.cpp @@ -587,6 +587,8 @@ void LlamaCppServer::load(const std::string& model_name, throw std::runtime_error("llama-server failed to start"); } + start_backend_watchdog("/health"); + LOG(DEBUG, "LlamaCpp") << "Model loaded on port " << get_backend_port() << std::endl; } diff --git a/src/cpp/server/backends/moonshine_server.cpp b/src/cpp/server/backends/moonshine_server.cpp index d3729d435f..dc2ff63482 100644 --- a/src/cpp/server/backends/moonshine_server.cpp +++ b/src/cpp/server/backends/moonshine_server.cpp @@ -184,6 +184,8 @@ void MoonshineServer::load(const std::string& model_name, throw std::runtime_error("moonshine-server failed to start or become ready"); } + start_backend_watchdog("/health"); + LOG(INFO, "MoonshineServer") << "Server is ready!" << std::endl; } diff --git a/src/cpp/server/backends/ryzenaiserver.cpp b/src/cpp/server/backends/ryzenaiserver.cpp index 6e250fa35a..5bbae1feb3 100644 --- a/src/cpp/server/backends/ryzenaiserver.cpp +++ b/src/cpp/server/backends/ryzenaiserver.cpp @@ -122,6 +122,8 @@ void RyzenAIServer::load(const std::string& model_name, throw std::runtime_error("RyzenAI-Server failed to start (check logs for details)"); } + start_backend_watchdog("/health"); + is_loaded_ = true; LOG(INFO, "RyzenAI") << "Model loaded on port " << get_backend_port() << std::endl; } diff --git a/src/cpp/server/backends/sd_server.cpp b/src/cpp/server/backends/sd_server.cpp index 734454c366..c98744d736 100644 --- a/src/cpp/server/backends/sd_server.cpp +++ b/src/cpp/server/backends/sd_server.cpp @@ -396,6 +396,8 @@ void SDServer::load(const std::string& model_name, throw std::runtime_error("sd-server failed to start or become ready"); } + start_backend_watchdog("/"); + LOG(INFO, "SDServer") << "Server is ready at http://127.0.0.1:" << get_backend_port() << std::endl; } diff --git a/src/cpp/server/backends/vllm_server.cpp b/src/cpp/server/backends/vllm_server.cpp index 4b998ac80b..3d3f54688a 100644 --- a/src/cpp/server/backends/vllm_server.cpp +++ b/src/cpp/server/backends/vllm_server.cpp @@ -229,6 +229,8 @@ void VLLMServer::load(const std::string& model_name, throw std::runtime_error(err); } + start_backend_watchdog("/health"); + LOG(DEBUG, "vLLM") << "Model loaded on port " << get_backend_port() << std::endl; } diff --git a/src/cpp/server/backends/whisper_server.cpp b/src/cpp/server/backends/whisper_server.cpp index 9f50da0207..553082b4cb 100644 --- a/src/cpp/server/backends/whisper_server.cpp +++ b/src/cpp/server/backends/whisper_server.cpp @@ -348,6 +348,8 @@ void WhisperServer::load(const std::string& model_name, throw std::runtime_error("whisper-server failed to start or become ready"); } + start_backend_watchdog("/health"); + LOG(INFO, "WhisperServer") << "Server is ready!" << std::endl; } diff --git a/src/cpp/server/router.cpp b/src/cpp/server/router.cpp index b3ec22c3b7..17686ba0e1 100644 --- a/src/cpp/server/router.cpp +++ b/src/cpp/server/router.cpp @@ -494,6 +494,20 @@ void Router::load_model(const std::string& model_name, effective_options.set_option("ctx_size", auto_ctx); } + // NPU models on memory-constrained systems: when auto-tune can only + // reserve the fallback ctx_size, the NPU driver's own runtime overhead + // will push the system past its memory limit during first inference, + // causing the backend to hang. Reject early with a clear error. + if (auto_ctx == AUTO_CTX_FALLBACK + && (model_info.device & DEVICE_NPU) + && model_info.size > 10.0) { + throw std::runtime_error( + "Not enough memory to load " + canonical_model_name + + " (" + std::to_string(static_cast(model_info.size)) + + " GB). The NPU driver needs additional working memory beyond" + + " model weights. Free up memory or try a smaller model."); + } + LOG(DEBUG, "Router") << "Effective settings: " << effective_options.to_log_string() << std::endl; // Create new backend server diff --git a/src/cpp/server/utils/http_client.cpp b/src/cpp/server/utils/http_client.cpp index 471f8040f5..998090fba0 100644 --- a/src/cpp/server/utils/http_client.cpp +++ b/src/cpp/server/utils/http_client.cpp @@ -356,7 +356,8 @@ HttpResponse HttpClient::post(const std::string& url, curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str()); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_body); - curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout_seconds); + curl_easy_setopt(curl, CURLOPT_TIMEOUT, + timeout_seconds > 0 ? timeout_seconds : default_timeout_seconds_.load()); curl_easy_setopt(curl, CURLOPT_USERAGENT, "lemon.cpp/1.0"); // Add custom headers @@ -492,7 +493,8 @@ HttpResponse HttpClient::post_stream(const std::string& url, curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str()); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, stream_write_callback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &callback_data); - curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout_seconds); + curl_easy_setopt(curl, CURLOPT_TIMEOUT, + timeout_seconds > 0 ? timeout_seconds : default_timeout_seconds_.load()); curl_easy_setopt(curl, CURLOPT_USERAGENT, "lemon.cpp/1.0"); // Add custom headers From a9ea45e41a264bf8107c330f2d39069db4603929 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Sat, 27 Jun 2026 20:54:31 -0300 Subject: [PATCH 06/14] =?UTF-8?q?fix(vulkan):=20prevent=20GPU=20hang=20on?= =?UTF-8?q?=20Windows=20=E2=80=94=20flash-attn=20off,=20GPU=20driver=20coo?= =?UTF-8?q?ldown,=20TDR=20warning?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three fixes for Vulkan backend hangs on Windows with AMD/Intel iGPUs (#2000): 1. Default --no-flash-attn on Windows+Vulkan. Flash attention on Vulkan Windows has caused reproducible hangs since llama.cpp b7064 (ggml#17297). Users can opt back in with --flash-attn in llamacpp_args. 2. 2-second GPU driver cooldown in ProcessManager::terminate() on Windows, matching the existing Linux behavior. Without this, orphaned Vulkan command buffers leave the GPU pegged at 90-100% after process termination (AMD Adrenalin known issue RN-RAD-WIN-22-11-1). 3. Log warning recommending TdrDelay=30 on Windows iGPU systems. The default 2-second TDR timeout is too short for LLM compute — a hung dispatch triggers VK_ERROR_DEVICE_LOST and a utilization stall. Additional upstream fixes needed in llama.cpp/ggml for complete coverage: - nodes_per_submit=1 on UMA devices (PR#24872) - Device-lost recovery (destroy-recreate-reload) - vkDeviceWaitIdle before vkDestroyDevice References: ggml#21724, ggml#17297, ggml#14105, mpv#16334 Closes #2000 Co-Authored-By: Claude --- src/cpp/server/backends/llamacpp_server.cpp | 25 +++++++++++++++++++ .../server/utils/platform/process_windows.cpp | 6 +++++ 2 files changed, 31 insertions(+) diff --git a/src/cpp/server/backends/llamacpp_server.cpp b/src/cpp/server/backends/llamacpp_server.cpp index 9c631fd658..fce7baba4e 100644 --- a/src/cpp/server/backends/llamacpp_server.cpp +++ b/src/cpp/server/backends/llamacpp_server.cpp @@ -378,6 +378,31 @@ void LlamaCppServer::load(const std::string& model_name, push_overridable_arg(args, llamacpp_args, "--no-mmap"); } + // Flash attention hangs on Windows Vulkan with AMD/Intel GPUs (ggml#17297). + // Default it off so the user doesn't get a hung backend on first run. + // Users can opt back in with --flash-attn in llamacpp_args. +#ifdef _WIN32 + if (llamacpp_backend == "vulkan") { + push_overridable_arg(args, llamacpp_args, "--no-flash-attn"); + LOG(INFO, "LlamaCpp") + << "Disabling flash attention by default on Windows Vulkan" + << " (override with --flash-attn in llamacpp_args)" << std::endl; + // Windows TDR (Timeout Detection & Recovery) defaults to 2 seconds, + // which is too short for LLM compute workloads on integrated GPUs. + // A hung dispatch exceeding 2s causes the driver to reset the GPU, + // producing VK_ERROR_DEVICE_LOST and a 90-100% utilization stall. + // Users can increase this via registry: TdrDelay=30 (DWORD) under + // HKLM\System\CurrentControlSet\Control\GraphicsDrivers + if (SystemInfo::get_has_igpu()) { + LOG(INFO, "LlamaCpp") + << "On Windows with an integrated GPU, consider increasing" + << " GPU timeout: set TdrDelay=30 (DWORD) in registry at" + << " HKLM\\System\\CurrentControlSet\\Control\\GraphicsDrivers" + << std::endl; + } + } +#endif + // Add embeddings support if the model supports it if (supports_embeddings) { LOG(INFO, "LlamaCpp") << "Model supports embeddings, adding --embeddings flag" << std::endl; diff --git a/src/cpp/server/utils/platform/process_windows.cpp b/src/cpp/server/utils/platform/process_windows.cpp index e1f7b05456..7b05a01ec7 100644 --- a/src/cpp/server/utils/platform/process_windows.cpp +++ b/src/cpp/server/utils/platform/process_windows.cpp @@ -358,6 +358,12 @@ class WindowsProcessPlatform : public ProcessPlatform { TerminateProcess(handle.handle, 0); WaitForSingleObject(handle.handle, 5000); // Wait up to 5 seconds CloseHandle(handle.handle); + // Give the GPU driver time to release resources before another + // process attempts to initialize the same device. Without this, + // orphaned Vulkan command buffers can leave the GPU pegged at + // 90-100% utilization after process termination (#2000). + // Matching the 2-second cooldown in the Linux path. + Sleep(2000); } } From c690983318e5de23321f10e1a96001e9be7ee953 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Sat, 27 Jun 2026 20:56:11 -0300 Subject: [PATCH 07/14] fix: make model load timeout configurable via global_timeout for all backends MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FastFlowLM had a hardcoded 5-minute timeout (max_attempts=300) that was not configurable. This prevented loading large models like GPT-OSS 120B that need more than 5 minutes for GPU offload. All other backends (llama.cpp, whisper, moonshine, ryzenai, sd, kokoro) used wait_for_ready(endpoint) with the hardcoded 600s default instead of passing timeout_seconds=0 to use the configurable global_timeout. Changes: - fastflowlm_server.cpp: read HttpClient::get_default_timeout() instead of hardcoded 300 — respects config.json global_timeout (default 600s) - llamacpp/whisper/moonshine/ryzenai/sd/kokoro server files: pass timeout_seconds=0 to wait_for_ready() to use global_timeout - vllm already used HttpClient::get_default_timeout() — unchanged Users can set global_timeout in config.json or via POST /api/v1/params {"global_timeout": 1200} for large models. Closes #674 Co-Authored-By: Claude --- src/cpp/server/backends/fastflowlm_server.cpp | 8 ++++++-- src/cpp/server/backends/kokoro_server.cpp | 2 +- src/cpp/server/backends/llamacpp_server.cpp | 2 +- src/cpp/server/backends/moonshine_server.cpp | 2 +- src/cpp/server/backends/ryzenaiserver.cpp | 2 +- src/cpp/server/backends/sd_server.cpp | 2 +- src/cpp/server/backends/whisper_server.cpp | 2 +- 7 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/cpp/server/backends/fastflowlm_server.cpp b/src/cpp/server/backends/fastflowlm_server.cpp index dc38928e3e..fe108eb573 100644 --- a/src/cpp/server/backends/fastflowlm_server.cpp +++ b/src/cpp/server/backends/fastflowlm_server.cpp @@ -249,9 +249,13 @@ bool FastFlowLMServer::wait_for_ready() { // FLM doesn't have a health endpoint, so we use /api/tags to check if it's up std::string tags_url = get_base_url() + "/api/tags"; - LOG(INFO, "FastFlowLM") << "Waiting for " + server_name_ + " to be ready..." << std::endl; + // Use the configurable global_timeout (default 600s) so users can + // increase it for large models like GPT-OSS 120B that take >5 min. + long timeout_seconds = utils::HttpClient::get_default_timeout(); + LOG(INFO, "FastFlowLM") << "Waiting for " + server_name_ + " to be ready" + << " (timeout: " << timeout_seconds << "s)..." << std::endl; - const int max_attempts = 300; // 5 minutes timeout (large models can take time to load) + const int max_attempts = static_cast(timeout_seconds); // 1 attempt per second for (int attempt = 0; attempt < max_attempts; ++attempt) { // Check if process is still running. If it already exited, consume and // reap the owned handle here so failed-start cleanup cannot later signal diff --git a/src/cpp/server/backends/kokoro_server.cpp b/src/cpp/server/backends/kokoro_server.cpp index 4548dad532..e48bc61cbe 100644 --- a/src/cpp/server/backends/kokoro_server.cpp +++ b/src/cpp/server/backends/kokoro_server.cpp @@ -142,7 +142,7 @@ void KokoroServer::load(const std::string& model_name, const ModelInfo& model_in LOG(INFO, "KokoroServer") << "Process started with PID: " << started_handle.pid << std::endl; // Wait for server to be ready - if (!wait_for_ready("/")) { + if (!wait_for_ready("/", 0)) { unload(); throw std::runtime_error("koko failed to start or become ready"); } diff --git a/src/cpp/server/backends/llamacpp_server.cpp b/src/cpp/server/backends/llamacpp_server.cpp index fce7baba4e..b8313309de 100644 --- a/src/cpp/server/backends/llamacpp_server.cpp +++ b/src/cpp/server/backends/llamacpp_server.cpp @@ -604,7 +604,7 @@ void LlamaCppServer::load(const std::string& model_name, process_executable, args, working_dir, inherit_llama_output, true, env_vars)); // Wait for server to be ready - if (!wait_for_ready("/health")) { + if (!wait_for_ready("/health", 0)) { const ProcessHandle handle = consume_process_handle_for_cleanup(); if (has_process_handle(handle)) { ProcessManager::stop_process(handle); diff --git a/src/cpp/server/backends/moonshine_server.cpp b/src/cpp/server/backends/moonshine_server.cpp index dc2ff63482..18afe24fb8 100644 --- a/src/cpp/server/backends/moonshine_server.cpp +++ b/src/cpp/server/backends/moonshine_server.cpp @@ -179,7 +179,7 @@ void MoonshineServer::load(const std::string& model_name, LOG(INFO, "MoonshineServer") << "Process started with PID: " << started_handle.pid << std::endl; // Wait for server to be ready - if (!wait_for_ready("/health")) { + if (!wait_for_ready("/health", 0)) { unload(); throw std::runtime_error("moonshine-server failed to start or become ready"); } diff --git a/src/cpp/server/backends/ryzenaiserver.cpp b/src/cpp/server/backends/ryzenaiserver.cpp index 5bbae1feb3..827eb18bac 100644 --- a/src/cpp/server/backends/ryzenaiserver.cpp +++ b/src/cpp/server/backends/ryzenaiserver.cpp @@ -114,7 +114,7 @@ void RyzenAIServer::load(const std::string& model_name, << started_handle.pid << std::endl; // Wait for server to be ready - if (!wait_for_ready("/health")) { + if (!wait_for_ready("/health", 0)) { const ProcessHandle handle = consume_process_handle_for_cleanup(); if (has_process_handle(handle)) { utils::ProcessManager::stop_process(handle); diff --git a/src/cpp/server/backends/sd_server.cpp b/src/cpp/server/backends/sd_server.cpp index c98744d736..00adeb9ec0 100644 --- a/src/cpp/server/backends/sd_server.cpp +++ b/src/cpp/server/backends/sd_server.cpp @@ -391,7 +391,7 @@ void SDServer::load(const std::string& model_name, LOG(INFO, "SDServer") << "Process started with PID: " << started_handle.pid << std::endl; // Wait for server to be ready - if (!wait_for_ready("/")) { + if (!wait_for_ready("/", 0)) { unload(); throw std::runtime_error("sd-server failed to start or become ready"); } diff --git a/src/cpp/server/backends/whisper_server.cpp b/src/cpp/server/backends/whisper_server.cpp index 553082b4cb..010cc6f5e3 100644 --- a/src/cpp/server/backends/whisper_server.cpp +++ b/src/cpp/server/backends/whisper_server.cpp @@ -343,7 +343,7 @@ void WhisperServer::load(const std::string& model_name, LOG(INFO, "WhisperServer") << "Process started with PID: " << started_handle.pid << std::endl; // Wait for server to be ready - if (!wait_for_ready("/health")) { + if (!wait_for_ready("/health", 0)) { unload(); throw std::runtime_error("whisper-server failed to start or become ready"); } From 8008a5ef8e4534c772e3e6bae0ab8fc48ac70f5c Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Sat, 27 Jun 2026 20:59:24 -0300 Subject: [PATCH 08/14] feat: add PR-Agent and Qodo Merge workflows for automated PR review - pr-agent-review.yml: DeepSeek-powered review on PR open/sync/reopen and @mention comments - qodo-merge.yml: Qodo Merge (qodo-ai/pr-agent) review on PR open/sync/reopen - Both gated on DEEPSEEK_API_KEY / QODO_API_KEY secrets being set Co-Authored-By: Claude --- .github/workflows/pr-agent-review.yml | 33 +++++++++++++++++++++++++++ .github/workflows/qodo-merge.yml | 20 ++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 .github/workflows/pr-agent-review.yml create mode 100644 .github/workflows/qodo-merge.yml diff --git a/.github/workflows/pr-agent-review.yml b/.github/workflows/pr-agent-review.yml new file mode 100644 index 0000000000..a46f8c0599 --- /dev/null +++ b/.github/workflows/pr-agent-review.yml @@ -0,0 +1,33 @@ +name: pr-agent-review + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + issue_comment: + types: [created] + +permissions: + contents: read + pull-requests: write + issues: write + +jobs: + pr_agent_job: + name: PR-Agent (DeepSeek) + runs-on: ubuntu-latest + if: ${{ github.event.sender.type != 'Bot' && secrets.DEEPSEEK_API_KEY != '' }} + steps: + - name: PR Agent review + uses: the-pr-agent/pr-agent@main + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} + config.model: "deepseek/deepseek-chat" + config.fallback_models: '["deepseek/deepseek-chat"]' + github_action_config.auto_review: "true" + github_action_config.auto_describe: "true" + github_action_config.auto_improve: "true" + pr_description.publish_labels: "true" + pr_description.publish_description_as: "suggestion" + pr_reviewer.require_score_review: "false" + pr_reviewer.num_code_suggestions: "4" diff --git a/.github/workflows/qodo-merge.yml b/.github/workflows/qodo-merge.yml new file mode 100644 index 0000000000..fb4e101411 --- /dev/null +++ b/.github/workflows/qodo-merge.yml @@ -0,0 +1,20 @@ +name: qodo-merge +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + +jobs: + qodo-merge: + if: ${{ secrets.QODO_API_KEY != "" }} + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + contents: read + steps: + - name: Qodo Merge Review + uses: qodo-ai/qodo-merge@main + env: + QODO_API_KEY: ${{ secrets.QODO_API_KEY }} + with: + github_token: ${{ secrets.GITHUB_TOKEN }} From 1fb483bb066e258abdf9b9ef2efef74409bbb5d2 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Sat, 27 Jun 2026 21:10:50 -0300 Subject: [PATCH 09/14] fix: NPU kernel safety check and download memory pressure guard (#1003) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Layer A — metrics_linux.cpp: Skip NPU sensor polling on kernels < 6.19.7. The amdxdna driver has known deadlock bugs (CVE-2026-23295) where sensor IOCTLs race with PM runtime suspend/resume, causing complete system freezes on Strix Halo. On unsafe kernels, get_npu_utilization() returns -1 immediately and logs a one-time warning advising a kernel upgrade. Layer B — model_manager.cpp + http_client: Cap download bandwidth when the total download size exceeds 50% of available system memory. On UMA architectures, download page-cache competes with NPU/GPU DMA buffers for the same RAM pool. A 70 GB model download can flood the page cache and starve the NPU driver, triggering OOM kills or system freezes. When memory is tight, downloads are throttled to ~50 MB/s via CURLOPT_MAX_RECV_SPEED_LARGE. Also adds max_recv_speed_bytes to DownloadOptions in http_client.h. Closes #1003 Co-Authored-By: Claude --- src/cpp/include/lemon/utils/http_client.h | 6 +++ src/cpp/server/model_manager.cpp | 38 +++++++++++++++ src/cpp/server/platform/metrics_linux.cpp | 58 +++++++++++++++++++++++ src/cpp/server/utils/http_client.cpp | 5 ++ 4 files changed, 107 insertions(+) diff --git a/src/cpp/include/lemon/utils/http_client.h b/src/cpp/include/lemon/utils/http_client.h index 62253dfced..02c7955a30 100644 --- a/src/cpp/include/lemon/utils/http_client.h +++ b/src/cpp/include/lemon/utils/http_client.h @@ -61,6 +61,12 @@ struct DownloadOptions { int low_speed_time = 0; // Seconds below low_speed_limit before timeout (disabled) int connect_timeout = 30; // Connection timeout in seconds + // Maximum download speed in bytes/sec (0 = unlimited). + // On systems with limited memory (e.g. UMA architectures where downloads + // compete with NPU/GPU DMA buffers for the same system RAM), capping + // download bandwidth prevents page-cache floods from triggering OOM. + size_t max_recv_speed_bytes = 0; + // Optional content verification. expected_hash accepts plain hex or // prefixed values like "sha256:", "sha1:", or // "git-sha1:". git-sha1 verifies the Git blob object id, i.e. diff --git a/src/cpp/server/model_manager.cpp b/src/cpp/server/model_manager.cpp index 6cd2a483ca..a879b1d7ea 100644 --- a/src/cpp/server/model_manager.cpp +++ b/src/cpp/server/model_manager.cpp @@ -3869,6 +3869,41 @@ void ModelManager::download_from_manifest(const json& manifest, std::map 0 && total_download_size > 0) { + size_t mem_available_bytes = static_cast(mem_available_kb) * 1024; + if (total_download_size > mem_available_bytes / 2) { + // Cap download at ~50 MB/s to avoid saturating page cache + download_speed_limit = 50 * 1024 * 1024; + LOG(WARNING, "ModelManager") + << "Download size (" << (total_download_size / (1024.0 * 1024.0 * 1024.0)) + << " GB) exceeds 50% of available memory (" + << (mem_available_bytes / (1024.0 * 1024.0 * 1024.0)) + << " GB). Throttling download to ~50 MB/s to prevent " + << "memory pressure. Consider pausing active NPU models " + << "before downloading large models." + << std::endl; + } + } + } + } + for (const auto& file_desc : manifest["files"]) { file_index++; std::string filename = file_desc["name"].get(); @@ -3937,6 +3972,9 @@ void ModelManager::download_from_manifest(const json& manifest, std::map 0) { + download_opts.max_recv_speed_bytes = download_speed_limit; + } if (file_desc.contains("hash") && file_desc["hash"].is_object()) { const auto& hash = file_desc["hash"]; if (hash.contains("algorithm") && hash["algorithm"].is_string() && diff --git a/src/cpp/server/platform/metrics_linux.cpp b/src/cpp/server/platform/metrics_linux.cpp index 5a321e4d76..42512d68b5 100644 --- a/src/cpp/server/platform/metrics_linux.cpp +++ b/src/cpp/server/platform/metrics_linux.cpp @@ -10,11 +10,52 @@ #include #include #include +#include namespace fs = std::filesystem; namespace lemon { +namespace { + +// Parse kernel version from /proc/version and compare against a minimum. +// Returns true if the running kernel is >= min_major.min_minor.min_patch. +// The amdxdna driver has known deadlock bugs (CVE-2026-23295, CVE-2026-43446) +// that cause complete system freezes when NPU sensor IOCTLs race with suspend/ +// resume. These were fixed in 6.19.7+. On older kernels, skip NPU monitoring +// to avoid triggering the deadlock. +bool kernel_at_least(int min_major, int min_minor, int min_patch) { + std::ifstream proc_version("/proc/version"); + if (!proc_version.is_open()) return false; + + std::string line; + std::getline(proc_version, line); + + const std::string tag = "version "; + size_t pos = line.find(tag); + if (pos == std::string::npos) return false; + pos += tag.size(); + size_t end = line.find(' ', pos); + if (end == std::string::npos) end = line.size(); + std::string kernel_str = line.substr(pos, end - pos); + + int major = 0, minor = 0, patch = 0; + if (sscanf(kernel_str.c_str(), "%d.%d.%d", &major, &minor, &patch) < 2) return false; + + if (major > min_major) return true; + if (major < min_major) return false; + if (minor > min_minor) return true; + if (minor < min_minor) return false; + return patch >= min_patch; +} + +bool is_npu_kernel_safe() { + // The amdxdna deadlock fixes landed in 6.19.7 + return kernel_at_least(6, 19, 7); +} + +} // anonymous namespace + class LinuxMetricsPlatform : public SystemMetricsPlatform { public: const char* get_platform_name() const override { @@ -190,6 +231,23 @@ class LinuxMetricsPlatform : public SystemMetricsPlatform { double get_npu_utilization() override { try { + // The amdxdna driver has known deadlock bugs (CVE-2026-23295) on + // kernels < 6.19.7. The sensor IOCTL below can race with PM runtime + // suspend and cause a complete system freeze. Skip NPU monitoring + // entirely on unsafe kernels rather than risk triggering the deadlock. + if (!is_npu_kernel_safe()) { + static bool warned = false; + if (!warned) { + warned = true; + LOG(WARNING, "metrics") << "NPU utilization monitoring disabled: " + << "running kernel is older than 6.19.7, which has known " + << "amdxdna driver deadlocks that can freeze the system. " + << "Upgrade to kernel 6.19.7+ or 7.0+ to enable NPU monitoring." + << std::endl; + } + return -1.0; + } + std::string accel_path = "/dev/accel/accel0"; if (!fs::exists(accel_path)) { return -1.0; diff --git a/src/cpp/server/utils/http_client.cpp b/src/cpp/server/utils/http_client.cpp index 998090fba0..392a739448 100644 --- a/src/cpp/server/utils/http_client.cpp +++ b/src/cpp/server/utils/http_client.cpp @@ -579,6 +579,11 @@ DownloadResult HttpClient::download_attempt(const std::string& url, curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, static_cast(options.low_speed_limit)); curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, static_cast(options.low_speed_time)); + if (options.max_recv_speed_bytes > 0) { + curl_easy_setopt(curl, CURLOPT_MAX_RECV_SPEED_LARGE, + static_cast(options.max_recv_speed_bytes)); + } + if (resume_from > 0) { curl_easy_setopt(curl, CURLOPT_RESUME_FROM_LARGE, static_cast(resume_from)); } From 5eb43e37652c55cb54f4b5e88d2a3a452dfd72e5 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Sat, 27 Jun 2026 21:16:27 -0300 Subject: [PATCH 10/14] fix: streaming inference timeout after 5 minutes on ROCm (#1792) post_stream() was applying default_timeout_seconds_ (300s/600s) when timeout_seconds=0 was passed. But the streaming call chain passes 0 meaning 'infinite timeout' (documented at wrapped_server.cpp:763). This caused long-running ROCm inference to fail at exactly the default timeout mark with 'CURL error: Timeout was reached', which the UI renders as 'Error: No content received from stream'. Fix: in post_stream(), pass 0L to CURLOPT_TIMEOUT when timeout_seconds is 0, which libcurl interprets as infinite. Callers who want a finite timeout should pass a positive value. Closes #1792 Co-Authored-By: Claude --- src/cpp/server/utils/http_client.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpp/server/utils/http_client.cpp b/src/cpp/server/utils/http_client.cpp index 392a739448..32e549b219 100644 --- a/src/cpp/server/utils/http_client.cpp +++ b/src/cpp/server/utils/http_client.cpp @@ -494,7 +494,7 @@ HttpResponse HttpClient::post_stream(const std::string& url, curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, stream_write_callback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &callback_data); curl_easy_setopt(curl, CURLOPT_TIMEOUT, - timeout_seconds > 0 ? timeout_seconds : default_timeout_seconds_.load()); + timeout_seconds > 0 ? timeout_seconds : 0L); curl_easy_setopt(curl, CURLOPT_USERAGENT, "lemon.cpp/1.0"); // Add custom headers From 891409c918f480a77e7e7e624185e2c5d47eb6c5 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Sat, 27 Jun 2026 21:20:42 -0300 Subject: [PATCH 11/14] fix: Vulkan VK_ERROR_DEVICE_LOST on Linux with large contexts (#2180) Two changes: 1. Reorder RECIPE_DEFS to prefer ROCm over Vulkan on Linux for AMD GPUs. On systems where both backends are available (HSA runtime + supported GPU arch), ROCm is now preferred. This avoids the Linux GPU scheduler timeout entirely, since ROCm compute submissions are not subject to the 2s RADV/amdgpu scheduler timeout. Vulkan remains as fallback for unsupported GPUs. 2. Cap Vulkan context size at 65536 on Linux. The amdgpu kernel scheduler has a ~2s timeout per compute submission (same as Windows TDR). Long prompt processing (e.g. 262K context) produces submissions that exceed this, causing VK_ERROR_DEVICE_LOST and a crash with the RADV message 'The CS has been cancelled because the context is lost.' Closes #2180 Co-Authored-By: Claude --- src/cpp/server/backends/llamacpp_server.cpp | 22 +++++++++++++++++++++ src/cpp/server/system_info.cpp | 6 +++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/cpp/server/backends/llamacpp_server.cpp b/src/cpp/server/backends/llamacpp_server.cpp index b8313309de..27aff48cdc 100644 --- a/src/cpp/server/backends/llamacpp_server.cpp +++ b/src/cpp/server/backends/llamacpp_server.cpp @@ -403,6 +403,28 @@ void LlamaCppServer::load(const std::string& model_name, } #endif + // Linux GPU scheduler (RADV/amdgpu) also has a ~2s timeout per compute + // submission. Long prompt processing with large contexts produces compute + // submissions that exceed this timeout, causing VK_ERROR_DEVICE_LOST + // ("The CS has been cancelled because the context is lost"). + // Cap context size for Vulkan on Linux and advise users to use ROCm instead. +#ifdef __linux__ + if (llamacpp_backend == "vulkan") { + const int64_t vulkan_max_ctx = 65536; + if (ctx_size > vulkan_max_ctx) { + LOG(WARNING, "LlamaCpp") + << "Capping context size at " << vulkan_max_ctx + << " for Vulkan backend on Linux (requested " << ctx_size + << "). Large contexts trigger GPU scheduler timeout" + << " (VK_ERROR_DEVICE_LOST). " + << "Use the ROCm backend for larger contexts — it does not" + << " have this limitation." + << std::endl; + ctx_size = vulkan_max_ctx; + } + } +#endif + // Add embeddings support if the model supports it if (supports_embeddings) { LOG(INFO, "LlamaCpp") << "Model supports embeddings, adding --embeddings flag" << std::endl; diff --git a/src/cpp/server/system_info.cpp b/src/cpp/server/system_info.cpp index f02d4c6199..d95473b3bf 100644 --- a/src/cpp/server/system_info.cpp +++ b/src/cpp/server/system_info.cpp @@ -434,13 +434,13 @@ static const std::vector RECIPE_DEFS = { {"llamacpp", "cuda", {"windows", "linux"}, { {"nvidia_gpu", {"sm_75", "sm_80", "sm_86", "sm_89", "sm_90", "sm_100", "sm_120", "sm_121"}}, }}, + {"llamacpp", "rocm", {"windows", "linux"}, { + {"amd_gpu", {"gfx1150", "gfx1151", "gfx1152", "gfx103X", "gfx110X", "gfx120X"}}, // STX iGPUs + RDNA2/3/4 dGPUs + }}, {"llamacpp", "vulkan", {"windows", "linux"}, { {"cpu", {"x86_64", "arm64"}}, {"amd_gpu", {}}, // all AMD GPU families }}, - {"llamacpp", "rocm", {"windows", "linux"}, { - {"amd_gpu", {"gfx1150", "gfx1151", "gfx1152", "gfx103X", "gfx110X", "gfx120X"}}, // STX iGPUs + RDNA2/3/4 dGPUs - }}, {"llamacpp", "cpu", {"windows", "linux"}, { {"cpu", {"x86_64", "arm64"}}, }}, From 99bcd1d586a64007dce46432e6202f1a427c3891 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Sat, 27 Jun 2026 21:31:25 -0300 Subject: [PATCH 12/14] feat: per-hardware max_loaded_models (#1705) Extend max_loaded_models to accept a JSON object with per-device caps alongside the existing integer format. When an object is provided, the router enforces per-hardware limits (cpu/gpu/npu) before the per-type LRU check, so users on heterogeneous systems like Strix Halo can express limits such as '1 GPU model, 1 NPU model, 2 CPU models'. Config format (backward compatible): Integer: { "max_loaded_models": 2 } Object: { "max_loaded_models": { "gpu": 1, "npu": 1, "total": 3 } } Omitted device keys default to unlimited (-1). Co-Authored-By: Claude --- src/cpp/include/lemon/router.h | 2 + src/cpp/include/lemon/runtime_config.h | 3 + src/cpp/server/router.cpp | 70 +++++++++++++++++++++++- src/cpp/server/runtime_config.cpp | 76 +++++++++++++++++++++++--- 4 files changed, 142 insertions(+), 9 deletions(-) diff --git a/src/cpp/include/lemon/router.h b/src/cpp/include/lemon/router.h index e98a8b11d2..2abc97c2c7 100644 --- a/src/cpp/include/lemon/router.h +++ b/src/cpp/include/lemon/router.h @@ -162,8 +162,10 @@ class Router { bool reload_model_after_watchdog_reset(const std::string& requested_model, const RecipeOptions& options); bool is_watchdog_reset_response(const json& response) const; int count_servers_by_type(ModelType type) const; + int count_servers_by_device(int device_bitmask) const; int count_pinned_servers_by_type(ModelType type) const; WrappedServer* find_lru_server_by_type(ModelType type) const; + WrappedServer* find_lru_server_by_device(int device_bitmask) const; bool has_npu_server() const; WrappedServer* find_npu_server() const; WrappedServer* find_npu_server_by_recipe(const std::string& recipe) const; diff --git a/src/cpp/include/lemon/runtime_config.h b/src/cpp/include/lemon/runtime_config.h index 9dbc422f68..cceaaba12a 100644 --- a/src/cpp/include/lemon/runtime_config.h +++ b/src/cpp/include/lemon/runtime_config.h @@ -30,6 +30,9 @@ class RuntimeConfig { bool no_broadcast() const; long global_timeout() const; int max_loaded_models() const; + int max_loaded_models_for_device(int device_bitmask) const; + int max_loaded_models_total() const; + bool max_loaded_models_is_object() const; std::string models_dir() const; int ctx_size() const; bool auto_evict() const; diff --git a/src/cpp/server/router.cpp b/src/cpp/server/router.cpp index 17686ba0e1..483d49cb75 100644 --- a/src/cpp/server/router.cpp +++ b/src/cpp/server/router.cpp @@ -158,6 +158,19 @@ int Router::count_servers_by_type(ModelType type) const { return count; } +int Router::count_servers_by_device(int device_bitmask) const { + int count = 0; + for (const auto& server : loaded_servers_) { + if (server->get_recipe_options().get_recipe() == "cloud") { + continue; + } + if (server->is_backend_alive() && (server->get_device_type() & device_bitmask)) { + count++; + } + } + return count; +} + WrappedServer* Router::find_lru_server_by_type(ModelType type) const { WrappedServer* lru = nullptr; @@ -181,6 +194,26 @@ WrappedServer* Router::find_lru_server_by_type(ModelType type) const { return lru; } +WrappedServer* Router::find_lru_server_by_device(int device_bitmask) const { + WrappedServer* lru = nullptr; + + for (const auto& server : loaded_servers_) { + if (server->get_recipe_options().get_recipe() == "cloud") { + continue; + } + if (server->is_backend_alive() && (server->get_device_type() & device_bitmask)) { + if (server->is_pinned()) { + continue; + } + if (!lru || server->get_last_access_time() < lru->get_last_access_time()) { + lru = server.get(); + } + } + } + + return lru; +} + bool Router::has_npu_server() const { for (const auto& server : loaded_servers_) { if (server->is_backend_alive() && (server->get_device_type() & DEVICE_NPU)) { @@ -466,11 +499,37 @@ void Router::load_model(const std::string& model_name, } } + bool is_cloud_load = (model_info.recipe == "cloud"); + + // PER-DEVICE SLOT CHECK (only when max_loaded_models is an object) + // This enforces per-hardware caps (e.g. max 1 GPU model, max 1 NPU model) + // before the per-type LRU check, so users can express heterogeneous limits. + if (!is_cloud_load && config_->max_loaded_models_is_object()) { + int device_bitmask = static_cast(model_info.device); + int device_limit = config_->max_loaded_models_for_device(device_bitmask); + if (device_limit != -1) { + int device_count = count_servers_by_device(device_bitmask); + if (device_count >= device_limit) { + WrappedServer* lru_dev = find_lru_server_by_device(device_bitmask); + if (lru_dev) { + LOG(INFO, "Router") << "Device slot limit reached for " + << device_type_to_string(model_info.device) + << " (" << device_count << "/" << device_limit + << "), evicting LRU: " << lru_dev->get_model_name() << std::endl; + evict_server(lru_dev); + } else { + is_loading_ = false; + load_cv_.notify_all(); + throw SlotsPinnedException(model_type_to_string(model_type)); + } + } + } + } + // LRU EVICTION CHECK (from spec: Least Recently Used Cache) // Skip eviction if unlimited (-1). Cloud-recipe loads also skip the // check entirely: they consume no local resources, so they have no // business kicking a warm local model out of memory. - bool is_cloud_load = (model_info.recipe == "cloud"); int current_count = count_servers_by_type(model_type); if (!is_cloud_load && max_models != -1 && current_count >= max_models) { WrappedServer* lru = find_lru_server_by_type(model_type); @@ -746,7 +805,7 @@ json Router::get_all_loaded_models() const { json Router::get_max_model_limits() const { int max = config_->max_loaded_models(); - return { + json result = { {"llm", max}, {"embedding", max}, {"reranking", max}, @@ -754,6 +813,13 @@ json Router::get_max_model_limits() const { {"image", max}, {"tts", max} }; + if (config_->max_loaded_models_is_object()) { + result["cpu"] = config_->max_loaded_models_for_device(1); // DEVICE_CPU + result["gpu"] = config_->max_loaded_models_for_device(2); // DEVICE_GPU + result["npu"] = config_->max_loaded_models_for_device(4); // DEVICE_NPU + result["total"] = config_->max_loaded_models_total(); + } + return result; } bool Router::is_model_loaded() const { diff --git a/src/cpp/server/runtime_config.cpp b/src/cpp/server/runtime_config.cpp index 261de4477e..b78c420731 100644 --- a/src/cpp/server/runtime_config.cpp +++ b/src/cpp/server/runtime_config.cpp @@ -219,7 +219,52 @@ long RuntimeConfig::global_timeout() const { int RuntimeConfig::max_loaded_models() const { std::shared_lock lock(mutex_); - return config_["max_loaded_models"].get(); + const auto& val = config_["max_loaded_models"]; + if (val.is_object()) { + if (val.contains("total") && val["total"].is_number_integer()) { + return val["total"].get(); + } + return -1; // no total key → unlimited + } + return val.get(); +} + +int RuntimeConfig::max_loaded_models_for_device(int device_bitmask) const { + std::shared_lock lock(mutex_); + const auto& val = config_["max_loaded_models"]; + if (!val.is_object()) { + // Integer format: all devices share the same limit. + return val.get(); + } + // Map bitmask to JSON key. + const char* key = nullptr; + if (device_bitmask == 1) key = "cpu"; // DEVICE_CPU + else if (device_bitmask == 2) key = "gpu"; // DEVICE_GPU + else if (device_bitmask == 4) key = "npu"; // DEVICE_NPU + else if (device_bitmask == 3) key = "cpu"; // CPU|GPU → use cpu (most restrictive common case not used) + else return -1; // multi-device or unknown: unlimited + + if (val.contains(key) && val[key].is_number_integer()) { + return val[key].get(); + } + return -1; // not specified → unlimited +} + +int RuntimeConfig::max_loaded_models_total() const { + std::shared_lock lock(mutex_); + const auto& val = config_["max_loaded_models"]; + if (val.is_object()) { + if (val.contains("total") && val["total"].is_number_integer()) { + return val["total"].get(); + } + return -1; + } + return val.get(); +} + +bool RuntimeConfig::max_loaded_models_is_object() const { + std::shared_lock lock(mutex_); + return config_["max_loaded_models"].is_object(); } std::string RuntimeConfig::models_dir() const { @@ -454,13 +499,30 @@ void RuntimeConfig::validate(const std::string& key, const json& value) const { throw std::invalid_argument("'global_timeout' must be positive"); } } else if (key == "max_loaded_models") { - if (!value.is_number_integer()) { - throw std::invalid_argument("'max_loaded_models' must be an integer"); - } - int m = value.get(); - if (m < -1 || m == 0) { + if (value.is_object()) { + static const std::vector device_keys = {"cpu", "gpu", "npu", "total"}; + for (const auto& dk : device_keys) { + if (!value.contains(dk)) continue; + const auto& dv = value[dk]; + if (!dv.is_number_integer()) { + throw std::invalid_argument( + "'max_loaded_models." + dk + "' must be an integer"); + } + int dm = dv.get(); + if (dm < -1 || dm == 0) { + throw std::invalid_argument( + "'max_loaded_models." + dk + "' must be -1 (unlimited) or a positive integer"); + } + } + } else if (value.is_number_integer()) { + int m = value.get(); + if (m < -1 || m == 0) { + throw std::invalid_argument( + "'max_loaded_models' must be -1 (unlimited) or a positive integer"); + } + } else { throw std::invalid_argument( - "'max_loaded_models' must be -1 (unlimited) or a positive integer"); + "'max_loaded_models' must be an integer or an object with per-device limits"); } } else if (key == "ctx_size") { if (!value.is_number_integer()) { From bb35f4f4a4ed8a17ba05ad29e354bdc575acd8d6 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Sat, 27 Jun 2026 21:40:15 -0300 Subject: [PATCH 13/14] fix: disable MTP speculative decoding for GLM-4 MoE models (#2451) GLM-4.5 GGUF files embed MTP layers (nextn_predict_layers > 0), so the GGUF capability scanner sets caps.mtp=true, which causes lemonade to enable speculative decoding (--spec-type draft-mtp). However, llama.cpp's glm4-moe.cpp graph builder asserts when constructing the draft model context because the GGUF was not converted with the multimodal metadata that the graph builder expects: glm4-moe.cpp:149: This GGUF does not support multimodal. Please reconvert it. Fix: strip the 'mtp' label for models whose GGUF architecture contains 'glm4' after capability labels are applied. This is a surgical workaround until upstream llama.cpp fixes the GLM-4 MoE draft context construction. Co-Authored-By: Claude --- src/cpp/server/model_manager.cpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/cpp/server/model_manager.cpp b/src/cpp/server/model_manager.cpp index a879b1d7ea..d235373523 100644 --- a/src/cpp/server/model_manager.cpp +++ b/src/cpp/server/model_manager.cpp @@ -236,6 +236,25 @@ static void populate_model_metadata(ModelInfo& info) { // type and break /embeddings or /rerank. if (info.type == ModelType::LLM) { apply_gguf_capability_labels(info.labels, info.gguf.caps); + + // GLM-4 MoE GGUF files embed MTP layers, but llama.cpp's + // glm4-moe graph builder asserts when constructing the + // draft model context (the GGUF was not converted with + // multimodal support that the graph builder expects). + // Disable MTP for these architectures until upstream + // llama.cpp fixes the conversion or the graph builder. + if (info.gguf.architecture.find("glm4") != std::string::npos) { + auto it = std::find(info.labels.begin(), info.labels.end(), "mtp"); + if (it != info.labels.end()) { + info.labels.erase(it); + LOG(INFO, "ModelManager") + << "Disabling MTP speculative decoding for " + << info.model_name + << " (GLM-4 MoE architecture — " + << "llama.cpp draft context crashes, glm4-moe.cpp:149)" + << std::endl; + } + } } } } From f6f01d4ef2d3c5fa810bb3c37db44f6b02a37208 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Sat, 27 Jun 2026 22:00:05 -0300 Subject: [PATCH 14/14] fix: add Conflicts/Replaces for lemonade-server-minimal in DEB packaging Without these dpkg refuses to overwrite files owned by the other package when upgrading between lemonade-server and lemonade-server-minimal, which both ship /usr/local/bin/lemonade-router and other shared files. Closes #677 Co-Authored-By: Claude --- CMakeLists.txt | 8 ++++++++ contrib/debian/control | 2 ++ 2 files changed, 10 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 311b19e3a8..acc9510b2c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1620,6 +1620,14 @@ if(UNIX AND NOT APPLE) # RPM specific variables defined within include(${CMAKE_CURRENT_SOURCE_DIR}/src/cpp/CPackRPM.cmake) + + # Ensure smooth upgrades from legacy/alternate package names that ship + # the same files. Without these dpkg refuses to overwrite files owned by + # the other package (e.g. upgrading lemonade-server to + # lemonade-server-minimal or vice versa). + set(CPACK_DEBIAN_PACKAGE_CONFLICTS "lemonade-server-minimal") + set(CPACK_DEBIAN_PACKAGE_REPLACES "lemonade-server-minimal") + include(CPack) endif() diff --git a/contrib/debian/control b/contrib/debian/control index 1093eed95a..36ee6eb6e1 100644 --- a/contrib/debian/control +++ b/contrib/debian/control @@ -51,6 +51,8 @@ Multi-Arch: foreign Depends: ${shlibs:Depends}, ${misc:Depends}, fonts-katex, unzip, libatomic1 Recommends: ffmpeg, libxrt-npu2, +Conflicts: lemonade-server-minimal +Replaces: lemonade-server-minimal Description: Local LLM serving with GPU and NPU acceleration server Lemonade helps users run local LLMs with the highest performance by configuring state-of-the-art inference engines for their NPUs and GPUs.