Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
caac78a
fix: make get_rocm_arch() prefer the best AMD GPU (dGPU > iGPU, more …
bong-water-water-bong Jun 27, 2026
654840f
fix: remove unused rocm_arch_score param, align comment, add selectio…
bong-water-water-bong Jun 27, 2026
17a0306
fix(vllm): set VLLM_TARGET_DEVICE=rocm to prevent dual-platform crash…
bong-water-water-bong Jun 27, 2026
4d5fc84
fix(llamacpp): hide CUDA devices from ROCm backend to fix hybrid-GPU …
bong-water-water-bong Jun 27, 2026
55d5133
fix: prevent NPU backend hangs — CURL timeout, watchdog for all backe…
bong-water-water-bong Jun 27, 2026
a9ea45e
fix(vulkan): prevent GPU hang on Windows — flash-attn off, GPU driver…
bong-water-water-bong Jun 27, 2026
c690983
fix: make model load timeout configurable via global_timeout for all …
bong-water-water-bong Jun 27, 2026
8008a5e
feat: add PR-Agent and Qodo Merge workflows for automated PR review
bong-water-water-bong Jun 27, 2026
1fb483b
fix: NPU kernel safety check and download memory pressure guard (#1003)
bong-water-water-bong Jun 28, 2026
5eb43e3
fix: streaming inference timeout after 5 minutes on ROCm (#1792)
bong-water-water-bong Jun 28, 2026
891409c
fix: Vulkan VK_ERROR_DEVICE_LOST on Linux with large contexts (#2180)
bong-water-water-bong Jun 28, 2026
99bcd1d
feat: per-hardware max_loaded_models (#1705)
bong-water-water-bong Jun 28, 2026
bb35f4f
fix: disable MTP speculative decoding for GLM-4 MoE models (#2451)
bong-water-water-bong Jun 28, 2026
f6f01d4
fix: add Conflicts/Replaces for lemonade-server-minimal in DEB packaging
bong-water-water-bong Jun 28, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 33 additions & 0 deletions .github/workflows/pr-agent-review.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
name: pr-agent-review

on:
pull_request:
types: [opened, synchronize, reopened, ready_for_review]
issue_comment:
types: [created]

permissions:
contents: read
pull-requests: write
issues: write

jobs:
pr_agent_job:
name: PR-Agent (DeepSeek)
runs-on: ubuntu-latest
if: ${{ github.event.sender.type != 'Bot' && secrets.DEEPSEEK_API_KEY != '' }}
steps:
- name: PR Agent review
uses: the-pr-agent/pr-agent@main
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
config.model: "deepseek/deepseek-chat"
config.fallback_models: '["deepseek/deepseek-chat"]'
github_action_config.auto_review: "true"
github_action_config.auto_describe: "true"
github_action_config.auto_improve: "true"
pr_description.publish_labels: "true"
pr_description.publish_description_as: "suggestion"
pr_reviewer.require_score_review: "false"
pr_reviewer.num_code_suggestions: "4"
20 changes: 20 additions & 0 deletions .github/workflows/qodo-merge.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
name: qodo-merge
on:
pull_request:
types: [opened, synchronize, reopened, ready_for_review]

jobs:
qodo-merge:
if: ${{ secrets.QODO_API_KEY != "" }}
runs-on: ubuntu-latest
permissions:
issues: write
pull-requests: write
contents: read
steps:
- name: Qodo Merge Review
uses: qodo-ai/qodo-merge@main
env:
QODO_API_KEY: ${{ secrets.QODO_API_KEY }}
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
8 changes: 8 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1620,6 +1620,14 @@ if(UNIX AND NOT APPLE)

# RPM specific variables defined within
include(${CMAKE_CURRENT_SOURCE_DIR}/src/cpp/CPackRPM.cmake)

# Ensure smooth upgrades from legacy/alternate package names that ship
# the same files. Without these dpkg refuses to overwrite files owned by
# the other package (e.g. upgrading lemonade-server to
# lemonade-server-minimal or vice versa).
set(CPACK_DEBIAN_PACKAGE_CONFLICTS "lemonade-server-minimal")
set(CPACK_DEBIAN_PACKAGE_REPLACES "lemonade-server-minimal")

include(CPack)
endif()

Expand Down
2 changes: 2 additions & 0 deletions contrib/debian/control
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ Multi-Arch: foreign
Depends: ${shlibs:Depends}, ${misc:Depends}, fonts-katex, unzip, libatomic1
Recommends: ffmpeg,
libxrt-npu2,
Conflicts: lemonade-server-minimal
Replaces: lemonade-server-minimal
Description: Local LLM serving with GPU and NPU acceleration server
Lemonade helps users run local LLMs with the highest performance by
configuring state-of-the-art inference engines for their NPUs and GPUs.
Expand Down
2 changes: 2 additions & 0 deletions src/cpp/include/lemon/router.h
Original file line number Diff line number Diff line change
Expand Up @@ -162,8 +162,10 @@ class Router {
bool reload_model_after_watchdog_reset(const std::string& requested_model, const RecipeOptions& options);
bool is_watchdog_reset_response(const json& response) const;
int count_servers_by_type(ModelType type) const;
int count_servers_by_device(int device_bitmask) const;
int count_pinned_servers_by_type(ModelType type) const;
WrappedServer* find_lru_server_by_type(ModelType type) const;
WrappedServer* find_lru_server_by_device(int device_bitmask) const;
bool has_npu_server() const;
WrappedServer* find_npu_server() const;
WrappedServer* find_npu_server_by_recipe(const std::string& recipe) const;
Expand Down
3 changes: 3 additions & 0 deletions src/cpp/include/lemon/runtime_config.h
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,9 @@ class RuntimeConfig {
bool no_broadcast() const;
long global_timeout() const;
int max_loaded_models() const;
int max_loaded_models_for_device(int device_bitmask) const;
int max_loaded_models_total() const;
bool max_loaded_models_is_object() const;
std::string models_dir() const;
int ctx_size() const;
bool auto_evict() const;
Expand Down
6 changes: 6 additions & 0 deletions src/cpp/include/lemon/utils/http_client.h
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,12 @@ struct DownloadOptions {
int low_speed_time = 0; // Seconds below low_speed_limit before timeout (disabled)
int connect_timeout = 30; // Connection timeout in seconds

// Maximum download speed in bytes/sec (0 = unlimited).
// On systems with limited memory (e.g. UMA architectures where downloads
// compete with NPU/GPU DMA buffers for the same system RAM), capping
// download bandwidth prevents page-cache floods from triggering OOM.
size_t max_recv_speed_bytes = 0;

// Optional content verification. expected_hash accepts plain hex or
// prefixed values like "sha256:<hex>", "sha1:<hex>", or
// "git-sha1:<hex>". git-sha1 verifies the Git blob object id, i.e.
Expand Down
8 changes: 6 additions & 2 deletions src/cpp/server/backends/fastflowlm_server.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -249,9 +249,13 @@ bool FastFlowLMServer::wait_for_ready() {
// FLM doesn't have a health endpoint, so we use /api/tags to check if it's up
std::string tags_url = get_base_url() + "/api/tags";

LOG(INFO, "FastFlowLM") << "Waiting for " + server_name_ + " to be ready..." << std::endl;
// Use the configurable global_timeout (default 600s) so users can
// increase it for large models like GPT-OSS 120B that take >5 min.
long timeout_seconds = utils::HttpClient::get_default_timeout();
LOG(INFO, "FastFlowLM") << "Waiting for " + server_name_ + " to be ready"
<< " (timeout: " << timeout_seconds << "s)..." << std::endl;

const int max_attempts = 300; // 5 minutes timeout (large models can take time to load)
const int max_attempts = static_cast<int>(timeout_seconds); // 1 attempt per second
for (int attempt = 0; attempt < max_attempts; ++attempt) {
// Check if process is still running. If it already exited, consume and
// reap the owned handle here so failed-start cleanup cannot later signal
Expand Down
4 changes: 3 additions & 1 deletion src/cpp/server/backends/kokoro_server.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -142,10 +142,12 @@ void KokoroServer::load(const std::string& model_name, const ModelInfo& model_in
LOG(INFO, "KokoroServer") << "Process started with PID: " << started_handle.pid << std::endl;

// Wait for server to be ready
if (!wait_for_ready("/")) {
if (!wait_for_ready("/", 0)) {
unload();
throw std::runtime_error("koko failed to start or become ready");
}

start_backend_watchdog("/");
}

void KokoroServer::unload() {
Expand Down
69 changes: 68 additions & 1 deletion src/cpp/server/backends/llamacpp_server.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -378,6 +378,53 @@ void LlamaCppServer::load(const std::string& model_name,
push_overridable_arg(args, llamacpp_args, "--no-mmap");
}

// Flash attention hangs on Windows Vulkan with AMD/Intel GPUs (ggml#17297).
// Default it off so the user doesn't get a hung backend on first run.
// Users can opt back in with --flash-attn in llamacpp_args.
#ifdef _WIN32
if (llamacpp_backend == "vulkan") {
push_overridable_arg(args, llamacpp_args, "--no-flash-attn");
LOG(INFO, "LlamaCpp")
<< "Disabling flash attention by default on Windows Vulkan"
<< " (override with --flash-attn in llamacpp_args)" << std::endl;
// Windows TDR (Timeout Detection & Recovery) defaults to 2 seconds,
// which is too short for LLM compute workloads on integrated GPUs.
// A hung dispatch exceeding 2s causes the driver to reset the GPU,
// producing VK_ERROR_DEVICE_LOST and a 90-100% utilization stall.
// Users can increase this via registry: TdrDelay=30 (DWORD) under
// HKLM\System\CurrentControlSet\Control\GraphicsDrivers
if (SystemInfo::get_has_igpu()) {
LOG(INFO, "LlamaCpp")
<< "On Windows with an integrated GPU, consider increasing"
<< " GPU timeout: set TdrDelay=30 (DWORD) in registry at"
<< " HKLM\\System\\CurrentControlSet\\Control\\GraphicsDrivers"
<< std::endl;
}
}
#endif

// Linux GPU scheduler (RADV/amdgpu) also has a ~2s timeout per compute
// submission. Long prompt processing with large contexts produces compute
// submissions that exceed this timeout, causing VK_ERROR_DEVICE_LOST
// ("The CS has been cancelled because the context is lost").
// Cap context size for Vulkan on Linux and advise users to use ROCm instead.
#ifdef __linux__
if (llamacpp_backend == "vulkan") {
const int64_t vulkan_max_ctx = 65536;
if (ctx_size > vulkan_max_ctx) {
LOG(WARNING, "LlamaCpp")
<< "Capping context size at " << vulkan_max_ctx
<< " for Vulkan backend on Linux (requested " << ctx_size
<< "). Large contexts trigger GPU scheduler timeout"
<< " (VK_ERROR_DEVICE_LOST). "
<< "Use the ROCm backend for larger contexts — it does not"
<< " have this limitation."
<< std::endl;
ctx_size = vulkan_max_ctx;
}
}
#endif

// Add embeddings support if the model supports it
if (supports_embeddings) {
LOG(INFO, "LlamaCpp") << "Model supports embeddings, adding --embeddings flag" << std::endl;
Expand Down Expand Up @@ -434,6 +481,24 @@ void LlamaCppServer::load(const std::string& model_name,

env_vars.push_back({"LD_LIBRARY_PATH", lib_path});
LOG(DEBUG, "LlamaCpp") << "Setting LD_LIBRARY_PATH=" << lib_path << std::endl;

// On hybrid AMD+NVIDIA systems, llama.cpp's ggml memory budgeting
// counts all GPU memory pools including the NVIDIA dGPU, then tries
// to allocate the full model on the ROCm device. Hiding CUDA devices
// keeps the NVIDIA GPU out of ggml's memory calculation while the
// ROCm backend discovers AMD GPUs through ROCr directly.
const char* existing_cuda_visible = std::getenv("CUDA_VISIBLE_DEVICES");
if (!existing_cuda_visible || existing_cuda_visible[0] == '\0') {
env_vars.push_back({"CUDA_VISIBLE_DEVICES", ""});
LOG(INFO, "LlamaCpp")
<< "Hiding CUDA devices from llama.cpp ROCm backend"
<< " to prevent hybrid-GPU memory budgeting skew"
<< std::endl;
} else {
LOG(INFO, "LlamaCpp")
<< "Respecting existing CUDA_VISIBLE_DEVICES="
<< existing_cuda_visible << std::endl;
}
} else if (is_llamacpp_cuda_backend(llamacpp_backend)) {
// The llama.cpp-builds Linux tarballs ship the bundled CUDA runtime
// (libcudart.so, libcublas.so, etc.) alongside llama-server, so add the
Expand Down Expand Up @@ -561,14 +626,16 @@ void LlamaCppServer::load(const std::string& model_name,
process_executable, args, working_dir, inherit_llama_output, true, env_vars));

// Wait for server to be ready
if (!wait_for_ready("/health")) {
if (!wait_for_ready("/health", 0)) {
const ProcessHandle handle = consume_process_handle_for_cleanup();
if (has_process_handle(handle)) {
ProcessManager::stop_process(handle);
}
throw std::runtime_error("llama-server failed to start");
}

start_backend_watchdog("/health");

LOG(DEBUG, "LlamaCpp") << "Model loaded on port " << get_backend_port() << std::endl;
}

Expand Down
4 changes: 3 additions & 1 deletion src/cpp/server/backends/moonshine_server.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -179,11 +179,13 @@ void MoonshineServer::load(const std::string& model_name,
LOG(INFO, "MoonshineServer") << "Process started with PID: " << started_handle.pid << std::endl;

// Wait for server to be ready
if (!wait_for_ready("/health")) {
if (!wait_for_ready("/health", 0)) {
unload();
throw std::runtime_error("moonshine-server failed to start or become ready");
}

start_backend_watchdog("/health");

LOG(INFO, "MoonshineServer") << "Server is ready!" << std::endl;
}

Expand Down
4 changes: 3 additions & 1 deletion src/cpp/server/backends/ryzenaiserver.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -114,14 +114,16 @@ void RyzenAIServer::load(const std::string& model_name,
<< started_handle.pid << std::endl;

// Wait for server to be ready
if (!wait_for_ready("/health")) {
if (!wait_for_ready("/health", 0)) {
const ProcessHandle handle = consume_process_handle_for_cleanup();
if (has_process_handle(handle)) {
utils::ProcessManager::stop_process(handle);
}
throw std::runtime_error("RyzenAI-Server failed to start (check logs for details)");
}

start_backend_watchdog("/health");

is_loaded_ = true;
LOG(INFO, "RyzenAI") << "Model loaded on port " << get_backend_port() << std::endl;
}
Expand Down
4 changes: 3 additions & 1 deletion src/cpp/server/backends/sd_server.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -391,11 +391,13 @@ void SDServer::load(const std::string& model_name,
LOG(INFO, "SDServer") << "Process started with PID: " << started_handle.pid << std::endl;

// Wait for server to be ready
if (!wait_for_ready("/")) {
if (!wait_for_ready("/", 0)) {
unload();
throw std::runtime_error("sd-server failed to start or become ready");
}

start_backend_watchdog("/");

LOG(INFO, "SDServer") << "Server is ready at http://127.0.0.1:" << get_backend_port() << std::endl;
}

Expand Down
6 changes: 6 additions & 0 deletions src/cpp/server/backends/vllm_server.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,10 @@ void VLLMServer::load(const std::string& model_name,
env_vars.push_back({"FLASH_ATTENTION_TRITON_AMD_ENABLE", "TRUE"});
// Prevent system/user Python packages from leaking into the bundled vLLM environment
env_vars.push_back({"PYTHONNOUSERSITE", "1"});
// vLLM platform detection sees both CUDA and ROCm on hybrid AMD+NVIDIA
// laptops and refuses to start. Tell the vLLM ROCm launcher to activate
// only the ROCm platform plugin, skipping the CUDA/NVML probe entirely.
env_vars.push_back({"VLLM_TARGET_DEVICE", "rocm"});

// Start process
bool inherit_output = (log_level_ == "info") || is_debug();
Expand All @@ -225,6 +229,8 @@ void VLLMServer::load(const std::string& model_name,
throw std::runtime_error(err);
}

start_backend_watchdog("/health");

LOG(DEBUG, "vLLM") << "Model loaded on port " << get_backend_port() << std::endl;
}

Expand Down
4 changes: 3 additions & 1 deletion src/cpp/server/backends/whisper_server.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -343,11 +343,13 @@ void WhisperServer::load(const std::string& model_name,
LOG(INFO, "WhisperServer") << "Process started with PID: " << started_handle.pid << std::endl;

// Wait for server to be ready
if (!wait_for_ready("/health")) {
if (!wait_for_ready("/health", 0)) {
unload();
throw std::runtime_error("whisper-server failed to start or become ready");
}

start_backend_watchdog("/health");

LOG(INFO, "WhisperServer") << "Server is ready!" << std::endl;
}

Expand Down
Loading