From bda58569d9afd04553deafec3918f796f33cdd28 Mon Sep 17 00:00:00 2001 From: Aman Gupta Date: Sun, 4 Jan 2026 13:45:02 +0100 Subject: [PATCH 1/2] CUDA: disable cuda graph when using n-cpu-moe --- ggml/src/ggml-cuda/ggml-cuda.cu | 1 + 1 file changed, 1 insertion(+) diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index f05d5562ba0..8f67ea8bcdb 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -3762,6 +3762,7 @@ static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend, if (cuda_ctx->cuda_graph->number_consecutive_updates >= 4) { cuda_ctx->cuda_graph->disable_due_to_too_many_updates = true; + cuda_ctx->cuda_graph->cuda_graphs_enabled = false; #ifndef NDEBUG GGML_LOG_DEBUG("%s: disabling CUDA graphs due to too many consecutive updates\n", __func__); #endif From d91f2f2f3b3dd9264b1a18668ffa9a5b534a3fe0 Mon Sep 17 00:00:00 2001 From: Aman Gupta Date: Sun, 4 Jan 2026 14:29:53 +0100 Subject: [PATCH 2/2] call ggml_cuda_set_device --- ggml/src/ggml-cuda/ggml-cuda.cu | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 8f67ea8bcdb..80d983f9eef 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -3696,6 +3696,7 @@ static void evaluate_and_capture_cuda_graph(ggml_backend_cuda_context * cuda_ctx } static bool ggml_cuda_set_cuda_graph_enabled(ggml_backend_cuda_context * cuda_ctx) { + #ifdef USE_CUDA_GRAPH static const bool disable_cuda_graphs_due_to_env = (getenv("GGML_CUDA_DISABLE_GRAPHS") != nullptr); @@ -3736,17 +3737,15 @@ static bool ggml_cuda_set_cuda_graph_enabled(ggml_backend_cuda_context * cuda_ct static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context; + ggml_cuda_set_device(cuda_ctx->device); + bool use_cuda_graph = false; bool cuda_graph_update_required = false; // graph_optimize calls set_cuda_graph_enabled, in-case it not called (i.e. graph_compute is directly called) // we call it here instead. #ifdef USE_CUDA_GRAPH - if (!cuda_ctx->cuda_graph) { - use_cuda_graph = ggml_cuda_set_cuda_graph_enabled(cuda_ctx); - } else { - use_cuda_graph = cuda_ctx->cuda_graph && cuda_ctx->cuda_graph->cuda_graphs_enabled; - } + use_cuda_graph = ggml_cuda_set_cuda_graph_enabled(cuda_ctx); if (use_cuda_graph) { cuda_graph_update_required = is_cuda_graph_update_required(cuda_ctx, cgraph);