From ccab10faa85060c67b0d6f05c468513362cb2205 Mon Sep 17 00:00:00 2001 From: Bhavik Sharda Date: Sat, 1 Aug 2026 01:58:20 +0530 Subject: [PATCH 1/2] * server: add spec-decode counters to /metrics endpoint --- tools/server/README.md | 4 +++ tools/server/server-context.cpp | 44 +++++++++++++++++++++++++++++++++ tools/server/server-task.cpp | 5 ++++ tools/server/server-task.h | 5 ++++ 4 files changed, 58 insertions(+) diff --git a/tools/server/README.md b/tools/server/README.md index f45c018972d2..a7bd9fd8770b 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -1074,6 +1074,10 @@ In *router mode* the query param `?model={model_id}` has to be set. This endpoin | `llamacpp:n_tokens_max` | Counter | High watermark of the context size observed. | | `llamacpp:n_decode_total` | Counter | Total Number of llama_decode() calls. | | `llamacpp:n_busy_slots_per_decode` | Gauge | Average number of busy slots per llama_decode() call. | +| `llamacpp:spec_decode_num_draft_tokens` | Counter | Total draft tokens generated (0 when spec-decode is off). | +| `llamacpp:spec_decode_num_accepted_tokens` | Counter | Total draft tokens accepted by the target model (0 when spec-decode is off). | +| `llamacpp:spec_decode_num_drafts` | Counter | Total speculative decoding verification steps (0 when spec-decode is off). | +| `llamacpp:spec_decode_num_accepted_tokens_per_pos` | Counter | Accepted tokens per draft position (labeled `position="N"`; absent when spec-decode is off or before the first completed speculative request). | ### POST `/slots/{id_slot}?action=save`: Save the prompt cache of the specified slot to a file. diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 4655b518e21f..43fee1925979 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -845,6 +845,11 @@ struct server_metrics { uint64_t n_decode_total = 0; uint64_t n_busy_slots_total = 0; + uint64_t n_draft_tokens_total = 0; + uint64_t n_draft_accepted_total = 0; + uint64_t n_draft_verif_steps_total = 0; + std::vector n_accepted_per_pos_total; + void init() { t_start = ggml_time_us(); } @@ -863,6 +868,17 @@ struct server_metrics { n_tokens_predicted += slot.n_decoded; t_tokens_generation += slot.t_token_generation; t_tokens_generation_total += slot.t_token_generation; + + n_draft_tokens_total += slot.n_draft_total; + n_draft_accepted_total += slot.n_draft_accepted; + n_draft_verif_steps_total += slot.n_draft_verif_steps; + + if (n_accepted_per_pos_total.size() < slot.n_accepted_per_pos.size()) { + n_accepted_per_pos_total.resize(slot.n_accepted_per_pos.size(), 0); + } + for (size_t i = 0; i < slot.n_accepted_per_pos.size(); i++) { + n_accepted_per_pos_total[i] += slot.n_accepted_per_pos[i]; + } } void on_decoded(const std::vector & slots) { @@ -2552,6 +2568,11 @@ struct server_context_impl { res->n_decode_total = metrics.n_decode_total; res->n_busy_slots_total = metrics.n_busy_slots_total; + res->n_draft_tokens_total = metrics.n_draft_tokens_total; + res->n_draft_accepted_total = metrics.n_draft_accepted_total; + res->n_draft_verif_steps_total = metrics.n_draft_verif_steps_total; + res->n_accepted_per_pos_total = metrics.n_accepted_per_pos_total; + if (task.metrics_reset_bucket) { metrics.reset_bucket(); } @@ -4441,6 +4462,18 @@ void server_routes::init_routes() { {"name", "n_tokens_max"}, {"help", "Largest observed n_tokens."}, {"value", res_task->n_tokens_max} + }, { + {"name", "spec_decode_num_draft_tokens"}, + {"help", "Total draft tokens generated"}, + {"value", res_task->n_draft_tokens_total} + }, { + {"name", "spec_decode_num_accepted_tokens"}, + {"help", "Total draft tokens accepted by the target model"}, + {"value", res_task->n_draft_accepted_total} + }, { + {"name", "spec_decode_num_drafts"}, + {"help", "Total speculative decoding verification steps"}, + {"value", res_task->n_draft_verif_steps_total} }}}, {"gauge", {{ {"name", "prompt_tokens_seconds"}, @@ -4482,6 +4515,17 @@ void server_routes::init_routes() { } } + // labeled counter: one time series per draft position + if (!res_task->n_accepted_per_pos_total.empty()) { + prometheus << "# HELP llamacpp:spec_decode_num_accepted_tokens_per_pos" + " Accepted tokens per draft position\n" + << "# TYPE llamacpp:spec_decode_num_accepted_tokens_per_pos counter\n"; + for (size_t i = 0; i < res_task->n_accepted_per_pos_total.size(); i++) { + prometheus << "llamacpp:spec_decode_num_accepted_tokens_per_pos{position=\"" + << i << "\"} " << res_task->n_accepted_per_pos_total[i] << "\n"; + } + } + res->headers["Process-Start-Time-Unix"] = std::to_string(res_task->t_start); res->content_type = "text/plain; version=0.0.4"; res->status = 200; diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp index 070f1ade241c..1ee67755307c 100644 --- a/tools/server/server-task.cpp +++ b/tools/server/server-task.cpp @@ -1560,6 +1560,11 @@ json server_task_result_metrics::to_json() { { "n_decode_total", n_decode_total }, { "n_busy_slots_total", n_busy_slots_total }, + { "n_draft_tokens_total", n_draft_tokens_total }, + { "n_draft_accepted_total", n_draft_accepted_total }, + { "n_draft_verif_steps_total", n_draft_verif_steps_total }, + { "n_accepted_per_pos_total", n_accepted_per_pos_total }, + { "slots", slots_data }, }; } diff --git a/tools/server/server-task.h b/tools/server/server-task.h index 411d91807955..6275ec7604b5 100644 --- a/tools/server/server-task.h +++ b/tools/server/server-task.h @@ -532,6 +532,11 @@ struct server_task_result_metrics : server_task_result { uint64_t n_decode_total = 0; uint64_t n_busy_slots_total = 0; + uint64_t n_draft_tokens_total = 0; + uint64_t n_draft_accepted_total = 0; + uint64_t n_draft_verif_steps_total = 0; + std::vector n_accepted_per_pos_total; + // while we can also use std::vector this requires copying the slot object which can be quite messy // therefore, we use json to temporarily store the slot.to_json() result json slots_data = json::array(); From 6192d877722ac43838fa83c05433a3590c2bc71f Mon Sep 17 00:00:00 2001 From: Bhavik Sharda Date: Mon, 3 Aug 2026 21:08:36 +0530 Subject: [PATCH 2/2] server: fixed review comments and now aligned param names exactly with vLLM. --- tools/server/README.md | 8 ++++---- tools/server/server-context.cpp | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tools/server/README.md b/tools/server/README.md index a7bd9fd8770b..ad087a4efdad 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -1074,10 +1074,10 @@ In *router mode* the query param `?model={model_id}` has to be set. This endpoin | `llamacpp:n_tokens_max` | Counter | High watermark of the context size observed. | | `llamacpp:n_decode_total` | Counter | Total Number of llama_decode() calls. | | `llamacpp:n_busy_slots_per_decode` | Gauge | Average number of busy slots per llama_decode() call. | -| `llamacpp:spec_decode_num_draft_tokens` | Counter | Total draft tokens generated (0 when spec-decode is off). | -| `llamacpp:spec_decode_num_accepted_tokens` | Counter | Total draft tokens accepted by the target model (0 when spec-decode is off). | -| `llamacpp:spec_decode_num_drafts` | Counter | Total speculative decoding verification steps (0 when spec-decode is off). | -| `llamacpp:spec_decode_num_accepted_tokens_per_pos` | Counter | Accepted tokens per draft position (labeled `position="N"`; absent when spec-decode is off or before the first completed speculative request). | +| `llamacpp:spec_decode_num_draft_tokens_total` | Counter | Total draft tokens generated (0 when spec-decode is off). | +| `llamacpp:spec_decode_num_accepted_tokens_total` | Counter | Total draft tokens accepted by the target model (0 when spec-decode is off). | +| `llamacpp:spec_decode_num_drafts_total` | Counter | Total speculative decoding verification steps (0 when spec-decode is off). | +| `llamacpp:spec_decode_num_accepted_tokens_per_pos_total` | Counter | Accepted tokens per draft position (labeled `position="N"`; absent when spec-decode is off or before the first completed speculative request). | ### POST `/slots/{id_slot}?action=save`: Save the prompt cache of the specified slot to a file. diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 43fee1925979..42213e243747 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -4463,15 +4463,15 @@ void server_routes::init_routes() { {"help", "Largest observed n_tokens."}, {"value", res_task->n_tokens_max} }, { - {"name", "spec_decode_num_draft_tokens"}, + {"name", "spec_decode_num_draft_tokens_total"}, {"help", "Total draft tokens generated"}, {"value", res_task->n_draft_tokens_total} }, { - {"name", "spec_decode_num_accepted_tokens"}, + {"name", "spec_decode_num_accepted_tokens_total"}, {"help", "Total draft tokens accepted by the target model"}, {"value", res_task->n_draft_accepted_total} }, { - {"name", "spec_decode_num_drafts"}, + {"name", "spec_decode_num_drafts_total"}, {"help", "Total speculative decoding verification steps"}, {"value", res_task->n_draft_verif_steps_total} }}}, @@ -4517,11 +4517,11 @@ void server_routes::init_routes() { // labeled counter: one time series per draft position if (!res_task->n_accepted_per_pos_total.empty()) { - prometheus << "# HELP llamacpp:spec_decode_num_accepted_tokens_per_pos" + prometheus << "# HELP llamacpp:spec_decode_num_accepted_tokens_per_pos_total" " Accepted tokens per draft position\n" - << "# TYPE llamacpp:spec_decode_num_accepted_tokens_per_pos counter\n"; + << "# TYPE llamacpp:spec_decode_num_accepted_tokens_per_pos_total counter\n"; for (size_t i = 0; i < res_task->n_accepted_per_pos_total.size(); i++) { - prometheus << "llamacpp:spec_decode_num_accepted_tokens_per_pos{position=\"" + prometheus << "llamacpp:spec_decode_num_accepted_tokens_per_pos_total{position=\"" << i << "\"} " << res_task->n_accepted_per_pos_total[i] << "\n"; } }