(feature) llamacpp-llm: dynamic tools

packages/qvac-lib-infer-llamacpp-llm/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## [0.14.0] - 2026-03-19
+
+### Added
+
+#### `tools_at_end` configuration for dynamic tool management in multi-turn conversations
+
+New `tools_at_end` configuration option (`"true"` or `"false"`, default: `"false"`) places tool definitions at the end of the prompt (after conversation history) instead of in the system prompt. This enables KV cache optimization for multi-turn conversations with dynamic tool sets, where tools change between turns. Currently supports Qwen3 models only.
+
+- **KV cache trimming**: After each turn, tools are automatically removed from the KV cache, preventing stale tool definitions from accumulating
+- **Conversation history reuse**: History tokens are preserved in cache, saving recomputation on long conversations
+- **Dynamic tool replacement**: Different tool sets can be used per turn without cache bloat from unused tools
+
 ## [0.13.0] - 2026-03-18
 
 ### Added

packages/qvac-lib-infer-llamacpp-llm/CMakeLists.txt

@@ -74,6 +74,7 @@ endif()
     ${PROJECT_SOURCE_DIR}/addon/src/utils/BackendSelection.cpp
     ${PROJECT_SOURCE_DIR}/addon/src/utils/ChatTemplateUtils.cpp
     ${PROJECT_SOURCE_DIR}/addon/src/utils/Qwen3ReasoningUtils.cpp
+    ${PROJECT_SOURCE_DIR}/addon/src/utils/Qwen3ToolsDynamicTemplate.cpp
     ${PROJECT_SOURCE_DIR}/addon/src/utils/QwenTemplate.cpp
   )
 
@@ -118,6 +119,7 @@ if(BUILD_CLI)
     ${PROJECT_SOURCE_DIR}/addon/src/utils/BackendSelection.cpp
     ${PROJECT_SOURCE_DIR}/addon/src/utils/ChatTemplateUtils.cpp
     ${PROJECT_SOURCE_DIR}/addon/src/utils/Qwen3ReasoningUtils.cpp
+    ${PROJECT_SOURCE_DIR}/addon/src/utils/Qwen3ToolsDynamicTemplate.cpp
     ${PROJECT_SOURCE_DIR}/addon/src/utils/QwenTemplate.cpp
   )
 

packages/qvac-lib-infer-llamacpp-llm/README.md

@@ -143,6 +143,7 @@ const config = {
 | presence_penalty  | float                                       | 0                            | Presence penalty for sampling                         |
 | frequency_penalty | float                                       | 0                            | Frequency penalty for sampling                        |
 | tools             | `"true"` or `"false"`                       | `"false"`                    | Enable tool calling with jinja templating             |
+| tools_at_end      | `"true"` or `"false"`                       | `"false"`                    | Place tools at end of prompt ([details](./docs/tools-at-end.md)) |
 | verbosity         | 0 – 3 (0=ERROR, 1=WARNING, 2=INFO, 3=DEBUG) | 0                            | Logging verbosity level                               |
 | n_discarded       | integer                                     | 0                            | Tokens to discard in sliding window context           |
 | main-gpu          | integer, `"integrated"`, or `"dedicated"`   | —                            | GPU selection for multi-GPU systems                   |
@@ -287,6 +288,8 @@ npm run quickstart
 -   [LoRA Finetuning](./examples/finetune/simple-lora-finetune.js) – Basic LoRA finetuning.
 -   [LoRA Finetuning Pause/Resume](./examples/finetune/simple-lora-finetune-pause-resume.js) – Pause and resume finetuning.
 -   [LoRA Inference](./examples/simple-lora-inference.js) – Inference with a finetuned LoRA adapter.
+-   [Bench Tools Placement](./examples/benchToolsPlacement.js) – Benchmarks standard vs `tools_at_end` placement across multi-turn conversations.
+-   [Test Tool Removal](./examples/testToolRemoval.js) – Demonstrates dynamic tool addition and removal between turns.
 
 ## OCR with Vision-Language Models
 

packages/qvac-lib-infer-llamacpp-llm/addon/src/model-interface/LlamaModel.cpp

@@ -308,7 +308,9 @@ void LlamaModel::init(bool acquireLock) {
 
   common_params params;
   std::optional<int> adrenoVersion;
-  commonParamsParse(modelPath, configFilemap, params, adrenoVersion);
+  bool toolsAtEnd = false;
+  commonParamsParse(
+      modelPath, configFilemap, params, adrenoVersion, toolsAtEnd);
 
   const std::string errorWhenFailed = toString(UnableToLoadModel);
   auto streamedFiles =
@@ -334,7 +336,8 @@ void LlamaModel::init(bool acquireLock) {
   snap->llmContext_ = createContext(
       std::string(constructionArgs_.projectionPath),
       params,
-      std::move(llamaInit));
+      std::move(llamaInit),
+      toolsAtEnd);
 
   if (snap->configuredNDiscarded_ > 0 && snap->llmContext_) {
     snap->llmContext_->setNDiscarded(snap->configuredNDiscarded_);
@@ -360,6 +363,14 @@ bool LlamaModel::isLoaded() {
   return static_cast<bool>(state_->llmContext_);
 }
 
+llama_pos LlamaModel::getNPastBeforeTools() const {
+  std::shared_lock lock(stateMtx_);
+  if (state_->llmContext_) {
+    return state_->llmContext_->dynamicToolsState().nPastBeforeTools();
+  }
+  return -1;
+}
+
 llama_context* LlamaModel::getContext() {
   if (!state_->llmContext_) {
     return nullptr;
@@ -504,6 +515,11 @@ std::string LlamaModel::processPromptImpl(const Prompt& prompt) {
   std::string out;
   ResolvedPrompt resolved = resolveChatAndTools(prompt.input);
 
+  if (resolved.shouldResetAfterInference &&
+      state_->llmContext_->getNPast() > 0) {
+    resetState(true);
+  }
+
   if (resolved.chatMsgs.empty() && resolved.tools.empty()) {
     QLOG_IF(
         Priority::INFO,
@@ -552,6 +568,18 @@ std::string LlamaModel::processPromptImpl(const Prompt& prompt) {
   if (!prompt.outputCallback) {
     out = oss.str();
   }
+  auto& dts = state_->llmContext_->dynamicToolsState();
+  if (dts.toolsAtEnd() && !resolved.tools.empty() &&
+      dts.nPastBeforeTools() > 0 &&
+      state_->llmContext_->getNPast() > dts.nPastBeforeTools()) {
+    state_->llmContext_->removeLastNTokens(
+        state_->llmContext_->getNPast() - dts.nPastBeforeTools());
+    dts.reset();
+    if (state_->llmContext_->getFirstMsgTokens() >
+        state_->llmContext_->getNPast()) {
+      state_->llmContext_->setFirstMsgTokens(state_->llmContext_->getNPast());
+    }
+  }
   if (resolved.shouldResetAfterInference) {
     resetState(false);
   }
@@ -589,7 +617,8 @@ qvac_lib_inference_addon_cpp::RuntimeStats LlamaModel::runtimeStats() const {
 void LlamaModel::commonParamsParse(
     const std::string& modelPath,
     std::unordered_map<std::string, std::string>& configFilemap,
-    common_params& params, std::optional<int>& outAdrenoVersion) {
+    common_params& params, std::optional<int>& outAdrenoVersion,
+    bool& outToolsAtEnd) {
 
   std::vector<std::string> configVector;
 
@@ -632,6 +661,26 @@ void LlamaModel::commonParamsParse(
     configFilemap.erase(iter);
   }
 
+  // parse tools_at_end flag from config
+  if (auto iter = configFilemap.find("tools_at_end");
+      iter != configFilemap.end()) {
+    std::string val = iter->second;
+    std::transform(val.begin(), val.end(), val.begin(), ::tolower);
+    outToolsAtEnd = (val == "true");
+    configFilemap.erase(iter);
+  }
+
+  if (outToolsAtEnd) {
+    auto arch = metadata_.tryGetString("general.architecture");
+    if (!arch.has_value() || arch.value() != "qwen3") {
+      QLOG_IF(
+          Priority::WARNING,
+          "[LlamaModel] tools_at_end is only supported for Qwen3 models, "
+          "ignoring\n");
+      outToolsAtEnd = false;
+    }
+  }
+
   auto deviceIt = configFilemap.find("device");
   if (deviceIt == configFilemap.end()) {
     std::string errorMsg =
@@ -968,12 +1017,14 @@ void LlamaModel::resetState(bool resetStats) {
 
 std::unique_ptr<LlmContext> LlamaModel::createContext(
     std::string&& projectionPath, common_params& params,
-    common_init_result&& llamaInit) {
+    common_init_result&& llamaInit, bool toolsAtEnd) {
   if (!projectionPath.empty()) {
     params.mmproj.path = std::move(projectionPath);
-    return std::make_unique<MtmdLlmContext>(params, std::move(llamaInit));
+    return std::make_unique<MtmdLlmContext>(
+        params, std::move(llamaInit), toolsAtEnd);
   }
-  return std::make_unique<TextLlmContext>(params, std::move(llamaInit));
+  return std::make_unique<TextLlmContext>(
+      params, std::move(llamaInit), toolsAtEnd);
 }
 
 bool LlamaModel::loadMedia(const std::vector<uint8_t>& input) {

packages/qvac-lib-infer-llamacpp-llm/addon/src/model-interface/LlamaModel.hpp

@@ -158,6 +158,14 @@ class LlamaModel : public IModel, public IModelAsyncLoad, public IModelCancel {
    */
   bool isLoaded();
 
+  /**
+   * Get the nPast position before tool evaluation.
+   * This is used to find the boundary in the KV cache after evaluating
+   * conversation tokens but before tool tokens.
+   * @return the nPast position, or -1 if not set.
+   */
+  llama_pos getNPastBeforeTools() const;
+
   void waitForLoadInitialization() final {
     std::shared_ptr<ReloadableState> localState;
     {
@@ -233,7 +241,8 @@ class LlamaModel : public IModel, public IModelAsyncLoad, public IModelCancel {
   void commonParamsParse(
       const std::string& modelPath,
       std::unordered_map<std::string, std::string>& configFilemap,
-      common_params& params, std::optional<int>& outAdrenoVersion);
+      common_params& params, std::optional<int>& outAdrenoVersion,
+      bool& outToolsAtEnd);
 
   /**
    * The Format prompt method. It formats the prompt json to chat messages.
@@ -246,7 +255,8 @@ class LlamaModel : public IModel, public IModelAsyncLoad, public IModelCancel {
   void resetState(bool resetStats = true);
   std::unique_ptr<LlmContext> createContext(
       std::string&& projectionPath, common_params& params,
-      common_init_result&& llamaInit);
+      common_init_result&& llamaInit, bool toolsAtEnd);
+
   bool loadMedia(const std::vector<uint8_t>& input);
 
   void setInitLoader(

packages/qvac-lib-infer-llamacpp-llm/addon/src/model-interface/LlmContext.hpp

@@ -84,32 +84,58 @@ class LlamaBatch {
   const llama_batch* operator->() const noexcept { return &batch_; }
 };
 
-struct ThreadPoolDeleter{
-    void operator()(ggml_threadpool* ptr) {
-      if (ptr != nullptr) {
-        auto* cpuDev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
-        if (cpuDev == nullptr) {
-          throw qvac_errors::StatusError(
-              ADDON_ID, toString(NoBackendFound), "no CPU backend found");
-        }
-        auto* reg = ggml_backend_dev_backend_reg(cpuDev);
-        void* procAddr =
-            ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_free");
-        if (procAddr == nullptr) {
-          throw qvac_errors::StatusError(
-              ADDON_ID,
-              toString(UnableToDeleteThreadPool),
-              "Failed to get ggml_threadpool_free function address");
-        }
-        // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
-        auto* ggmlThreadpoolFreeFn =
-            reinterpret_cast<decltype(ggml_threadpool_free)*>(procAddr);
-        ggmlThreadpoolFreeFn(ptr);
+struct ThreadPoolDeleter {
+  void operator()(ggml_threadpool* ptr) {
+    if (ptr != nullptr) {
+      auto* cpuDev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
+      if (cpuDev == nullptr) {
+        throw qvac_errors::StatusError(
+            ADDON_ID, toString(NoBackendFound), "no CPU backend found");
+      }
+      auto* reg = ggml_backend_dev_backend_reg(cpuDev);
+      void* procAddr =
+          ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_free");
+      if (procAddr == nullptr) {
+        throw qvac_errors::StatusError(
+            ADDON_ID,
+            toString(UnableToDeleteThreadPool),
+            "Failed to get ggml_threadpool_free function address");
       }
+      // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
+      auto* ggmlThreadpoolFreeFn =
+          reinterpret_cast<decltype(ggml_threadpool_free)*>(procAddr);
+      ggmlThreadpoolFreeFn(ptr);
     }
+  }
 };
 using ThreadPoolPtr = std::unique_ptr<ggml_threadpool, ThreadPoolDeleter>;
 
+class DynamicToolsState {
+public:
+  void setToolsAtEnd(bool v) { toolsAtEnd_ = v; }
+  [[nodiscard]] bool toolsAtEnd() const { return toolsAtEnd_; }
+  [[nodiscard]] llama_pos nPastBeforeTools() const { return nPastBeforeTools_; }
+  void setNPastBeforeTools(llama_pos pos) { nPastBeforeTools_ = pos; }
+  void recordToolBoundary(llama_pos nPast, llama_pos totalTokens) {
+    if (toolsAtEnd_ && nConversationOnlyTokens_ > 0) {
+      nPastBeforeTools_ = nPast - (totalTokens - nConversationOnlyTokens_);
+    }
+  }
+  void setConversationOnlyTokens(llama_pos n) { nConversationOnlyTokens_ = n; }
+  [[nodiscard]] llama_pos conversationOnlyTokens() const {
+    return nConversationOnlyTokens_;
+  }
+  void reset() {
+    nConversationOnlyTokens_ = 0;
+    nPastBeforeTools_ = -1;
+  }
+
+private:
+  bool toolsAtEnd_ = false;
+  llama_pos nConversationOnlyTokens_ = 0;
+  llama_pos nPastBeforeTools_ = -1;
+};
+
 class LlmContext { // NOLINT(cppcoreguidelines-special-member-functions)
 public:
   LlmContext() = default;
@@ -211,6 +237,11 @@ class LlmContext { // NOLINT(cppcoreguidelines-special-member-functions)
    */
   virtual void setNDiscarded(llama_pos nDiscarded) = 0;
 
+  DynamicToolsState& dynamicToolsState() { return dynamicToolsState_; }
+  [[nodiscard]] const DynamicToolsState& dynamicToolsState() const {
+    return dynamicToolsState_;
+  }
+
   /**
    * Get the number of context slides (discards) that have occurred.
    */
@@ -276,6 +307,7 @@ class LlmContext { // NOLINT(cppcoreguidelines-special-member-functions)
    *
    */
   virtual void resetMedia() {};
-};
-
 
+private:
+  DynamicToolsState dynamicToolsState_;
+};