From 804d667a55ebbc277a244846135856dfc00d8c03 Mon Sep 17 00:00:00 2001 From: IC Date: Thu, 16 Apr 2026 07:38:10 +0000 Subject: [PATCH 1/7] feat: add dynamic backend loading to nmtcpp GGML backend for GPU acceleration --- packages/qvac-lib-infer-nmtcpp/CMakeLists.txt | 17 ++- .../NmtLazyInitializeBackend.cpp | 111 ++++++++++++++++++ .../NmtLazyInitializeBackend.hpp | 80 +++++++++++++ .../model-interface/PivotTranslationModel.cpp | 11 ++ .../model-interface/PivotTranslationModel.hpp | 4 + .../src/model-interface/TranslationModel.cpp | 13 ++ .../src/model-interface/TranslationModel.hpp | 4 + packages/qvac-lib-infer-nmtcpp/index.js | 5 + 8 files changed, 244 insertions(+), 1 deletion(-) create mode 100644 packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/NmtLazyInitializeBackend.cpp create mode 100644 packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/NmtLazyInitializeBackend.hpp diff --git a/packages/qvac-lib-infer-nmtcpp/CMakeLists.txt b/packages/qvac-lib-infer-nmtcpp/CMakeLists.txt index 9f730f8b29..99494d5b2c 100644 --- a/packages/qvac-lib-infer-nmtcpp/CMakeLists.txt +++ b/packages/qvac-lib-infer-nmtcpp/CMakeLists.txt @@ -134,7 +134,19 @@ if(USE_BERGAMOT) find_package(bergamot-translator CONFIG REQUIRED) endif() -add_bare_module(qvac-lib-infer-nmtcpp EXPORTS) +bare_target(bare_target_value) +bare_module_target("." unused_target NAME module_name VERSION unused_version) +set(BACKENDS_SUBDIR_VALUE "${bare_target_value}/${module_name}") +message("Building qvac-lib-infer-nmtcpp with BACKENDS_SUBDIR='${BACKENDS_SUBDIR_VALUE}'") + +set(BACKEND_DL_LIBS "") +if((ANDROID OR UNIX) AND NOT APPLE) + foreach(_backend ${GGML_AVAILABLE_BACKENDS}) + list(APPEND BACKEND_DL_LIBS INSTALL TARGET ggml::${_backend}) + endforeach() +endif() + +add_bare_module(qvac-lib-infer-nmtcpp EXPORTS ${BACKEND_DL_LIBS}) if(CMAKE_SYSTEM_NAME STREQUAL "Linux") target_link_options(${qvac-lib-infer-nmtcpp}_module PRIVATE -Wl,--exclude-libs,ALL) @@ -154,6 +166,7 @@ target_sources( ${PROJECT_SOURCE_DIR}/addon/src/model-interface/nmt_graph_encoder.cpp ${PROJECT_SOURCE_DIR}/addon/src/model-interface/nmt_beam_search.cpp ${PROJECT_SOURCE_DIR}/addon/src/model-interface/nmt_utils.cpp + ${PROJECT_SOURCE_DIR}/addon/src/model-interface/NmtLazyInitializeBackend.cpp ) # Add bergamot source files if enabled @@ -185,6 +198,8 @@ target_compile_definitions( JS_LOGGER ) +target_compile_definitions(${qvac-lib-infer-nmtcpp} PRIVATE BACKENDS_SUBDIR="${BACKENDS_SUBDIR_VALUE}") + # Add bergamot compile definition if enabled if(USE_BERGAMOT) target_compile_definitions( diff --git a/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/NmtLazyInitializeBackend.cpp b/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/NmtLazyInitializeBackend.cpp new file mode 100644 index 0000000000..97e60c2596 --- /dev/null +++ b/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/NmtLazyInitializeBackend.cpp @@ -0,0 +1,111 @@ +#include "NmtLazyInitializeBackend.hpp" + +#include +#include + +#include + +#include "qvac-lib-inference-addon-cpp/Logger.hpp" + +using namespace qvac_lib_inference_addon_cpp::logger; + +std::mutex NmtLazyInitializeBackend::g_initMutex; +bool NmtLazyInitializeBackend::g_initialized = false; +std::string NmtLazyInitializeBackend::g_recordedBackendsDir; +int NmtLazyInitializeBackend::g_refCount = 0; + +bool NmtLazyInitializeBackend::initialize( + const std::string& backendsDir, const std::string& openclCacheDir) { + std::lock_guard lock(g_initMutex); + + if (g_initialized) { + if (!backendsDir.empty() && !g_recordedBackendsDir.empty() && + backendsDir != g_recordedBackendsDir) { + QLOG( + Priority::WARNING, + "Backend already initialized with different backendsDir. " + "Previously initialized at: " + + g_recordedBackendsDir + ", requested: " + backendsDir); + } + return false; + } + + if (!backendsDir.empty()) { + g_recordedBackendsDir = backendsDir; + } + +#ifdef __ANDROID__ + if (!openclCacheDir.empty()) { + auto oclCachePath = + (std::filesystem::path(openclCacheDir) / "opencl-cache").string(); + setenv("GGML_OPENCL_CACHE_DIR", oclCachePath.c_str(), /*overwrite=*/1); + } +#endif + + if (!backendsDir.empty()) { + std::filesystem::path backendsDirPath(backendsDir); +#ifdef BACKENDS_SUBDIR + std::filesystem::path subdirPath(BACKENDS_SUBDIR); + backendsDirPath = backendsDirPath / subdirPath; + backendsDirPath = backendsDirPath.lexically_normal(); +#endif + QLOG( + Priority::INFO, + "Loading backends from directory: " + backendsDirPath.string()); + ggml_backend_load_all_from_path(backendsDirPath.string().c_str()); + } else { + QLOG(Priority::DEBUG, "Loading backends using default path"); + ggml_backend_load_all(); + } + + g_initialized = true; + return true; +} + +void NmtLazyInitializeBackend::incrementRefCount() { + std::lock_guard lock(g_initMutex); + g_refCount++; +} + +void NmtLazyInitializeBackend::decrementRefCount() { + std::lock_guard lock(g_initMutex); + if (g_refCount > 0) { + g_refCount--; + if (g_refCount == 0 && g_initialized) { + QLOG( + Priority::DEBUG, "Resetting backend state (reference count reached zero)"); + g_initialized = false; + g_recordedBackendsDir.clear(); + } + } +} + +NmtBackendsHandle::NmtBackendsHandle( + const std::string& backendsDir, const std::string& openclCacheDir) + : ownsHandle_(true) { + NmtLazyInitializeBackend::initialize(backendsDir, openclCacheDir); + NmtLazyInitializeBackend::incrementRefCount(); +} + +NmtBackendsHandle::~NmtBackendsHandle() { + if (ownsHandle_) { + NmtLazyInitializeBackend::decrementRefCount(); + } +} + +NmtBackendsHandle::NmtBackendsHandle(NmtBackendsHandle&& other) noexcept + : ownsHandle_(other.ownsHandle_) { + other.ownsHandle_ = false; +} + +NmtBackendsHandle& +NmtBackendsHandle::operator=(NmtBackendsHandle&& other) noexcept { + if (this != &other) { + if (ownsHandle_) { + NmtLazyInitializeBackend::decrementRefCount(); + } + ownsHandle_ = other.ownsHandle_; + other.ownsHandle_ = false; + } + return *this; +} diff --git a/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/NmtLazyInitializeBackend.hpp b/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/NmtLazyInitializeBackend.hpp new file mode 100644 index 0000000000..6b7916e01c --- /dev/null +++ b/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/NmtLazyInitializeBackend.hpp @@ -0,0 +1,80 @@ +#pragma once + +#include +#include + +/** + * Lazy initialization class for NMT GGML backend. + * Ensures backend is initialized only once (even when instantiating multiple + * TranslationModel objects) and tracks the backends directory. + */ +class NmtLazyInitializeBackend { +public: + /** + * Initialize the backend lazily. + * @param backendsDir - path to the backends directory (optional). + * If empty, uses default backend loading. + * @param openclCacheDir - writable directory for OpenCL kernel cache + * (optional). + * @return true if initialization was successful, false if already + * initialized. + */ + static bool initialize( + const std::string& backendsDir = "", + const std::string& openclCacheDir = ""); + + /** + * Increment the reference count. + */ + static void incrementRefCount(); + + /** + * Decrement the reference count and reset state if count reaches zero. + */ + static void decrementRefCount(); + +private: + static std::mutex g_initMutex; + static bool g_initialized; + static std::string g_recordedBackendsDir; + static int g_refCount; +}; + +/** + * RAII handle for NMT backend initialization. + * Increments reference count on construction and decrements on destruction. + * When the last handle is destroyed, the backend state is reset. + */ +class NmtBackendsHandle { +public: + /** + * No-op default constructor (does not own a handle). + */ + NmtBackendsHandle() : ownsHandle_(false) {} + + /** + * Construct a handle and increment the reference count. + * @param backendsDir - optional path to the backends directory. + * @param openclCacheDir - writable directory for OpenCL kernel cache + * (optional). + */ + explicit NmtBackendsHandle( + const std::string& backendsDir, + const std::string& openclCacheDir = ""); + + /** + * Destructor decrements reference count and may reset backend state. + */ + ~NmtBackendsHandle(); + + // Non-copyable + NmtBackendsHandle(const NmtBackendsHandle&) = delete; + NmtBackendsHandle& operator=(const NmtBackendsHandle&) = delete; + + // Movable + NmtBackendsHandle(NmtBackendsHandle&&) noexcept; + NmtBackendsHandle& operator=(NmtBackendsHandle&&) noexcept; + +private: + bool ownsHandle_; +}; diff --git a/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/PivotTranslationModel.cpp b/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/PivotTranslationModel.cpp index 7f678e0547..957dc55bb8 100644 --- a/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/PivotTranslationModel.cpp +++ b/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/PivotTranslationModel.cpp @@ -17,6 +17,17 @@ PivotTranslationModel::PivotTranslationModel( secondModel_(std::make_unique(secondModelPath)), stopTranslation_(false) { + // Initialize backends before sub-models load (do NOT erase keys) + std::string backendsDir; + if (auto it = firstModelConfig.find("backendsdir"); it != firstModelConfig.end()) { + backendsDir = std::get(it->second); + } + std::string openclCacheDir; + if (auto it = firstModelConfig.find("openclcachedir"); it != firstModelConfig.end()) { + openclCacheDir = std::get(it->second); + } + backendsHandle_.emplace(backendsDir, openclCacheDir); + firstModel_->setConfig(std::move(firstModelConfig)); secondModel_->setConfig(std::move(secondModelConfig)); diff --git a/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/PivotTranslationModel.hpp b/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/PivotTranslationModel.hpp index 9d8f5382bb..5302c216cc 100644 --- a/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/PivotTranslationModel.hpp +++ b/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/PivotTranslationModel.hpp @@ -3,11 +3,13 @@ #include #include #include +#include #include #include #include #include +#include "NmtLazyInitializeBackend.hpp" #include "TranslationModel.hpp" #include "qvac-lib-inference-addon-cpp/ModelInterfaces.hpp" @@ -71,6 +73,8 @@ class PivotTranslationModel config_; mutable std::atomic stopTranslation_ = false; + + std::optional backendsHandle_; }; } // namespace qvac_lib_inference_addon_nmt diff --git a/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/TranslationModel.cpp b/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/TranslationModel.cpp index 833165ca4a..771f50f43b 100644 --- a/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/TranslationModel.cpp +++ b/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/TranslationModel.cpp @@ -88,6 +88,19 @@ void TranslationModel::unload() { } void TranslationModel::load() { + // Extract backend loading config and initialize backends before any model loading + std::string backendsDir; + if (auto it = config_.find("backendsdir"); it != config_.end()) { + backendsDir = std::get(it->second); + config_.erase(it); + } + std::string openclCacheDir; + if (auto it = config_.find("openclcachedir"); it != config_.end()) { + openclCacheDir = std::get(it->second); + config_.erase(it); + } + backendsHandle_.emplace(backendsDir, openclCacheDir); + QLOG( qvac_lib_inference_addon_cpp::logger::Priority::INFO, "[TRANSLATION MODEL] modelPath_: " + modelPath_); diff --git a/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/TranslationModel.hpp b/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/TranslationModel.hpp index 95a09e91d8..d0dd91c269 100644 --- a/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/TranslationModel.hpp +++ b/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/TranslationModel.hpp @@ -2,10 +2,12 @@ #include #include +#include #include #include #include +#include "NmtLazyInitializeBackend.hpp" #include "nmt.hpp" #ifdef HAVE_BERGAMOT #include "bergamot.hpp" @@ -99,6 +101,8 @@ class TranslationModel : public qvac_lib_inference_addon_cpp::model::IModel, pub std::unordered_map> config_; + + std::optional backendsHandle_; }; } // namespace qvac_lib_inference_addon_nmt diff --git a/packages/qvac-lib-infer-nmtcpp/index.js b/packages/qvac-lib-infer-nmtcpp/index.js index 2cb5e7982c..c86cdb247c 100644 --- a/packages/qvac-lib-infer-nmtcpp/index.js +++ b/packages/qvac-lib-infer-nmtcpp/index.js @@ -1,5 +1,6 @@ 'use strict' +const path = require('path') const QvacLogger = require('@qvac/logging') const { QvacResponse, createJobHandler, exclusiveRunQueue } = require('@qvac/infer-base') @@ -221,6 +222,10 @@ class TranslationNmtcpp { async _load () { const { use_gpu: useGpu, ...otherConfig } = this._config + if (!otherConfig.backendsDir) { + otherConfig.backendsDir = path.join(__dirname, 'prebuilds') + } + const configurationParams = { path: this._files.model, config: otherConfig From 78a8f7918fd771b0b4572375fdcb282c441fe2d7 Mon Sep 17 00:00:00 2001 From: IC Date: Thu, 16 Apr 2026 07:46:40 +0000 Subject: [PATCH 2/7] test: add NmtLazyInitializeBackend unit tests Co-Authored-By: Claude Sonnet 4.6 --- .../addon/tests/CMakeLists.txt | 2 + .../tests/test_nmt_lazy_init_backend.cpp | 178 ++++++++++++++++++ 2 files changed, 180 insertions(+) create mode 100644 packages/qvac-lib-infer-nmtcpp/addon/tests/test_nmt_lazy_init_backend.cpp diff --git a/packages/qvac-lib-infer-nmtcpp/addon/tests/CMakeLists.txt b/packages/qvac-lib-infer-nmtcpp/addon/tests/CMakeLists.txt index 4bd8ceaa84..8e072786ac 100644 --- a/packages/qvac-lib-infer-nmtcpp/addon/tests/CMakeLists.txt +++ b/packages/qvac-lib-infer-nmtcpp/addon/tests/CMakeLists.txt @@ -10,6 +10,7 @@ set(TEST_SOURCES translation_tests.cpp nmt_model_wrapper_test_indic.cpp nmt_test.cpp + test_nmt_lazy_init_backend.cpp ${CMAKE_SOURCE_DIR}/addon/src/model-interface/TranslationModel.cpp ${CMAKE_SOURCE_DIR}/addon/src/model-interface/nmt.cpp ${CMAKE_SOURCE_DIR}/addon/src/model-interface/nmt_tokenization.cpp @@ -19,6 +20,7 @@ set(TEST_SOURCES ${CMAKE_SOURCE_DIR}/addon/src/model-interface/nmt_graph_encoder.cpp ${CMAKE_SOURCE_DIR}/addon/src/model-interface/nmt_beam_search.cpp ${CMAKE_SOURCE_DIR}/addon/src/model-interface/nmt_utils.cpp + ${CMAKE_SOURCE_DIR}/addon/src/model-interface/NmtLazyInitializeBackend.cpp ) # Add Bergamot validation tests if Bergamot is enabled diff --git a/packages/qvac-lib-infer-nmtcpp/addon/tests/test_nmt_lazy_init_backend.cpp b/packages/qvac-lib-infer-nmtcpp/addon/tests/test_nmt_lazy_init_backend.cpp new file mode 100644 index 0000000000..5b55589735 --- /dev/null +++ b/packages/qvac-lib-infer-nmtcpp/addon/tests/test_nmt_lazy_init_backend.cpp @@ -0,0 +1,178 @@ +#include +#include + +#include + +#include "model-interface/NmtLazyInitializeBackend.hpp" + +namespace fs = std::filesystem; + +class NmtLazyInitializeBackendTest : public ::testing::Test { +protected: + std::string getTestBackendsDir() { +#ifdef TEST_BINARY_DIR + return std::string(TEST_BINARY_DIR); +#else + return (fs::current_path() / "build" / "addon" / "tests").string(); +#endif + } + + void TearDown() override { + // Ensure the singleton is reset between tests. We increment once so + // g_refCount is at least 1, then drain with enough decrements to + // guarantee the count reaches zero and g_initialized is cleared. + // incrementRefCount / decrementRefCount are no-ops when not initialized, + // and decrementRefCount resets state when the count reaches zero. + NmtLazyInitializeBackend::incrementRefCount(); + for (int i = 0; i < 17; ++i) { + NmtLazyInitializeBackend::decrementRefCount(); + } + } +}; + +TEST_F(NmtLazyInitializeBackendTest, InitializeOnceReturnsTrueSecondReturnsFalse) { + bool result1 = NmtLazyInitializeBackend::initialize(""); + EXPECT_TRUE(result1); + + bool result2 = NmtLazyInitializeBackend::initialize(""); + EXPECT_FALSE(result2); +} + +TEST_F(NmtLazyInitializeBackendTest, InitializeWithBackendsDirDoesNotThrow) { + std::string backendsDir = getTestBackendsDir(); + EXPECT_NO_THROW({ + bool result = NmtLazyInitializeBackend::initialize(backendsDir); + (void)result; + }); +} + +TEST_F(NmtLazyInitializeBackendTest, InitializeIdempotencyReturnsFalseOnSecondCall) { + std::string backendsDir = getTestBackendsDir(); + + bool result1 = NmtLazyInitializeBackend::initialize(backendsDir); + bool result2 = NmtLazyInitializeBackend::initialize(backendsDir); + EXPECT_FALSE(result2); +} + +TEST_F(NmtLazyInitializeBackendTest, RefCountIncrementAndDecrementDoNotThrow) { + NmtLazyInitializeBackend::initialize(""); + + EXPECT_NO_THROW({ + NmtLazyInitializeBackend::incrementRefCount(); + NmtLazyInitializeBackend::incrementRefCount(); + NmtLazyInitializeBackend::decrementRefCount(); + NmtLazyInitializeBackend::decrementRefCount(); + NmtLazyInitializeBackend::decrementRefCount(); + NmtLazyInitializeBackend::decrementRefCount(); + }); +} + +TEST_F(NmtLazyInitializeBackendTest, RefCountReachingZeroResetsInitializedState) { + NmtLazyInitializeBackend::initialize(""); + + NmtLazyInitializeBackend::incrementRefCount(); + NmtLazyInitializeBackend::incrementRefCount(); + NmtLazyInitializeBackend::decrementRefCount(); + NmtLazyInitializeBackend::decrementRefCount(); + NmtLazyInitializeBackend::decrementRefCount(); + NmtLazyInitializeBackend::decrementRefCount(); + + // After refcount reaches zero, g_initialized is reset: a new initialize + // should return true again. + bool canReinitialize = NmtLazyInitializeBackend::initialize(""); + EXPECT_TRUE(canReinitialize); +} + +TEST_F(NmtLazyInitializeBackendTest, DifferentBackendsDirWarningStillReturnsFalse) { + std::string backendsDir = getTestBackendsDir(); + + bool result1 = NmtLazyInitializeBackend::initialize(backendsDir); + EXPECT_TRUE(result1); + + // Initialize with a different directory: should log a warning and return + // false (already initialized). + bool result2 = NmtLazyInitializeBackend::initialize("/different/path"); + EXPECT_FALSE(result2); + + // Clean up — decrement so the singleton resets for subsequent tests. + NmtLazyInitializeBackend::decrementRefCount(); + NmtLazyInitializeBackend::decrementRefCount(); +} + +TEST_F(NmtLazyInitializeBackendTest, DecrementRefCountWhenNotInitializedDoesNotCrash) { + EXPECT_NO_THROW({ + NmtLazyInitializeBackend::decrementRefCount(); + NmtLazyInitializeBackend::decrementRefCount(); + }); +} + +TEST_F(NmtLazyInitializeBackendTest, NmtBackendsHandleConstructionDoesNotThrow) { + std::string backendsDir = getTestBackendsDir(); + EXPECT_NO_THROW({ NmtBackendsHandle handle(backendsDir); }); +} + +TEST_F(NmtLazyInitializeBackendTest, NmtBackendsHandleEmptyDirDoesNotThrow) { + EXPECT_NO_THROW({ NmtBackendsHandle handle(""); }); +} + +TEST_F(NmtLazyInitializeBackendTest, NmtBackendsHandleMoveConstruction) { + std::string backendsDir = getTestBackendsDir(); + + { + NmtBackendsHandle handle1(backendsDir); + EXPECT_NO_THROW({ NmtBackendsHandle handle2(std::move(handle1)); }); + } +} + +TEST_F(NmtLazyInitializeBackendTest, NmtBackendsHandleMoveAssignment) { + std::string backendsDir = getTestBackendsDir(); + + { + NmtBackendsHandle handle1(backendsDir); + NmtBackendsHandle handle2(""); + EXPECT_NO_THROW({ handle2 = std::move(handle1); }); + } +} + +TEST_F(NmtLazyInitializeBackendTest, NmtBackendsHandleSelfAssignment) { + std::string backendsDir = getTestBackendsDir(); + + { + NmtBackendsHandle handle(backendsDir); + EXPECT_NO_THROW({ handle = std::move(handle); }); + } +} + +TEST_F(NmtLazyInitializeBackendTest, MultipleNmtBackendsHandlesDoNotThrow) { + std::string backendsDir = getTestBackendsDir(); + EXPECT_NO_THROW({ + NmtBackendsHandle handle1(backendsDir); + NmtBackendsHandle handle2(backendsDir); + NmtBackendsHandle handle3(backendsDir); + }); +} + +TEST_F(NmtLazyInitializeBackendTest, NmtBackendsHandleDefaultConstructorIsNoOp) { + // Default-constructed handle does not own the backend; destroying it should + // not decrement the reference count or touch backend state. + NmtLazyInitializeBackend::initialize(""); + NmtLazyInitializeBackend::incrementRefCount(); + + EXPECT_NO_THROW({ + NmtBackendsHandle noopHandle; + // noopHandle goes out of scope here without affecting refcount + }); + + // The owning refcount is still live — decrement explicitly to clean up. + NmtLazyInitializeBackend::decrementRefCount(); + NmtLazyInitializeBackend::decrementRefCount(); +} + +TEST_F(NmtLazyInitializeBackendTest, NmtBackendsHandleMoveTransfersOwnership) { + std::string backendsDir = getTestBackendsDir(); + EXPECT_NO_THROW({ + NmtBackendsHandle handle1(backendsDir); + NmtBackendsHandle handle2(std::move(handle1)); + // handle1 no longer owns; handle2 does. Destroying handle2 decrements once. + }); +} From 5101536b42b763865cdac30970c45780e7b24f96 Mon Sep 17 00:00:00 2001 From: IC Date: Thu, 16 Apr 2026 07:59:29 +0000 Subject: [PATCH 3/7] fix: use bare-path instead of path in nmtcpp index.js Bare runtime does not have Node.js built-in 'path' module. All other files in the package use 'bare-path'. Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/qvac-lib-infer-nmtcpp/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/qvac-lib-infer-nmtcpp/index.js b/packages/qvac-lib-infer-nmtcpp/index.js index c86cdb247c..4f70823c8b 100644 --- a/packages/qvac-lib-infer-nmtcpp/index.js +++ b/packages/qvac-lib-infer-nmtcpp/index.js @@ -1,6 +1,6 @@ 'use strict' -const path = require('path') +const path = require('bare-path') const QvacLogger = require('@qvac/logging') const { QvacResponse, createJobHandler, exclusiveRunQueue } = require('@qvac/infer-base') From dd724f9ce0fbe5322bbdc1430d2fce3020758901 Mon Sep 17 00:00:00 2001 From: IC Date: Thu, 16 Apr 2026 08:18:31 +0000 Subject: [PATCH 4/7] style: apply clang-format to NmtLazyInitializeBackend files Co-Authored-By: Claude Opus 4.6 (1M context) --- .../NmtLazyInitializeBackend.cpp | 3 ++- .../NmtLazyInitializeBackend.hpp | 3 +-- .../tests/test_nmt_lazy_init_backend.cpp | 24 +++++++++++++------ 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/NmtLazyInitializeBackend.cpp b/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/NmtLazyInitializeBackend.cpp index 97e60c2596..7aece0575e 100644 --- a/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/NmtLazyInitializeBackend.cpp +++ b/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/NmtLazyInitializeBackend.cpp @@ -73,7 +73,8 @@ void NmtLazyInitializeBackend::decrementRefCount() { g_refCount--; if (g_refCount == 0 && g_initialized) { QLOG( - Priority::DEBUG, "Resetting backend state (reference count reached zero)"); + Priority::DEBUG, + "Resetting backend state (reference count reached zero)"); g_initialized = false; g_recordedBackendsDir.clear(); } diff --git a/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/NmtLazyInitializeBackend.hpp b/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/NmtLazyInitializeBackend.hpp index 6b7916e01c..b54e2b0a3a 100644 --- a/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/NmtLazyInitializeBackend.hpp +++ b/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/NmtLazyInitializeBackend.hpp @@ -59,8 +59,7 @@ class NmtBackendsHandle { * (optional). */ explicit NmtBackendsHandle( - const std::string& backendsDir, - const std::string& openclCacheDir = ""); + const std::string& backendsDir, const std::string& openclCacheDir = ""); /** * Destructor decrements reference count and may reset backend state. diff --git a/packages/qvac-lib-infer-nmtcpp/addon/tests/test_nmt_lazy_init_backend.cpp b/packages/qvac-lib-infer-nmtcpp/addon/tests/test_nmt_lazy_init_backend.cpp index 5b55589735..c161d1dacc 100644 --- a/packages/qvac-lib-infer-nmtcpp/addon/tests/test_nmt_lazy_init_backend.cpp +++ b/packages/qvac-lib-infer-nmtcpp/addon/tests/test_nmt_lazy_init_backend.cpp @@ -30,7 +30,8 @@ class NmtLazyInitializeBackendTest : public ::testing::Test { } }; -TEST_F(NmtLazyInitializeBackendTest, InitializeOnceReturnsTrueSecondReturnsFalse) { +TEST_F( + NmtLazyInitializeBackendTest, InitializeOnceReturnsTrueSecondReturnsFalse) { bool result1 = NmtLazyInitializeBackend::initialize(""); EXPECT_TRUE(result1); @@ -46,7 +47,9 @@ TEST_F(NmtLazyInitializeBackendTest, InitializeWithBackendsDirDoesNotThrow) { }); } -TEST_F(NmtLazyInitializeBackendTest, InitializeIdempotencyReturnsFalseOnSecondCall) { +TEST_F( + NmtLazyInitializeBackendTest, + InitializeIdempotencyReturnsFalseOnSecondCall) { std::string backendsDir = getTestBackendsDir(); bool result1 = NmtLazyInitializeBackend::initialize(backendsDir); @@ -67,7 +70,8 @@ TEST_F(NmtLazyInitializeBackendTest, RefCountIncrementAndDecrementDoNotThrow) { }); } -TEST_F(NmtLazyInitializeBackendTest, RefCountReachingZeroResetsInitializedState) { +TEST_F( + NmtLazyInitializeBackendTest, RefCountReachingZeroResetsInitializedState) { NmtLazyInitializeBackend::initialize(""); NmtLazyInitializeBackend::incrementRefCount(); @@ -83,7 +87,9 @@ TEST_F(NmtLazyInitializeBackendTest, RefCountReachingZeroResetsInitializedState) EXPECT_TRUE(canReinitialize); } -TEST_F(NmtLazyInitializeBackendTest, DifferentBackendsDirWarningStillReturnsFalse) { +TEST_F( + NmtLazyInitializeBackendTest, + DifferentBackendsDirWarningStillReturnsFalse) { std::string backendsDir = getTestBackendsDir(); bool result1 = NmtLazyInitializeBackend::initialize(backendsDir); @@ -99,14 +105,17 @@ TEST_F(NmtLazyInitializeBackendTest, DifferentBackendsDirWarningStillReturnsFals NmtLazyInitializeBackend::decrementRefCount(); } -TEST_F(NmtLazyInitializeBackendTest, DecrementRefCountWhenNotInitializedDoesNotCrash) { +TEST_F( + NmtLazyInitializeBackendTest, + DecrementRefCountWhenNotInitializedDoesNotCrash) { EXPECT_NO_THROW({ NmtLazyInitializeBackend::decrementRefCount(); NmtLazyInitializeBackend::decrementRefCount(); }); } -TEST_F(NmtLazyInitializeBackendTest, NmtBackendsHandleConstructionDoesNotThrow) { +TEST_F( + NmtLazyInitializeBackendTest, NmtBackendsHandleConstructionDoesNotThrow) { std::string backendsDir = getTestBackendsDir(); EXPECT_NO_THROW({ NmtBackendsHandle handle(backendsDir); }); } @@ -152,7 +161,8 @@ TEST_F(NmtLazyInitializeBackendTest, MultipleNmtBackendsHandlesDoNotThrow) { }); } -TEST_F(NmtLazyInitializeBackendTest, NmtBackendsHandleDefaultConstructorIsNoOp) { +TEST_F( + NmtLazyInitializeBackendTest, NmtBackendsHandleDefaultConstructorIsNoOp) { // Default-constructed handle does not own the backend; destroying it should // not decrement the reference count or touch backend state. NmtLazyInitializeBackend::initialize(""); From 31ad6a8ab93a436245706b6e7fb0dbdd28fa07b4 Mon Sep 17 00:00:00 2001 From: IC Date: Thu, 16 Apr 2026 09:01:06 +0000 Subject: [PATCH 5/7] style: apply clang-format to modified TranslationModel and PivotTranslationModel Co-Authored-By: Claude Opus 4.6 (1M context) --- .../model-interface/PivotTranslationModel.cpp | 6 +++-- .../src/model-interface/TranslationModel.cpp | 23 +++++++++++-------- .../src/model-interface/TranslationModel.hpp | 14 +++++++---- 3 files changed, 26 insertions(+), 17 deletions(-) diff --git a/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/PivotTranslationModel.cpp b/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/PivotTranslationModel.cpp index 957dc55bb8..9f9c771f4c 100644 --- a/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/PivotTranslationModel.cpp +++ b/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/PivotTranslationModel.cpp @@ -19,11 +19,13 @@ PivotTranslationModel::PivotTranslationModel( // Initialize backends before sub-models load (do NOT erase keys) std::string backendsDir; - if (auto it = firstModelConfig.find("backendsdir"); it != firstModelConfig.end()) { + if (auto it = firstModelConfig.find("backendsdir"); + it != firstModelConfig.end()) { backendsDir = std::get(it->second); } std::string openclCacheDir; - if (auto it = firstModelConfig.find("openclcachedir"); it != firstModelConfig.end()) { + if (auto it = firstModelConfig.find("openclcachedir"); + it != firstModelConfig.end()) { openclCacheDir = std::get(it->second); } backendsHandle_.emplace(backendsDir, openclCacheDir); diff --git a/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/TranslationModel.cpp b/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/TranslationModel.cpp index 771f50f43b..095adc7075 100644 --- a/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/TranslationModel.cpp +++ b/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/TranslationModel.cpp @@ -49,10 +49,12 @@ BackendType TranslationModel::detectBackendType(const std::string& modelPath) { std::string filename = entry.path().filename().string(); // Check for bergamot model signatures if (filename.find(".intgemm") != std::string::npos || - (filename.find("vocab.") != std::string::npos && filename.find(".spm") != std::string::npos)) { + (filename.find("vocab.") != std::string::npos && + filename.find(".spm") != std::string::npos)) { QLOG( qvac_lib_inference_addon_cpp::logger::Priority::INFO, - "[TRANSLATION MODEL] Detected Bergamot backend based on model files"); + "[TRANSLATION MODEL] Detected Bergamot backend based on model " + "files"); return BackendType::BERGAMOT; } } @@ -62,14 +64,16 @@ BackendType TranslationModel::detectBackendType(const std::string& modelPath) { if (pathStr.find(".intgemm") != std::string::npos) { QLOG( qvac_lib_inference_addon_cpp::logger::Priority::INFO, - "[TRANSLATION MODEL] Detected Bergamot backend based on model filename"); + "[TRANSLATION MODEL] Detected Bergamot backend based on model " + "filename"); return BackendType::BERGAMOT; } } } catch (const std::exception& e) { QLOG( qvac_lib_inference_addon_cpp::logger::Priority::WARNING, - "[TRANSLATION MODEL] Error during backend detection: " + std::string(e.what())); + "[TRANSLATION MODEL] Error during backend detection: " + + std::string(e.what())); } #endif @@ -88,7 +92,8 @@ void TranslationModel::unload() { } void TranslationModel::load() { - // Extract backend loading config and initialize backends before any model loading + // Extract backend loading config and initialize backends before any model + // loading std::string backendsDir; if (auto it = config_.find("backendsdir"); it != config_.end()) { backendsDir = std::get(it->second); @@ -348,10 +353,7 @@ std::any TranslationModel::process(const std::any& input) { } } -void TranslationModel::cancel() const -{ - reset(); -} +void TranslationModel::cancel() const { reset(); } std::string TranslationModel::processString(const std::string& text) { #ifdef HAVE_BERGAMOT if (backendType_ == BackendType::BERGAMOT) { @@ -559,7 +561,8 @@ TranslationModel::getConfig() const { } void TranslationModel::setConfig( - std::unordered_map> config) { + std::unordered_map> + config) { config_ = std::move(config); updateConfig(); } diff --git a/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/TranslationModel.hpp b/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/TranslationModel.hpp index d0dd91c269..0e4e507ed7 100644 --- a/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/TranslationModel.hpp +++ b/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/TranslationModel.hpp @@ -24,7 +24,9 @@ enum class BackendType { #endif }; -class TranslationModel : public qvac_lib_inference_addon_cpp::model::IModel, public qvac_lib_inference_addon_cpp::model::IModelCancel { +class TranslationModel + : public qvac_lib_inference_addon_cpp::model::IModel, + public qvac_lib_inference_addon_cpp::model::IModelCancel { public: TranslationModel() {}; @@ -40,14 +42,14 @@ class TranslationModel : public qvac_lib_inference_addon_cpp::model::IModel, pub void unload(); - void reload(); + void reload(); void reset() const; void setUseGpu(bool useGpu); std::unordered_map> - getConfig() const; + getConfig() const; bool isLoaded() const; @@ -89,10 +91,12 @@ class TranslationModel : public qvac_lib_inference_addon_cpp::model::IModel, pub BackendType backendType_ = BackendType::GGML; - mutable std::unique_ptr nmtCtx_{nullptr, nmt_free}; + mutable std::unique_ptr nmtCtx_{ + nullptr, nmt_free}; #ifdef HAVE_BERGAMOT - std::unique_ptr bergamotCtx_{nullptr, bergamot_free}; + std::unique_ptr bergamotCtx_{ + nullptr, bergamot_free}; #endif mutable bool isFirstSentence_ = true; From b029b7e35acbef392184821391f67c1b2292bf4a Mon Sep 17 00:00:00 2001 From: IC Date: Fri, 17 Apr 2026 05:26:01 +0000 Subject: [PATCH 6/7] fix: preserve nmtcpp backend config across reload and harden variant extraction Stop erasing backendsdir/openclcachedir from config_ in TranslationModel::load so reload() re-initializes the backend with the original path instead of falling back to the default loader. Without this, GPU backends disappeared silently after any model reload on Android. Use std::get_if for backendsdir/openclcachedir extraction in TranslationModel and PivotTranslationModel; on type mismatch log a warning and fall through instead of propagating bad_variant_access out of the load path. Default backendsDir in JS only when the field is missing (=== undefined) rather than on any falsy value, so callers passing "" to force the default system backend loader are respected. Refs QVAC-17129. --- .../model-interface/PivotTranslationModel.cpp | 13 +++++++--- .../src/model-interface/TranslationModel.cpp | 25 ++++++++++++++----- packages/qvac-lib-infer-nmtcpp/index.js | 2 +- 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/PivotTranslationModel.cpp b/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/PivotTranslationModel.cpp index 9f9c771f4c..6886e66043 100644 --- a/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/PivotTranslationModel.cpp +++ b/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/PivotTranslationModel.cpp @@ -17,16 +17,23 @@ PivotTranslationModel::PivotTranslationModel( secondModel_(std::make_unique(secondModelPath)), stopTranslation_(false) { - // Initialize backends before sub-models load (do NOT erase keys) + // Initialize backends before sub-models load. Keys are read from the first + // model's config only — both sub-models share one process-wide backend, so + // the second model's backendsdir/openclcachedir (if any) are ignored. Keys + // are not erased so sub-model load() calls can re-read them. std::string backendsDir; if (auto it = firstModelConfig.find("backendsdir"); it != firstModelConfig.end()) { - backendsDir = std::get(it->second); + if (const auto* value = std::get_if(&it->second)) { + backendsDir = *value; + } } std::string openclCacheDir; if (auto it = firstModelConfig.find("openclcachedir"); it != firstModelConfig.end()) { - openclCacheDir = std::get(it->second); + if (const auto* value = std::get_if(&it->second)) { + openclCacheDir = *value; + } } backendsHandle_.emplace(backendsDir, openclCacheDir); diff --git a/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/TranslationModel.cpp b/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/TranslationModel.cpp index 095adc7075..03ef35dd0e 100644 --- a/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/TranslationModel.cpp +++ b/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/TranslationModel.cpp @@ -92,17 +92,30 @@ void TranslationModel::unload() { } void TranslationModel::load() { - // Extract backend loading config and initialize backends before any model - // loading + // Read backend loading config and initialize backends before any model + // loading. Keys are preserved in config_ so reload() can re-initialize with + // the same backends directory. std::string backendsDir; if (auto it = config_.find("backendsdir"); it != config_.end()) { - backendsDir = std::get(it->second); - config_.erase(it); + if (const auto* value = std::get_if(&it->second)) { + backendsDir = *value; + } else { + QLOG( + qvac_lib_inference_addon_cpp::logger::Priority::WARNING, + "[TRANSLATION MODEL] 'backendsdir' config value is not a string; " + "ignoring"); + } } std::string openclCacheDir; if (auto it = config_.find("openclcachedir"); it != config_.end()) { - openclCacheDir = std::get(it->second); - config_.erase(it); + if (const auto* value = std::get_if(&it->second)) { + openclCacheDir = *value; + } else { + QLOG( + qvac_lib_inference_addon_cpp::logger::Priority::WARNING, + "[TRANSLATION MODEL] 'openclcachedir' config value is not a string; " + "ignoring"); + } } backendsHandle_.emplace(backendsDir, openclCacheDir); diff --git a/packages/qvac-lib-infer-nmtcpp/index.js b/packages/qvac-lib-infer-nmtcpp/index.js index 4f70823c8b..ca27a2789a 100644 --- a/packages/qvac-lib-infer-nmtcpp/index.js +++ b/packages/qvac-lib-infer-nmtcpp/index.js @@ -222,7 +222,7 @@ class TranslationNmtcpp { async _load () { const { use_gpu: useGpu, ...otherConfig } = this._config - if (!otherConfig.backendsDir) { + if (otherConfig.backendsDir === undefined) { otherConfig.backendsDir = path.join(__dirname, 'prebuilds') } From 0b732e4bb81849264d659d19fecdd9a3983ad38a Mon Sep 17 00:00:00 2001 From: IC Date: Fri, 17 Apr 2026 10:01:54 +0000 Subject: [PATCH 7/7] fix: drop GGML backend init from PivotTranslationModel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PivotTranslationModel only uses Bergamot — GGML dynamic backend loading is for the IndicTrans (GGML) path, which is a different inference engine. The earlier backendsHandle_ initialization in the pivot constructor was initializing GGML backends even when none were used. Remove the backend config extraction block, the backendsHandle_ member and the NmtLazyInitializeBackend include. Sub-model TranslationModel instances still handle their own backend init on load() for the GGML path. Refs QVAC-17129. --- .../model-interface/PivotTranslationModel.cpp | 21 ------------------- .../model-interface/PivotTranslationModel.hpp | 4 ---- 2 files changed, 25 deletions(-) diff --git a/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/PivotTranslationModel.cpp b/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/PivotTranslationModel.cpp index 6886e66043..36e70212c5 100644 --- a/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/PivotTranslationModel.cpp +++ b/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/PivotTranslationModel.cpp @@ -16,27 +16,6 @@ PivotTranslationModel::PivotTranslationModel( firstModel_(std::make_unique(firstModelPath)), secondModel_(std::make_unique(secondModelPath)), stopTranslation_(false) { - - // Initialize backends before sub-models load. Keys are read from the first - // model's config only — both sub-models share one process-wide backend, so - // the second model's backendsdir/openclcachedir (if any) are ignored. Keys - // are not erased so sub-model load() calls can re-read them. - std::string backendsDir; - if (auto it = firstModelConfig.find("backendsdir"); - it != firstModelConfig.end()) { - if (const auto* value = std::get_if(&it->second)) { - backendsDir = *value; - } - } - std::string openclCacheDir; - if (auto it = firstModelConfig.find("openclcachedir"); - it != firstModelConfig.end()) { - if (const auto* value = std::get_if(&it->second)) { - openclCacheDir = *value; - } - } - backendsHandle_.emplace(backendsDir, openclCacheDir); - firstModel_->setConfig(std::move(firstModelConfig)); secondModel_->setConfig(std::move(secondModelConfig)); diff --git a/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/PivotTranslationModel.hpp b/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/PivotTranslationModel.hpp index 5302c216cc..9d8f5382bb 100644 --- a/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/PivotTranslationModel.hpp +++ b/packages/qvac-lib-infer-nmtcpp/addon/src/model-interface/PivotTranslationModel.hpp @@ -3,13 +3,11 @@ #include #include #include -#include #include #include #include #include -#include "NmtLazyInitializeBackend.hpp" #include "TranslationModel.hpp" #include "qvac-lib-inference-addon-cpp/ModelInterfaces.hpp" @@ -73,8 +71,6 @@ class PivotTranslationModel config_; mutable std::atomic stopTranslation_ = false; - - std::optional backendsHandle_; }; } // namespace qvac_lib_inference_addon_nmt