diff --git a/CMakeLists.txt b/CMakeLists.txt index 6c1564be7b..92a3ff0136 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,6 +61,7 @@ option(SHERPA_ONNX_ENABLE_SANITIZER "Whether to enable ubsan and asan" OFF) option(SHERPA_ONNX_BUILD_C_API_EXAMPLES "Whether to enable C API examples" ${SUGGEST_BUILD_BINARIES}) option(SHERPA_ONNX_ENABLE_RKNN "Whether to build for RKNN NPU " OFF) option(SHERPA_ONNX_ENABLE_ASCEND_NPU "Whether to build for Ascend NPU " OFF) +option(SHERPA_ONNX_ENABLE_QNN "Whether to build for Qualcomm NPU" OFF) set(SHERPA_ONNX_LINUX_ARM64_GPU_ONNXRUNTIME_VERSION "1.11.0" CACHE STRING "Used only for Linux ARM64 GPU. Set to 1.11.0 if you use CUDA 10.2 and cudnn8. Set it to 1.16.0 if you use CUDA 11.4 and cudnn8. Set it to 1.18.0 if you use CUDA 12.2 and cudnn8. Set it to 1.18.1 if you use CUDA 12.6 and cudnn9") @@ -179,6 +180,7 @@ message(STATUS "SHERPA_ONNX_ENABLE_SANITIZER: ${SHERPA_ONNX_ENABLE_SANITIZER}") message(STATUS "SHERPA_ONNX_BUILD_C_API_EXAMPLES: ${SHERPA_ONNX_BUILD_C_API_EXAMPLES}") message(STATUS "SHERPA_ONNX_ENABLE_RKNN: ${SHERPA_ONNX_ENABLE_RKNN}") message(STATUS "SHERPA_ONNX_ENABLE_ASCEND_NPU: ${SHERPA_ONNX_ENABLE_ASCEND_NPU}") +message(STATUS "SHERPA_ONNX_ENABLE_QNN: ${SHERPA_ONNX_ENABLE_QNN}") message(STATUS "SHERPA_ONNX_LINK_D3D: ${SHERPA_ONNX_LINK_D3D}") if(BUILD_SHARED_LIBS OR SHERPA_ONNX_ENABLE_JNI) @@ -337,6 +339,44 @@ if(SHERPA_ONNX_ENABLE_ASCEND_NPU) message(STATUS "Build with Ascend NPU") endif() +if(SHERPA_ONNX_ENABLE_QNN) + if(NOT DEFINED ENV{QNN_SDK_ROOT}) + message(FATAL_ERROR "\ + Please specify the installation directory of the QNN SDK toolkit. + For instance, if it is installed in + + /mnt/sdb/open-source/qairt/2.33.0.250327 + + You can run + + source /mnt/sdb/open-source/qairt/2.33.0.250327/bin/envsetup.sh + + which will give you the following output + + [INFO] AISW SDK environment set + [INFO] QNN_SDK_ROOT: /mnt/sdb/open-source/qairt/2.33.0.250327 + [INFO] SNPE_ROOT: /mnt/sdb/open-source/qairt/2.33.0.250327 + + Then run + + echo $QNN_SDK_ROOT + + It should print: + + /mnt/sdb/open-source/qairt/2.33.0.250327 + + You can choose a version of QNN SDK by yourself. You don't need + to use 2.33.0.250327 + ") + endif() + + set(QNN_SDK_ROOT $ENV{QNN_SDK_ROOT}) + + if(NOT EXISTS ${QNN_SDK_ROOT}/include/QNN/QnnInterface.h) + message(FATAL_ERROR "${QNN_SDK_ROOT}/include/QNN/QnnInterface.h does not exist") + endif() +endif() + if(UNIX AND NOT APPLE AND NOT SHERPA_ONNX_ENABLE_WASM AND NOT CMAKE_SYSTEM_NAME STREQUAL Android AND NOT CMAKE_SYSTEM_NAME STREQUAL OHOS) check_include_file_cxx(alsa/asoundlib.h SHERPA_ONNX_HAS_ALSA) if(SHERPA_ONNX_HAS_ALSA) diff --git a/build-android-arm64-v8a.sh b/build-android-arm64-v8a.sh index 7de2e68152..1491c8ba0b 100755 --- a/build-android-arm64-v8a.sh +++ b/build-android-arm64-v8a.sh @@ -101,6 +101,10 @@ if [ -z $SHERPA_ONNX_ENABLE_RKNN ]; then SHERPA_ONNX_ENABLE_RKNN=OFF fi +if [ -z $SHERPA_ONNX_ENABLE_QNN ]; then + SHERPA_ONNX_ENABLE_QNN=OFF +fi + if [ $SHERPA_ONNX_ENABLE_RKNN == ON ]; then rknn_version=2.2.0 if [ ! -d ./librknnrt-android ]; then @@ -130,6 +134,10 @@ if [ -z $SHERPA_ONNX_ENABLE_C_API ]; then SHERPA_ONNX_ENABLE_C_API=OFF fi +if [ -z $SHERPA_ONNX_ANDROID_PLATFORM ]; then + SHERPA_ONNX_ANDROID_PLATFORM=android-21 +fi + if [ -z $SHERPA_ONNX_ENABLE_JNI ]; then SHERPA_ONNX_ENABLE_JNI=ON fi @@ -153,8 +161,9 @@ cmake -DCMAKE_TOOLCHAIN_FILE="$ANDROID_NDK/build/cmake/android.toolchain.cmake" -DSHERPA_ONNX_ENABLE_C_API=$SHERPA_ONNX_ENABLE_C_API \ -DCMAKE_INSTALL_PREFIX=./install \ -DSHERPA_ONNX_ENABLE_RKNN=$SHERPA_ONNX_ENABLE_RKNN \ + -DSHERPA_ONNX_ENABLE_QNN=$SHERPA_ONNX_ENABLE_QNN \ -DANDROID_ABI="arm64-v8a" \ - -DANDROID_PLATFORM=android-21 .. + -DANDROID_PLATFORM=$SHERPA_ONNX_ANDROID_PLATFORM .. # By default, it links to libc++_static.a # -DANDROID_STL=c++_shared \ diff --git a/sherpa-onnx/csrc/CMakeLists.txt b/sherpa-onnx/csrc/CMakeLists.txt index d44e8e8a64..577048ec9a 100644 --- a/sherpa-onnx/csrc/CMakeLists.txt +++ b/sherpa-onnx/csrc/CMakeLists.txt @@ -200,6 +200,14 @@ if(SHERPA_ONNX_ENABLE_ASCEND_NPU) ) endif() +if(SHERPA_ONNX_ENABLE_QNN) + list(APPEND sources + ./qnn/qnn-backend.cc + ./qnn/qnn-model.cc + ./qnn/utils.cc + ) +endif() + if(SHERPA_ONNX_ENABLE_TTS) list(APPEND sources character-lexicon.cc @@ -314,6 +322,10 @@ if(SHERPA_ONNX_ENABLE_ASCEND_NPU) ) endif() +if(SHERPA_ONNX_ENABLE_QNN) + target_include_directories(sherpa-onnx-core PRIVATE ${QNN_SDK_ROOT}/include/QNN) +endif() + if(TARGET onnxruntime) target_link_libraries(sherpa-onnx-core onnxruntime) else() diff --git a/sherpa-onnx/csrc/qnn/macros.h b/sherpa-onnx/csrc/qnn/macros.h new file mode 100644 index 0000000000..58c65cf37a --- /dev/null +++ b/sherpa-onnx/csrc/qnn/macros.h @@ -0,0 +1,19 @@ +// sherpa-onnx/csrc/qnn/macros.h +// +// Copyright 2025 Xiaomi Corporation + +#ifndef SHERPA_ONNX_CSRC_QNN_MACROS_H_ +#define SHERPA_ONNX_CSRC_QNN_MACROS_H_ + +#include "sherpa-onnx/csrc/macros.h" + +#define SHERPA_ONNX_QNN_CHECK(ret, msg, ...) \ + do { \ + if (ret != QNN_SUCCESS) { \ + SHERPA_ONNX_LOGE("Return code is: %d", static_cast(ret)); \ + SHERPA_ONNX_LOGE(msg, ##__VA_ARGS__); \ + SHERPA_ONNX_EXIT(-1); \ + } \ + } while (0) + +#endif // SHERPA_ONNX_CSRC_QNN_MACROS_H_ diff --git a/sherpa-onnx/csrc/qnn/qnn-backend.cc b/sherpa-onnx/csrc/qnn/qnn-backend.cc new file mode 100644 index 0000000000..8ed9c2818f --- /dev/null +++ b/sherpa-onnx/csrc/qnn/qnn-backend.cc @@ -0,0 +1,251 @@ +// sherpa-onnx/csrc/qnn/qnn-backend.cc +// +// Copyright (c) 2025 Xiaomi Corporation + +#include "sherpa-onnx/csrc/qnn/qnn-backend.h" + +#include +#include + +#include +#include +#include +#include + +#include "QnnInterface.h" +#include "System/QnnSystemInterface.h" +#include "sherpa-onnx/csrc/qnn/macros.h" +#include "sherpa-onnx/csrc/qnn/utils.h" + +namespace sherpa_onnx { + +class QnnBackend::Impl { + public: + explicit Impl(const std::string &backend_lib) { + bool ok = InitQnnInterface(backend_lib); + if (!ok) { + SHERPA_ONNX_LOGE("Failed to init qnn interface from '%s'", + backend_lib.c_str()); + return; + } + + InitLog(); + InitBackend(); + InitDevice(); + + is_initialized_ = true; + } + + ~Impl() { + if (context_handle_) { + auto ret = qnn_interface_.contextFree(context_handle_, nullptr); + SHERPA_ONNX_QNN_CHECK(ret, "Failed to call contextFree"); + } + + if (device_handle_) { + auto ret = qnn_interface_.deviceFree(device_handle_); + SHERPA_ONNX_QNN_CHECK(ret, "Failed to call deviceFree"); + } + + if (backend_handle_) { + auto ret = qnn_interface_.backendFree(backend_handle_); + SHERPA_ONNX_QNN_CHECK(ret, "Failed to call backendFree"); + } + + if (log_handle_) { + auto ret = qnn_interface_.logFree(log_handle_); + SHERPA_ONNX_QNN_CHECK(ret, "Failed to call logFree"); + } + } + + void InitContext() { + if (context_handle_) { + SHERPA_ONNX_LOGE("context handle is already initialized"); + return; + } + + auto ret = qnn_interface_.contextCreate(backend_handle_, device_handle_, + context_config_, &context_handle_); + SHERPA_ONNX_QNN_CHECK(ret, "Failed to call contextCreate"); + } + + void InitContext(Qnn_ContextHandle_t t) { context_handle_ = t; } + + Qnn_LogHandle_t LogHandle() const { return log_handle_; } + + Qnn_BackendHandle_t BackendHandle() const { return backend_handle_; } + + Qnn_DeviceHandle_t DeviceHandle() const { return device_handle_; } + + Qnn_ContextHandle_t ContextHandle() const { return context_handle_; } + + QNN_INTERFACE_VER_TYPE QnnInterface() const { return qnn_interface_; } + + QnnLog_Level_t LogLevel() const { return log_level_; } + + bool IsInitialized() const { return is_initialized_; } + + private: + bool InitQnnInterface(const std::string &backend_lib) { + backend_lib_handle_ = std::unique_ptr( + dlopen(backend_lib.c_str(), RTLD_NOW | RTLD_LOCAL), &dlclose); + if (!backend_lib_handle_) { + SHERPA_ONNX_LOGE("Failed to dlopen '%s'. Error is: '%s'", + backend_lib.c_str(), dlerror()); + return false; + } + SHERPA_ONNX_LOGE("loaded %s", backend_lib.c_str()); + + const char *symbol = "QnnInterface_getProviders"; + auto get_interface_providers = + reinterpret_cast( + dlsym(backend_lib_handle_.get(), symbol)); + if (!get_interface_providers) { + SHERPA_ONNX_LOGE("Failed to dlsym for '%s'. Error is: '%s'", symbol, + dlerror()); + return false; + } + SHERPA_ONNX_LOGE("Got %s", symbol); + + const QnnInterface_t **interface_providers = nullptr; + uint32_t num_providers = 0; + + auto ret = get_interface_providers(&interface_providers, &num_providers); + SHERPA_ONNX_QNN_CHECK(ret, "Failed to call get_interface_providers"); + + if (!interface_providers) { + SHERPA_ONNX_LOGE("interface_providers is nullptr"); + return false; + } + + if (num_providers == 0) { + SHERPA_ONNX_LOGE("Number of providers is 0"); + return false; + } + + bool found_valid_interface = false; + + if (debug_) { + SHERPA_ONNX_LOGE("QNN_API_VERSION_MAJOR: %d", QNN_API_VERSION_MAJOR); + SHERPA_ONNX_LOGE("QNN_API_VERSION_MINOR: %d", QNN_API_VERSION_MINOR); + SHERPA_ONNX_LOGE("QNN_API_VERSION_PATCH: %d", QNN_API_VERSION_PATCH); + } + + for (size_t idx = 0; idx < num_providers; ++idx) { + auto p = interface_providers[idx]; + + if (debug_) { + std::ostringstream os; + os << "---" << idx << "----\n"; + os << "backendId: " << p->backendId << "\n"; + os << "coreApiVersion.major: " << p->apiVersion.coreApiVersion.major + << "\n"; + os << "coreApiVersion.minor: " << p->apiVersion.coreApiVersion.minor + << "\n"; + os << "coreApiVersion.patch: " << p->apiVersion.coreApiVersion.patch + << "\n"; + + os << "backendApiVersion.major: " + << p->apiVersion.backendApiVersion.major << "\n"; + os << "backendApiVersion.minor: " + << p->apiVersion.backendApiVersion.minor << "\n"; + os << "backendApiVersion.patch: " + << p->apiVersion.backendApiVersion.patch << "\n"; + SHERPA_ONNX_LOGE("%s", os.str().c_str()); + } + + qnn_interface_ = p->QNN_INTERFACE_VER_NAME; + found_valid_interface = true; + break; + } + + if (!found_valid_interface) { + SHERPA_ONNX_LOGE("Failed to find valid interface"); + return false; + } + + if (debug_) { + const char *build_id = nullptr; + ret = qnn_interface_.backendGetBuildId(&build_id); + SHERPA_ONNX_QNN_CHECK(ret, "Failed to call backendGetBuildId()"); + + SHERPA_ONNX_LOGE("backend build ID: %s", build_id); + } + + return true; + } + + void InitLog() { + auto ret = qnn_interface_.logCreate(LogCallback, log_level_, &log_handle_); + SHERPA_ONNX_QNN_CHECK(ret, "Failed to call logCreate"); + } + + void InitBackend() { + auto ret = qnn_interface_.backendCreate(log_handle_, backend_config_, + &backend_handle_); + SHERPA_ONNX_QNN_CHECK(ret, "Failed to call backendCreate"); + } + + void InitDevice() { + auto ret = + qnn_interface_.deviceCreate(log_handle_, nullptr, &device_handle_); + SHERPA_ONNX_QNN_CHECK(ret, "Failed to call deviceCreate"); + } + + private: + bool debug_ = true; + std::unique_ptr backend_lib_handle_{nullptr, + &dlclose}; + + QNN_INTERFACE_VER_TYPE qnn_interface_; + + QnnLog_Level_t log_level_ = QNN_LOG_LEVEL_WARN; + // QnnLog_Level_t log_level_ = QNN_LOG_LEVEL_INFO; + // QnnLog_Level_t log_level_ = QNN_LOG_LEVEL_VERBOSE; + + Qnn_LogHandle_t log_handle_ = nullptr; + + const QnnBackend_Config_t **backend_config_ = nullptr; + Qnn_BackendHandle_t backend_handle_ = nullptr; + + Qnn_DeviceHandle_t device_handle_ = nullptr; + + Qnn_ContextHandle_t context_handle_ = nullptr; + const QnnContext_Config_t **context_config_ = nullptr; + bool is_initialized_ = false; +}; + +QnnBackend::~QnnBackend() = default; + +QnnBackend::QnnBackend(const std::string &backend_lib) + : impl_(std::make_unique(backend_lib)) {} + +void QnnBackend::InitContext() const { impl_->InitContext(); } + +void QnnBackend::InitContext(Qnn_ContextHandle_t context_handle) const { + impl_->InitContext(context_handle); +} + +Qnn_LogHandle_t QnnBackend::LogHandle() const { return impl_->LogHandle(); } + +Qnn_BackendHandle_t QnnBackend::BackendHandle() const { + return impl_->BackendHandle(); +} + +Qnn_DeviceHandle_t QnnBackend::DeviceHandle() const { + return impl_->DeviceHandle(); +} + +Qnn_ContextHandle_t QnnBackend::ContextHandle() const { + return impl_->ContextHandle(); +} + +QNN_INTERFACE_VER_TYPE QnnBackend::QnnInterface() const { + return impl_->QnnInterface(); +} + +QnnLog_Level_t QnnBackend::LogLevel() const { return impl_->LogLevel(); } + +bool QnnBackend::IsInitialized() const { return impl_->IsInitialized(); } + +} // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/qnn/qnn-backend.h b/sherpa-onnx/csrc/qnn/qnn-backend.h new file mode 100644 index 0000000000..62049a699c --- /dev/null +++ b/sherpa-onnx/csrc/qnn/qnn-backend.h @@ -0,0 +1,36 @@ +// sherpa-onnx/csrc/qnn/qnn-backend.h +// +// Copyright (c) 2025 Xiaomi Corporation +#ifndef SHERPA_ONNX_CSRC_QNN_QNN_BACKEND_H_ +#define SHERPA_ONNX_CSRC_QNN_QNN_BACKEND_H_ + +#include +#include + +#include "QnnInterface.h" + +namespace sherpa_onnx { + +class QnnBackend { + public: + explicit QnnBackend(const std::string &backend_lib); + ~QnnBackend(); + + void InitContext() const; + void InitContext(Qnn_ContextHandle_t context_handle) const; + Qnn_LogHandle_t LogHandle() const; + Qnn_BackendHandle_t BackendHandle() const; + Qnn_DeviceHandle_t DeviceHandle() const; + Qnn_ContextHandle_t ContextHandle() const; + QNN_INTERFACE_VER_TYPE QnnInterface() const; + QnnLog_Level_t LogLevel() const; + bool IsInitialized() const; + + private: + class Impl; + std::unique_ptr impl_; +}; + +} // namespace sherpa_onnx + +#endif // SHERPA_ONNX_CSRC_QNN_QNN_BACKEND_H_ diff --git a/sherpa-onnx/csrc/qnn/qnn-model.cc b/sherpa-onnx/csrc/qnn/qnn-model.cc new file mode 100644 index 0000000000..d3c53788e9 --- /dev/null +++ b/sherpa-onnx/csrc/qnn/qnn-model.cc @@ -0,0 +1,665 @@ +// sherpa-onnx/csrc/qnn/qnn-model.h +// +// Copyright (c) 2025 Xiaomi Corporation + +#include "sherpa-onnx/csrc/qnn/qnn-model.h" + +#include + +#include +#include +#include +#include +#include +#include + +#include "sherpa-onnx/csrc/qnn/macros.h" +#include "sherpa-onnx/csrc/qnn/qnn-backend.h" +#include "sherpa-onnx/csrc/qnn/utils.h" + +namespace sherpa_onnx { + +class QnnModel::Impl { + public: + Impl(const std::string &model_so, const QnnBackend *backend) + : backend_(backend) { + bool ok = InitModel(model_so); + if (!ok) { + SHERPA_ONNX_LOGE("Failed to load '%s'", model_so.c_str()); + return; + } + + ok = InitSymbols(); + if (!ok) { + SHERPA_ONNX_LOGE("Failed to get model symbols from '%s'", + model_so.c_str()); + return; + } + + InitGraph(); + + PostInit(); + } + + Impl(const std::string &binary_context_file, const std::string &system_lib, + const QnnBackend *backend, BinaryContextTag) + : backend_(backend) { + bool ok = LoadSystemLib(binary_context_file, system_lib); + if (!ok) { + return; + } + + PostInit(); + } + + bool LoadSystemLib(const std::string &binary_context_file, + const std::string &system_lib) { + system_lib_handle_ = std::unique_ptr( + dlopen(system_lib.c_str(), RTLD_NOW | RTLD_LOCAL), &dlclose); + if (!system_lib_handle_) { + SHERPA_ONNX_LOGE("Failed to dlopen '%s'. Error is: '%s'", + system_lib.c_str(), dlerror()); + return false; + } + SHERPA_ONNX_LOGE("loaded %s", system_lib.c_str()); + + auto get_system_interface_providers = + reinterpret_cast( + dlsym(system_lib_handle_.get(), "QnnSystemInterface_getProviders")); + + if (!get_system_interface_providers) { + SHERPA_ONNX_LOGE("Failed to get QnnSystemInterface_getProviders"); + return false; + } + + const QnnSystemInterface_t **system_interface_providers = nullptr; + uint32_t num_providers = 0; + if (get_system_interface_providers(&system_interface_providers, + &num_providers) != QNN_SUCCESS) { + SHERPA_ONNX_LOGE("Failed to get system interface providers."); + return false; + } + + if (!system_interface_providers) { + SHERPA_ONNX_LOGE( + "Failed to get system interface providers: null " + "interface providers received."); + return false; + } + + if (!num_providers) { + SHERPA_ONNX_LOGE( + "Failed to get interface providers: 0 interface providers."); + return false; + } + + for (uint32_t i = 0; i < num_providers; ++i) { + if (debug_) { + SHERPA_ONNX_LOGE("QNN_SYSTEM_API_VERSION_MAJOR: %d", + static_cast(QNN_SYSTEM_API_VERSION_MAJOR)); + SHERPA_ONNX_LOGE("QNN_SYSTEM_API_VERSION_MINOR: %d", + static_cast(QNN_SYSTEM_API_VERSION_MINOR)); + SHERPA_ONNX_LOGE( + "systemApiVersion.major: %d", + static_cast( + system_interface_providers[i]->systemApiVersion.major)); + SHERPA_ONNX_LOGE( + "systemApiVersion.minor: %d", + static_cast( + system_interface_providers[i]->systemApiVersion.minor)); + } + + qnn_system_interface_ = + system_interface_providers[i]->QNN_SYSTEM_INTERFACE_VER_NAME; + } + + // read file into a buffer + std::vector buffer = ReadFile(binary_context_file); + + QnnSystemContext_Handle_t sys_ctx_handle = nullptr; + if (qnn_system_interface_.systemContextCreate(&sys_ctx_handle) != + QNN_SUCCESS) { + SHERPA_ONNX_LOGE("Could not create system handle."); + return false; + } + + const QnnSystemContext_BinaryInfo_t *binary_info = nullptr; + Qnn_ContextBinarySize_t binary_info_size = 0; + + auto ret = qnn_system_interface_.systemContextGetBinaryInfo( + sys_ctx_handle, static_cast(buffer.data()), buffer.size(), + &binary_info, &binary_info_size); + if (ret != QNN_SUCCESS) { + SHERPA_ONNX_LOGE("Failed to get context binary info from '%s'", + binary_context_file.c_str()); + + qnn_system_interface_.systemContextFree(sys_ctx_handle); + return false; + } + + const GraphConfigInfo **graph_configs_info = nullptr; + + uint32_t graph_configs_info_count = 0; + GraphInfo **graphs_info = nullptr; + uint32_t graphs_count = 0; + + if (!CopyMetadataToGraphsInfo(binary_info, graphs_info, graphs_count)) { + SHERPA_ONNX_LOGE("Failed to call CopyMetadataToGraphsInfo"); + + qnn_system_interface_.systemContextFree(sys_ctx_handle); + return false; + } + + qnn_system_interface_.systemContextFree(sys_ctx_handle); + + auto free_graphs_info = [&graphs_info, &graphs_count] { + for (uint32_t i = 0; i < graphs_count; ++i) { + for (uint32_t k = 0; k < graphs_info[i]->num_input_tensors; ++k) { + FreeTensor(&graphs_info[i]->input_tensors[k]); + } + + for (uint32_t k = 0; k < graphs_info[i]->num_output_tensors; ++k) { + FreeTensor(&graphs_info[i]->output_tensors[k]); + } + + free(graphs_info[i]->input_tensors); + free(graphs_info[i]->output_tensors); + + free(graphs_info[i]->graph_name); + } + + free(graphs_info[0]); + free(graphs_info); + }; + + if (graphs_count > 1) { + SHERPA_ONNX_LOGE("Only the first graph is used"); + } + + Qnn_ContextHandle_t context_handle = nullptr; + + if (backend_->QnnInterface().contextCreateFromBinary( + backend_->BackendHandle(), backend_->DeviceHandle(), + context_config_, static_cast(buffer.data()), buffer.size(), + &context_handle, nullptr) != QNN_SUCCESS) { + free_graphs_info(); + SHERPA_ONNX_LOGE("Could not create context from binary."); + return false; + } + + backend_->InitContext(context_handle); + + if (backend_->QnnInterface().graphRetrieve( + context_handle, (*graphs_info)[0].graph_name, + &((*graphs_info)[0].graph)) != QNN_SUCCESS) { + free_graphs_info(); + SHERPA_ONNX_LOGE("Unable to retrieve graph handle for graph %d", 0); + return false; + } + + graph_handle_ = (*graphs_info)[0].graph; + + InitInputTensors((*graphs_info)[0]); + InitOutputTensors((*graphs_info)[0]); + + free_graphs_info(); + + return true; + } + + ~Impl() = default; + + bool SaveBinaryContext(const std::string &filename) { + auto qnn_interface = backend_->QnnInterface(); + + if (!qnn_interface.contextGetBinarySize || + !qnn_interface.contextGetBinary) { + SHERPA_ONNX_LOGE( + "contextGetBinarySizeFnHandle or " + "contextGetBinaryFnHandle is nullptr."); + return false; + } + + uint64_t required_buffer_size{0}; + auto ret = qnn_interface.contextGetBinarySize(backend_->ContextHandle(), + &required_buffer_size); + SHERPA_ONNX_QNN_CHECK(ret, "Failed to call contextGetBinarySize"); + + if (debug_) { + SHERPA_ONNX_LOGE("context binary size: %.3f MB", + static_cast(required_buffer_size) / 1024 / 1024); + } + std::vector saveBuffer(required_buffer_size); + uint64_t writtenBufferSize{0}; + + ret = qnn_interface.contextGetBinary( + backend_->ContextHandle(), reinterpret_cast(saveBuffer.data()), + required_buffer_size, &writtenBufferSize); + + SHERPA_ONNX_QNN_CHECK(ret, "Failed to call contextGetBinary"); + + if (required_buffer_size < writtenBufferSize) { + SHERPA_ONNX_LOGE( + "Illegal written buffer size %d bytes. Cannot exceed " + "allocated memory of %d bytes", + static_cast(writtenBufferSize), + static_cast(required_buffer_size)); + return false; + } + std::ofstream ofs(filename, std::ios::binary | std::ios::trunc); + ofs.write(reinterpret_cast(saveBuffer.data()), + saveBuffer.size()); + + if (!ofs) { + SHERPA_ONNX_LOGE("Failed to write '%s'", filename.c_str()); + return false; + } + + return true; + } + + const std::vector &InputTensorNames() const { + return input_tensor_names_; + } + + const std::vector &OutputTensorNames() const { + return output_tensor_names_; + } + + std::vector TensorShape(const std::string &name) const { + std::vector shape; + + if (!HasTensor(name)) { + SHERPA_ONNX_LOGE("No such tensor '%s'", name.c_str()); + return shape; + } + + auto t = name2tensor_.at(name); + + shape = {t->v1.dimensions, t->v1.dimensions + t->v1.rank}; + + return shape; + } + + int32_t TensorSizeInBytes(const std::string &name) const { + if (!HasTensor(name)) { + return 0; + } + + return name2tensor_.at(name)->v1.clientBuf.dataSize; + } + + bool HasTensor(const std::string &name) const { + return name2tensor_.count(name); + } + + bool SetInputTensorData(const std::string &name, const float *p, int32_t n) { + if (!HasTensor(name)) { + SHERPA_ONNX_LOGE("No such tensor '%s'", name.c_str()); + return false; + } + + auto t = name2tensor_.at(name); + if (t->v1.dataType != QNN_DATATYPE_UFIXED_POINT_16) { + SHERPA_ONNX_LOGE( + "tensor '%s' should be of type " + "QNN_DATATYPE_UFIXED_POINT_16, but it is %s", + name.c_str(), TensorDataTypeToString(t->v1.dataType).c_str()); + return false; + } + + if (t->v1.quantizeParams.quantizationEncoding != + QNN_QUANTIZATION_ENCODING_SCALE_OFFSET) { + SHERPA_ONNX_LOGE( + "tensor '%s' should be quantized with " + "QNN_QUANTIZATION_ENCODING_SCALE_OFFSET, but it is %s", + name.c_str(), + QuantizationEncodingToString( + t->v1.quantizeParams.quantizationEncoding) + .c_str()); + return false; + } + + if (n * sizeof(uint16_t) != t->v1.clientBuf.dataSize) { + SHERPA_ONNX_LOGE("tensor '%s' expects %d bytes, but you provide %d bytes", + name.c_str(), + static_cast(t->v1.clientBuf.dataSize), + static_cast(n * sizeof(uint16_t))); + return false; + } + + FillData(t, p, n); + SHERPA_ONNX_LOGE("set %s", name.c_str()); + + return true; + } + + bool SetInputTensorData(const std::string &name, const int32_t *p, + int32_t n) { + if (!HasTensor(name)) { + SHERPA_ONNX_LOGE("No such tensor '%s'", name.c_str()); + return false; + } + + auto t = name2tensor_.at(name); + if (t->v1.dataType != QNN_DATATYPE_INT_32) { + SHERPA_ONNX_LOGE( + "tensor '%s' should be of type " + "QNN_DATATYPE_INT_32, but it is %s", + name.c_str(), TensorDataTypeToString(t->v1.dataType).c_str()); + return false; + } + + if (n * sizeof(int32_t) != t->v1.clientBuf.dataSize) { + SHERPA_ONNX_LOGE("tensor '%s' expects %d bytes, but you provide %d bytes", + name.c_str(), + static_cast(t->v1.clientBuf.dataSize), + static_cast(n * sizeof(int32_t))); + return false; + } + + FillData(t, p, n); + SHERPA_ONNX_LOGE("set %s", name.c_str()); + + return true; + } + + std::vector GetOutputTensorData(const std::string &name) { + if (!HasTensor(name)) { + SHERPA_ONNX_LOGE("No such tensor '%s'", name.c_str()); + return {}; + } + + auto t = name2tensor_.at(name); + if (t->v1.dataType != QNN_DATATYPE_UFIXED_POINT_16) { + SHERPA_ONNX_LOGE( + "tensor '%s' should be of type " + "QNN_DATATYPE_UFIXED_POINT_16, but it is %s", + name.c_str(), TensorDataTypeToString(t->v1.dataType).c_str()); + return {}; + } + + if (t->v1.quantizeParams.quantizationEncoding != + QNN_QUANTIZATION_ENCODING_SCALE_OFFSET) { + SHERPA_ONNX_LOGE( + "tensor '%s' should be quantized with " + "QNN_QUANTIZATION_ENCODING_SCALE_OFFSET, but it is %s", + name.c_str(), + QuantizationEncodingToString( + t->v1.quantizeParams.quantizationEncoding) + .c_str()); + return {}; + } + + int32_t n = t->v1.clientBuf.dataSize / sizeof(uint16_t); + std::vector ans(n); + + GetData(t, ans.data(), n); + + return ans; + } + + bool Run() { + std::vector input_tensors_raw; + std::vector output_tensors_raw; + + input_tensors_raw.reserve(input_tensors_.size()); + output_tensors_raw.reserve(output_tensors_.size()); + + for (const auto &p : input_tensors_) { + input_tensors_raw.push_back(*p); + } + + for (const auto &p : output_tensors_) { + output_tensors_raw.push_back(*p); + } + + auto ret = backend_->QnnInterface().graphExecute( + graph_handle_, input_tensors_raw.data(), input_tensors_raw.size(), + output_tensors_raw.data(), output_tensors_raw.size(), nullptr, nullptr); + SHERPA_ONNX_QNN_CHECK(ret, "Failed to run graphExecute"); + + return true; + } + + bool IsInitialized() const { return is_initialized_; } + + private: + void PostInit() { + AllocateBuffer(); + SetupPointers(); + + is_initialized_ = true; + } + + bool InitModel(const std::string &model_so) { + model_lib_handle_ = std::unique_ptr( + dlopen(model_so.c_str(), RTLD_NOW | RTLD_LOCAL), &dlclose); + if (!model_lib_handle_) { + SHERPA_ONNX_LOGE("Failed to dlopen '%s'. Error is: '%s'", + model_so.c_str(), dlerror()); + return false; + } + SHERPA_ONNX_LOGE("loaded %s", model_so.c_str()); + + return true; + } + + bool InitSymbols() { + const char *symbol = "QnnModel_composeGraphs"; + + compose_graphs_fn_handle_ = reinterpret_cast( + dlsym(model_lib_handle_.get(), symbol)); + if (!compose_graphs_fn_handle_) { + SHERPA_ONNX_LOGE("Failed to dlsym for '%s'. Error is: '%s'", symbol, + dlerror()); + return false; + } + + symbol = "QnnModel_freeGraphsInfo"; + free_graph_info_fn_handle_ = reinterpret_cast( + dlsym(model_lib_handle_.get(), symbol)); + if (!free_graph_info_fn_handle_) { + SHERPA_ONNX_LOGE("Failed to dlsym for '%s'. Error is: '%s'", symbol, + dlerror()); + return false; + } + return true; + } + + void InitGraph() { + const GraphConfigInfo **graph_configs_info = nullptr; + + uint32_t graph_configs_info_count = 0; + GraphInfo **graphs_info = nullptr; + uint32_t graphs_count = 0; + + auto ret = compose_graphs_fn_handle_( + backend_->BackendHandle(), backend_->QnnInterface(), + backend_->ContextHandle(), graph_configs_info, graph_configs_info_count, + &graphs_info, &graphs_count, debug_, LogCallback, backend_->LogLevel()); + SHERPA_ONNX_QNN_CHECK(ret, "Failed to call compose_graphs_fn_handle_"); + + SHERPA_ONNX_LOGE("graphs_count: %d", (int32_t)graphs_count); + + for (uint32_t i = 0; i < graphs_count; ++i) { + if (debug_) { + SHERPA_ONNX_LOGE( + "Finalizing graph %d/%d: '%s'", static_cast(i), + static_cast(graphs_count), (*graphs_info)[i].graph_name); + } + ret = backend_->QnnInterface().graphFinalize((*graphs_info)[i].graph, + nullptr, nullptr); + SHERPA_ONNX_QNN_CHECK(ret, "Failed to call graph_finalize"); + } + + if (graphs_count > 1) { + SHERPA_ONNX_LOGE("We only use the first graph: %s", + (*graphs_info)[0].graph_name); + } + + InitInputTensors((*graphs_info)[0]); + InitOutputTensors((*graphs_info)[0]); + + graph_handle_ = (*graphs_info)[0].graph; + } + + void InitInputTensors(GraphInfo graph) { + input_tensors_.reserve(graph.num_input_tensors); + input_tensor_names_.reserve(graph.num_input_tensors); + + for (uint32_t i = 0; i < graph.num_input_tensors; ++i) { + SHERPA_ONNX_LOGE("input %d", (int)i); + auto p = TensorPtr(new Qnn_Tensor_t(QNN_TENSOR_INIT), &FreeTensor); + + CopyTensorInfo(graph.input_tensors[i], *p); + PrintTensor(p->v2); + + std::string name = p->v1.name; + name2tensor_[name] = p.get(); + input_tensor_names_.push_back(std::move(name)); + + input_tensors_.push_back(std::move(p)); + } + } + + void InitOutputTensors(GraphInfo graph) { + output_tensors_.reserve(graph.num_output_tensors); + output_tensor_names_.reserve(graph.num_output_tensors); + for (uint32_t i = 0; i < graph.num_output_tensors; ++i) { + auto p = TensorPtr(new Qnn_Tensor_t(QNN_TENSOR_INIT), &FreeTensor); + + CopyTensorInfo(graph.output_tensors[i], *p); + + SHERPA_ONNX_LOGE("output %d", (int)i); + PrintTensor(p->v2); + + std::string name = p->v1.name; + name2tensor_[name] = p.get(); + output_tensor_names_.push_back(std::move(name)); + + output_tensors_.push_back(std::move(p)); + } + } + + void AllocateBuffer() { + uint32_t n = 0; + for (const auto &p : name2tensor_) { + n += p.second->v1.clientBuf.dataSize; + } + + if (debug_) { + SHERPA_ONNX_LOGE("Allocate %d bytes, or %.3f MB", static_cast(n), + static_cast(n) / 1024 / 1024); + } + + buffer_.resize(n); + } + + void SetupPointers() { + uint8_t *p = buffer_.data(); + uint32_t n = 0; + for (auto &t : input_tensors_) { + t->v1.clientBuf.data = p; + p += t->v1.clientBuf.dataSize; + } + + for (auto &t : output_tensors_) { + t->v1.clientBuf.data = p; + p += t->v1.clientBuf.dataSize; + } + + if (debug_) { + if (p == buffer_.data() + buffer_.size()) { + SHERPA_ONNX_LOGE("Setup pointers successfully."); + } else { + SHERPA_ONNX_LOGE("Bad things happened in setting up pointers."); + } + } + } + + private: + bool debug_ = true; + std::unique_ptr model_lib_handle_{nullptr, + &dlclose}; + + std::unique_ptr system_lib_handle_{nullptr, + &dlclose}; + + QNN_SYSTEM_INTERFACE_VER_TYPE qnn_system_interface_; + + ComposeGraphsFnHandleType compose_graphs_fn_handle_ = nullptr; + FreeGraphInfoFnHandleType free_graph_info_fn_handle_ = nullptr; + + std::vector input_tensors_; + std::vector output_tensors_; + + std::vector input_tensor_names_; + std::vector output_tensor_names_; + + std::unordered_map name2tensor_; + + std::vector buffer_; + const QnnBackend *backend_ = nullptr; + + Qnn_GraphHandle_t graph_handle_ = nullptr; + + const QnnContext_Config_t **context_config_ = nullptr; + bool is_initialized_ = false; +}; + +QnnModel::~QnnModel() = default; + +QnnModel::QnnModel(const std::string &model_so, const QnnBackend *backend) + : impl_(std::make_unique(model_so, backend)) {} + +QnnModel::QnnModel(const std::string &binary_context_file, + const std::string &system_lib, const QnnBackend *backend, + BinaryContextTag tag) + : impl_(std::make_unique(binary_context_file, system_lib, backend, + tag)) {} + +bool QnnModel::SaveBinaryContext(const std::string &filename) const { + return impl_->SaveBinaryContext(filename); +} + +const std::vector &QnnModel::InputTensorNames() const { + return impl_->InputTensorNames(); +} + +const std::vector &QnnModel::OutputTensorNames() const { + return impl_->OutputTensorNames(); +} + +std::vector QnnModel::TensorShape(const std::string &name) const { + return impl_->TensorShape(name); +} + +int32_t QnnModel::TensorSizeInBytes(const std::string &name) const { + return impl_->TensorSizeInBytes(name); +} + +bool QnnModel::HasTensor(const std::string &name) const { + return impl_->HasTensor(name); +} + +bool QnnModel::SetInputTensorData(const std::string &name, const float *p, + int32_t n) const { + return impl_->SetInputTensorData(name, p, n); +} + +bool QnnModel::SetInputTensorData(const std::string &name, const int32_t *p, + int32_t n) const { + return impl_->SetInputTensorData(name, p, n); +} + +std::vector QnnModel::GetOutputTensorData( + const std::string &name) const { + return impl_->GetOutputTensorData(name); +} + +bool QnnModel::Run() const { return impl_->Run(); } + +bool QnnModel::IsInitialized() const { return impl_->IsInitialized(); } + +} // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/qnn/qnn-model.h b/sherpa-onnx/csrc/qnn/qnn-model.h new file mode 100644 index 0000000000..d4cbfa3dfe --- /dev/null +++ b/sherpa-onnx/csrc/qnn/qnn-model.h @@ -0,0 +1,55 @@ +// sherpa-onnx/csrc/qnn/qnn-model.h +// +// Copyright (c) 2025 Xiaomi Corporation +#ifndef SHERPA_ONNX_CSRC_QNN_QNN_MODEL_H_ +#define SHERPA_ONNX_CSRC_QNN_QNN_MODEL_H_ + +#include +#include +#include + +#include "QnnInterface.h" + +namespace sherpa_onnx { + +class QnnBackend; + +struct BinaryContextTag {}; + +class QnnModel { + public: + QnnModel(const std::string &model_so, const QnnBackend *backend); + QnnModel(const std::string &binary_context_file, + const std::string &system_lib, const QnnBackend *backend, + BinaryContextTag tag); + ~QnnModel(); + + bool SaveBinaryContext(const std::string &filename) const; + + const std::vector &InputTensorNames() const; + const std::vector &OutputTensorNames() const; + + std::vector TensorShape(const std::string &name) const; + int32_t TensorSizeInBytes(const std::string &name) const; + + bool HasTensor(const std::string &name) const; + + bool SetInputTensorData(const std::string &name, const float *p, + int32_t n) const; + + bool SetInputTensorData(const std::string &name, const int32_t *p, + int32_t n) const; + + std::vector GetOutputTensorData(const std::string &name) const; + + bool Run() const; + bool IsInitialized() const; + + private: + class Impl; + std::unique_ptr impl_; +}; + +} // namespace sherpa_onnx + +#endif // SHERPA_ONNX_CSRC_QNN_QNN_MODEL_H_ diff --git a/sherpa-onnx/csrc/qnn/utils.cc b/sherpa-onnx/csrc/qnn/utils.cc new file mode 100644 index 0000000000..20c498f718 --- /dev/null +++ b/sherpa-onnx/csrc/qnn/utils.cc @@ -0,0 +1,546 @@ +// sherpa-onnx/csrc/qnn/utils.h +// +// Copyright (c) 2025 Xiaomi Corporation +#include "sherpa-onnx/csrc/qnn/utils.h" + +#include +#include + +#include +#include +#include +#include +#include + +#include "sherpa-onnx/csrc/qnn/macros.h" + +#define SHERPA_ONNX_TO_STRING(s) \ + case s: \ + return #s + +std::string TensorTypeToString(Qnn_TensorType_t t) { + switch (t) { + SHERPA_ONNX_TO_STRING(QNN_TENSOR_TYPE_APP_WRITE); + SHERPA_ONNX_TO_STRING(QNN_TENSOR_TYPE_APP_READ); + SHERPA_ONNX_TO_STRING(QNN_TENSOR_TYPE_APP_READWRITE); + SHERPA_ONNX_TO_STRING(QNN_TENSOR_TYPE_NATIVE); + SHERPA_ONNX_TO_STRING(QNN_TENSOR_TYPE_STATIC); + SHERPA_ONNX_TO_STRING(QNN_TENSOR_TYPE_NULL); + SHERPA_ONNX_TO_STRING(QNN_TENSOR_TYPE_UPDATEABLE_STATIC); + SHERPA_ONNX_TO_STRING(QNN_TENSOR_TYPE_UPDATEABLE_NATIVE); + SHERPA_ONNX_TO_STRING(QNN_TENSOR_TYPE_UPDATEABLE_APP_WRITE); + SHERPA_ONNX_TO_STRING(QNN_TENSOR_TYPE_UPDATEABLE_APP_READ); + SHERPA_ONNX_TO_STRING(QNN_TENSOR_TYPE_UPDATEABLE_APP_READWRITE); + SHERPA_ONNX_TO_STRING(QNN_TENSOR_TYPE_OPTIONAL_APP_WRITE); + SHERPA_ONNX_TO_STRING(QNN_TENSOR_TYPE_OPTIONAL_APP_READ); + SHERPA_ONNX_TO_STRING(QNN_TENSOR_TYPE_OPTIONAL_APP_READWRITE); + SHERPA_ONNX_TO_STRING(QNN_TENSOR_TYPE_UNDEFINED); + } + return "Unknown"; +} + +std::string QuantizationEncodingToString(Qnn_QuantizationEncoding_t q) { + switch (q) { + SHERPA_ONNX_TO_STRING(QNN_QUANTIZATION_ENCODING_SCALE_OFFSET); + SHERPA_ONNX_TO_STRING(QNN_QUANTIZATION_ENCODING_AXIS_SCALE_OFFSET); + SHERPA_ONNX_TO_STRING(QNN_QUANTIZATION_ENCODING_BW_SCALE_OFFSET); + SHERPA_ONNX_TO_STRING(QNN_QUANTIZATION_ENCODING_BW_AXIS_SCALE_OFFSET); + SHERPA_ONNX_TO_STRING(QNN_QUANTIZATION_ENCODING_BLOCK); + SHERPA_ONNX_TO_STRING(QNN_QUANTIZATION_ENCODING_BLOCKWISE_EXPANSION); + SHERPA_ONNX_TO_STRING(QNN_QUANTIZATION_ENCODING_VECTOR); + SHERPA_ONNX_TO_STRING(QNN_QUANTIZATION_ENCODING_UNDEFINED); + } + return "Unknown"; +} + +std::string TensorDataTypeToString(Qnn_DataType_t t) { + switch (t) { + SHERPA_ONNX_TO_STRING(QNN_DATATYPE_INT_8); + SHERPA_ONNX_TO_STRING(QNN_DATATYPE_INT_16); + SHERPA_ONNX_TO_STRING(QNN_DATATYPE_INT_32); + SHERPA_ONNX_TO_STRING(QNN_DATATYPE_INT_64); + SHERPA_ONNX_TO_STRING(QNN_DATATYPE_UINT_8); + SHERPA_ONNX_TO_STRING(QNN_DATATYPE_UINT_16); + SHERPA_ONNX_TO_STRING(QNN_DATATYPE_UINT_32); + SHERPA_ONNX_TO_STRING(QNN_DATATYPE_UINT_64); + SHERPA_ONNX_TO_STRING(QNN_DATATYPE_FLOAT_16); + SHERPA_ONNX_TO_STRING(QNN_DATATYPE_FLOAT_32); + SHERPA_ONNX_TO_STRING(QNN_DATATYPE_FLOAT_64); + SHERPA_ONNX_TO_STRING(QNN_DATATYPE_SFIXED_POINT_4); + SHERPA_ONNX_TO_STRING(QNN_DATATYPE_SFIXED_POINT_8); + SHERPA_ONNX_TO_STRING(QNN_DATATYPE_SFIXED_POINT_16); + SHERPA_ONNX_TO_STRING(QNN_DATATYPE_SFIXED_POINT_32); + SHERPA_ONNX_TO_STRING(QNN_DATATYPE_UFIXED_POINT_4); + SHERPA_ONNX_TO_STRING(QNN_DATATYPE_UFIXED_POINT_8); + SHERPA_ONNX_TO_STRING(QNN_DATATYPE_UFIXED_POINT_16); + SHERPA_ONNX_TO_STRING(QNN_DATATYPE_UFIXED_POINT_32); + SHERPA_ONNX_TO_STRING(QNN_DATATYPE_BOOL_8); + SHERPA_ONNX_TO_STRING(QNN_DATATYPE_STRING); + SHERPA_ONNX_TO_STRING(QNN_DATATYPE_UNDEFINED); + } + return "unknown"; +} + +std::string TensorMemTypeToString(Qnn_TensorMemType_t t) { + switch (t) { + SHERPA_ONNX_TO_STRING(QNN_TENSORMEMTYPE_RAW); + SHERPA_ONNX_TO_STRING(QNN_TENSORMEMTYPE_MEMHANDLE); + SHERPA_ONNX_TO_STRING(QNN_TENSORMEMTYPE_RETRIEVE_RAW); + SHERPA_ONNX_TO_STRING(QNN_TENSORMEMTYPE_UNDEFINED); + } + return "Unknown"; +} + +#undef SHERPA_ONNX_TO_STRING + +// quantized = float / scale - offset; +void FillData(Qnn_Tensor_t *t, const float *data, int32_t n) { + float scale = t->v1.quantizeParams.scaleOffsetEncoding.scale; + int32_t offset = t->v1.quantizeParams.scaleOffsetEncoding.offset; + + size_t bit_width = 16; + double true_bit_width_max = pow(2, bit_width) - 1; + double encoding_min = offset * scale; + double encoding_max = (true_bit_width_max + offset) * scale; + double encoding_range = encoding_max - encoding_min; + + uint16_t *out = reinterpret_cast(t->v1.clientBuf.data); + + for (size_t i = 0; i < n; ++i) { + int32_t quantized_value = + round(true_bit_width_max * (data[i] - encoding_min) / encoding_range); + + if (quantized_value < 0) { + quantized_value = 0; + } else if (quantized_value > static_cast(true_bit_width_max)) { + quantized_value = static_cast(true_bit_width_max); + } + out[i] = static_cast(quantized_value); + } +} + +void FillData(Qnn_Tensor_t *t, const int32_t *data, int32_t n) { + int32_t *out = reinterpret_cast(t->v1.clientBuf.data); + std::copy(data, data + n, out); +} + +void GetData(const Qnn_Tensor_t *t, float *data, int32_t n) { + double scale = t->v1.quantizeParams.scaleOffsetEncoding.scale; + double offset = t->v1.quantizeParams.scaleOffsetEncoding.offset; + + const uint16_t *p = reinterpret_cast(t->v1.clientBuf.data); + for (int32_t i = 0; i < n; ++i) { + double quantizedValue = static_cast(p[i]); + data[i] = (quantizedValue + offset) * scale; + } +} + +static void FreeTensorV1(Qnn_Tensor_t *t) { + free(const_cast(t->v1.name)); + + delete[] t->v1.dimensions; +} + +static void FreeTensorV2(Qnn_Tensor_t *t) { + free(const_cast(t->v2.name)); + + delete[] t->v2.dimensions; + delete[] t->v2.isDynamicDimensions; +} + +void FreeTensor(Qnn_Tensor_t *t) { + if (t->version == QNN_TENSOR_VERSION_1) { + FreeTensorV1(t); + } else if (t->version == QNN_TENSOR_VERSION_2) { + FreeTensorV2(t); + } else { + SHERPA_ONNX_LOGE("Unknown tensor version: %d", t->version); + } +} + +uint32_t GetSizeInBytes(const uint32_t *dimensions, uint32_t n, + Qnn_DataType_t type) { + if (n == 0) { + return 0; + } + + auto count = std::accumulate(dimensions, dimensions + n, 1, + std::multiplies()); + + uint32_t b = 1; + switch (type) { + case QNN_DATATYPE_INT_8: + b = 1; + break; + case QNN_DATATYPE_INT_16: + b = 2; + break; + case QNN_DATATYPE_INT_32: + b = 4; + break; + case QNN_DATATYPE_INT_64: + b = 8; + break; + case QNN_DATATYPE_UINT_8: + b = 1; + break; + case QNN_DATATYPE_UINT_16: + b = 2; + break; + case QNN_DATATYPE_UINT_32: + b = 4; + break; + case QNN_DATATYPE_UINT_64: + b = 8; + break; + case QNN_DATATYPE_FLOAT_16: + b = 2; + break; + case QNN_DATATYPE_FLOAT_32: + b = 4; + break; + case QNN_DATATYPE_FLOAT_64: + b = 8; + break; + case QNN_DATATYPE_SFIXED_POINT_8: + b = 1; + break; + case QNN_DATATYPE_SFIXED_POINT_16: + b = 2; + break; + case QNN_DATATYPE_SFIXED_POINT_32: + b = 4; + break; + case QNN_DATATYPE_UFIXED_POINT_8: + b = 1; + break; + case QNN_DATATYPE_UFIXED_POINT_16: + b = 2; + break; + case QNN_DATATYPE_UFIXED_POINT_32: + b = 4; + break; + case QNN_DATATYPE_BOOL_8: + b = 1; + break; + default: + SHERPA_ONNX_LOGE("Unsupported data type: %s", + TensorDataTypeToString(type).c_str()); + break; + } + + return count * b; +} + +template +void CopyDimensions(const T *src, uint32_t n, T **dst) { + if (!src || n == 0) { + *dst = nullptr; + return; + } + + *dst = new T[n]; + std::copy(src, src + n, *dst); +} + +static void CopyQuantizeParams(const Qnn_QuantizeParams_t &src, + Qnn_QuantizeParams_t &dst) { // NOLINT + dst.encodingDefinition = src.encodingDefinition; + dst.quantizationEncoding = src.quantizationEncoding; + + switch (src.quantizationEncoding) { + case QNN_QUANTIZATION_ENCODING_SCALE_OFFSET: + dst.scaleOffsetEncoding = src.scaleOffsetEncoding; + break; + case QNN_QUANTIZATION_ENCODING_UNDEFINED: + // do nothing in this case + break; + default: + SHERPA_ONNX_LOGE( + "Unsupported quantizationEncoding: %s", + QuantizationEncodingToString(src.quantizationEncoding).c_str()); + } +} + +static void CopyTensorInfoV1(const Qnn_Tensor_t &src, + Qnn_Tensor_t &dst) { // NOLINT + dst.version = src.version; + dst.v1.id = src.v1.id; + if (src.v1.name) { + dst.v1.name = strdup(src.v1.name); + } else { + dst.v1.name = strdup(""); + } + + dst.v1.type = src.v1.type; + dst.v1.dataFormat = src.v1.dataFormat; + dst.v1.dataType = src.v1.dataType; + + CopyQuantizeParams(src.v1.quantizeParams, dst.v1.quantizeParams); + + dst.v1.rank = src.v1.rank; + + CopyDimensions(src.v1.dimensions, src.v1.rank, &dst.v1.dimensions); + + dst.v1.memType = src.v1.memType; + if (dst.v1.memType != QNN_TENSORMEMTYPE_RAW) { + SHERPA_ONNX_LOGE("Unsupported mem type: %s", + TensorMemTypeToString(dst.v1.memType).c_str()); + } else { + dst.v1.clientBuf.data = nullptr; + dst.v1.clientBuf.dataSize = + GetSizeInBytes(dst.v1.dimensions, dst.v1.rank, dst.v1.dataType); + } +} + +static void CopyTensorInfoV2(const Qnn_Tensor_t &src, + Qnn_Tensor_t &dst) { // NOLINT + dst.version = src.version; + dst.v2.id = src.v2.id; + if (src.v2.name) { + dst.v2.name = strdup(src.v2.name); + } else { + dst.v2.name = strdup(""); + } + + dst.v2.type = src.v2.type; + dst.v2.dataFormat = src.v2.dataFormat; + dst.v2.dataType = src.v2.dataType; + + CopyQuantizeParams(src.v2.quantizeParams, dst.v2.quantizeParams); + + dst.v2.rank = src.v2.rank; + + CopyDimensions(src.v2.dimensions, src.v2.rank, &dst.v2.dimensions); + + dst.v2.memType = src.v2.memType; + if (dst.v2.memType != QNN_TENSORMEMTYPE_RAW) { + SHERPA_ONNX_LOGE("Unsupported mem type: %s", + TensorMemTypeToString(dst.v2.memType).c_str()); + } else { + dst.v2.clientBuf.data = nullptr; + dst.v2.clientBuf.dataSize = + GetSizeInBytes(dst.v2.dimensions, dst.v2.rank, dst.v2.dataType); + } + + CopyDimensions(src.v2.isDynamicDimensions, src.v2.rank, + &dst.v2.isDynamicDimensions); + + dst.v2.sparseParams.type = src.v2.sparseParams.type; + dst.v2.sparseParams.hybridCoo.numSpecifiedElements = + src.v2.sparseParams.hybridCoo.numSpecifiedElements; + dst.v2.sparseParams.hybridCoo.numSparseDimensions = + src.v2.sparseParams.hybridCoo.numSparseDimensions; + dst.v2.isProduced = src.v2.isProduced; +} + +void CopyTensorInfo(const Qnn_Tensor_t &src, Qnn_Tensor_t &dst) { // NOLINT + if (src.version == QNN_TENSOR_VERSION_1) { + CopyTensorInfoV1(src, dst); + } else if (src.version == QNN_TENSOR_VERSION_2) { + CopyTensorInfoV2(src, dst); + } else { + SHERPA_ONNX_LOGE("Unknown tensor version: %d", dst.version); + } +} + +void LogCallback(const char *fmt, QnnLog_Level_t level, uint64_t timestamp, + va_list args) { + std::string s; + switch (level) { + case QNN_LOG_LEVEL_ERROR: + s = "ERROR"; + break; + case QNN_LOG_LEVEL_WARN: + s = "WARN"; + break; + case QNN_LOG_LEVEL_INFO: + s = "INFO"; + break; + case QNN_LOG_LEVEL_DEBUG: + s = "DEBUG"; + break; + case QNN_LOG_LEVEL_VERBOSE: + s = "VERBOSE"; + break; + case QNN_LOG_LEVEL_MAX: + s = "UNKNOWN"; + break; + } + + double ms = timestamp / 1000000.0; + fprintf(stdout, "%8.1fms [%-7s] ", ms, s.c_str()); + vfprintf(stdout, fmt, args); +} + +void PrintTensor(Qnn_TensorV2_t t) { + std::ostringstream os; + os << " id: " << t.id << "\n"; + os << " name: " << t.name << "\n"; + os << " type: " << TensorTypeToString(t.type) << "\n"; + os << " data format: " << t.dataFormat << "\n"; + os << " data type: " << TensorDataTypeToString(t.dataType) << "\n"; + os << " quantize info: \n"; + auto qp = t.quantizeParams; + os << " encodingDefinition: " << std::hex << "0x" << qp.encodingDefinition + << std::dec << "\n"; + os << " quantizationEncoding: " + << QuantizationEncodingToString(qp.quantizationEncoding) << "\n"; + if (qp.quantizationEncoding == QNN_QUANTIZATION_ENCODING_SCALE_OFFSET) { + Qnn_ScaleOffset_t s = qp.scaleOffsetEncoding; + os << " scale: " << s.scale << "\n"; + os << " offset: " << s.offset << "\n"; + } + os << " rank: " << t.rank << "\n"; + os << " dimensions: "; + for (int32_t i = 0; i < t.rank; ++i) { + os << t.dimensions[i] << ", "; + if (i + 1 == t.rank) { + os << "\n"; + } + } + os << " memType: " << TensorMemTypeToString(t.memType) << "\n"; + if (t.memType == QNN_TENSORMEMTYPE_RAW) { + os << " memType raw data size: " << t.clientBuf.dataSize << "\n"; + } + os << " isDynamicDimensions: " + << ((t.isDynamicDimensions != nullptr) ? "True" : "False") << "\n"; + os << " isProduced: " << static_cast(t.isProduced) << "\n"; + + SHERPA_ONNX_LOGE("%s", os.str().c_str()); +} + +static bool CopyGraphsInfoV3(const QnnSystemContext_GraphInfoV3_t *src, + GraphInfo *dst) { + if (src->graphName) { + dst->graph_name = strdup(src->graphName); + } else { + dst->graph_name = strdup(""); + } + + dst->input_tensors = nullptr; + dst->num_input_tensors = 0; + + if (src->graphInputs) { + dst->input_tensors = reinterpret_cast( + calloc(src->numGraphInputs, sizeof(Qnn_Tensor_t))); + + for (uint32_t i = 0; i < src->numGraphInputs; ++i) { + dst->input_tensors[i] = QNN_TENSOR_INIT; + + CopyTensorInfo(src->graphInputs[i], dst->input_tensors[i]); + } + + dst->num_input_tensors = src->numGraphInputs; + } + + dst->output_tensors = nullptr; + dst->num_output_tensors = 0; + + if (src->graphOutputs) { + dst->output_tensors = reinterpret_cast( + calloc(src->numGraphOutputs, sizeof(Qnn_Tensor_t))); + + for (uint32_t i = 0; i < src->numGraphOutputs; ++i) { + dst->output_tensors[i] = QNN_TENSOR_INIT; + + CopyTensorInfo(src->graphOutputs[i], dst->output_tensors[i]); + } + + dst->num_output_tensors = src->numGraphOutputs; + } + + return true; +} + +static bool CopyGraphsInfo(const QnnSystemContext_GraphInfo_t *graphs_input, + uint32_t num_graphs, + GraphInfo **&graphs_info) { // NOLINT + if (num_graphs == 0) { + SHERPA_ONNX_LOGE("empty graphs"); + graphs_info = nullptr; + return false; + } + + SHERPA_ONNX_LOGE("version: %d", (int)graphs_input[0].version); + + // remember to free graphs_info + graphs_info = + reinterpret_cast(calloc(num_graphs, sizeof(GraphInfo *))); + + GraphInfo *graph_info_arr = + reinterpret_cast(calloc(num_graphs, sizeof(GraphInfo))); + + if (!graphs_info || !graph_info_arr) { + SHERPA_ONNX_LOGE("Failure to allocate memory for *graphInfo"); + return false; + } + + for (uint32_t i = 0; i < num_graphs; ++i) { + switch (graphs_input[i].version) { + case QNN_SYSTEM_CONTEXT_GRAPH_INFO_VERSION_1: + SHERPA_ONNX_LOGE("Unsupported version: %d", + static_cast(graphs_input[i].version)); + return false; + + case QNN_SYSTEM_CONTEXT_GRAPH_INFO_VERSION_2: + SHERPA_ONNX_LOGE("Unsupported version: %d", + static_cast(graphs_input[i].version)); + return false; + + case QNN_SYSTEM_CONTEXT_GRAPH_INFO_VERSION_3: { + bool ok = + CopyGraphsInfoV3(&graphs_input[i].graphInfoV3, &graph_info_arr[i]); + if (!ok) { + SHERPA_ONNX_LOGE("Failed to copy graphs info v3"); + } + graphs_info[i] = graph_info_arr + i; + + break; + } + + default: + SHERPA_ONNX_LOGE("Unsupported version: %d", + static_cast(graphs_input[i].version)); + return false; + } + } + + return true; +} + +bool CopyMetadataToGraphsInfo(const QnnSystemContext_BinaryInfo_t *binary_info, + GraphInfo **&graphs_info, // NOLINT + uint32_t &graphs_count) { // NOLINT + graphs_count = 0; + + switch (binary_info->version) { + case QNN_SYSTEM_CONTEXT_BINARY_INFO_VERSION_1: { + SHERPA_ONNX_LOGE("Unsupported binary context version: %d", + binary_info->version); + return false; + } + case QNN_SYSTEM_CONTEXT_BINARY_INFO_VERSION_2: { + SHERPA_ONNX_LOGE("Unsupported binary context version: %d", + binary_info->version); + return false; + } + case QNN_SYSTEM_CONTEXT_BINARY_INFO_VERSION_3: { + bool ok = CopyGraphsInfo(binary_info->contextBinaryInfoV3.graphs, + binary_info->contextBinaryInfoV3.numGraphs, + graphs_info); + + if (!ok) { + SHERPA_ONNX_LOGE("Failed while copying graphs Info v3."); + return false; + } + graphs_count = binary_info->contextBinaryInfoV3.numGraphs; + return true; + } + default: { + SHERPA_ONNX_LOGE("Unsupported binary context version: %d", + binary_info->version); + return false; + } + } +} diff --git a/sherpa-onnx/csrc/qnn/utils.h b/sherpa-onnx/csrc/qnn/utils.h new file mode 100644 index 0000000000..53663ecd5e --- /dev/null +++ b/sherpa-onnx/csrc/qnn/utils.h @@ -0,0 +1,94 @@ +// sherpa-onnx/csrc/qnn/utils.h +// +// Copyright (c) 2025 Xiaomi Corporation +#ifndef SHERPA_ONNX_CSRC_QNN_UTILS_H_ +#define SHERPA_ONNX_CSRC_QNN_UTILS_H_ +#include + +#include +#include +#include +#include + +#include "QnnInterface.h" +#include "System/QnnSystemInterface.h" +#include "sherpa-onnx/csrc/macros.h" + +template +std::vector ReadFile(const std::string &filename) { + FILE *fp = fopen(filename.c_str(), "rb"); + if (!fp) { + SHERPA_ONNX_LOGE("Failed to open '%s'", filename.c_str()); + return {}; + } + + fseek(fp, 0, SEEK_END); + int32_t n = ftell(fp); + fseek(fp, 0, SEEK_SET); + + std::vector ans(n / sizeof(T)); + fread(ans.data(), sizeof(T), ans.size(), fp); + fclose(fp); + + return ans; +} + +void PrintTensor(Qnn_TensorV2_t t); + +// float -> uint16_t +void FillData(Qnn_Tensor_t *t, const float *data, int32_t n); + +// int32_t -> int32_t +void FillData(Qnn_Tensor_t *t, const int32_t *data, int32_t n); + +// uint16_t -> float +void GetData(const Qnn_Tensor_t *t, float *data, int32_t n); + +void FreeTensor(Qnn_Tensor_t *t); + +using TensorPtr = std::unique_ptr; + +void CopyTensorInfo(const Qnn_Tensor_t &src, Qnn_Tensor_t &dst); // NOLINT + +std::string QuantizationEncodingToString(Qnn_QuantizationEncoding_t q); + +std::string TensorDataTypeToString(Qnn_DataType_t t); + +using QnnInterfaceGetProvidersFnType = Qnn_ErrorHandle_t (*)( + const QnnInterface_t ***provider_list, uint32_t *num_providers); + +using QnnSystemInterfaceGetProvidersFnType = Qnn_ErrorHandle_t (*)( + const QnnSystemInterface_t ***provider_list, uint32_t *num_providers); + +struct GraphInfo { + Qnn_GraphHandle_t graph; + char *graph_name; + Qnn_Tensor_t *input_tensors; + uint32_t num_input_tensors; + Qnn_Tensor_t *output_tensors; + uint32_t num_output_tensors; +}; + +struct GraphConfigInfo { + char *graph_name; + const QnnGraph_Config_t **graph_configs; +}; + +using ComposeGraphsFnHandleType = Qnn_ErrorHandle_t (*)( + Qnn_BackendHandle_t backend_handle, QNN_INTERFACE_VER_TYPE interface, + Qnn_ContextHandle_t context_handle, + const GraphConfigInfo **graphs_config_info, + const uint32_t num_graphs_config_info, GraphInfo ***graphs_info, + uint32_t *num_graphs_info, bool debug, QnnLog_Callback_t logCallback, + QnnLog_Level_t max_log_level); + +using FreeGraphInfoFnHandleType = + Qnn_ErrorHandle_t (*)(GraphInfo ***, uint32_t num_graphs_info); + +void LogCallback(const char *fmt, QnnLog_Level_t level, uint64_t timestamp, + va_list args); + +bool CopyMetadataToGraphsInfo(const QnnSystemContext_BinaryInfo_t *binary_info, + GraphInfo **&graphs_info, // NOLINT + uint32_t &graphs_count); // NOLINT +#endif // SHERPA_ONNX_CSRC_QNN_UTILS_H_