Begin to add qnn C API

CMakeLists.txt

@@ -61,6 +61,7 @@ option(SHERPA_ONNX_ENABLE_SANITIZER "Whether to enable ubsan and asan" OFF)
 option(SHERPA_ONNX_BUILD_C_API_EXAMPLES "Whether to enable C API examples" ${SUGGEST_BUILD_BINARIES})
 option(SHERPA_ONNX_ENABLE_RKNN "Whether to build for RKNN NPU " OFF)
 option(SHERPA_ONNX_ENABLE_ASCEND_NPU "Whether to build for Ascend NPU " OFF)
+option(SHERPA_ONNX_ENABLE_QNN "Whether to build for Qualcomm NPU" OFF)
 
 set(SHERPA_ONNX_LINUX_ARM64_GPU_ONNXRUNTIME_VERSION "1.11.0" CACHE STRING "Used only for Linux ARM64 GPU. Set to 1.11.0 if you use CUDA 10.2 and cudnn8. Set it to 1.16.0 if you use CUDA 11.4 and cudnn8. Set it to 1.18.0 if you use CUDA 12.2 and cudnn8. Set it to 1.18.1 if you use CUDA 12.6 and cudnn9")
 
@@ -179,6 +180,7 @@ message(STATUS "SHERPA_ONNX_ENABLE_SANITIZER: ${SHERPA_ONNX_ENABLE_SANITIZER}")
 message(STATUS "SHERPA_ONNX_BUILD_C_API_EXAMPLES: ${SHERPA_ONNX_BUILD_C_API_EXAMPLES}")
 message(STATUS "SHERPA_ONNX_ENABLE_RKNN: ${SHERPA_ONNX_ENABLE_RKNN}")
 message(STATUS "SHERPA_ONNX_ENABLE_ASCEND_NPU: ${SHERPA_ONNX_ENABLE_ASCEND_NPU}")
+message(STATUS "SHERPA_ONNX_ENABLE_QNN: ${SHERPA_ONNX_ENABLE_QNN}")
 message(STATUS "SHERPA_ONNX_LINK_D3D: ${SHERPA_ONNX_LINK_D3D}")
 
 if(BUILD_SHARED_LIBS OR SHERPA_ONNX_ENABLE_JNI)
@@ -337,6 +339,44 @@ if(SHERPA_ONNX_ENABLE_ASCEND_NPU)
   message(STATUS "Build with Ascend NPU")
 endif()
 
+if(SHERPA_ONNX_ENABLE_QNN)
+  if(NOT DEFINED ENV{QNN_SDK_ROOT})
+      message(FATAL_ERROR "\
+      Please specify the installation directory of the QNN SDK toolkit.
+      For instance, if it is installed in
+
+        /mnt/sdb/open-source/qairt/2.33.0.250327
+
+      You can run
+
+        source /mnt/sdb/open-source/qairt/2.33.0.250327/bin/envsetup.sh
+
+      which will give you the following output
+
+      [INFO] AISW SDK environment set
+      [INFO] QNN_SDK_ROOT: /mnt/sdb/open-source/qairt/2.33.0.250327
+      [INFO] SNPE_ROOT: /mnt/sdb/open-source/qairt/2.33.0.250327
+
+      Then run
+
+        echo $QNN_SDK_ROOT
+
+      It should print:
+
+        /mnt/sdb/open-source/qairt/2.33.0.250327
+
+      You can choose a version of QNN SDK by yourself. You don't need
+      to use 2.33.0.250327
+      ")
+  endif()
+
+  set(QNN_SDK_ROOT $ENV{QNN_SDK_ROOT})
+
+  if(NOT EXISTS ${QNN_SDK_ROOT}/include/QNN/QnnInterface.h)
+    message(FATAL_ERROR "${QNN_SDK_ROOT}/include/QNN/QnnInterface.h does not exist")
+  endif()
+endif()
+
 if(UNIX AND NOT APPLE AND NOT SHERPA_ONNX_ENABLE_WASM AND NOT CMAKE_SYSTEM_NAME STREQUAL Android AND NOT CMAKE_SYSTEM_NAME STREQUAL OHOS)
   check_include_file_cxx(alsa/asoundlib.h SHERPA_ONNX_HAS_ALSA)
   if(SHERPA_ONNX_HAS_ALSA)

build-android-arm64-v8a.sh

@@ -101,6 +101,10 @@ if [ -z $SHERPA_ONNX_ENABLE_RKNN ]; then
   SHERPA_ONNX_ENABLE_RKNN=OFF
 fi
 
+if [ -z $SHERPA_ONNX_ENABLE_QNN ]; then
+  SHERPA_ONNX_ENABLE_QNN=OFF
+fi
+
 if [ $SHERPA_ONNX_ENABLE_RKNN == ON ]; then
   rknn_version=2.2.0
   if [ ! -d ./librknnrt-android ]; then
@@ -130,6 +134,10 @@ if [ -z $SHERPA_ONNX_ENABLE_C_API ]; then
   SHERPA_ONNX_ENABLE_C_API=OFF
 fi
 
+if [ -z $SHERPA_ONNX_ANDROID_PLATFORM ]; then
+  SHERPA_ONNX_ANDROID_PLATFORM=android-21
+fi
+
 if [ -z $SHERPA_ONNX_ENABLE_JNI ]; then
   SHERPA_ONNX_ENABLE_JNI=ON
 fi
@@ -153,8 +161,9 @@ cmake -DCMAKE_TOOLCHAIN_FILE="$ANDROID_NDK/build/cmake/android.toolchain.cmake"
     -DSHERPA_ONNX_ENABLE_C_API=$SHERPA_ONNX_ENABLE_C_API \
     -DCMAKE_INSTALL_PREFIX=./install \
     -DSHERPA_ONNX_ENABLE_RKNN=$SHERPA_ONNX_ENABLE_RKNN \
+    -DSHERPA_ONNX_ENABLE_QNN=$SHERPA_ONNX_ENABLE_QNN \
     -DANDROID_ABI="arm64-v8a" \
-    -DANDROID_PLATFORM=android-21 ..
+    -DANDROID_PLATFORM=$SHERPA_ONNX_ANDROID_PLATFORM ..
 
     # By default, it links to libc++_static.a
     # -DANDROID_STL=c++_shared \

sherpa-onnx/csrc/CMakeLists.txt

@@ -200,6 +200,14 @@ if(SHERPA_ONNX_ENABLE_ASCEND_NPU)
   )
 endif()
 
+if(SHERPA_ONNX_ENABLE_QNN)
+  list(APPEND sources
+    ./qnn/qnn-backend.cc
+    ./qnn/qnn-model.cc
+    ./qnn/utils.cc
+  )
+endif()
+
 if(SHERPA_ONNX_ENABLE_TTS)
   list(APPEND sources
     character-lexicon.cc
@@ -314,6 +322,10 @@ if(SHERPA_ONNX_ENABLE_ASCEND_NPU)
     )
 endif()
 
+if(SHERPA_ONNX_ENABLE_QNN)
+  target_include_directories(sherpa-onnx-core PRIVATE ${QNN_SDK_ROOT}/include/QNN)
+endif()
+
 if(TARGET onnxruntime)
   target_link_libraries(sherpa-onnx-core onnxruntime)
 else()

sherpa-onnx/csrc/qnn/macros.h

@@ -0,0 +1,19 @@
+// sherpa-onnx/csrc/qnn/macros.h
+//
+// Copyright      2025  Xiaomi Corporation
+
+#ifndef SHERPA_ONNX_CSRC_QNN_MACROS_H_
+#define SHERPA_ONNX_CSRC_QNN_MACROS_H_
+
+#include "sherpa-onnx/csrc/macros.h"
+
+#define SHERPA_ONNX_QNN_CHECK(ret, msg, ...)                             \
+  do {                                                                   \
+    if (ret != QNN_SUCCESS) {                                            \
+      SHERPA_ONNX_LOGE("Return code is: %d", static_cast<int32_t>(ret)); \
+      SHERPA_ONNX_LOGE(msg, ##__VA_ARGS__);                              \
+      SHERPA_ONNX_EXIT(-1);                                              \
+    }                                                                    \
+  } while (0)
+
+#endif  // SHERPA_ONNX_CSRC_QNN_MACROS_H_

sherpa-onnx/csrc/qnn/qnn-backend.cc

@@ -0,0 +1,251 @@
+// sherpa-onnx/csrc/qnn/qnn-backend.cc
+//
+// Copyright (c)  2025  Xiaomi Corporation
+
+#include "sherpa-onnx/csrc/qnn/qnn-backend.h"
+
+#include <dlfcn.h>
+#include <stdio.h>
+
+#include <cstdint>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include "QnnInterface.h"
+#include "System/QnnSystemInterface.h"
+#include "sherpa-onnx/csrc/qnn/macros.h"
+#include "sherpa-onnx/csrc/qnn/utils.h"
+
+namespace sherpa_onnx {
+
+class QnnBackend::Impl {
+ public:
+  explicit Impl(const std::string &backend_lib) {
+    bool ok = InitQnnInterface(backend_lib);
+    if (!ok) {
+      SHERPA_ONNX_LOGE("Failed to init qnn interface from '%s'",
+                       backend_lib.c_str());
+      return;
+    }
+
+    InitLog();
+    InitBackend();
+    InitDevice();
+
+    is_initialized_ = true;
+  }
+
+  ~Impl() {
+    if (context_handle_) {
+      auto ret = qnn_interface_.contextFree(context_handle_, nullptr);
+      SHERPA_ONNX_QNN_CHECK(ret, "Failed to call contextFree");
+    }
+
+    if (device_handle_) {
+      auto ret = qnn_interface_.deviceFree(device_handle_);
+      SHERPA_ONNX_QNN_CHECK(ret, "Failed to call deviceFree");
+    }
+
+    if (backend_handle_) {
+      auto ret = qnn_interface_.backendFree(backend_handle_);
+      SHERPA_ONNX_QNN_CHECK(ret, "Failed to call backendFree");
+    }
+
+    if (log_handle_) {
+      auto ret = qnn_interface_.logFree(log_handle_);
+      SHERPA_ONNX_QNN_CHECK(ret, "Failed to call logFree");
+    }
+  }
+
+  void InitContext() {
+    if (context_handle_) {
+      SHERPA_ONNX_LOGE("context handle is already initialized");
+      return;
+    }
+
+    auto ret = qnn_interface_.contextCreate(backend_handle_, device_handle_,
+                                            context_config_, &context_handle_);
+    SHERPA_ONNX_QNN_CHECK(ret, "Failed to call contextCreate");
+  }
+
+  void InitContext(Qnn_ContextHandle_t t) { context_handle_ = t; }
+
+  Qnn_LogHandle_t LogHandle() const { return log_handle_; }
+
+  Qnn_BackendHandle_t BackendHandle() const { return backend_handle_; }
+
+  Qnn_DeviceHandle_t DeviceHandle() const { return device_handle_; }
+
+  Qnn_ContextHandle_t ContextHandle() const { return context_handle_; }
+
+  QNN_INTERFACE_VER_TYPE QnnInterface() const { return qnn_interface_; }
+
+  QnnLog_Level_t LogLevel() const { return log_level_; }
+
+  bool IsInitialized() const { return is_initialized_; }
+
+ private:
+  bool InitQnnInterface(const std::string &backend_lib) {
+    backend_lib_handle_ = std::unique_ptr<void, decltype(&dlclose)>(
+        dlopen(backend_lib.c_str(), RTLD_NOW | RTLD_LOCAL), &dlclose);
+    if (!backend_lib_handle_) {
+      SHERPA_ONNX_LOGE("Failed to dlopen '%s'. Error is: '%s'",
+                       backend_lib.c_str(), dlerror());
+      return false;
+    }
+    SHERPA_ONNX_LOGE("loaded %s", backend_lib.c_str());
+
+    const char *symbol = "QnnInterface_getProviders";
+    auto get_interface_providers =
+        reinterpret_cast<QnnInterfaceGetProvidersFnType>(
+            dlsym(backend_lib_handle_.get(), symbol));
+    if (!get_interface_providers) {
+      SHERPA_ONNX_LOGE("Failed to dlsym for '%s'. Error is: '%s'", symbol,
+                       dlerror());
+      return false;
+    }
+    SHERPA_ONNX_LOGE("Got %s", symbol);
+
+    const QnnInterface_t **interface_providers = nullptr;
+    uint32_t num_providers = 0;
+
+    auto ret = get_interface_providers(&interface_providers, &num_providers);
+    SHERPA_ONNX_QNN_CHECK(ret, "Failed to call get_interface_providers");
+
+    if (!interface_providers) {
+      SHERPA_ONNX_LOGE("interface_providers is nullptr");
+      return false;
+    }
+
+    if (num_providers == 0) {
+      SHERPA_ONNX_LOGE("Number of providers is 0");
+      return false;
+    }
+
+    bool found_valid_interface = false;
+
+    if (debug_) {
+      SHERPA_ONNX_LOGE("QNN_API_VERSION_MAJOR: %d", QNN_API_VERSION_MAJOR);
+      SHERPA_ONNX_LOGE("QNN_API_VERSION_MINOR: %d", QNN_API_VERSION_MINOR);
+      SHERPA_ONNX_LOGE("QNN_API_VERSION_PATCH: %d", QNN_API_VERSION_PATCH);
+    }
+
+    for (size_t idx = 0; idx < num_providers; ++idx) {
+      auto p = interface_providers[idx];
+
+      if (debug_) {
+        std::ostringstream os;
+        os << "---" << idx << "----\n";
+        os << "backendId: " << p->backendId << "\n";
+        os << "coreApiVersion.major: " << p->apiVersion.coreApiVersion.major
+           << "\n";
+        os << "coreApiVersion.minor: " << p->apiVersion.coreApiVersion.minor
+           << "\n";
+        os << "coreApiVersion.patch: " << p->apiVersion.coreApiVersion.patch
+           << "\n";
+
+        os << "backendApiVersion.major: "
+           << p->apiVersion.backendApiVersion.major << "\n";
+        os << "backendApiVersion.minor: "
+           << p->apiVersion.backendApiVersion.minor << "\n";
+        os << "backendApiVersion.patch: "
+           << p->apiVersion.backendApiVersion.patch << "\n";
+        SHERPA_ONNX_LOGE("%s", os.str().c_str());
+      }
+
+      qnn_interface_ = p->QNN_INTERFACE_VER_NAME;
+      found_valid_interface = true;
+      break;
+    }
+
+    if (!found_valid_interface) {
+      SHERPA_ONNX_LOGE("Failed to find valid interface");
+      return false;
+    }
+
+    if (debug_) {
+      const char *build_id = nullptr;
+      ret = qnn_interface_.backendGetBuildId(&build_id);
+      SHERPA_ONNX_QNN_CHECK(ret, "Failed to call backendGetBuildId()");
+
+      SHERPA_ONNX_LOGE("backend build ID: %s", build_id);
+    }
+
+    return true;
+  }
+
+  void InitLog() {
+    auto ret = qnn_interface_.logCreate(LogCallback, log_level_, &log_handle_);
+    SHERPA_ONNX_QNN_CHECK(ret, "Failed to call logCreate");
+  }
+
+  void InitBackend() {
+    auto ret = qnn_interface_.backendCreate(log_handle_, backend_config_,
+                                            &backend_handle_);
+    SHERPA_ONNX_QNN_CHECK(ret, "Failed to call backendCreate");
+  }
+
+  void InitDevice() {
+    auto ret =
+        qnn_interface_.deviceCreate(log_handle_, nullptr, &device_handle_);
+    SHERPA_ONNX_QNN_CHECK(ret, "Failed to call deviceCreate");
+  }
+
+ private:
+  bool debug_ = true;
+  std::unique_ptr<void, decltype(&dlclose)> backend_lib_handle_{nullptr,
+                                                                &dlclose};
+
+  QNN_INTERFACE_VER_TYPE qnn_interface_;
+
+  QnnLog_Level_t log_level_ = QNN_LOG_LEVEL_WARN;
+  // QnnLog_Level_t log_level_ = QNN_LOG_LEVEL_INFO;
+  // QnnLog_Level_t log_level_ = QNN_LOG_LEVEL_VERBOSE;
+
+  Qnn_LogHandle_t log_handle_ = nullptr;
+
+  const QnnBackend_Config_t **backend_config_ = nullptr;
+  Qnn_BackendHandle_t backend_handle_ = nullptr;
+
+  Qnn_DeviceHandle_t device_handle_ = nullptr;
+
+  Qnn_ContextHandle_t context_handle_ = nullptr;
+  const QnnContext_Config_t **context_config_ = nullptr;
+  bool is_initialized_ = false;
+};
+
+QnnBackend::~QnnBackend() = default;
+
+QnnBackend::QnnBackend(const std::string &backend_lib)
+    : impl_(std::make_unique<Impl>(backend_lib)) {}
+
+void QnnBackend::InitContext() const { impl_->InitContext(); }
+
+void QnnBackend::InitContext(Qnn_ContextHandle_t context_handle) const {
+  impl_->InitContext(context_handle);
+}
+
+Qnn_LogHandle_t QnnBackend::LogHandle() const { return impl_->LogHandle(); }
+
+Qnn_BackendHandle_t QnnBackend::BackendHandle() const {
+  return impl_->BackendHandle();
+}
+
+Qnn_DeviceHandle_t QnnBackend::DeviceHandle() const {
+  return impl_->DeviceHandle();
+}
+
+Qnn_ContextHandle_t QnnBackend::ContextHandle() const {
+  return impl_->ContextHandle();
+}
+
+QNN_INTERFACE_VER_TYPE QnnBackend::QnnInterface() const {
+  return impl_->QnnInterface();
+}
+
+QnnLog_Level_t QnnBackend::LogLevel() const { return impl_->LogLevel(); }
+
+bool QnnBackend::IsInitialized() const { return impl_->IsInitialized(); }
+
+}  // namespace sherpa_onnx