pipeline done

Author: ZillaRU

.clang-format

@@ -94,7 +94,7 @@ if(UNIX AND NOT APPLE AND NOT SHERPA_ONNX_ENABLE_WASM AND NOT CMAKE_SYSTEM_NAME
 Could not find alsa/asoundlib.h !
 We won't build sherpa-onnx-alsa
 To fix that, please do:
-  (1) sudo apt-get install alsa-utils libasound2-dev
+  (1) sudo apt-get update -y && sudo apt-get install -y alsa-utils libasound2-dev
   (2) rm -rf build
   (3) re-try
   ")

.flake8

@@ -29,6 +29,8 @@ set(sources
   online-transducer-model.cc
   online-zipformer-transducer-model.cc
   onnx-utils.cc
+  cvi-utils.cc
+  onnx-to-cvi.cc
   packed-sequence.cc
   pad-sequence.cc
   parse-options.cc
@@ -70,6 +72,12 @@ endif()
 
 target_link_libraries(sherpa-onnx-core -lm)
 
+set(CVI_LIBS ${CVI_LIBS} cviruntime cvikernel)
+if(NOT CMAKE_CROSSCOMPILING)
+  set(CVI_LIBS ${CVI_LIBS} cvicmodel)
+endif()
+
+target_link_libraries(sherpa-onnx-core ${CVI_LIBS})
 
 if(SHERPA_ONNX_ENABLE_TTS)
   target_link_libraries(sherpa-onnx-core piper_phonemize)
@@ -137,11 +145,6 @@ if(SHERPA_ONNX_ENABLE_BINARY)
   )
 endif()
 
-set(CVI_LIBS ${CVI_LIBS} cviruntime cvikernel)
-if(NOT CMAKE_CROSSCOMPILING)
-  set(CVI_LIBS ${CVI_LIBS} cvicmodel)
-endif()
-
 if(SHERPA_ONNX_HAS_ALSA AND SHERPA_ONNX_ENABLE_BINARY)
   add_executable(sherpa-onnx-alsa sherpa-onnx-alsa.cc alsa.cc)
 
@@ -151,7 +154,6 @@ if(SHERPA_ONNX_HAS_ALSA AND SHERPA_ONNX_ENABLE_BINARY)
 
   foreach(exe IN LISTS exes)
     target_link_libraries(${exe} sherpa-onnx-core)
-    target_link_libraries(${exe} ${CVI_LIBS})
   endforeach()
 
   foreach(exe IN LISTS exes)

.github/scripts/.gitignore

@@ -0,0 +1,42 @@
+#include "sherpa-onnx/csrc/cvi-utils.h"
+
+#include <algorithm>
+#include <fstream>
+#include <sstream>
+#include <string>
+
+#include "cviruntime.h"  // NOLINT
+#include "cvitpu_debug.h"
+
+
+#define EXIT_IF_ERROR(cond, statement)                                                   \
+  if ((cond)) {                                                                          \
+    TPU_LOG_ERROR("%s\n", statement);                                                         \
+    exit(1);                                                                             \
+  }
+
+void GetInputOutPutInfo(CVI_MODEL_HANDLE model) {
+    CVI_TENSOR *input_tensors, *output_tensors;
+    int32_t input_num, output_num;
+    CVI_RC ret = CVI_NN_GetInputOutputTensors(model, &input_tensors, &input_num, &output_tensors, &output_num);
+
+    EXIT_IF_ERROR(ret != CVI_RC_SUCCESS, "failed to get inputs & outputs from model");
+    
+    // print the inputs & outputs's information
+    if (1) {
+        TPU_LOG_INFO("Inputs:\n");
+        for (int i = 0; i < input_num; ++i) {
+        auto &tensor = input_tensors[i];
+        TPU_LOG_INFO("  [%d] %s <%d,%d,%d,%d>,%s\n",
+                    i, tensor.name, tensor.shape.dim[0], tensor.shape.dim[1], tensor.shape.dim[2],
+                    tensor.shape.dim[3], formatToStr(tensor.fmt));
+        }
+        TPU_LOG_INFO("Outputs:\n");
+        for (int i = 0; i < output_num; ++i) {
+        auto &tensor = output_tensors[i];
+        TPU_LOG_INFO("  [%d] %s <%d,%d,%d,%d>,%s\n",
+                    i, tensor.name, tensor.shape.dim[0], tensor.shape.dim[1], tensor.shape.dim[2],
+                    tensor.shape.dim[3], formatToStr(tensor.fmt));
+        }
+    }
+}

.gitignore

@@ -3,86 +3,68 @@
 
 #include <cassert>
 #include <ostream>
-#include <string>
+#include <cstring>
 #include <utility>
 #include <vector>
-
+#include <algorithm>
 #include "cviruntime.h"  // NOLINT
 
-namespace sherpa_onnx {
-
-/**
- * Get the input names of a model.
- *
- * @param model A cviruntime model handle.
- * @param input_names. On return, it contains the input names of the model.
- * @param input_names_ptr. On return, input_names_ptr[i] contains
- *                         input_names[i].c_str()
- */
-void GetInputNames(CVI_MODEL_HANDLE model, std::vector<std::string> *input_names,
-                   std::vector<const char *> *input_names_ptr);
-
-/**
- * Get the output names of a model.
- *
- * @param model A cviruntime model handle.
- * @param output_names. On return, it contains the output names of the model.
- * @param output_names_ptr. On return, output_names_ptr[i] contains
- *                         output_names[i].c_str()
- */
-void GetOutputNames(CVI_MODEL_HANDLE model, std::vector<std::string> *output_names,
-                    std::vector<const char *> *output_names_ptr);
 
-/**
- * Get the output frame of Encoder
- *
- * @param allocator allocator of cviruntime
- * @param encoder_out encoder out tensor
- * @param t frame_index
- */
-CVI_TENSOR GetEncoderOutFrame(CVI_MODEL_HANDLE model, CVI_TENSOR *encoder_out,
-                              int32_t t);
+static const char* formatToStr(CVI_FMT fmt) {
+  switch(fmt) {
+    case CVI_FMT_FP32:  return "fp32";
+    case CVI_FMT_INT32:  return "i32";
+    case CVI_FMT_UINT32: return "u32";
+    case CVI_FMT_BF16:   return "bf16";
+    case CVI_FMT_INT16:  return "i16";
+    case CVI_FMT_UINT16: return "u16";
+    case CVI_FMT_INT8:   return "i8";
+    case CVI_FMT_UINT8:  return "u8";
+    default:
+      TPU_LOG_FATAL("unknown fmt:%d\n", fmt);
+  }
+  return nullptr;
+}
 
-/**
- * Print model metadata.
- *
- * @param os The output stream.
- * @param model The cviruntime model handle.
- */
-void PrintModelMetadata(std::ostream &os, CVI_MODEL_HANDLE model);
+static void copyFp32ToFp32(float *src, float *dst, int count) {
+  memcpy(dst, src, count * sizeof(float));
+}
 
-// Return a deep copy of v
-CVI_TENSOR Clone(CVI_MODEL_HANDLE model, const CVI_TENSOR *v);
+static void copyFp64ToFp32(double *src, float *dst, int count) {
+  std::transform(src, src + count, dst, [](double x) { return static_cast<float>(x); });
+}
 
-// Return a shallow copy
-CVI_TENSOR View(const CVI_TENSOR *v);
+static void copyI64ToFp32(int64_t *src, float *dst, int count) {
+  // use bitcasting to convert int64 to float, this is much faster than shifting
+  memcpy(dst, reinterpret_cast<float*>(src), count * sizeof(float));
+}
 
-// Print a 1-D tensor to stderr
-void Print1D(const CVI_TENSOR *v);
+static void copyI64ToU16(int64_t *src, uint16_t *dst, int count) {
+  std::transform(src, src + count, dst, [](int64_t x) { return static_cast<uint16_t>(x); });
+}
 
-// Print a 2-D tensor to stderr
-template <typename T = float>
-void Print2D(const CVI_TENSOR *v);
+static void copyI32ToFp32(int *src, float *dst, int count) {
+  std::transform(src, src + count, dst, [](int x) { return static_cast<float>(x); });
+}
 
-// Print a 3-D tensor to stderr
-void Print3D(const CVI_TENSOR *v);
+static void copyI32ToU16(int *src, uint16_t *dst, int count) {
+  memcpy(dst, reinterpret_cast<uint16_t*>(src), count * sizeof(uint16_t));
+}
 
-// Print a 4-D tensor to stderr
-void Print4D(const CVI_TENSOR *v);
+static void copyI32ToU8(int *src, uint8_t *dst, int count) {
+  memcpy(dst, reinterpret_cast<uint8_t*>(src), count * sizeof(uint8_t));
+}
 
-template <typename T = float>
-void Fill(CVI_TENSOR *tensor, T value) {
-    auto n = CVI_NN_TensorCount(tensor);
-    auto p = CVI_NN_TensorPtr(tensor);
-    std::fill(p, p + n * sizeof(T), value);
+static void copyI32ToI8(int *src, int8_t *dst, int count) {
+  memcpy(dst, reinterpret_cast<int8_t*>(src), count * sizeof(int8_t));
 }
 
-std::vector<CVI_TENSOR> ReadFile(const std::string &filename);
 
-// TODO(fangjun): Document it
-CVI_TENSOR Repeat(CVI_MODEL_HANDLE model, const CVI_TENSOR *cur_encoder_out,
-                  const std::vector<int32_t> &hyps_num_split);
 
-}  // namespace sherpa_onnx
+/**
+ * Get the input and output names and shapes of a model.
+ * @param model A cviruntime model handle.
+*/
+void GetInputOutPutInfo(CVI_MODEL_HANDLE model);
 
 #endif  // SHERPA_CVIRTL_UTILS_H_

CMakeLists.txt

@@ -62,34 +62,7 @@ static ModelType GetModelType(char *model_data, size_t model_data_length,
 
 std::unique_ptr<OnlineTransducerModel> OnlineTransducerModel::Create(
     const OnlineModelConfig &config) {
-  if (!config.model_type.empty()) {
-    const auto &model_type = config.model_type;
-    if (model_type == "zipformer") {
-      return std::make_unique<OnlineZipformerTransducerModel>(config);
-    } else {
-      SHERPA_ONNX_LOGE(
-          "Invalid model_type: %s. Trying to load the model to get its type",
-          model_type.c_str());
-    }
-  }
-  ModelType model_type = ModelType::kUnknown;
-
-  {
-    auto buffer = ReadFile(config.transducer.encoder);
-
-    model_type = GetModelType(buffer.data(), buffer.size(), config.debug);
-  }
-
-  switch (model_type) {
-    case ModelType::kZipformer:
-      return std::make_unique<OnlineZipformerTransducerModel>(config);
-    case ModelType::kUnknown:
-      SHERPA_ONNX_LOGE("Unknown model type in online transducer!");
-      return nullptr;
-  }
-
-  // unreachable code
-  return nullptr;
+  return std::make_unique<OnlineZipformerTransducerModel>(config);
 }
 
 Ort::Value OnlineTransducerModel::BuildDecoderInput(

CPPLINT.cfg

@@ -20,6 +20,9 @@
 #include "sherpa-onnx/csrc/session.h"
 #include "sherpa-onnx/csrc/text-utils.h"
 #include "sherpa-onnx/csrc/unbind.h"
+#include "cviruntime.h"
+#include "cvi-utils.h"
+#include "onnx-to-cvi.h"
 
 namespace sherpa_onnx {
 
@@ -345,41 +348,44 @@ OnlineZipformerTransducerModel::RunEncoder(Ort::Value features,
   // get input / output tensors
   CVI_TENSOR *input_tensors, *output_tensors;
   int32_t input_num, output_num;
-  CVI_NN_GetInputOutputTensors(encoder_sess_, &input_tensors, input_num, &output_tensors, output_num);
+  CVI_NN_GetInputOutputTensors(encoder_sess_, &input_tensors, &input_num, &output_tensors, &output_num);
   printf("[encoder] input num: %d, output num: %d\n", input_num, output_num);
 
   LoadOrtValuesToCviTensors(encoder_inputs, input_tensors, input_num);
 
-  CVI_NN_Forward(encoder_sess_, &input_tensors, &input_num, &output_tensors, &output_num);
+  CVI_NN_Forward(encoder_sess_, input_tensors, input_num, output_tensors, output_num);
 
-  std::vector<Ort::Value> next_states = GetOrtValuesFromCviTensors(output_tensors+1, output_num, allocator_);
+  std::vector<Ort::Value> next_states = GetOrtValuesFromCviTensors(output_tensors+1, output_num-1);
 
-  return {std::move(ConvertCviTensorToOrtValue(output_tensors[0], allocator_), std::move(next_states)};
+  return {std::move(GetOrtValueFromCviTensor(output_tensors[0])), std::move(next_states)};
 }
 
 Ort::Value OnlineZipformerTransducerModel::RunDecoder(Ort::Value decoder_input) {
   // get input / output tensors
   CVI_TENSOR *input_tensors, *output_tensors;
   int32_t input_num, output_num;
-  CVI_NN_GetInputOutputTensors(encoder_sess_, &input_tensors, input_num, &output_tensors, output_num);
+  CVI_NN_GetInputOutputTensors(decoder_sess_, &input_tensors, &input_num, &output_tensors, &output_num);
   printf("[Decoder] input num: %d, output num: %d\n", input_num, output_num);
-
-  LoadOrtValuesToCviTensors(encoder_inputs, input_tensors, input_num);
-  CVI_NN_Forward(decoder_sess_, &input_tensors, &input_num, &output_tensors, &output_num);
-  return std::move(ConvertCviTensorToOrtValue(output_tensors[0], allocator_);
+  std::vector<Ort::Value> temp = {};
+  temp.push_back(std::move(decoder_input));
+  LoadOrtValuesToCviTensors(temp, input_tensors, input_num);
+  CVI_NN_Forward(decoder_sess_, input_tensors, input_num, output_tensors, output_num);
+  return std::move(GetOrtValueFromCviTensor(output_tensors[0]));
 }
 
 Ort::Value OnlineZipformerTransducerModel::RunJoiner(Ort::Value encoder_out,
                                                      Ort::Value decoder_out) {
   // get input / output tensors
   CVI_TENSOR *input_tensors, *output_tensors;
   int32_t input_num, output_num;
-  CVI_NN_GetInputOutputTensors(joiner_sess_, &input_tensors, input_num, &output_tensors, output_num);
+  CVI_NN_GetInputOutputTensors(joiner_sess_, &input_tensors, &input_num, &output_tensors, &output_num);
   printf("[Decoder] input num: %d, output num: %d\n", input_num, output_num);
-  std::vector<Ort::Value> temp = {std::move(encoder_out), std::move(decoder_out)};
+  std::vector<Ort::Value> temp = {};
+  temp.push_back(std::move(encoder_out));
+  temp.push_back(std::move(decoder_out));
   LoadOrtValuesToCviTensors(temp, input_tensors, input_num);
-  CVI_NN_Forward(joiner_sess_, &input_tensors, &input_num, &output_tensors, &output_num);
-  return std::move(ConvertCviTensorToOrtValue(output_tensors[0], allocator_)
+  CVI_NN_Forward(joiner_sess_, input_tensors, input_num, output_tensors, output_num);
+  return std::move(GetOrtValueFromCviTensor(output_tensors[0]));
 }
 
 }  // namespace sherpa_onnx

LICENSE

@@ -9,7 +9,7 @@
 #include <utility>
 #include <vector>
 
-#include "onnx-to-cvi.h"
+#include "cviruntime.h"
 #include "sherpa-onnx/csrc/online-model-config.h"
 #include "sherpa-onnx/csrc/online-transducer-model.h"
 
@@ -54,9 +54,9 @@ class OnlineZipformerTransducerModel : public OnlineTransducerModel {
   Ort::SessionOptions sess_opts_;
   Ort::AllocatorWithDefaultOptions allocator_;
 
-  std::unique_ptr<Ort::Session> encoder_sess_;
-  std::unique_ptr<Ort::Session> decoder_sess_;
-  std::unique_ptr<Ort::Session> joiner_sess_;
+  CVI_MODEL_HANDLE encoder_sess_ = nullptr;
+  CVI_MODEL_HANDLE decoder_sess_ = nullptr;
+  CVI_MODEL_HANDLE joiner_sess_ = nullptr;
 
   std::vector<std::string> encoder_input_names_;
   std::vector<const char *> encoder_input_names_ptr_;