diff --git a/.github/workflows/export-sense-voice-to-ascend-npu.yaml b/.github/workflows/export-sense-voice-to-ascend-npu.yaml new file mode 100644 index 0000000000..bdabfdfd5f --- /dev/null +++ b/.github/workflows/export-sense-voice-to-ascend-npu.yaml @@ -0,0 +1,216 @@ +name: export-sense-voice-to-ascend-npu + +on: + push: + branches: + - export-sense-voice-ascend + workflow_dispatch: + +concurrency: + group: export-sense-voice-to-ascend-npu-${{ github.ref }} + cancel-in-progress: true + +jobs: + export-sense-voice-to-ascend-npu: + if: github.repository_owner == 'k2-fsa' || github.repository_owner == 'csukuangfj' + name: ${{ matrix.framework }} + runs-on: ${{ matrix.os }} + container: + # image: ascendai/cann:latest + # image: ascendai/cann:8.1.rc1-910b-ubuntu22.04-py3.10 + # see https://hub.docker.com/r/gpustack/ascendai-cann/tags?name=8.0 + image: gpustack/ascendai-cann:8.0.RC3-910b-ubuntu20.04-py3.9 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python-version: ["3.8"] + framework: ["FunASR", "WSYue-ASR"] + + steps: + - uses: actions/checkout@v4 + + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Show Python + shell: bash + run: | + python3 --version + + - name: Install curl + shell: bash + run: apt-get update && apt-get install -y curl bzip2 + + - name: Verify environment + shell: bash + run: | + ls -lh /usr/local/Ascend/ascend-toolkit/set_env.sh + + find /usr/local/Ascend -name "libascend*.so" 2>/dev/null + + + source /usr/local/Ascend/ascend-toolkit/set_env.sh + export LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/latest/x86_64-linux/devlib/linux/x86_64:$LD_LIBRARY_PATH + + echo "CANN environment:" + which atc || echo "atc not found" + atc --help + + - name: Install Python dependencies + shell: bash + run: | + python3 -m pip install "numpy<2" \ + onnx==1.17.0 \ + torch==2.0.0+cpu -f https://download.pytorch.org/whl/torch \ + attrs psutil scipy decorator cloudpickle ml-dtypes tornado \ + sentencepiece \ + pyyaml + + - name: Run SenseVoice from FunAsr + if: matrix.framework == 'FunASR' + shell: bash + run: | + cd scripts/sense-voice/ascend-npu + + curl -SL -O https://hf-mirror.com/FunAudioLLM/SenseVoiceSmall/resolve/main/am.mvn + curl -SL -O https://hf-mirror.com/FunAudioLLM/SenseVoiceSmall/resolve/main/model.pt + curl -SL -O https://hf-mirror.com/FunAudioLLM/SenseVoiceSmall/resolve/main/chn_jpn_yue_eng_ko_spectok.bpe.model + + curl -SL -O https://huggingface.co/csukuangfj/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/resolve/main/test_wavs/en.wav + curl -SL -O https://huggingface.co/csukuangfj/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/resolve/main/test_wavs/ja.wav + curl -SL -O https://huggingface.co/csukuangfj/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/resolve/main/test_wavs/ko.wav + curl -SL -O https://huggingface.co/csukuangfj/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/resolve/main/test_wavs/yue.wav + curl -SL -O https://huggingface.co/csukuangfj/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/resolve/main/test_wavs/zh.wav + + rm -f README.md || true + + curl -SL -O https://huggingface.co/csukuangfj/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/resolve/main/README.md + curl -SL -O https://huggingface.co/csukuangfj/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/resolve/main/LICENSE + + echo "export to onnx" + + python3 ./export_onnx.py + + ls -lh *.onnx + + source /usr/local/Ascend/ascend-toolkit/set_env.sh + export LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/latest/x86_64-linux/devlib/linux/x86_64:$LD_LIBRARY_PATH + + atc --model=./model.onnx \ + --framework=5 \ + --host_env_os=linux \ + --host_env_cpu=aarch64 \ + --output=model \ + --input_format=ND \ + --input_shape="x:1,-1,560;prompt:4" \ + --soc_version="Ascend910B" + + ls -lh *.om + + echo "collect results" + d=sherpa-onnx-ascend-910B-sense-voice-zh-en-ja-ko-yue-2024-07-17 + + mkdir -p $d + mkdir -p $d/test_wavs + + cp -v README.md $d + cp -v LICENSE $d + cp -v model_linux_aarch64.om $d/model.om + cp -v tokens.txt $d + cp -v test_om.py $d + cp -v *.wav $d/test_wavs + ls -lh $d + tar cjfv $d.tar.bz2 $d + ls -lh *.tar.bz2 + rm -rf $d + + echo "----show---" + ls -lh *.tar.bz2 + + mv *.tar.bz2 ../../.. + + - name: Run SenseVoice from WSYue-ASR + if: matrix.framework == 'WSYue-ASR' + shell: bash + run: | + cd scripts/sense-voice/ascend-npu + + curl -SL -O https://huggingface.co/ASLP-lab/WSYue-ASR/resolve/main/sensevoice_small_yue/model.pt + + curl -SL -O https://hf-mirror.com/FunAudioLLM/SenseVoiceSmall/resolve/main/am.mvn + curl -SL -O https://hf-mirror.com/FunAudioLLM/SenseVoiceSmall/resolve/main/chn_jpn_yue_eng_ko_spectok.bpe.model + + curl -SL -O https://huggingface.co/csukuangfj/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/resolve/main/test_wavs/en.wav + curl -SL -O https://huggingface.co/csukuangfj/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/resolve/main/test_wavs/yue.wav + curl -SL -O https://huggingface.co/csukuangfj/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/resolve/main/test_wavs/zh.wav + + for i in $(seq 0 17); do + curl -SL -O https://huggingface.co/csukuangfj/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2025-09-09/resolve/main/test_wavs/yue-$i.wav + done + + rm -f README.md || true + + curl -SL -O https://huggingface.co/csukuangfj/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2025-09-09/resolve/main/README.md + + echo "export to onnx" + python3 ./export_onnx.py + + ls -lh *.onnx + + source /usr/local/Ascend/ascend-toolkit/set_env.sh + export LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/latest/x86_64-linux/devlib/linux/x86_64:$LD_LIBRARY_PATH + + atc --model=./model.onnx \ + --framework=5 \ + --host_env_os=linux \ + --host_env_cpu=aarch64 \ + --output=model \ + --input_format=ND \ + --input_shape="x:1,-1,560;prompt:4" \ + --soc_version="Ascend910B" + + ls -lh *.om + + echo "collect results" + d=sherpa-onnx-ascend-910B-sense-voice-zh-en-ja-ko-yue-2025-09-09 + + mkdir -p $d + mkdir -p $d/test_wavs + + cp -v README.md $d + cp -v model_linux_aarch64.om $d/model.om + cp -v tokens.txt $d + cp -v test_om.py $d + cp -v *.wav $d/test_wavs + ls -lh $d + tar cjfv $d.tar.bz2 $d + ls -lh *.tar.bz2 + rm -rf $d + + echo "----show---" + ls -lh *.tar.bz2 + + mv *.tar.bz2 ../../.. + + - name: Release + if: github.repository_owner == 'csukuangfj' + uses: svenstaro/upload-release-action@v2 + with: + file_glob: true + file: ./*.tar.bz2 + overwrite: true + repo_name: k2-fsa/sherpa-onnx + repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} + tag: asr-models + + - name: Release + if: github.repository_owner == 'k2-fsa' + uses: svenstaro/upload-release-action@v2 + with: + file_glob: true + file: ./*.tar.bz2 + overwrite: true + tag: asr-models diff --git a/scripts/sense-voice/ascend-npu/export_onnx.py b/scripts/sense-voice/ascend-npu/export_onnx.py new file mode 100755 index 0000000000..596b50b397 --- /dev/null +++ b/scripts/sense-voice/ascend-npu/export_onnx.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +# Copyright 2025 Xiaomi Corp. (authors: Fangjun Kuang) + +from typing import List, Tuple + +import sentencepiece as spm +import torch + +from torch_model import SenseVoiceSmall + + +def load_cmvn(filename) -> Tuple[List[float], List[float]]: + neg_mean = None + inv_stddev = None + + with open(filename) as f: + for line in f: + if not line.startswith(""): + continue + t = line.split()[3:-1] + + if neg_mean is None: + neg_mean = list(map(lambda x: float(x), t)) + else: + inv_stddev = list(map(lambda x: float(x), t)) + + return neg_mean, inv_stddev + + +def generate_tokens(sp): + with open("tokens.txt", "w", encoding="utf-8") as f: + for i in range(sp.vocab_size()): + f.write(f"{sp.id_to_piece(i)} {i}\n") + print("saved to tokens.txt") + + +class ModelWrapper(torch.nn.Module): + def __init__(self, m): + super().__init__() + self.m = m + + def forward(self, x, prompt): + logits = self.m(x[None], prompt)[0] + part1 = logits[:4] + part2 = logits[4:] + part1 = part1.reshape(4, 25055) + part2 = part2.reshape(x.size(0), 25055) + return part1, part2 + + +@torch.no_grad() +def main(): + sp = spm.SentencePieceProcessor() + sp.load("./chn_jpn_yue_eng_ko_spectok.bpe.model") + generate_tokens(sp) + + print("loading model") + + state_dict = torch.load("./model.pt", map_location="cpu") + if "state_dict" in state_dict: + state_dict = state_dict["state_dict"] + + neg_mean, inv_stddev = load_cmvn("./am.mvn") + + neg_mean = torch.tensor(neg_mean, dtype=torch.float32) + inv_stddev = torch.tensor(inv_stddev, dtype=torch.float32) + + model = SenseVoiceSmall(neg_mean=neg_mean, inv_stddev=inv_stddev) + model.load_state_dict(state_dict) + model.eval() + del state_dict + + model = ModelWrapper(model) + model.eval() + + x = torch.randn(1, 93, 560, dtype=torch.float32) + + language = 3 + text_norm = 15 + prompt = torch.tensor([language, 1, 2, text_norm], dtype=torch.int32) + + opset_version = 14 + filename = "model.onnx" + torch.onnx.export( + model.m, + (x, prompt), + filename, + opset_version=opset_version, + input_names=["x", "prompt"], + output_names=["logits"], + dynamic_axes={ + "x": {0: "N", 1: "T"}, + "logits": {0: "N", 1: "T_4"}, + }, + ) + print(f"saved to {filename}") + + +if __name__ == "__main__": + torch.manual_seed(20251018) + main() diff --git a/scripts/sense-voice/ascend-npu/test_om.py b/scripts/sense-voice/ascend-npu/test_om.py new file mode 100755 index 0000000000..9415768cd9 --- /dev/null +++ b/scripts/sense-voice/ascend-npu/test_om.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +# Copyright 2025 Xiaomi Corp. (authors: Fangjun Kuang) + +import argparse +from typing import Tuple + +import kaldi_native_fbank as knf +import numpy as np +import soundfile as sf +import torch +from ais_bench.infer.interface import InferSession + + +class OmModel: + def __init__(self): + self.model = InferSession(device_id=0, model_path="./model.om", debug=False) + + print("---model---") + for i in self.model.get_inputs(): + print(i.name, i.datatype, i.shape) + + print("-----") + + for i in self.model.get_outputs(): + print(i.name, i.datatype, i.shape) + + def __call__(self, x, prompt=None, language=None, text_norm=None): + if language is not None and text_norm is not None: + x_len = np.array([x.shape[1]], dtype=np.int32) + print(x.shape, x_len, language, text_norm) + return self.model.infer( + [x, x_len, language, text_norm], mode="dymshape", custom_sizes=10000000 + )[0][0] + else: + return self.model.infer( + [x, prompt], mode="dymshape", custom_sizes=10000000 + )[0][0] + + return self.model.infer([x, prompt], mode="static", custom_sizes=10000000)[0][0] + logits_part1, logits_part2 = self.model.infer( + [x, prompt], mode="dymshape", custom_sizes=10000000 + ) + logits = np.concatenate([logits_part1, logits_part2], axis=0) + return logits + + +def load_audio(filename: str) -> Tuple[np.ndarray, int]: + data, sample_rate = sf.read( + filename, + always_2d=True, + dtype="float32", + ) + data = data[:, 0] # use only the first channel + samples = np.ascontiguousarray(data) + return samples, sample_rate + + +def load_tokens(filename): + ans = dict() + i = 0 + with open(filename, encoding="utf-8") as f: + for line in f: + ans[i] = line.strip().split()[0] + i += 1 + return ans + + +def compute_feat( + samples, + sample_rate, + window_size: int = 7, # lfr_m + window_shift: int = 6, # lfr_n +): + opts = knf.FbankOptions() + opts.frame_opts.dither = 0 + opts.frame_opts.snip_edges = False + opts.frame_opts.window_type = "hamming" + opts.frame_opts.samp_freq = sample_rate + opts.mel_opts.num_bins = 80 + + online_fbank = knf.OnlineFbank(opts) + online_fbank.accept_waveform(sample_rate, (samples * 32768).tolist()) + online_fbank.input_finished() + + features = np.stack( + [online_fbank.get_frame(i) for i in range(online_fbank.num_frames_ready)] + ) + assert features.data.contiguous is True + assert features.dtype == np.float32, features.dtype + + T = (features.shape[0] - window_size) // window_shift + 1 + features = np.lib.stride_tricks.as_strided( + features, + shape=(T, features.shape[1] * window_size), + strides=((window_shift * features.shape[1]) * 4, 4), + ) + + return np.copy(features) + + +def main(): + samples, sample_rate = load_audio("./test_wavs/zh.wav") + if sample_rate != 16000: + import librosa + + samples = librosa.resample(samples, orig_sr=sample_rate, target_sr=16000) + sample_rate = 16000 + + model = OmModel() + + features = compute_feat( + samples=samples, + sample_rate=sample_rate, + ) + print("features.shape", features.shape) + + language_auto = 0 + language_zh = 3 + language_en = 4 + language_yue = 7 + language_ya = 11 + language_ko = 12 + language_nospeech = 13 + + language = language_auto + + with_itn = 14 + without_itn = 15 + + text_norm = with_itn + + prompt = np.array([language, 1, 2, text_norm], dtype=np.int32) + # language = np.array([language], dtype=np.int32) + # text_norm = np.array([text_norm], dtype=np.int32) + + print("prompt", prompt.shape) + + logits = model( + x=features[None], + prompt=prompt, + # language=language, + ##text_norm=text_norm, + ) + print("logits.shape", logits.shape, type(logits)) + + idx = logits.argmax(axis=-1) + print(idx) + print(len(idx)) + prev = -1 + ids = [] + for i in idx: + if i != prev: + ids.append(i) + prev = i + ids = [i for i in ids if i != 0] + print(ids) + + tokens = load_tokens("./tokens.txt") + text = "".join([tokens[i] for i in ids]) + + text = text.replace("▁", " ") + print(text) + + +if __name__ == "__main__": + main() diff --git a/scripts/sense-voice/ascend-npu/torch_model.py b/scripts/sense-voice/ascend-npu/torch_model.py new file mode 120000 index 0000000000..2ed3ee1086 --- /dev/null +++ b/scripts/sense-voice/ascend-npu/torch_model.py @@ -0,0 +1 @@ +../rknn/torch_model.py \ No newline at end of file