Add multilingual Nemotron-3.5 streaming ASR support

.github/workflows/export-nemotron-3.5-asr-streaming-0.6b.yaml

@@ -0,0 +1,185 @@
+name: export-nemotron-3-5-asr-streaming-06b
+
+on:
+  push:
+    branches:
+      - export-nemotron-3-5-asr-streaming
+  workflow_dispatch:
+
+concurrency:
+  group: export-nemotron-3-5-asr-streaming-to-onnx-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  export-nemotron-3-5-asr-streaming-0-6b-to-onnx:
+    if: github.repository_owner == 'k2-fsa' || github.repository_owner == 'csukuangfj'
+    name: nemotron-3-5-asr-streaming-0-6b-to-onnx
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest]
+        python-version: ["3.10"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install NeMo
+        shell: bash
+        run: |
+          # nemotron-3.5-asr-streaming-0.6b requires EncDecRNNTBPEModelWithPrompt,
+          # which is not in the stable nemo-toolkit release yet; the model card
+          # instructs installing NeMo from main.
+          BRANCH='main'
+          pip install Cython packaging
+          pip install "nemo_toolkit[asr] @ git+https://github.com/NVIDIA/NeMo.git@$BRANCH"
+          pip install onnxruntime ipython sentencepiece
+          pip install kaldi-native-fbank
+          pip install soundfile librosa
+
+      - name: Run
+        shell: bash
+        run: |
+          cd scripts/nemo/nemotron-3.5-asr-streaming-0.6b
+
+          python3 ./export_onnx.py
+
+          ls -lh
+          echo "---"
+
+          ls -lh */
+
+          echo "---"
+
+      - name: Collect results
+        shell: bash
+        run: |
+          src=scripts/nemo/nemotron-3.5-asr-streaming-0.6b
+
+          for chunk in 80 160 560 1120; do
+            d=sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-${chunk}ms-2026-06-11
+            mkdir -p $d
+
+            cp -av $src/$chunk/encoder.onnx $d/
+            cp -av $src/$chunk/encoder.data $d/
+            cp -av $src/$chunk/decoder.onnx $d/
+            cp -av $src/$chunk/joiner.onnx $d/
+            cp -av $src/tokens.txt $d/
+            cat >$d/README.md <<EOF
+            # Introduction
+            This model is from https://huggingface.co/nvidia/nemotron-3.5-asr-streaming-0.6b
+
+            The chunk size is ${chunk}ms for the exported ONNX model.
+
+            Use per-stream language strings such as "en", "ja", or "auto".
+          EOF
+
+            ls -lh $d
+
+            d=sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-${chunk}ms-int8-2026-06-11
+            mkdir -p $d
+
+            cp -av $src/${chunk}/encoder.int8.onnx $d/
+            cp -av $src/${chunk}/decoder.int8.onnx $d/
+            cp -av $src/${chunk}/joiner.int8.onnx $d/
+            cp -av $src/tokens.txt $d/
+            cat >$d/README.md <<EOF
+            # Introduction
+            This model is from https://huggingface.co/nvidia/nemotron-3.5-asr-streaming-0.6b
+
+            The chunk size is ${chunk}ms for the exported ONNX model.
+
+            Use per-stream language strings such as "en", "ja", or "auto".
+          EOF
+
+            ls -lh $d
+          done
+
+      - name: Download test waves
+        shell: bash
+        run: |
+          mkdir test_wavs
+          pushd test_wavs
+          curl -SL -O https://hf-mirror.com/csukuangfj/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/resolve/main/test_wavs/en.wav
+          curl -SL -O https://hf-mirror.com/csukuangfj/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/resolve/main/test_wavs/ja.wav
+          popd
+
+          models=(
+            sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-80ms-int8-2026-06-11
+            sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-160ms-int8-2026-06-11
+            sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11
+            sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-1120ms-int8-2026-06-11
+            sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-80ms-2026-06-11
+            sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-160ms-2026-06-11
+            sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-2026-06-11
+            sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-1120ms-2026-06-11
+          )
+          for m in ${models[@]}; do
+            cp -av test_wavs $m
+            tar cjvf $m.tar.bz2 $m
+          done
+
+          ls -lh *.tar.bz2
+
+      - name: Release
+        uses: svenstaro/upload-release-action@v2
+        with:
+          file_glob: true
+          file: sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-*-int8-2026-06-11.tar.bz2
+          overwrite: true
+          repo_name: k2-fsa/sherpa-onnx
+          repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }}
+          tag: asr-models
+
+      - name: Publish to huggingface
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        uses: nick-fields/retry@v3
+        with:
+          max_attempts: 20
+          timeout_seconds: 200
+          shell: bash
+          command: |
+            git config --global user.email "csukuangfj@gmail.com"
+            git config --global user.name "Fangjun Kuang"
+
+            models=(
+              sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-80ms-int8-2026-06-11
+              sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-160ms-int8-2026-06-11
+              sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11
+              sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-1120ms-int8-2026-06-11
+              sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-80ms-2026-06-11
+              sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-160ms-2026-06-11
+              sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-2026-06-11
+              sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-1120ms-2026-06-11
+            )
+
+            for m in ${models[@]}; do
+              if [ ! -d $m ]; then
+                echo "skip $m"
+                continue
+              fi
+
+              rm -rf huggingface
+              export GIT_LFS_SKIP_SMUDGE=1
+              export GIT_CLONE_PROTECTION_ACTIVE=false
+              git clone https://csukuangfj2:$HF_TOKEN@huggingface.co/csukuangfj2/$m huggingface
+              cp -av $m/* huggingface
+              cd huggingface
+              git lfs track "*.onnx"
+              git lfs track "*.data"
+              git lfs track "*.wav"
+              git status
+              git add .
+              git status
+              git commit -m "first commit"
+              git push https://csukuangfj2:$HF_TOKEN@huggingface.co/csukuangfj2/$m main
+              cd ..
+            done
+
+            rm -rf huggingface

c-api-examples/streaming-nemotron-c-api.c

@@ -67,6 +67,10 @@ int32_t main() {
 
   const SherpaOnnxOnlineStream *stream =
       SherpaOnnxCreateOnlineStream(recognizer);
+  // Multilingual Nemotron models use the generic stream option "language".
+  // For example: SherpaOnnxOnlineStreamSetOption(stream, "language", "ja");
+  // Empty/unset means auto. English-only Nemotron ignores this option.
+  SherpaOnnxOnlineStreamSetOption(stream, "language", "en");
 
   const SherpaOnnxDisplay *display = SherpaOnnxCreateDisplay(50);
   int32_t segment_id = 0;

cxx-api-examples/streaming-nemotron-cxx-api.cc

@@ -63,6 +63,10 @@ int32_t main() {
   const auto begin = std::chrono::steady_clock::now();
 
   OnlineStream stream = recognizer.CreateStream();
+  // Multilingual Nemotron models use the generic stream option "language".
+  // For example: stream.SetOption("language", "ja");
+  // Empty/unset means auto. English-only Nemotron ignores this option.
+  stream.SetOption("language", "en");
   stream.AcceptWaveform(wave.sample_rate, wave.samples.data(),
                         wave.samples.size());
   stream.InputFinished();

flutter-examples/streaming_asr/lib/streaming_asr.dart

@@ -64,6 +64,7 @@ class _StreamingAsrScreenState extends State<StreamingAsrScreen> {
       sherpa_onnx.initBindings();
       _recognizer = await createOnlineRecognizer();
       _stream = _recognizer?.createStream();
+      // Multilingual Nemotron: _stream?.setOption(key: 'language', value: 'ja');
 
       _isInitialized = true;
     }

flutter/sherpa_onnx/lib/src/online_stream.dart

@@ -67,5 +67,25 @@ class OnlineStream {
     SherpaOnnxBindings.onlineStreamInputFinished?.call(ptr);
   }
 
+  /// Set a string option on the underlying stream.
+  ///
+  /// For multilingual Nemotron models, use `key: 'language'` with values such
+  /// as `ja` or `auto`.
+  void setOption({required String key, required String value}) {
+    if (SherpaOnnxBindings.onlineStreamSetOption == null) {
+      throw Exception("Please initialize sherpa-onnx first");
+    }
+
+    if (ptr == nullptr) {
+      return;
+    }
+
+    final pKey = key.toNativeUtf8();
+    final pValue = value.toNativeUtf8();
+    SherpaOnnxBindings.onlineStreamSetOption?.call(ptr, pKey, pValue);
+    calloc.free(pKey);
+    calloc.free(pValue);
+  }
+
   Pointer<SherpaOnnxOnlineStream> ptr;
 }

flutter/sherpa_onnx/lib/src/sherpa_onnx_bindings.dart

@@ -1795,6 +1795,20 @@ typedef OnlineStreamInputFinishedNative =
 typedef OnlineStreamInputFinished =
     void Function(Pointer<SherpaOnnxOnlineStream>);
 
+typedef OnlineStreamSetOptionNative =
+    Void Function(
+      Pointer<SherpaOnnxOnlineStream>,
+      Pointer<Utf8>,
+      Pointer<Utf8>,
+    );
+
+typedef OnlineStreamSetOption =
+    void Function(
+      Pointer<SherpaOnnxOnlineStream>,
+      Pointer<Utf8>,
+      Pointer<Utf8>,
+    );
+
 typedef SherpaOnnxSpeakerEmbeddingExtractorIsReadyNative =
     Int32 Function(
       Pointer<SherpaOnnxSpeakerEmbeddingExtractor>,
@@ -2043,6 +2057,8 @@ class SherpaOnnxBindings {
 
   static OnlineStreamInputFinished? onlineStreamInputFinished;
 
+  static OnlineStreamSetOption? onlineStreamSetOption;
+
   static SherpaOnnxSpeakerEmbeddingExtractorIsReady?
   speakerEmbeddingExtractorIsReady;
 
@@ -2749,6 +2765,12 @@ class SherpaOnnxBindings {
         )
         .asFunction();
 
+    onlineStreamSetOption ??= dynamicLibrary
+        .lookup<NativeFunction<OnlineStreamSetOptionNative>>(
+          'SherpaOnnxOnlineStreamSetOption',
+        )
+        .asFunction();
+
     speakerEmbeddingExtractorIsReady ??= dynamicLibrary
         .lookup<
           NativeFunction<SherpaOnnxSpeakerEmbeddingExtractorIsReadyNative>

scripts/nemo/nemotron-3.5-asr-streaming-0.6b/README.md

@@ -0,0 +1,64 @@
+# Multilingual Nemotron-3.5 Streaming ASR
+
+This directory exports the NVIDIA NeMo model
+`nvidia/nemotron-3.5-asr-streaming-0.6b` to sherpa-onnx streaming transducer
+packages.
+
+The exporter writes the same package layout as
+`nvidia/nemotron-speech-streaming-en-0.6b`:
+
+- `encoder.onnx`
+- `encoder.data`
+- `decoder.onnx`
+- `joiner.onnx`
+- int8 variants for all three ONNX graphs
+- `tokens.txt` converted from the model's SentencePiece tokenizer
+
+The encoder metadata contains `prompt_dictionary` and `auto_prompt_id`. Users
+set the per-stream language as a string; the numerical prompt id is internal.
+An empty language string and `auto` use the model's auto-detect prompt.
+
+## Export
+
+As of June 2026, stable NeMo releases lack `EncDecRNNTBPEModelWithPrompt`,
+so install NeMo from git main.
+
+```bash
+pip install Cython packaging
+pip install "nemo_toolkit[asr] @ git+https://github.com/NVIDIA/NeMo.git@main"
+pip install onnxruntime ipython sentencepiece
+pip install kaldi-native-fbank
+pip install soundfile librosa
+
+python3 ./export_onnx.py
+```
+
+The script exports 80ms, 160ms, 560ms, and 1120ms chunk sizes.
+
+## Decode
+
+Forced language:
+
+```bash
+./build/bin/sherpa-onnx \
+  --encoder=./sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11/encoder.int8.onnx \
+  --decoder=./sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11/decoder.int8.onnx \
+  --joiner=./sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11/joiner.int8.onnx \
+  --tokens=./sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11/tokens.txt \
+  --language=ja \
+  ./sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11/test_wavs/ja.wav
+```
+
+Auto language:
+
+```bash
+./build/bin/sherpa-onnx \
+  --encoder=./sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11/encoder.int8.onnx \
+  --decoder=./sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11/decoder.int8.onnx \
+  --joiner=./sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11/joiner.int8.onnx \
+  --tokens=./sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11/tokens.txt \
+  --language=auto \
+  ./sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11/test_wavs/ja.wav
+```
+
+The same auto behavior is used when `--language` is omitted.