Split sherpa-onnx Python package

.github/workflows/build-wheels-aarch64.yaml

@@ -5,6 +5,12 @@ on:
     branches:
       - wheel
   workflow_dispatch:
+    inputs:
+      publish_sherpa_onnx_bin:
+        description: "Publish sherpa-onnx-bin"
+        required: false
+        default: "true"
+        type: boolean
 
 env:
   SHERPA_ONNX_IS_IN_GITHUB_ACTIONS: 1
@@ -14,16 +20,286 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  core:
+    name: core
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-24.04-arm]
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Update version
+        shell: bash
+        run: |
+          ./new-release.sh
+          git diff .
+
+      - name: Display PWD
+        shell: bash
+        run: |
+          echo "pwd: $PWD"
+          ls -lh
+          du -h -d1 .
+
+      - name: Build sherpa-onnx
+        uses: addnab/docker-run-action@v3
+        with:
+            image: quay.io/pypa/manylinux2014_aarch64
+            options: |
+              --volume ${{ github.workspace }}/:/home/runner/work/sherpa-onnx/sherpa-onnx
+            shell: bash
+            run: |
+              uname -a
+              gcc --version
+              cmake --version
+              cat /etc/*release
+              id
+              pwd
+
+              cd /home/runner/work/sherpa-onnx/sherpa-onnx
+
+              find /opt -name "python*"
+
+              echo "--------------------"
+              export PATH=/opt/_internal/cpython-3.10.18/bin:$PATH
+              which python3
+              python3 --version
+
+              python3 -m venv my
+
+              source ./my/bin/activate
+
+              python3 -m pip install setuptools wheel twine
+
+              git clone --depth 1 --branch v1.2.12 https://github.com/alsa-project/alsa-lib
+              pushd alsa-lib
+              ./gitcompile
+              popd
+
+              export CPLUS_INCLUDE_PATH=$PWD/alsa-lib/include:$CPLUS_INCLUDE_PATH
+              export SHERPA_ONNX_ALSA_LIB_DIR=$PWD/alsa-lib/src/.libs
+
+              mkdir build
+              pushd build
+
+              cmake \
+                -D SHERPA_ONNX_ENABLE_TTS=ON \
+                -D CMAKE_BUILD_TYPE=Release \
+                -D BUILD_SHARED_LIBS=ON \
+                -D SHERPA_ONNX_BUILD_C_API_EXAMPLES=OFF \
+                -D CMAKE_INSTALL_PREFIX=./install \
+                ..
+
+              make -j2
+              make install
+
+              ls -lh lib
+              ls -lh bin
+
+              echo "----"
+              ls -lh install/lib
+
+              rm -fv install/lib/libcargs.so
+
+              echo "----"
+              ls -lh install/bin
+
+              echo 'sherpa-onnx-core'
+              mkdir -p ../scripts/wheel/sherpa-onnx-core/sherpa_onnx/lib
+              cp -v ./install/lib/lib*.so ../scripts/wheel/sherpa-onnx-core/sherpa_onnx/lib
+
+              mkdir -p ../scripts/wheel/sherpa-onnx-core/sherpa_onnx/include/sherpa-onnx/c-api
+              cp -v ./install/include/sherpa-onnx/c-api/*.h ../scripts/wheel/sherpa-onnx-core/sherpa_onnx/include/sherpa-onnx/c-api
+
+              pushd ../scripts/wheel/sherpa-onnx-core
+              python3 setup.py bdist_wheel --plat-name=manylinux2014_aarch64
+
+              ls -lh dist
+
+              popd
+
+              echo 'sherpa-onnx-bin'
+
+              mkdir -p ../scripts/wheel/sherpa-onnx-bin/bin
+              cp -v ./install/bin/sherpa-onnx* ../scripts/wheel/sherpa-onnx-bin/bin
+
+              pushd ../scripts/wheel/sherpa-onnx-bin
+              python3 setup.py bdist_wheel --plat-name=manylinux2014_aarch64
+
+              ls -lh dist
+
+              popd
+
+      - name: Collect wheels
+        shell: bash
+        run: |
+          sudo chown -R $USER ./scripts/wheel
+          mkdir wheelhouse
+          cp -v ./scripts/wheel/sherpa-onnx-core/dist/*.whl ./wheelhouse
+          cp -v ./scripts/wheel/sherpa-onnx-bin/dist/*.whl ./wheelhouse
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wheels-core-linux-aarch64
+          path: ./wheelhouse/*.whl
+
+      - name: Show wheels
+        shell: bash
+        run: |
+          sudo chown -R $USER ./scripts/wheel
+          ls -lh ./scripts/wheel/sherpa-onnx-core/dist
+          ls -lh ./scripts/wheel/sherpa-onnx-bin/dist
+
+          unzip -l ./scripts/wheel/sherpa-onnx-core/dist/*.whl
+          echo "---"
+          unzip -l ./scripts/wheel/sherpa-onnx-bin/dist/*.whl
+
+      - name: Install patchelf
+        shell: bash
+        run: |
+          sudo apt-get update -q
+          sudo apt-get install -q -y patchelf
+          patchelf --help
+
+      - name: Patch wheels
+        shell: bash
+        run: |
+          mkdir ./wheels
+          sudo ./scripts/wheel/patch_wheel.py --in-dir ./wheelhouse --out-dir ./wheels
+
+          ls -lh ./wheels/
+          rm -rf ./wheelhouse
+          mv ./wheels ./wheelhouse
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wheels-core-linux-aarch64-patched
+          path: ./wheelhouse/*.whl
+
+  test:
+    name: test
+    needs: [core]
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-24.04-arm]
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Retrieve artifact from Linux x64
+        uses: actions/download-artifact@v4
+        with:
+          name: wheels-core-linux-aarch64-patched
+          path: /tmp/wheels
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Show
+        shell: bash
+        run: |
+          ls -lh /tmp/wheels
+
+      - name: Install
+        shell: bash
+        run: |
+          python3 -m pip install /tmp/wheels/*.whl
+
+      - name: Show version
+        shell: bash
+        run: |
+          sherpa-onnx-version
+
+      - name: Show help
+        shell: bash
+        run: |
+          sherpa-onnx --help
+
+          echo "---"
+
+          ls -lh $(which sherpa-onnx)
+          file $(which sherpa-onnx)
+          readelf -d $(which sherpa-onnx)
+
+          ldd $(which sherpa-onnx)
+
+          sherpa-onnx-offline --help
+
+          echo "---"
+
+          sherpa-onnx-vad --help
+
+      - name: Publish to huggingface
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        uses: nick-fields/retry@v3
+        with:
+          max_attempts: 20
+          timeout_seconds: 200
+          shell: bash
+          command: |
+            git config --global user.email "csukuangfj@gmail.com"
+            git config --global user.name "Fangjun Kuang"
+
+            rm -rf huggingface
+            export GIT_LFS_SKIP_SMUDGE=1
+            export GIT_CLONE_PROTECTION_ACTIVE=false
+
+            SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt  | cut -d " " -f 2  | cut -d '"' -f 2)
+            echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION"
+
+            d=cpu/$SHERPA_ONNX_VERSION
+
+            git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface
+            cd huggingface
+            git fetch
+            git pull
+            git merge -m "merge remote" --ff origin main
+
+            mkdir -p $d
+
+            cp -v /tmp/wheels/*.whl $d/
+
+            git status
+            git add .
+            git commit -m "add more wheels"
+            git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-wheels main
+
+      - name: Publish wheels to PyPI ${{ github.event.inputs.publish_sherpa_onnx_bin }}
+        if: ${{ (github.event.inputs.publish_sherpa_onnx_bin || 'true') == 'true' }}
+        env:
+          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+        shell: bash
+        run: |
+          python3 -m pip install --upgrade pip
+          python3 -m pip install wheel twine==5.0.0 setuptools
+
+          twine upload /tmp/wheels/*.whl
+
+
   build_wheels_aarch64:
+    needs: [core, test]
     name: ${{ matrix.manylinux }} ${{ matrix.python-version }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         # see https://github.com/pypa/cibuildwheel/issues/2257
         # we don't use qemu from now on
-        os: [ubuntu-22.04-arm]
-        python-version: ["cp38", "cp39", "cp310", "cp311", "cp312", "cp313"]
+        os: [ubuntu-24.04-arm]
+        python-version: ["cp38", "cp39", "cp310", "cp311", "cp312", "cp313", "cp314"]
         manylinux: [manylinux2014] #, manylinux_2_28]
 
     steps:
@@ -38,7 +314,7 @@ jobs:
       # see https://cibuildwheel.readthedocs.io/en/stable/changelog/
       # for a list of versions
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.21.3
+        uses: pypa/cibuildwheel@v3.1.4
         env:
           CIBW_BEFORE_ALL: |
             git clone --depth 1 --branch v1.2.12 https://github.com/alsa-project/alsa-lib
@@ -48,7 +324,15 @@ jobs:
             echo "PWD"
             ls -lh /project/alsa-lib/src/.libs
 
-          CIBW_ENVIRONMENT: CPLUS_INCLUDE_PATH=/project/alsa-lib/include:$CPLUS_INCLUDE_PATH SHERPA_ONNX_ALSA_LIB_DIR=/project/alsa-lib/src/.libs LD_LIBRARY_PATH=/project/build/bdist.linux-x86_64/wheel/sherpa_onnx/lib:$SHERPA_ONNX_ALSA_LIB_DIR SHERPA_ONNX_MAKE_ARGS="VERBOSE=1" SHERPA_ONNX_ENABLE_ALSA=1
+          CIBW_ENVIRONMENT: >
+            SHERPA_ONNX_SPLIT_PYTHON_PACKAGE=ON
+            CPLUS_INCLUDE_PATH=/project/alsa-lib/include:$CPLUS_INCLUDE_PATH
+            SHERPA_ONNX_ALSA_LIB_DIR=/project/alsa-lib/src/.libs
+            LD_LIBRARY_PATH=/project/build/bdist.linux-aarch64/wheel/sherpa_onnx/lib:$SHERPA_ONNX_ALSA_LIB_DIR
+            SHERPA_ONNX_MAKE_ARGS="VERBOSE=1"
+            SHERPA_ONNX_ENABLE_ALSA=1
+            SHERPA_ONNX_CMAKE_ARGS="-DSHERPA_ONNX_ENABLE_BINARY=OFF -DSHERPA_ONNX_BUILD_C_API_EXAMPLES=OFF -DSHERPA_ONNX_ENABLE_C_API=OFF -DSHERPA_ONNX_ENABLE_WEBSOCKET=OFF"
+
           CIBW_BUILD: "${{ matrix.python-version}}-* "
           CIBW_SKIP: "cp27-* cp35-* cp36-* *-win32 pp* *-musllinux* *-manylinux_i686"
           CIBW_BUILD_VERBOSITY: 3
@@ -57,28 +341,38 @@ jobs:
           CIBW_MANYLINUX_AARCH64_IMAGE: quay.io/pypa/${{ matrix.manylinux }}_aarch64
           # From onnxruntime >= 1.17.0, it drops support for CentOS 7.0 and it supports only manylinux_2_28.
           # manylinux_2_24 is no longer supported
+          CIBW_REPAIR_WHEEL_COMMAND: >
+            auditwheel repair -w {dest_dir}
+            --exclude libonnxruntime.so
+            {wheel}
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wheel-${{ matrix.python-version }}-${{ matrix.manylinux }}-linux-aarch64
+          path: ./wheelhouse/*.whl
 
       - name: Display wheels
         shell: bash
         run: |
           ls -lh ./wheelhouse/
 
-      - name: Install patchelf
+      - name: Show wheels
         shell: bash
         run: |
-          sudo apt-get update -q
-          sudo apt-get install -q -y patchelf
-          patchelf --help
+          ls -lh wheelhouse/*.whl
 
-      - name: Patch wheels
-        shell: bash
-        run: |
-          mkdir ./wheels
-          sudo ./scripts/wheel/patch_wheel.py --in-dir ./wheelhouse --out-dir ./wheels
+          unzip -l wheelhouse/*.whl
 
-          ls -lh ./wheels/
-          rm -rf ./wheelhouse
-          mv ./wheels ./wheelhouse
+          echo "---"
+
+          mkdir t
+          cp wheelhouse/*.whl ./t
+          cd ./t
+          unzip ./*.whl
+          ls -lh
+          echo "---"
+
+          readelf -d sherpa_onnx/lib/*.so
 
       - name: Publish to huggingface
         env:
@@ -116,11 +410,6 @@ jobs:
             git commit -m "add more wheels"
             git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-wheels main
 
-      - uses: actions/upload-artifact@v4
-        with:
-          name: wheel-${{ matrix.python-version }}-${{ matrix.manylinux }}
-          path: ./wheelhouse/*.whl
-
       - name: Publish wheels to PyPI
         env:
           TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}