From 85dd6d060f7b4e22baa8dbbc49b58c02c1b53505 Mon Sep 17 00:00:00 2001 From: Viet-Anh Nguyen Date: Sun, 7 Jun 2026 09:38:34 +0700 Subject: [PATCH] Add PhoWhisper-Small Vietnamese ASR model Signed-off-by: Viet-Anh Nguyen --- .../models/phowhisper_small/README.md | 71 ++ .../models/phowhisper_small/__init__.py | 13 + .../models/phowhisper_small/code-gen.yaml | 12 + .../models/phowhisper_small/conftest.py | 38 ++ .../models/phowhisper_small/demo.py | 15 + .../models/phowhisper_small/export.py | 638 ++++++++++++++++++ .../models/phowhisper_small/info.yaml | 34 + .../models/phowhisper_small/model.py | 25 + .../models/phowhisper_small/requirements.txt | 6 + .../models/phowhisper_small/test.py | 23 + .../models/phowhisper_small/test_generated.py | 391 +++++++++++ 11 files changed, 1266 insertions(+) create mode 100644 src/qai_hub_models/models/phowhisper_small/README.md create mode 100644 src/qai_hub_models/models/phowhisper_small/__init__.py create mode 100644 src/qai_hub_models/models/phowhisper_small/code-gen.yaml create mode 100644 src/qai_hub_models/models/phowhisper_small/conftest.py create mode 100644 src/qai_hub_models/models/phowhisper_small/demo.py create mode 100644 src/qai_hub_models/models/phowhisper_small/export.py create mode 100644 src/qai_hub_models/models/phowhisper_small/info.yaml create mode 100644 src/qai_hub_models/models/phowhisper_small/model.py create mode 100644 src/qai_hub_models/models/phowhisper_small/requirements.txt create mode 100644 src/qai_hub_models/models/phowhisper_small/test.py create mode 100644 src/qai_hub_models/models/phowhisper_small/test_generated.py diff --git a/src/qai_hub_models/models/phowhisper_small/README.md b/src/qai_hub_models/models/phowhisper_small/README.md new file mode 100644 index 000000000..28b51c2b3 --- /dev/null +++ b/src/qai_hub_models/models/phowhisper_small/README.md @@ -0,0 +1,71 @@ +# [PhoWhisper-Small: Vietnamese-specialized automatic speech recognition (ASR) model, fine-tuned from Whisper-Small for accurate on-device transcription of Vietnamese speech](https://aihub.qualcomm.com/models/phowhisper_small) + +PhoWhisper-Small is a Vietnamese fine-tune of OpenAI's Whisper-Small ASR (Automatic Speech Recognition) model. It shares the same transformer encoder-decoder architecture as Whisper-Small but is trained on a large, diverse Vietnamese speech corpus covering multiple dialects, substantially improving Vietnamese transcription accuracy over the generic multilingual checkpoint. It fills the gap of having no Vietnamese-specialized ASR model in the catalog. Like Whisper-Small, it has been optimized for edge inference by replacing Multi-Head Attention (MHA) with Single-Head Attention (SHA) and linear layers with convolutional (conv) layers, and it transcribes audio clips up to 30 seconds long. Time to the first token is the encoder's latency, while time to each additional token is decoder's latency, where we assume a max decoded length specified below. + +On the Vietnamese FLEURS test set, PhoWhisper-Small attains a word error rate (WER) of 11.0%, compared with 21.4% for the generic Whisper-Small checkpoint. + +This is based on the implementation of PhoWhisper found [here](https://github.com/VinAIResearch/PhoWhisper). +This repository contains scripts for optimized on-device export suitable to run on Qualcomm® devices. More details on model performance across various devices, can be found [here](https://aihub.qualcomm.com/models/phowhisper_small). + +Qualcomm AI Hub Models uses [Qualcomm AI Hub Workbench](https://workbench.aihub.qualcomm.com) to compile, profile, and evaluate this model. [Sign up](https://myaccount.qualcomm.com/signup) to run these models on a hosted Qualcomm® device. + +## Setup +### 1. Install System-Level Dependencies +#### Linux +```bash +sudo apt install ffmpeg libportaudio2 +``` + +#### Windows +``` +winget install ffmpeg +``` + +### 2. Install the package +Install the package via pip: +```bash +# NOTE: 3.10 <= PYTHON_VERSION < 3.14 is supported. +pip install "qai-hub-models[phowhisper-small]" +``` + +### 3. Configure Qualcomm® AI Hub Workbench +Sign-in to [Qualcomm® AI Hub Workbench](https://workbench.aihub.qualcomm.com/) with your +Qualcomm® ID. Once signed in navigate to `Account -> Settings -> API Token`. + +With this API token, you can configure your client to run models on the cloud +hosted devices. +```bash +qai-hub configure --api_token API_TOKEN +``` +Navigate to [docs](https://workbench.aihub.qualcomm.com/docs/) for more information. + +## Run CLI Demo +Run the following simple CLI demo to verify the model is working end to end: + +```bash +python -m qai_hub_models.models.phowhisper_small.demo +``` +More details on the CLI tool can be found with the `--help` option. See +[demo.py](demo.py) for sample usage of the model including pre/post processing +scripts. Please refer to our [general instructions on using +models](../../../#getting-started) for more usage instructions. + +## Export for on-device deployment +To run the model on Qualcomm® devices, you must export the model for use with an edge runtime such as +TensorFlow Lite, ONNX Runtime, or Qualcomm AI Engine Direct. Use the following command to export the model: +```bash +python -m qai_hub_models.models.phowhisper_small.export --device "Samsung Galaxy S25 (Family)" +``` +Additional options are documented with the `--help` option. + +## License +* The license for the original implementation of PhoWhisper can be found + [here](https://github.com/VinAIResearch/PhoWhisper/blob/main/LICENSE). + +## References +* [PhoWhisper: Automatic Speech Recognition for Vietnamese](https://arxiv.org/abs/2406.02555) +* [Source Model Implementation](https://github.com/VinAIResearch/PhoWhisper) + +## Community +* Join [our AI Hub Slack community](https://aihub.qualcomm.com/community/slack) to collaborate, post questions and learn more about on-device AI. +* For questions or feedback please [reach out to us](mailto:ai-hub-support@qti.qualcomm.com). diff --git a/src/qai_hub_models/models/phowhisper_small/__init__.py b/src/qai_hub_models/models/phowhisper_small/__init__.py new file mode 100644 index 000000000..a919c55c7 --- /dev/null +++ b/src/qai_hub_models/models/phowhisper_small/__init__.py @@ -0,0 +1,13 @@ +# --------------------------------------------------------------------- +# Copyright (c) 2025 Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause +# --------------------------------------------------------------------- + +from qai_hub_models.models._shared.hf_whisper.app import ( + HfWhisperApp as App, +) + +from .model import MODEL_ID +from .model import PhoWhisperSmall as Model + +__all__ = ["MODEL_ID", "App", "Model"] diff --git a/src/qai_hub_models/models/phowhisper_small/code-gen.yaml b/src/qai_hub_models/models/phowhisper_small/code-gen.yaml new file mode 100644 index 000000000..05e4a7040 --- /dev/null +++ b/src/qai_hub_models/models/phowhisper_small/code-gen.yaml @@ -0,0 +1,12 @@ +is_collection_model: true +requires_aot_prepare: true +readme_install_system_deps: | + #### Linux + ```bash + sudo apt install ffmpeg libportaudio2 + ``` + + #### Windows + ``` + winget install ffmpeg + ``` diff --git a/src/qai_hub_models/models/phowhisper_small/conftest.py b/src/qai_hub_models/models/phowhisper_small/conftest.py new file mode 100644 index 000000000..7c8521c19 --- /dev/null +++ b/src/qai_hub_models/models/phowhisper_small/conftest.py @@ -0,0 +1,38 @@ +# --------------------------------------------------------------------- +# Copyright (c) 2025 Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause +# --------------------------------------------------------------------- +# THIS FILE WAS AUTO-GENERATED. DO NOT EDIT MANUALLY. + +import gc +import warnings + +import pytest +import torch.jit._trace + +from qai_hub_models.models.phowhisper_small import Model +from qai_hub_models.scorecard.utils.testing import make_cached_from_pretrained_fixture + + +def pytest_configure(config: pytest.Config) -> None: + # pytest is unable to figure out how to silence several PyTorch warning types from pyproject.toml settings, + # so we apply a manual warning filter here instead. + warnings.filterwarnings(action="ignore", category=torch.jit._trace.TracerWarning) + warnings.filterwarnings(action="ignore", category=UserWarning, module="torch.*") + warnings.filterwarnings(action="ignore", category=FutureWarning, module="torch.*") + warnings.filterwarnings( + action="ignore", category=DeprecationWarning, module="torch.*" + ) + + +# Instantiate the model only once for all tests. +# Mock from_pretrained to always return the initialized model. +# This speeds up tests and limits memory leaks. +cached_from_pretrained = make_cached_from_pretrained_fixture( + Model, skip_clone_repo=False +) + + +@pytest.fixture(scope="module", autouse=True) +def ensure_gc() -> None: + gc.collect() diff --git a/src/qai_hub_models/models/phowhisper_small/demo.py b/src/qai_hub_models/models/phowhisper_small/demo.py new file mode 100644 index 000000000..ce0b09407 --- /dev/null +++ b/src/qai_hub_models/models/phowhisper_small/demo.py @@ -0,0 +1,15 @@ +# --------------------------------------------------------------------- +# Copyright (c) 2025 Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause +# --------------------------------------------------------------------- + +from qai_hub_models.models._shared.hf_whisper.demo import hf_whisper_demo +from qai_hub_models.models.phowhisper_small.model import MODEL_ID, PhoWhisperSmall + + +def main(is_test: bool = False) -> None: + hf_whisper_demo(PhoWhisperSmall, MODEL_ID, is_test) + + +if __name__ == "__main__": + main() diff --git a/src/qai_hub_models/models/phowhisper_small/export.py b/src/qai_hub_models/models/phowhisper_small/export.py new file mode 100644 index 000000000..880dea6db --- /dev/null +++ b/src/qai_hub_models/models/phowhisper_small/export.py @@ -0,0 +1,638 @@ +# --------------------------------------------------------------------- +# Copyright (c) 2025 Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause +# --------------------------------------------------------------------- +# THIS FILE WAS AUTO-GENERATED. DO NOT EDIT MANUALLY. + + +from __future__ import annotations + +import os +import shutil +import tempfile +import warnings +from pathlib import Path +from typing import Any + +import qai_hub as hub + +from qai_hub_models import Precision, TargetRuntime +from qai_hub_models.common import SampleInputsType +from qai_hub_models.configs.model_metadata import ( + ChipsetAttributes, + ModelFileMetadata, + ModelMetadata, + merge_input_metadata, + merge_output_metadata, +) +from qai_hub_models.configs.tool_versions import ToolVersions +from qai_hub_models.models.phowhisper_small import MODEL_ID, App, Model +from qai_hub_models.utils import quantization as quantization_utils +from qai_hub_models.utils.args import ( + export_parser, + get_component_input_spec_kwargs, + get_export_model_name, + get_model_kwargs, +) +from qai_hub_models.utils.asset_loaders import ASSET_CONFIG +from qai_hub_models.utils.base_model import PretrainedCollectionModel +from qai_hub_models.utils.compare import torch_inference +from qai_hub_models.utils.export_result import CollectionExportResult, ComponentGroup +from qai_hub_models.utils.export_without_hub_access import export_without_hub_access +from qai_hub_models.utils.input_spec import InputSpec, to_hub_input_specs +from qai_hub_models.utils.onnx.helpers import download_and_unzip_workbench_onnx_model +from qai_hub_models.utils.path_helpers import get_next_free_path +from qai_hub_models.utils.printing import ( + print_inference_metrics, + print_profile_metrics_from_job, + print_tool_versions, +) +from qai_hub_models.utils.qai_hub_helpers import ( + assert_success_and_get_target_models, + can_access_qualcomm_ai_hub, +) + + +def quantize_model( + precision: Precision | dict[str, Precision], + model: PretrainedCollectionModel, + model_name: str, + onnx_models: ComponentGroup[hub.Model], + num_calibration_samples: int | None, + extra_options: str = "", + input_specs: dict[str, InputSpec] | None = None, + components: list[str] | None = None, +) -> ComponentGroup[hub.client.QuantizeJob]: + component_precisions = ( + model.get_mixed_precisions(precision) + if isinstance(precision, Precision) + else precision + ) + quantize_jobs: ComponentGroup[hub.client.QuantizeJob] = ComponentGroup() + input_specs = input_specs or model.get_input_spec() + for component_name in components or Model.component_class_names: + component_precision = component_precisions[component_name] + + if component_precision != Precision.float: + print(f"Quantizing {component_name}.") + if ( + not component_precision.activations_type + or not component_precision.weights_type + ): + raise ValueError( + "Quantization is only supported if both weights and activations are quantized." + ) + + calibration_data = quantization_utils.get_calibration_data( + model, + input_specs, + num_calibration_samples, + component_name=component_name, + app=App, + ) + quantize_jobs[component_name] = hub.submit_quantize_job( + model=onnx_models[component_name], + calibration_data=calibration_data, + activations_dtype=component_precision.activations_type, + weights_dtype=component_precision.weights_type, + name=f"{model_name}_{component_name}", + options=model.get_component_hub_quantize_options( + component_name, component_precision, extra_options + ), + ) + return quantize_jobs + + +def upload_model( + model: PretrainedCollectionModel, + input_specs: dict[str, InputSpec] | None = None, + components: list[str] | None = None, +) -> ComponentGroup[hub.Model]: + all_input_specs = input_specs or model.get_input_spec() + uploaded: ComponentGroup[hub.Model] = ComponentGroup() + for name in components or Model.component_class_names: + spec = all_input_specs[name] + with tempfile.TemporaryDirectory() as tmpdir: + uploaded[name] = hub.upload_model( + str(model.serialize_component(name, tmpdir, spec)) + ) + return uploaded + + +def compile_model( + model: PretrainedCollectionModel, + model_name: str, + device: hub.Device, + target_runtime: TargetRuntime, + precision: Precision, + source_models: ComponentGroup[hub.Model], + input_specs: dict[str, InputSpec] | None = None, + components: list[str] | None = None, + extra_options: str = "", +) -> ComponentGroup[hub.client.CompileJob]: + compile_jobs: ComponentGroup[hub.client.CompileJob] = ComponentGroup() + all_input_specs = input_specs or model.get_input_spec() + for component_name in components or Model.component_class_names: + input_spec = all_input_specs[component_name] + + model_compile_options = model.get_component_hub_compile_options( + component_name, target_runtime, precision, extra_options, device + ) + print(f"Optimizing model {component_name} to run on-device") + compile_jobs[component_name] = hub.submit_compile_job( + model=source_models[component_name], + input_specs=to_hub_input_specs(input_spec), + device=device, + name=f"{model_name}_{component_name}", + options=model_compile_options, + ) + return compile_jobs + + +def link_model( + compiled_models: ComponentGroup[hub.Model], + device: hub.Device, + model_name: str, + model: PretrainedCollectionModel, + target_runtime: TargetRuntime, + extra_options: str = "", +) -> ComponentGroup[hub.client.LinkJob]: + """Link compiled DLCs to context binary for AOT.""" + assert target_runtime.is_aot_compiled, ( + f"link_model() requires an AOT runtime, got {target_runtime}" + ) + link_jobs: ComponentGroup[hub.client.LinkJob] = ComponentGroup() + for component_name, compiled_model in compiled_models.items(): + link_options = model.get_component_hub_link_options( + component_name, target_runtime, extra_options + ) + print(f"Linking {component_name} to context binary") + link_jobs[component_name] = hub.submit_link_job( + [compiled_model], + device=device, + name=f"{model_name}_{component_name}", + options=link_options, + ) + return link_jobs + + +def profile_model( + model_name: str, + device: hub.Device, + options: ComponentGroup[str], + target_models: ComponentGroup[hub.Model], + components: list[str] | None = None, +) -> ComponentGroup[hub.client.ProfileJob]: + profile_jobs: ComponentGroup[hub.client.ProfileJob] = ComponentGroup() + for component_name in components or Model.component_class_names: + print(f"Profiling model {component_name} on a hosted device.") + profile_jobs[component_name] = hub.submit_profile_job( + model=target_models[component_name], + device=device, + name=f"{model_name}_{component_name}", + options=options.get(component_name, ""), + ) + return profile_jobs + + +def inference_model( + inputs: ComponentGroup[SampleInputsType], + model_name: str, + device: hub.Device, + options: ComponentGroup[str], + target_models: ComponentGroup[hub.Model], + components: list[str] | None = None, +) -> ComponentGroup[hub.client.InferenceJob]: + inference_jobs: ComponentGroup[hub.client.InferenceJob] = ComponentGroup() + for component_name in components or Model.component_class_names: + print( + f"Running inference for {component_name} on a hosted device with example inputs." + ) + inference_jobs[component_name] = hub.submit_inference_job( + model=target_models[component_name], + inputs=inputs[component_name], + device=device, + name=f"{model_name}_{component_name}", + options=options.get(component_name, ""), + ) + return inference_jobs + + +def download_model( + output_dir: os.PathLike | str, + model: PretrainedCollectionModel, + runtime: TargetRuntime, + precision: Precision, + tool_versions: ToolVersions, + target_models: ComponentGroup[hub.Model], + zip_assets: bool, + hub_device: hub.Device | None = None, +) -> Path: + output_folder_name = os.path.basename(output_dir) + output_path = get_next_free_path(output_dir) + + with tempfile.TemporaryDirectory() as tmpdir: + dst_path = Path(tmpdir) / output_folder_name + dst_path.mkdir() + + # Download models and capture filenames, then generate metadata + model_file_metadata = {} + for component_name, target_model in target_models.items(): + if target_model.model_type == hub.SourceModelType.ONNX: + onnx_result = download_and_unzip_workbench_onnx_model( + target_model, dst_path, component_name + ) + model_file_name = onnx_result.onnx_graph_name + else: + downloaded_path = target_model.download( + os.path.join(dst_path, component_name) + ) + model_file_name = os.path.basename(downloaded_path) + + # Generate metadata using the actual downloaded filename + model_file_metadata[model_file_name] = ModelFileMetadata.from_hub_model( + target_model + ) + # Merge semantic metadata from get_input_spec() + merge_input_metadata( + model_file_metadata[model_file_name], + model.get_component_input_spec(component_name), + ) + merge_output_metadata( + model_file_metadata[model_file_name], + model.get_component_output_spec(component_name), + ) + + # Extract and save metadata alongside downloaded model + metadata_path = dst_path / "metadata.json" + model_metadata = ModelMetadata( + model_id=MODEL_ID, + model_name="PhoWhisper-Small", + runtime=runtime, + precision=precision, + tool_versions=tool_versions, + model_files=model_file_metadata, + chipset_attributes=ChipsetAttributes.from_hub_device(hub_device) + if runtime.is_aot_compiled + else None, + ) + + # Dump supplementary files into the model folder + model.write_supplementary_files(dst_path, model_metadata) + + model_metadata.to_json(metadata_path) + if zip_assets: + output_path = Path( + shutil.make_archive( + str(output_path), + "zip", + root_dir=tmpdir, + base_dir=output_folder_name, + ) + ) + else: + shutil.move(dst_path, output_path) + + return output_path + + +def export_model( + device: hub.Device, + components: list[str] | None = None, + precision: Precision = Precision.float, + num_calibration_samples: int | None = None, + quantized_model_id: dict[str, str] | None = None, + skip_compiling: bool = False, + skip_profiling: bool = False, + skip_inferencing: bool = False, + skip_downloading: bool = False, + skip_summary: bool = False, + output_dir: str | None = None, + target_runtime: TargetRuntime = TargetRuntime.QNN_CONTEXT_BINARY, + compile_options: str = "", + quantize_options: str = "", + profile_options: str = "", + fetch_static_assets: str | None = None, + zip_assets: bool = False, + **additional_model_kwargs: Any, +) -> CollectionExportResult: + """ + This function executes the following recipe: + + 1. Instantiates a PyTorch model and converts it to a traced TorchScript format + 2. Converts the PyTorch model to ONNX and quantizes the ONNX model. + 3. Compiles the model to an asset that can be run on device + 4. Profiles the model performance on a real device + 5. Inferences the model on sample inputs + 6. Extracts relevant tool (eg. SDK) versions used to compile and profile this model + 7. Downloads the model asset to the local directory + 8. Summarizes the results from profiling and inference + + Each of the last 6 steps can be optionally skipped using the input options. + + Parameters + ---------- + device + Device for which to export the model (e.g., hub.Device("Samsung Galaxy S25")). + Full list of available devices can be found by running `hub.get_devices()`. + components + List of sub-components of the model that will be exported. + Each component is compiled and profiled separately. + Defaults to all components of the CollectionModel if not specified. + precision + The precision to which this model should be quantized. + Quantization is skipped if the precision is float. + num_calibration_samples + The number of calibration data samples + to use for quantization. If not set, uses the default number + specified by the dataset. If model doesn't have a calibration dataset + specified, this must be None. + quantized_model_id + A quantized ONNX hub model id, skips quantizing model. + skip_compiling + If set, skips compiling of model to format that can run on device. + skip_profiling + If set, skips profiling of compiled model on real devices. + skip_inferencing + If set, skips computing on-device outputs from sample data. + skip_downloading + If set, skips downloading of compiled model. + skip_summary + If set, skips waiting for and summarizing results + from profiling and inference. + output_dir + Directory to store generated assets (e.g. compiled model). + Defaults to `/export_assets`. + target_runtime + Which on-device runtime to target. Default is TFLite. + compile_options + Additional options to pass when submitting the compile job. + quantize_options + Additional options to pass when submitting the quantize job. + profile_options + Additional options to pass when submitting the profile job. + fetch_static_assets + If set, known assets are fetched from the given version rather than re-computing them. Can be passed as "latest" or "v". + zip_assets + If set, zip the assets after downloading. + **additional_model_kwargs + Additional optional kwargs used to customize + `model_cls.from_pretrained` and per-component `get_input_spec` + + Returns + ------- + CollectionExportResult + * A CompileJob object containing metadata about the compile job submitted to hub (None if compiling skipped). + * An InferenceJob containing metadata about the inference job (None if inferencing skipped). + * A ProfileJob containing metadata about the profile job (None if profiling skipped). + * A QuantizeJob object containing metadata about the quantize job submitted to hub + * The path to the downloaded model folder (or zip), or None if one or more of: skip_downloading is True, fetch_static_assets is set, or AI Hub Workbench is not accessible + """ + model_name = get_export_model_name( + Model, MODEL_ID, precision, additional_model_kwargs + ) + + output_path = Path(output_dir or Path.cwd() / "export_assets") + component_arg = components + components = components or Model.component_class_names + for component_name in components: + if component_name not in Model.component_class_names: + raise ValueError(f"Invalid component {component_name}.") + if fetch_static_assets or not can_access_qualcomm_ai_hub(): + static_model_path = export_without_hub_access( + MODEL_ID, + device, + skip_profiling, + skip_inferencing, + skip_downloading, + skip_summary, + output_path, + target_runtime, + precision, + quantize_options + compile_options + profile_options, + component_arg, + qaihm_version_tag=fetch_static_assets, + ) + return CollectionExportResult(download_path=static_model_path) + + hub_device = hub.get_devices( + name=device.name, attributes=device.attributes, os=device.os + )[-1] + chipset_attr = next( + (attr for attr in hub_device.attributes if "chipset" in attr), None + ) + chipset = chipset_attr.split(":")[-1] if chipset_attr else None + + # 1. Instantiates a PyTorch model and converts it to a traced TorchScript format + model = Model.from_pretrained( + **get_model_kwargs(Model, dict(**additional_model_kwargs, precision=precision)) + ) + input_specs: ComponentGroup[InputSpec] = ComponentGroup( + { + name: model.components[name].get_input_spec( + **get_component_input_spec_kwargs(Model, name, additional_model_kwargs) + ) + for name in components + } + ) + source_models_to_compile = upload_model(model, input_specs, components) + + # 2. Converts the PyTorch model to ONNX and quantizes the ONNX model. + quantize_jobs: ComponentGroup[hub.client.QuantizeJob] | None = None + quantized_models: ComponentGroup[hub.Model] | None = None + if precision != Precision.float: + if quantized_model_id: + quantized_models = ComponentGroup( + { + component: hub_model + for component in components + if (hub_model := hub.get_model(quantized_model_id[component])) + is not None + } + ) + else: + component_precisions = model.get_mixed_precisions(precision) + onnx_compile_result = compile_model( + model, + model_name, + device, + TargetRuntime.ONNX, + precision, + source_models_to_compile, + input_specs=input_specs, + components=[ + c + for c, p in component_precisions.items() + if c in components and p != Precision.float + ], + ) + onnx_models = assert_success_and_get_target_models(onnx_compile_result) + quantize_jobs = quantize_model( + component_precisions, + model, + model_name, + onnx_models, + num_calibration_samples, + quantize_options, + input_specs, + components, + ) + if skip_compiling: + return CollectionExportResult(quantize_jobs=quantize_jobs) + quantized_models = assert_success_and_get_target_models(quantize_jobs) + + # 3. Compiles the model to an asset that can be run on device + if quantized_models: + source_models_to_compile |= quantized_models + compile_result = compile_model( + model, + model_name, + device, + target_runtime, + precision, + source_models_to_compile, + input_specs=input_specs, + components=components, + extra_options=compile_options, + ) + + link_result: ComponentGroup[hub.client.LinkJob] | None = None + target_models: ComponentGroup[hub.Model] + if target_runtime.uses_hub_link: + compiled_models = assert_success_and_get_target_models(compile_result) + link_result = link_model( + compiled_models, + device, + model_name, + model, + target_runtime, + ) + target_models = assert_success_and_get_target_models(link_result or compile_result) + + # 4. Profiles the model performance on a real device + profile_result: ComponentGroup[hub.client.ProfileJob] | None = None + if not skip_profiling: + profile_result = profile_model( + model_name, + device, + model.get_hub_profile_options(target_runtime, profile_options), + target_models, + components, + ) + + # 5. Inferences the model on sample inputs + inference_result: ComponentGroup[hub.client.InferenceJob] | None = None + if not skip_inferencing: + inference_result = inference_model( + model.sample_inputs( + input_specs=input_specs, + use_channel_last_format=target_runtime.channel_last_native_execution, + ), + model_name, + device, + model.get_hub_profile_options(target_runtime, profile_options), + target_models, + components, + ) + + # 6. Extracts relevant tool (eg. SDK) versions used to compile and profile this model + tool_versions: ToolVersions | None = None + tool_versions_are_from_device_job = False + if not skip_summary or not skip_downloading: + profile_job = ( + next(iter(profile_result.values()), None) if profile_result else None + ) + inference_job = ( + next(iter(inference_result.values()), None) if inference_result else None + ) + compile_job = next(iter(compile_result.values()), None) + if profile_job is not None and profile_job.wait(): + tool_versions = ToolVersions.from_job(profile_job) + tool_versions_are_from_device_job = True + elif inference_job is not None and inference_job.wait(): + tool_versions = ToolVersions.from_job(inference_job) + tool_versions_are_from_device_job = True + elif compile_job and compile_job.wait(): + tool_versions = ToolVersions.from_job(compile_job) + + # 7. Downloads the model asset to the local directory + downloaded_model_path: Path | None = None + if not skip_downloading and tool_versions is not None: + model_directory = output_path / ASSET_CONFIG.get_release_asset_name( + MODEL_ID, target_runtime, precision, chipset + ) + downloaded_model_path = download_model( + model_directory, + model, + target_runtime, + precision, + tool_versions, + target_models, + zip_assets, + hub_device=hub_device, + ) + + # 8. Summarizes the results from profiling and inference + if not skip_summary and profile_result is not None: + for profile_job in profile_result.values(): + assert profile_job.wait().success, "Job failed: " + profile_job.url + profile_data: dict[str, Any] = profile_job.download_profile() + print_profile_metrics_from_job(profile_job, profile_data) + + if not skip_summary and inference_result is not None: + for component_name in components: + component = model.components[component_name] + inference_job = inference_result[component_name] + sample_inputs = component.sample_inputs( + input_specs[component_name], use_channel_last_format=False + ) + torch_out = torch_inference( + component, + sample_inputs, + return_channel_last_output=target_runtime.channel_last_native_execution, + ) + assert inference_job.wait().success, "Job failed: " + inference_job.url + ij_output = inference_job.download_output_data() + assert ij_output is not None + print_inference_metrics( + inference_job, ij_output, torch_out, component.get_output_names() + ) + + if not skip_summary: + print_tool_versions(tool_versions, tool_versions_are_from_device_job) + + if downloaded_model_path: + print(f"{model_name} was saved to {downloaded_model_path}\n") + + return CollectionExportResult( + quantize_jobs=quantize_jobs if precision != Precision.float else None, + compile_jobs=compile_result, + link_jobs=link_result, + profile_jobs=profile_result, + inference_jobs=inference_result, + download_path=downloaded_model_path, + tool_versions=tool_versions, + ) + + +def main() -> None: + warnings.filterwarnings("ignore") + supported_precision_runtimes: dict[Precision, list[TargetRuntime]] = { + Precision.float: [ + TargetRuntime.QNN_CONTEXT_BINARY, + TargetRuntime.PRECOMPILED_QNN_ONNX, + ], + } + + parser = export_parser( + model_cls=Model, + export_fn=export_model, + supported_precision_runtimes=supported_precision_runtimes, + default_export_device="Samsung Galaxy S25 (Family)", + ) + args = parser.parse_args() + export_model(**vars(args)) + + +if __name__ == "__main__": + main() diff --git a/src/qai_hub_models/models/phowhisper_small/info.yaml b/src/qai_hub_models/models/phowhisper_small/info.yaml new file mode 100644 index 000000000..883c5263e --- /dev/null +++ b/src/qai_hub_models/models/phowhisper_small/info.yaml @@ -0,0 +1,34 @@ +name: PhoWhisper-Small +id: phowhisper_small +status: pending +headline: Vietnamese-specialized automatic speech recognition (ASR) model, fine-tuned from Whisper-Small for accurate on-device transcription of Vietnamese speech. +domain: Audio +description: PhoWhisper-Small is a Vietnamese fine-tune of OpenAI's Whisper-Small ASR (Automatic Speech Recognition) model. It shares the same transformer encoder-decoder architecture as Whisper-Small but is trained on a large, diverse Vietnamese speech corpus covering multiple dialects, substantially improving Vietnamese transcription accuracy over the generic multilingual checkpoint. It fills the gap of having no Vietnamese-specialized ASR model in the catalog. Like Whisper-Small, it has been optimized for edge inference by replacing Multi-Head Attention (MHA) with Single-Head Attention (SHA) and linear layers with convolutional (conv) layers, and it transcribes audio clips up to 30 seconds long. Time to the first token is the encoder's latency, while time to each additional token is decoder's latency, where we assume a max decoded length specified below. +use_case: Speech Recognition +tags: [] +applicable_scenarios: +- Voice Assistants +- Accessibility +- Meeting Transcription +related_models: +- whisper_small +form_factors: +- Phone +- Tablet +has_static_banner: false +has_animated_banner: false +dataset: [] +technical_details: + Model checkpoint: vinai/PhoWhisper-small + Input resolution: 80x3000 (30 seconds audio) + Max decoded sequence length: 200 tokens + Number of parameters (encoder): 102M + Model size (encoder) (float): 391 MB + Number of parameters (decoder): 139M + Model size (decoder) (float): 533 MB + Vietnamese WER (FLEURS): 11.0% (vs 21.4% for generic Whisper-Small) +license_type: bsd-3-clause +research_paper: https://arxiv.org/abs/2406.02555 +research_paper_title: 'PhoWhisper: Automatic Speech Recognition for Vietnamese' +source_repo: https://github.com/VinAIResearch/PhoWhisper +license: https://github.com/VinAIResearch/PhoWhisper/blob/main/LICENSE diff --git a/src/qai_hub_models/models/phowhisper_small/model.py b/src/qai_hub_models/models/phowhisper_small/model.py new file mode 100644 index 000000000..e52bae9b8 --- /dev/null +++ b/src/qai_hub_models/models/phowhisper_small/model.py @@ -0,0 +1,25 @@ +# --------------------------------------------------------------------- +# Copyright (c) 2025 Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause +# --------------------------------------------------------------------- + +from __future__ import annotations + +from qai_hub_models.models._shared.hf_whisper.model import ( + HfWhisper, + HfWhisperDecoder, + HfWhisperEncoder, +) +from qai_hub_models.utils.base_model import CollectionModel + +MODEL_ID = __name__.split(".")[-2] +MODEL_ASSET_VERSION = 1 +WHISPER_VERSION = "vinai/PhoWhisper-small" + + +@CollectionModel.add_component(HfWhisperEncoder, "encoder") +@CollectionModel.add_component(HfWhisperDecoder, "decoder") +class PhoWhisperSmall(HfWhisper): + @classmethod + def get_hf_whisper_version(cls) -> str: + return WHISPER_VERSION diff --git a/src/qai_hub_models/models/phowhisper_small/requirements.txt b/src/qai_hub_models/models/phowhisper_small/requirements.txt new file mode 100644 index 000000000..73dfdcba5 --- /dev/null +++ b/src/qai_hub_models/models/phowhisper_small/requirements.txt @@ -0,0 +1,6 @@ +jiwer==3.0.3 +transformers==4.56.2 +audio2numpy==0.1.2 +scipy>=1.8.1,<=1.17.1 # 1.8.1 is for AIMET +sounddevice==0.5.2 +pyaudio==0.2.14; sys_platform == 'win32' # provides portaudio libraries on Windows diff --git a/src/qai_hub_models/models/phowhisper_small/test.py b/src/qai_hub_models/models/phowhisper_small/test.py new file mode 100644 index 000000000..de06c233a --- /dev/null +++ b/src/qai_hub_models/models/phowhisper_small/test.py @@ -0,0 +1,23 @@ +# --------------------------------------------------------------------- +# Copyright (c) 2025 Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause +# --------------------------------------------------------------------- + +from qai_hub_models.models._shared.hf_whisper.test_utils import ( + run_test_transcribe, + run_test_wrapper_numerics, +) +from qai_hub_models.models.phowhisper_small.demo import main as demo_main +from qai_hub_models.models.phowhisper_small.model import PhoWhisperSmall + + +def test_numerics() -> None: + run_test_wrapper_numerics(PhoWhisperSmall) + + +def test_transcribe() -> None: + run_test_transcribe(PhoWhisperSmall) + + +def test_demo() -> None: + demo_main(is_test=True) diff --git a/src/qai_hub_models/models/phowhisper_small/test_generated.py b/src/qai_hub_models/models/phowhisper_small/test_generated.py new file mode 100644 index 000000000..73a3eb50c --- /dev/null +++ b/src/qai_hub_models/models/phowhisper_small/test_generated.py @@ -0,0 +1,391 @@ +# --------------------------------------------------------------------- +# Copyright (c) 2025 Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause +# --------------------------------------------------------------------- +# THIS FILE WAS AUTO-GENERATED. DO NOT EDIT MANUALLY. + +from __future__ import annotations + +import os +from collections.abc import Generator +from pathlib import Path + +import numpy as np +import pytest +import qai_hub as hub +import torch + +import qai_hub_models.models.phowhisper_small as _model_module +from qai_hub_models import Precision, TargetRuntime +from qai_hub_models.models.phowhisper_small import MODEL_ID, Model +from qai_hub_models.models.phowhisper_small.export import ( + compile_model, + export_model, + inference_model, + link_model, + profile_model, + quantize_model, + upload_model, +) +from qai_hub_models.scorecard import ( + ScorecardCompilePath, + ScorecardDevice, + ScorecardProfilePath, +) +from qai_hub_models.scorecard.errors import CachedScorecardJobError +from qai_hub_models.scorecard.execution_helpers import ( + get_compile_parameterized_pytest_config, + get_evaluation_parameterized_pytest_config, + get_export_parameterized_pytest_config, + get_link_parameterized_pytest_config, + get_profile_parameterized_pytest_config, + get_quantize_parameterized_pytest_config, + needs_pre_quantize_compile, + pytest_device_idfn, +) +from qai_hub_models.scorecard.utils.testing import skip_invalid_runtime_device +from qai_hub_models.scorecard.utils.testing_export_eval import ( + accuracy_on_sample_inputs_via_export, + compile_via_export, + export_test_e2e, + inference_via_export, + link_via_export, + on_device_inference_for_accuracy_validation, + pre_quantize_compile_via_export, + profile_via_export, + quantize_via_export, + split_and_group_accuracy_validation_output_batches, + torch_inference_for_accuracy_validation, + torch_inference_for_accuracy_validation_outputs, +) +from qai_hub_models.utils.input_spec import InputSpec +from qai_hub_models.utils.validation import perform_runtime_model_validation + +# All runtime + precision pairs that are enabled for testing and are compatibile with this model. +# NOTE: +# Certain supported pairs may be excluded from this list if they are not enabled for testing. +# For example, models that allow JIT (on-device) compile will not test AOT runtimes; we assume that if it works on JIT it will work on AOT. +ENABLED_PRECISION_RUNTIMES: dict[Precision, list[TargetRuntime]] = { + Precision.float: [ + TargetRuntime.QNN_CONTEXT_BINARY, + TargetRuntime.PRECOMPILED_QNN_ONNX, + ], +} + + +# All runtime + precision pairs that are enabled for testing and have no known failure reasons. +# NOTE: +# Certain supported pairs may be excluded from this list if they are not enabled for testing. +# For example, models that allow JIT (on-device) compile will not test AOT runtimes; we assume that if it works on JIT it will work on AOT. +PASSING_PRECISION_RUNTIMES: dict[Precision, list[TargetRuntime]] = { + Precision.float: [ + TargetRuntime.QNN_CONTEXT_BINARY, + TargetRuntime.PRECOMPILED_QNN_ONNX, + ], +} + + +EVAL_DEVICE = ScorecardDevice.get("Samsung Galaxy S25 (Family)") +HAS_EVAL_DATASET = len(Model.get_eval_dataset_classes()) > 0 + + +@pytest.mark.compile +def test_runtime_model_validation() -> None: + perform_runtime_model_validation( + Model, MODEL_ID, getattr(_model_module, "App", None) + ) + + +@pytest.mark.pre_quantize_compile +@pytest.mark.skipif( + not needs_pre_quantize_compile( + MODEL_ID, ENABLED_PRECISION_RUNTIMES, PASSING_PRECISION_RUNTIMES + ), + reason="Model does not require pre-quantize compile step", +) +def test_pre_quantize_compile() -> None: + pre_quantize_compile_via_export( + compile_model, + MODEL_ID, + Model.from_pretrained(), + upload_model, + ) + + +@pytest.mark.parametrize( + "precision", + get_quantize_parameterized_pytest_config( + MODEL_ID, ENABLED_PRECISION_RUNTIMES, PASSING_PRECISION_RUNTIMES + ), + ids=pytest_device_idfn, +) +@pytest.mark.quantize +def test_quantize(precision: Precision) -> None: + try: + quantize_via_export( + quantize_model, + MODEL_ID, + Model.from_pretrained(), + precision, + ) + except CachedScorecardJobError as e: + pytest.skip(str(e)) + + +ALL_COMPONENTS = Model.component_class_names + + +@pytest.mark.parametrize( + ("precision", "scorecard_path", "device"), + get_compile_parameterized_pytest_config( + MODEL_ID, ENABLED_PRECISION_RUNTIMES, PASSING_PRECISION_RUNTIMES + ), + ids=pytest_device_idfn, +) +@pytest.mark.compile +def test_compile( + precision: Precision, scorecard_path: ScorecardCompilePath, device: ScorecardDevice +) -> None: + skip_invalid_runtime_device(Model, scorecard_path.runtime, device) + try: + compile_via_export( + compile_model, + MODEL_ID, + Model.from_pretrained(), + precision, + scorecard_path, + device, + upload_model=upload_model, + ) + except CachedScorecardJobError as e: + pytest.skip(str(e)) + + +@pytest.mark.parametrize( + ("precision", "scorecard_path", "device"), + get_link_parameterized_pytest_config( + MODEL_ID, ENABLED_PRECISION_RUNTIMES, PASSING_PRECISION_RUNTIMES + ), + ids=pytest_device_idfn, +) +@pytest.mark.link +def test_link( + precision: Precision, scorecard_path: ScorecardCompilePath, device: ScorecardDevice +) -> None: + skip_invalid_runtime_device(Model, scorecard_path.runtime, device) + try: + link_via_export( + link_model, + MODEL_ID, + Model.from_pretrained(), + precision, + scorecard_path, + device, + ) + except CachedScorecardJobError as e: + pytest.skip(str(e)) + + +@pytest.mark.parametrize( + ("precision", "scorecard_path", "device"), + get_profile_parameterized_pytest_config( + MODEL_ID, ENABLED_PRECISION_RUNTIMES, PASSING_PRECISION_RUNTIMES + ), + ids=pytest_device_idfn, +) +@pytest.mark.profile +def test_profile( + precision: Precision, scorecard_path: ScorecardProfilePath, device: ScorecardDevice +) -> None: + skip_invalid_runtime_device(Model, scorecard_path.runtime, device) + try: + profile_via_export( + profile_model, + MODEL_ID, + Model.from_pretrained(), + precision, + scorecard_path, + device, + ) + except CachedScorecardJobError as e: + pytest.skip(str(e)) + + +@pytest.mark.parametrize( + ("precision", "scorecard_path", "device"), + get_evaluation_parameterized_pytest_config( + MODEL_ID, + EVAL_DEVICE, + ENABLED_PRECISION_RUNTIMES, + PASSING_PRECISION_RUNTIMES, + ), + ids=pytest_device_idfn, +) +@pytest.mark.inference +def test_inference( + precision: Precision, scorecard_path: ScorecardProfilePath, device: ScorecardDevice +) -> None: + skip_invalid_runtime_device(Model, scorecard_path.runtime, device) + try: + if HAS_EVAL_DATASET: + on_device_inference_for_accuracy_validation( + Model, + Model.get_eval_dataset_classes()[0], + MODEL_ID, + precision, + scorecard_path, + device, + ) + else: + inference_via_export( + inference_model, + MODEL_ID, + Model.from_pretrained(), + precision, + scorecard_path, + device, + ) + except CachedScorecardJobError as e: + pytest.skip(str(e)) + + +@pytest.mark.inference +def test_val_data_torch() -> None: + if not HAS_EVAL_DATASET: + return + torch_inference_for_accuracy_validation( + Model.from_pretrained(), Model.get_eval_dataset_classes()[0], MODEL_ID + ) + + +@pytest.fixture(scope="module") +def torch_val_outputs() -> list[np.ndarray]: + """ + Because the below method downloads a dataset over the internet, + it is called in a fixture so it can be reused. + """ + if not HAS_EVAL_DATASET: + return [] + return torch_inference_for_accuracy_validation_outputs(MODEL_ID) + + +@pytest.fixture(scope="module") +def torch_evaluate_mock_outputs( + torch_val_outputs: list[np.ndarray], +) -> list[torch.Tensor | tuple[torch.Tensor, ...]]: + """ + Because the below method does some memory movement, + it is called in a fixture so its output can be reused. + """ + if not HAS_EVAL_DATASET: + return [] + return split_and_group_accuracy_validation_output_batches(torch_val_outputs) + + +@pytest.mark.parametrize( + ("precision", "scorecard_path", "device"), + get_evaluation_parameterized_pytest_config( + MODEL_ID, + EVAL_DEVICE, + ENABLED_PRECISION_RUNTIMES, + PASSING_PRECISION_RUNTIMES, + ), + ids=pytest_device_idfn, +) +@pytest.mark.compute_device_accuracy +def test_val_accuracy( + precision: Precision, + scorecard_path: ScorecardProfilePath, + device: ScorecardDevice, + torch_val_outputs: list[np.ndarray], + torch_evaluate_mock_outputs: list[torch.Tensor | tuple[torch.Tensor, ...]], +) -> None: + try: + accuracy_on_sample_inputs_via_export( + export_model, + MODEL_ID, + Model.from_pretrained(), + precision, + scorecard_path, + device, + ) + except CachedScorecardJobError as e: + pytest.skip(str(e)) + + +@pytest.mark.parametrize( + ("precision", "scorecard_path", "device"), + get_export_parameterized_pytest_config( + MODEL_ID, + EVAL_DEVICE, + ENABLED_PRECISION_RUNTIMES, + PASSING_PRECISION_RUNTIMES, + requires_aot_prepare=True, + ), + ids=pytest_device_idfn, +) +@pytest.mark.export +def test_export( + precision: Precision, scorecard_path: ScorecardProfilePath, device: ScorecardDevice +) -> None: + skip_invalid_runtime_device(Model, scorecard_path.runtime, device) + try: + export_test_e2e( + export_model, + Model, + MODEL_ID, + precision, + scorecard_path, + device, + ALL_COMPONENTS, + ) + except CachedScorecardJobError as e: + pytest.skip(str(e)) + + +# Cache serialize() and hub.upload_model() across the module so the same +# (component, graph, input_spec) is serialized once and the resulting bytes are +# uploaded once -- matters most for multi-GB AIMET LLM bundles. +@pytest.fixture(scope="module", autouse=True) +def cached_serialize_for_export( + tmp_path_factory: pytest.TempPathFactory, +) -> Generator[pytest.MonkeyPatch, None, None]: + cache_dir = tmp_path_factory.mktemp("serialize_cache") + with pytest.MonkeyPatch.context() as mp: + model_cache: dict[str, Path] = {} + upload_cache: dict[str, hub.Model] = {} + + real_upload_model = hub.upload_model + + def _cached_upload_model( + model: hub.client.SourceModel | str, + name: str | None = None, + project: str | hub.client.Project | None = None, + ) -> hub.Model: + key = str(model) + cached = upload_cache.get(key) + if cached is None: + cached = real_upload_model(model, name, project) + upload_cache[key] = cached + return cached + + mp.setattr(hub, "upload_model", _cached_upload_model) + serialize_component = Model.serialize_component + + def _cached_serialize_component( + self: Model, + component_name: str, + output_dir: str | os.PathLike, + input_spec: InputSpec | None = None, + ) -> Path: + model_key = component_name + str(input_spec) + cached = model_cache.get(model_key) + if not cached: + cached = serialize_component( + self, component_name, cache_dir, input_spec + ) + model_cache[model_key] = cached + return cached + + mp.setattr(Model, "serialize_component", _cached_serialize_component) + yield mp