From e016ed74336dd64e58d1e3049288720e8f3e410c Mon Sep 17 00:00:00 2001 From: Raju Date: Wed, 15 Apr 2026 12:39:11 +0530 Subject: [PATCH 01/19] QVAC-17057 feat: add bci-whispercpp package for BCI neural signal transcription Add a new @qvac/bci-whispercpp addon that transcribes brain-computer interface neural signals into text using a modified whisper.cpp backend. This POC includes: - C++ native addon with BCI model inference (NeuralProcessor, BCIModel, BCIConfig) built on the qvac addon-cpp framework - CMake + vcpkg build system with whisper-cpp overlay ports carrying BCI-specific patches (variable conv1 kernel, windowed attention) - JavaScript API: BCIWhispercpp class with batch transcribeFile/transcribe - Integration tests for load/destroy and batch transcription - Example script and model conversion tooling - WER utility for accuracy measurement Streaming transcription will be added in a follow-up PR (QVAC-17062). Made-with: Cursor --- packages/bci-whispercpp/.gitignore | 9 + packages/bci-whispercpp/CMakeLists.txt | 124 +++++ packages/bci-whispercpp/LICENSE | 179 +++++++ packages/bci-whispercpp/NOTICE | 23 + packages/bci-whispercpp/README.md | 196 ++++++++ .../addon/src/addon/AddonJs.hpp | 160 ++++++ .../addon/src/addon/BCIErrors.hpp | 25 + .../addon/src/js-interface/JSAdapter.cpp | 129 +++++ .../addon/src/js-interface/JSAdapter.hpp | 48 ++ .../addon/src/js-interface/binding.cpp | 39 ++ .../addon/src/model-interface/BCITypes.hpp | 28 ++ .../src/model-interface/bci/BCIConfig.cpp | 150 ++++++ .../src/model-interface/bci/BCIConfig.hpp | 44 ++ .../src/model-interface/bci/BCIModel.cpp | 347 +++++++++++++ .../src/model-interface/bci/BCIModel.hpp | 130 +++++ .../model-interface/bci/NeuralProcessor.cpp | 241 +++++++++ .../model-interface/bci/NeuralProcessor.hpp | 62 +++ .../bci-whispercpp/addon/tests/test_core.cpp | 102 ++++ packages/bci-whispercpp/bci.js | 300 ++++++++++++ packages/bci-whispercpp/binding.js | 1 + packages/bci-whispercpp/configChecker.js | 82 ++++ .../examples/transcribe-neural.js | 105 ++++ packages/bci-whispercpp/index.d.ts | 100 ++++ packages/bci-whispercpp/index.js | 172 +++++++ packages/bci-whispercpp/lib/error.js | 76 +++ packages/bci-whispercpp/lib/wer.js | 40 ++ packages/bci-whispercpp/package.json | 77 +++ .../bci-whispercpp/scripts/convert-model.py | 459 ++++++++++++++++++ .../bci-whispercpp/scripts/download-models.sh | 22 + .../test/fixtures/manifest.json | 54 +++ .../test/integration/bci-addon.test.js | 69 +++ .../test/integration/helpers.js | 34 ++ .../bci-whispercpp/vcpkg-configuration.json | 17 + .../qvac-lint-cpp/portfile.cmake | 7 + .../vcpkg-overlays/qvac-lint-cpp/vcpkg.json | 5 + .../whisper-cpp/0001-fix-vcpkg-build.patch | 277 +++++++++++ ...0002-fix-apple-silicon-cross-compile.patch | 15 + .../0003-bci-variable-conv1-kernel.patch | 28 ++ .../0004-bci-windowed-attention.patch | 97 ++++ .../vcpkg-overlays/whisper-cpp/portfile.cmake | 57 +++ .../vcpkg-overlays/whisper-cpp/vcpkg.json | 18 + packages/bci-whispercpp/vcpkg.json | 18 + 42 files changed, 4166 insertions(+) create mode 100644 packages/bci-whispercpp/.gitignore create mode 100644 packages/bci-whispercpp/CMakeLists.txt create mode 100644 packages/bci-whispercpp/LICENSE create mode 100644 packages/bci-whispercpp/NOTICE create mode 100644 packages/bci-whispercpp/README.md create mode 100644 packages/bci-whispercpp/addon/src/addon/AddonJs.hpp create mode 100644 packages/bci-whispercpp/addon/src/addon/BCIErrors.hpp create mode 100644 packages/bci-whispercpp/addon/src/js-interface/JSAdapter.cpp create mode 100644 packages/bci-whispercpp/addon/src/js-interface/JSAdapter.hpp create mode 100644 packages/bci-whispercpp/addon/src/js-interface/binding.cpp create mode 100644 packages/bci-whispercpp/addon/src/model-interface/BCITypes.hpp create mode 100644 packages/bci-whispercpp/addon/src/model-interface/bci/BCIConfig.cpp create mode 100644 packages/bci-whispercpp/addon/src/model-interface/bci/BCIConfig.hpp create mode 100644 packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp create mode 100644 packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.hpp create mode 100644 packages/bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.cpp create mode 100644 packages/bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.hpp create mode 100644 packages/bci-whispercpp/addon/tests/test_core.cpp create mode 100644 packages/bci-whispercpp/bci.js create mode 100644 packages/bci-whispercpp/binding.js create mode 100644 packages/bci-whispercpp/configChecker.js create mode 100644 packages/bci-whispercpp/examples/transcribe-neural.js create mode 100644 packages/bci-whispercpp/index.d.ts create mode 100644 packages/bci-whispercpp/index.js create mode 100644 packages/bci-whispercpp/lib/error.js create mode 100644 packages/bci-whispercpp/lib/wer.js create mode 100644 packages/bci-whispercpp/package.json create mode 100644 packages/bci-whispercpp/scripts/convert-model.py create mode 100755 packages/bci-whispercpp/scripts/download-models.sh create mode 100644 packages/bci-whispercpp/test/fixtures/manifest.json create mode 100644 packages/bci-whispercpp/test/integration/bci-addon.test.js create mode 100644 packages/bci-whispercpp/test/integration/helpers.js create mode 100644 packages/bci-whispercpp/vcpkg-configuration.json create mode 100644 packages/bci-whispercpp/vcpkg-overlays/qvac-lint-cpp/portfile.cmake create mode 100644 packages/bci-whispercpp/vcpkg-overlays/qvac-lint-cpp/vcpkg.json create mode 100644 packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/0001-fix-vcpkg-build.patch create mode 100644 packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/0002-fix-apple-silicon-cross-compile.patch create mode 100644 packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/0003-bci-variable-conv1-kernel.patch create mode 100644 packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/0004-bci-windowed-attention.patch create mode 100644 packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/portfile.cmake create mode 100644 packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/vcpkg.json create mode 100644 packages/bci-whispercpp/vcpkg.json diff --git a/packages/bci-whispercpp/.gitignore b/packages/bci-whispercpp/.gitignore new file mode 100644 index 0000000000..33aefedf56 --- /dev/null +++ b/packages/bci-whispercpp/.gitignore @@ -0,0 +1,9 @@ +node_modules/ +build/ +prebuilds/ +models/ +package-lock.json +test/fixtures/*.bin +.clang-format +.clang-tidy +.valgrind.supp diff --git a/packages/bci-whispercpp/CMakeLists.txt b/packages/bci-whispercpp/CMakeLists.txt new file mode 100644 index 0000000000..dfb91051d8 --- /dev/null +++ b/packages/bci-whispercpp/CMakeLists.txt @@ -0,0 +1,124 @@ +cmake_minimum_required(VERSION 3.25) + +option(BUILD_TESTING "Build tests" OFF) + +if(BUILD_TESTING) + list(APPEND VCPKG_MANIFEST_FEATURES "tests") +endif() + +find_package(cmake-bare REQUIRED PATHS node_modules/cmake-bare) +find_package(cmake-vcpkg REQUIRED PATHS node_modules/cmake-vcpkg) + +set(VCPKG_OVERLAY_PORTS "${CMAKE_CURRENT_SOURCE_DIR}/vcpkg-overlays;${VCPKG_OVERLAY_PORTS}") + +project(bci-whispercpp CXX C) + +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + add_compile_options(-stdlib=libc++) + add_link_options(-stdlib=libc++ -static-libstdc++) +endif() + +find_path(QVAC_LIB_INFERENCE_ADDON_CPP_INCLUDE_DIRS "qvac-lib-inference-addon-cpp/ModelInterfaces.hpp") +find_package(whisper CONFIG REQUIRED) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_EXTENSIONS OFF) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) + +if(CMAKE_BUILD_TYPE STREQUAL "Debug") + add_definitions(-D_DEBUG) +endif() + +if(WIN32) + add_definitions(-DNOMINMAX -DWIN32_LEAN_AND_MEAN -DNOGDI) +endif() + +add_bare_module(bci-whispercpp EXPORTS) + +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + target_link_options(${bci-whispercpp}_module PRIVATE -Wl,--exclude-libs,ALL) +endif() + +target_sources( + ${bci-whispercpp} + PRIVATE + ${PROJECT_SOURCE_DIR}/addon/src/js-interface/binding.cpp + ${PROJECT_SOURCE_DIR}/addon/src/js-interface/JSAdapter.cpp + ${PROJECT_SOURCE_DIR}/addon/src/model-interface/bci/BCIConfig.cpp + ${PROJECT_SOURCE_DIR}/addon/src/model-interface/bci/BCIModel.cpp + ${PROJECT_SOURCE_DIR}/addon/src/model-interface/bci/NeuralProcessor.cpp +) + +target_include_directories( + ${bci-whispercpp} + PRIVATE + ${PROJECT_SOURCE_DIR}/addon + ${PROJECT_SOURCE_DIR}/addon/src + ${CMAKE_BINARY_DIR}/_bare/node_modules/bare-headers/include + ${QVAC_LIB_INFERENCE_ADDON_CPP_INCLUDE_DIRS} +) + +target_link_libraries( + ${bci-whispercpp} + PRIVATE + whisper::whisper +) + +target_compile_definitions(${bci-whispercpp} PUBLIC JS_LOGGER) + +if(WIN32) + target_link_libraries( + ${bci-whispercpp} + PRIVATE + msvcrt.lib + ) +endif() + +if(BUILD_TESTING) + find_package(GTest REQUIRED) + + set(CORE_SRCS + ${PROJECT_SOURCE_DIR}/addon/src/model-interface/bci/BCIConfig.cpp + ${PROJECT_SOURCE_DIR}/addon/src/model-interface/bci/BCIModel.cpp + ${PROJECT_SOURCE_DIR}/addon/src/model-interface/bci/NeuralProcessor.cpp + ) + + add_library(bci-core STATIC ${CORE_SRCS}) + + target_link_libraries(bci-core PRIVATE + whisper::whisper + ) + + target_include_directories(bci-core PRIVATE + ${PROJECT_SOURCE_DIR}/addon/ + ${PROJECT_SOURCE_DIR}/addon/src/ + ${CMAKE_BINARY_DIR}/_bare/node_modules/bare-headers/include + ${QVAC_LIB_INFERENCE_ADDON_CPP_INCLUDE_DIRS} + ) + + add_executable( + test-bci-core + ${PROJECT_SOURCE_DIR}/addon/tests/test_core.cpp + ) + + target_include_directories(test-bci-core PRIVATE + ${PROJECT_SOURCE_DIR}/addon/ + ${PROJECT_SOURCE_DIR}/addon/src/ + ${PROJECT_SOURCE_DIR}/addon/src/model-interface + ${PROJECT_SOURCE_DIR}/addon/src/model-interface/bci/ + ${PROJECT_SOURCE_DIR}/addon/tests/ + ${CMAKE_BINARY_DIR}/_bare/node_modules/bare-headers/include + ${QVAC_LIB_INFERENCE_ADDON_CPP_INCLUDE_DIRS} + ) + + target_link_libraries(test-bci-core PRIVATE + bci-core + whisper::whisper + GTest::gtest_main + GTest::gmock + ) + + set_target_properties(test-bci-core PROPERTIES + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/addon/tests + ) +endif() diff --git a/packages/bci-whispercpp/LICENSE b/packages/bci-whispercpp/LICENSE new file mode 100644 index 0000000000..7d199ae333 --- /dev/null +++ b/packages/bci-whispercpp/LICENSE @@ -0,0 +1,179 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + +Copyright 2026 Tether Data, S.A. de C.V. diff --git a/packages/bci-whispercpp/NOTICE b/packages/bci-whispercpp/NOTICE new file mode 100644 index 0000000000..3df664bfac --- /dev/null +++ b/packages/bci-whispercpp/NOTICE @@ -0,0 +1,23 @@ +@qvac/bci-whispercpp +Copyright 2026 Tether Data, S.A. de C.V. + +This product includes third-party components under their +respective licenses. @qvac/bci-whispercpp itself is licensed under +Apache-2.0; bundled dependencies are governed by the licenses +listed below. + +========================================================================= +Third-Party Software Licenses +========================================================================= + +--- MIT --- + + whisper.cpp + https://github.com/ggerganov/whisper.cpp + Copyright (c) 2023-2024 Georgi Gerganov + +--- MIT --- + + ggml + https://github.com/ggerganov/ggml + Copyright (c) 2023-2024 Georgi Gerganov diff --git a/packages/bci-whispercpp/README.md b/packages/bci-whispercpp/README.md new file mode 100644 index 0000000000..e19812caf3 --- /dev/null +++ b/packages/bci-whispercpp/README.md @@ -0,0 +1,196 @@ +# @qvac/bci-whispercpp + +Brain-Computer Interface (BCI) neural signal transcription addon for qvac, powered by [whisper.cpp](https://github.com/tetherto/whisper.cpp). + +Transcribes multi-channel neural signals (e.g., 512-channel microelectrode array recordings) into text using a BCI-trained whisper model running natively via GGML. Output matches the Python BrainWhisperer reference model exactly. + +## Architecture + +``` +Neural Signal (512ch, 20ms bins) + │ + ▼ +┌──────────────────────────────┐ +│ NeuralProcessor (C++) │ +│ - Gaussian smoothing │ std=2, kernel=100 +│ - Day-specific projection │ low-rank (A·B) + month + softsign +│ - Pad to 3000 frames │ mel-major layout for whisper.cpp +└──────────────┬───────────────┘ + │ mel features (512 × 3000) + ▼ +┌──────────────────────────────┐ +│ whisper.cpp (patched) │ +│ - conv1 (k=7, 512→384) │ BCI-trained embedder weights +│ - conv2 (k=3, stride=2) │ +│ - Positional encoding │ learned time PE + sinusoidal day PE +│ - 6-layer encoder │ windowed attention (w=57) on layers 0–3 +│ - 4-layer decoder (LoRA) │ beam search, length_penalty=0.14 +└──────────────┬───────────────┘ + │ + ▼ + Text output +``` + +## Results + +Native GGML inference matches the Python BrainWhisperer reference on all test samples: + +| Sample | Ground Truth | GGML Native Output | Python Reference | +|--------|-------------|-------------------|-----------------| +| 0 | "You can see the code at this point as well." | "You can see the good at this point as well." | "you can see the good at this point as well" | +| 1 | "How does it keep the cost down?" | "How does it keep the cost said?" | "how does it keep the cost said" | +| 2 | "Not too controversial." | "Not too controversial." | "not too controversial" | +| 3 | "The jury and a judge work together on it." | "The jury and a judge work together on it." | "the jury and a judge work together on it" | +| 4 | "Were quite vocal about it." | "We're quite vocal about it." | "we're quite vocal about it" | + +## Neural Signal Format + +Binary files with the following layout: + +| Offset | Type | Description | +|--------|-----------|------------------------------------------------------| +| 0 | uint32 | Number of timesteps | +| 4 | uint32 | Number of channels | +| 8 | float32[] | Feature data (row-major: `features[t * channels + c]`) | + +Each timestep represents a 20ms bin of neural activity. Channels correspond to individual electrodes in a microelectrode array (typically 512 channels). + +## Installation + +```bash +cd packages/bci-whispercpp +npm install +VCPKG_ROOT=/path/to/vcpkg npm run build +``` + +### Prerequisites + +- **Bare runtime** >= 1.19.0 +- **CMake** >= 3.25 +- **vcpkg** with `VCPKG_ROOT` environment variable set + +### Model Conversion + +Convert a trained BrainWhisperer checkpoint. This produces **two files**, both required for inference: + +| File | Size | Description | +|------|------|-------------| +| `ggml-bci-windowed.bin` | ~84 MB | GGML model: whisper encoder/decoder (LoRA-merged), tokenizer, positional embedding, windowed attention header | +| `bci-embedder.bin` | ~24 MB | Day projection weights: low-rank A·B matrices per recording day, month projections, session-to-day mapping | + +```bash +python3 scripts/convert-model.py \ + --checkpoint /path/to/epoch=93-val_wer=0.0910.ckpt +``` + +Both files are written to `models/` by default. All flags are optional: + +| Flag | Default | Description | +|------|---------|-------------| +| `--output` | `models/ggml-bci-windowed.bin` | GGML model output path | +| `--embedder-output` | `models/bci-embedder.bin` | Embedder weights output path | +| `--day-idx` | `1` | Day index for baked positional embedding | +| `--window-size` | `57` | Windowed attention size (0 to disable) | +| `--last-window-layer` | `3` | Last encoder layer with windowed attention | +| `--f32` | off | Use f32 for all tensors (avoids f16 precision loss, ~2x larger) | + +**Important:** Both files must be in the same directory at runtime. The C++ addon looks for `bci-embedder.bin` next to the GGML model file and will fail if it is missing. + +## Usage + +### Low-level API (BCIInterface) + +```javascript +const { BCIInterface } = require('@qvac/bci-whispercpp/bci') +const binding = require('@qvac/bci-whispercpp/binding') + +const config = { + contextParams: { model: '/path/to/ggml-bci.bin' }, + whisperConfig: { language: 'en', temperature: 0.0 }, + miscConfig: { caption_enabled: false }, + bciConfig: { day_idx: 1 } +} + +const onOutput = (addon, event, jobId, data, error) => { + if (event === 'Output') console.log('Segment:', data.text) + if (event === 'JobEnded') console.log('Done:', data) + if (event === 'Error') console.error('Error:', error) +} + +const model = new BCIInterface(binding, config, onOutput) +await model.activate() + +// Batch mode — pass entire signal at once +const neuralData = fs.readFileSync('signal.bin') +await model.runJob({ input: new Uint8Array(neuralData) }) + +// Streaming mode — send chunks then signal end +await model.append({ type: 'neural', input: chunk1 }) +await model.append({ type: 'neural', input: chunk2 }) +await model.append({ type: 'end of job' }) + +await model.destroyInstance() +``` + +## Testing + +### Integration Tests + +```bash +WHISPER_MODEL_PATH=./models/ggml-bci-windowed.bin npm run test:integration +``` + +### C++ Unit Tests + +```bash +VCPKG_ROOT=/path/to/vcpkg npm run test:cpp +``` + +## Configuration + +### whisperConfig + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `language` | string | `"en"` | Language code | +| `temperature` | number | `0.0` | Sampling temperature | +| `n_threads` | number | `0` (auto) | Number of threads | + +### bciConfig + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `day_idx` | number | `0` | Session day index for day-specific projection | + +### contextParams + +| Parameter | Type | Description | +|-----------|------|-------------| +| `model` | string | **Required.** Path to BCI GGML model file | +| `use_gpu` | boolean | Enable GPU acceleration | +| `flash_attn` | boolean | Enable flash attention | + +## whisper.cpp Patches + +The package includes a vcpkg overlay with 4 patches applied to whisper.cpp: + +| Patch | Description | +|-------|-------------| +| 0001 | Fix vcpkg build | +| 0002 | Fix Apple Silicon cross-compilation | +| 0003 | Variable conv1 kernel size (read `n_audio_conv1_kernel` from model header) | +| 0004 | Windowed attention mask, window size/layer params in header, BCI-specific SOS tokens | + +## Platform Support + +| Platform | Architecture | Status | +|----------|-------------|--------| +| macOS | arm64 (Apple Silicon) | Tested | +| Linux | x64 | Feasible (same build system as transcription-whispercpp) | +| Windows | x64 | Feasible (whisper.cpp supports MSVC) | +| Android | arm64 | Feasible (NDK toolchain) | +| iOS | arm64 | Feasible (Xcode toolchain) | + +## License + +Apache-2.0 diff --git a/packages/bci-whispercpp/addon/src/addon/AddonJs.hpp b/packages/bci-whispercpp/addon/src/addon/AddonJs.hpp new file mode 100644 index 0000000000..f5d8f7c40d --- /dev/null +++ b/packages/bci-whispercpp/addon/src/addon/AddonJs.hpp @@ -0,0 +1,160 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "model-interface/BCITypes.hpp" +#include "model-interface/bci/BCIModel.hpp" +#include "src/js-interface/JSAdapter.hpp" + +namespace qvac_lib_inference_addon_bci { + +namespace js = qvac_lib_inference_addon_cpp::js; +using qvac_lib_inference_addon_cpp::OutputQueue; + +inline void disableWhisperLogs( + enum ggml_log_level, const char*, void*) {} + +inline BCIConfig +createBCIConfig(js_env_t* env, const js::Object& configurationParams) { + JSAdapter adapter; + return adapter.loadFromJSObject(configurationParams, env); +} + +struct JsTranscriptOutputHandler + : qvac_lib_inference_addon_cpp::out_handl::JsBaseOutputHandler { + JsTranscriptOutputHandler() + : qvac_lib_inference_addon_cpp::out_handl::JsBaseOutputHandler< + Transcript>([this](const Transcript& output) -> js_value_t* { + auto jsTranscript = js::Object::create(this->env_); + jsTranscript.setProperty( + this->env_, "text", js::String::create(this->env_, output.text)); + jsTranscript.setProperty( + this->env_, "toAppend", + js::Boolean::create(this->env_, output.toAppend)); + jsTranscript.setProperty( + this->env_, "start", + js::Number::create(this->env_, output.start)); + jsTranscript.setProperty( + this->env_, "end", + js::Number::create(this->env_, output.end)); + jsTranscript.setProperty( + this->env_, "id", + js::Number::create(this->env_, static_cast(output.id))); + return jsTranscript; + }) {} +}; + +struct JsTranscriptArrayOutputHandler + : qvac_lib_inference_addon_cpp::out_handl::JsBaseOutputHandler< + std::vector> { + JsTranscriptArrayOutputHandler() + : qvac_lib_inference_addon_cpp::out_handl::JsBaseOutputHandler< + std::vector>( + [this](const std::vector& output) -> js_value_t* { + auto jsOutput = js::Array::create(this->env_); + for (size_t i = 0; i < output.size(); ++i) { + auto jsTranscript = js::Object::create(this->env_); + jsTranscript.setProperty( + this->env_, "text", + js::String::create(this->env_, output[i].text)); + jsTranscript.setProperty( + this->env_, "toAppend", + js::Boolean::create(this->env_, output[i].toAppend)); + jsTranscript.setProperty( + this->env_, "start", + js::Number::create(this->env_, output[i].start)); + jsTranscript.setProperty( + this->env_, "end", + js::Number::create(this->env_, output[i].end)); + jsTranscript.setProperty( + this->env_, "id", + js::Number::create( + this->env_, static_cast(output[i].id))); + jsOutput.set(this->env_, i, jsTranscript); + } + return jsOutput; + }) {} +}; + +inline js_value_t* createInstance(js_env_t* env, js_callback_info_t* info) try { + using namespace qvac_lib_inference_addon_cpp; + using namespace std; + + whisper_log_set(disableWhisperLogs, nullptr); + JsArgsParser args(env, info); + auto configurationParams = args.getJsObject(1, "configurationParams"); + + unique_ptr model = + make_unique(createBCIConfig(env, configurationParams)); + + out_handl::OutputHandlers outputHandlers; + outputHandlers.add(make_shared()); + outputHandlers.add(make_shared()); + unique_ptr callback = make_unique( + env, + args.get(0, "jsHandle"), + args.getFunction(2, "outputCallback"), + std::move(outputHandlers)); + + auto addon = make_unique(env, std::move(callback), std::move(model)); + return JsInterface::createInstance(env, std::move(addon)); +} +JSCATCH + +inline js_value_t* runJob(js_env_t* env, js_callback_info_t* info) try { + using namespace qvac_lib_inference_addon_cpp; + using namespace std; + + JsArgsParser args(env, info); + AddonJs& instance = JsInterface::getInstance(env, args.get(0, "instance")); + auto [type, jsInput] = JsInterface::getInput(args); + + if (type != "neural") { + throw qvac_errors::StatusError( + qvac_errors::general_error::InvalidArgument, + "Unknown input type: " + type + " (expected 'neural')"); + } + + vector neuralBytes = + js::TypedArray(env, jsInput).as>(env); + return instance.runJob(std::any(std::move(neuralBytes))); +} +JSCATCH + +inline js_value_t* reload(js_env_t* env, js_callback_info_t* info) try { + using namespace qvac_lib_inference_addon_cpp; + using namespace std; + + JsArgsParser args(env, info); + AddonJs& instance = JsInterface::getInstance(env, args.get(0, "instance")); + auto configurationParams = args.getJsObject(1, "configurationParams"); + BCIConfig config = createBCIConfig(env, configurationParams); + + return js::JsAsyncTask::run( + env, + [addonCpp = instance.addonCpp, config = std::move(config)]() mutable { + auto* bciModel = + dynamic_cast(&addonCpp->model.get()); + if (bciModel == nullptr) { + throw std::runtime_error("Invalid model type for reload"); + } + bciModel->setConfig(config); + }); +} +JSCATCH + +} // namespace qvac_lib_inference_addon_bci diff --git a/packages/bci-whispercpp/addon/src/addon/BCIErrors.hpp b/packages/bci-whispercpp/addon/src/addon/BCIErrors.hpp new file mode 100644 index 0000000000..5711fb5c53 --- /dev/null +++ b/packages/bci-whispercpp/addon/src/addon/BCIErrors.hpp @@ -0,0 +1,25 @@ +#pragma once + +#include +#include + +#include "qvac-lib-inference-addon-cpp/Errors.hpp" + +namespace qvac_lib_inference_addon_bci::errors { +constexpr const char* ADDON_ID = "BCI"; +} // namespace qvac_lib_inference_addon_bci::errors + +namespace qvac_errors { +namespace bci_error { +enum class Code : std::uint8_t { + InvalidNeuralSignal, + UnsupportedSignalFormat, + ProcessingFailed, +}; + +inline qvac_errors::StatusError +makeStatus(Code /*code*/, const std::string& message) { + return qvac_errors::StatusError("BCI", "BCIError", message); +} +} // namespace bci_error +} // namespace qvac_errors diff --git a/packages/bci-whispercpp/addon/src/js-interface/JSAdapter.cpp b/packages/bci-whispercpp/addon/src/js-interface/JSAdapter.cpp new file mode 100644 index 0000000000..58e60eeb47 --- /dev/null +++ b/packages/bci-whispercpp/addon/src/js-interface/JSAdapter.cpp @@ -0,0 +1,129 @@ +#include "JSAdapter.hpp" + +#include +#include +#include + +#include + +using namespace qvac_lib_inference_addon_cpp::js; + +namespace qvac_lib_inference_addon_bci { + +namespace { + +auto getPropertyNames(js_env_t* env, Object object) -> Array { + js_value_t* propertyNames; + JS(js_get_property_names(env, object, &propertyNames)); + return Array::fromValue(propertyNames); +} + +auto getValueType(js_env_t* env, js_value_t* value) -> js_value_type_t { + js_value_type_t valueType; + JS(js_typeof(env, value, &valueType)); + return valueType; +} + +template +void addConfigParam( + std::map& cfg, std::string&& key, T&& value) { + if (auto e = cfg.try_emplace(std::move(key), std::forward(value)); + !e.second) { + std::ostringstream oss; + oss << "key '" << key << "' already exists"; + throw std::runtime_error{oss.str()}; + } +} + +} // namespace + +void JSAdapter::loadMap( + Object jsObject, js_env_t* env, + std::map& output) { + + auto names = getPropertyNames(env, jsObject); + auto namesSize = names.size(env); + for (auto i = 0; i < namesSize; ++i) { + auto key = names.get(env, i); + auto value = jsObject.getProperty(env, key); + switch (getValueType(env, value)) { + case js_boolean: + addConfigParam( + output, + key.as(env), + Boolean::fromValue(value).as(env)); + break; + case js_number: + addConfigParam( + output, + key.as(env), + Number::fromValue(value).as(env)); + break; + case js_string: + addConfigParam( + output, + key.as(env), + String::fromValue(value).as(env)); + break; + case js_object: + continue; + case js_function: + continue; + default: + throw qvac_errors::StatusError( + qvac_errors::general_error::InvalidArgument, + "Invalid type for key: " + key.as(env) + + " is not supported"); + } + } +} + +BCIConfig JSAdapter::loadFromJSObject(Object jsObject, js_env_t* env) { + BCIConfig config; + + auto whisperConfigObj = + jsObject.getOptionalProperty(env, "whisperConfig"); + if (whisperConfigObj.has_value()) { + loadMap(whisperConfigObj.value(), env, config.whisperMainCfg); + } + + auto contextParamsObj = + jsObject.getOptionalProperty(env, "contextParams"); + if (contextParamsObj.has_value()) { + loadContextParams(contextParamsObj.value(), env, config); + } + + auto miscConfigObj = + jsObject.getOptionalProperty(env, "miscConfig"); + if (miscConfigObj.has_value()) { + loadMiscParams(miscConfigObj.value(), env, config); + } + + auto bciConfigObj = + jsObject.getOptionalProperty(env, "bciConfig"); + if (bciConfigObj.has_value()) { + loadBCIParams(bciConfigObj.value(), env, config); + } + + return config; +} + +BCIConfig JSAdapter::loadContextParams( + Object contextParamsObj, js_env_t* env, BCIConfig& config) { + loadMap(contextParamsObj, env, config.whisperContextCfg); + return config; +} + +BCIConfig JSAdapter::loadMiscParams( + Object miscParamsObj, js_env_t* env, BCIConfig& config) { + loadMap(miscParamsObj, env, config.miscConfig); + return config; +} + +BCIConfig JSAdapter::loadBCIParams( + Object bciParamsObj, js_env_t* env, BCIConfig& config) { + loadMap(bciParamsObj, env, config.bciConfig); + return config; +} + +} // namespace qvac_lib_inference_addon_bci diff --git a/packages/bci-whispercpp/addon/src/js-interface/JSAdapter.hpp b/packages/bci-whispercpp/addon/src/js-interface/JSAdapter.hpp new file mode 100644 index 0000000000..9b5b18b7c8 --- /dev/null +++ b/packages/bci-whispercpp/addon/src/js-interface/JSAdapter.hpp @@ -0,0 +1,48 @@ +#pragma once + +#include +#include +#include + +#include + +#include "addon/BCIErrors.hpp" +#include "model-interface/bci/BCIConfig.hpp" +#include "qvac-lib-inference-addon-cpp/Errors.hpp" + +namespace qvac_lib_inference_addon_cpp::js { +class Object; +} + +namespace qvac_lib_inference_addon_bci { + +class JSAdapter { +public: + JSAdapter() = default; + + auto loadFromJSObject( + qvac_lib_inference_addon_cpp::js::Object jsObject, js_env_t* env) + -> BCIConfig; + + auto loadContextParams( + qvac_lib_inference_addon_cpp::js::Object contextParamsObj, js_env_t* env, + BCIConfig& config) + -> BCIConfig; + + auto loadMiscParams( + qvac_lib_inference_addon_cpp::js::Object miscParamsObj, js_env_t* env, + BCIConfig& config) + -> BCIConfig; + + auto loadBCIParams( + qvac_lib_inference_addon_cpp::js::Object bciParamsObj, js_env_t* env, + BCIConfig& config) + -> BCIConfig; + +private: + void loadMap( + qvac_lib_inference_addon_cpp::js::Object jsObject, js_env_t* env, + std::map& output); +}; + +} // namespace qvac_lib_inference_addon_bci diff --git a/packages/bci-whispercpp/addon/src/js-interface/binding.cpp b/packages/bci-whispercpp/addon/src/js-interface/binding.cpp new file mode 100644 index 0000000000..3a9a90072c --- /dev/null +++ b/packages/bci-whispercpp/addon/src/js-interface/binding.cpp @@ -0,0 +1,39 @@ +#include + +#include "src/addon/AddonJs.hpp" + +// NOLINTBEGIN(cppcoreguidelines-macro-usage,readability-function-cognitive-complexity,modernize-use-trailing-return-type,readability-identifier-naming) +auto qvac_lib_inference_addon_bci_exports( + js_env_t* env, + js_value_t* exports) + -> js_value_t* { // NOLINT(readability-identifier-naming) + +#define V(name, fn) \ + { \ + js_value_t* val; \ + if (js_create_function(env, name, -1, fn, nullptr, &val) != 0) { \ + return nullptr; \ + } \ + if (js_set_named_property(env, exports, name, val) != 0) { \ + return nullptr; \ + } \ + } + + V("createInstance", qvac_lib_inference_addon_bci::createInstance) + V("runJob", qvac_lib_inference_addon_bci::runJob) + V("reload", qvac_lib_inference_addon_bci::reload) + V("loadWeights", qvac_lib_inference_addon_cpp::JsInterface::loadWeights) + V("activate", qvac_lib_inference_addon_cpp::JsInterface::activate) + V("cancel", qvac_lib_inference_addon_cpp::JsInterface::cancel) + V("destroyInstance", + qvac_lib_inference_addon_cpp::JsInterface::destroyInstance) + V("setLogger", qvac_lib_inference_addon_cpp::JsInterface::setLogger) + V("releaseLogger", qvac_lib_inference_addon_cpp::JsInterface::releaseLogger) +#undef V + + return exports; +} + +BARE_MODULE( + qvac_lib_inference_addon_bci, qvac_lib_inference_addon_bci_exports) +// NOLINTEND(cppcoreguidelines-macro-usage,readability-function-cognitive-complexity,modernize-use-trailing-return-type,readability-identifier-naming) diff --git a/packages/bci-whispercpp/addon/src/model-interface/BCITypes.hpp b/packages/bci-whispercpp/addon/src/model-interface/BCITypes.hpp new file mode 100644 index 0000000000..900ee86d97 --- /dev/null +++ b/packages/bci-whispercpp/addon/src/model-interface/BCITypes.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include +#include +#include +#include + +namespace qvac_lib_inference_addon_bci { + +struct Transcript { + std::string text; + bool toAppend; + float start; + float end; + size_t id; + + Transcript() : toAppend{false}, start(-1.0F), end(-1.0F), id{0} {} + + explicit Transcript(std::string_view strView) + : text{strView}, toAppend{false}, start{-1.0F}, end{-1.0F}, id{0} {} +}; + +struct NeuralSignalHeader { + uint32_t numTimesteps; + uint32_t numChannels; +}; + +} // namespace qvac_lib_inference_addon_bci diff --git a/packages/bci-whispercpp/addon/src/model-interface/bci/BCIConfig.cpp b/packages/bci-whispercpp/addon/src/model-interface/bci/BCIConfig.cpp new file mode 100644 index 0000000000..5a80272db4 --- /dev/null +++ b/packages/bci-whispercpp/addon/src/model-interface/bci/BCIConfig.cpp @@ -0,0 +1,150 @@ +#include "BCIConfig.hpp" + +#include +#include + +namespace qvac_lib_inference_addon_bci { + +std::string convertVariantToString(const JSValueVariant& value) { + return std::visit( + [](const auto& v) -> std::string { + using T = std::decay_t; + if constexpr (std::is_same_v) { + return "null"; + } else if constexpr (std::is_same_v) { + return std::to_string(v); + } else if constexpr (std::is_same_v) { + std::ostringstream oss; + oss << v; + return oss.str(); + } else if constexpr (std::is_same_v) { + return v; + } else if constexpr (std::is_same_v) { + return v ? "true" : "false"; + } + return "unknown"; + }, + value); +} + +const HandlersMap& getWhisperMainHandlers() { + static const HandlersMap handlers = { + {"language", + [](whisper_full_params& /*p*/, const JSValueVariant& /*v*/) { + // Language is handled separately in toWhisperFullParams via + // BCIConfig::lang_ to avoid static-local lifetime issues. + }}, + {"n_threads", + [](whisper_full_params& p, const JSValueVariant& v) { + if (auto* i = std::get_if(&v)) { + p.n_threads = *i; + } + }}, + {"translate", + [](whisper_full_params& p, const JSValueVariant& v) { + if (auto* b = std::get_if(&v)) { + p.translate = *b; + } + }}, + {"no_timestamps", + [](whisper_full_params& p, const JSValueVariant& v) { + if (auto* b = std::get_if(&v)) { + p.no_timestamps = *b; + } + }}, + {"single_segment", + [](whisper_full_params& p, const JSValueVariant& v) { + if (auto* b = std::get_if(&v)) { + p.single_segment = *b; + } + }}, + {"temperature", + [](whisper_full_params& p, const JSValueVariant& v) { + if (auto* d = std::get_if(&v)) { + p.temperature = static_cast(*d); + } + }}, + {"suppress_nst", + [](whisper_full_params& p, const JSValueVariant& v) { + if (auto* b = std::get_if(&v)) { + p.suppress_nst = *b; + } + }}, + {"duration_ms", + [](whisper_full_params& p, const JSValueVariant& v) { + if (auto* i = std::get_if(&v)) { + p.duration_ms = *i; + } + }}, + }; + return handlers; +} + +const HandlersMap& getWhisperContextHandlers() { + static const HandlersMap handlers = { + {"use_gpu", + [](whisper_context_params& p, const JSValueVariant& v) { + if (auto* b = std::get_if(&v)) { + p.use_gpu = *b; + } + }}, + {"flash_attn", + [](whisper_context_params& p, const JSValueVariant& v) { + if (auto* b = std::get_if(&v)) { + p.flash_attn = *b; + } + }}, + }; + return handlers; +} + +whisper_full_params toWhisperFullParams(BCIConfig& bciConfig) { + whisper_full_params params = whisper_full_default_params( + WHISPER_SAMPLING_BEAM_SEARCH); + + // BCI defaults matching the Python notebook's decode settings + params.beam_search.beam_size = 4; + params.suppress_nst = false; + params.suppress_blank = false; + params.temperature = 0.0F; + params.no_timestamps = true; + params.single_segment = true; + params.no_context = true; + params.length_penalty = 0.14F; + params.max_initial_ts = 0; + + const auto& handlers = getWhisperMainHandlers(); + for (const auto& [key, value] : bciConfig.whisperMainCfg) { + auto it = handlers.find(key); + if (it != handlers.end()) { + it->second(params, value); + } + } + + // Set language from config-owned storage so the pointer outlives params + auto langIt = bciConfig.whisperMainCfg.find("language"); + if (langIt != bciConfig.whisperMainCfg.end()) { + if (auto* s = std::get_if(&langIt->second)) { + bciConfig.lang_ = *s; + params.language = bciConfig.lang_.c_str(); + } + } + + return params; +} + +whisper_context_params toWhisperContextParams(const BCIConfig& bciConfig) { + whisper_context_params params = whisper_context_default_params(); + + const auto& handlers = getWhisperContextHandlers(); + for (const auto& [key, value] : bciConfig.whisperContextCfg) { + auto it = handlers.find(key); + if (it != handlers.end()) { + it->second(params, value); + } + } + + return params; +} + +} // namespace qvac_lib_inference_addon_bci diff --git a/packages/bci-whispercpp/addon/src/model-interface/bci/BCIConfig.hpp b/packages/bci-whispercpp/addon/src/model-interface/bci/BCIConfig.hpp new file mode 100644 index 0000000000..df1b0ac75c --- /dev/null +++ b/packages/bci-whispercpp/addon/src/model-interface/bci/BCIConfig.hpp @@ -0,0 +1,44 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace qvac_lib_inference_addon_bci { + +using JSValueVariant = + std::variant; + +template +using HandlerFunction = std::function; + +template +using HandlersMap = std::unordered_map>; + +struct BCIConfig { + std::map miscConfig; + std::map whisperMainCfg; + std::map whisperContextCfg; + std::map bciConfig; + + // Owned storage for string values that whisper_full_params references by + // pointer (e.g. p.language = lang_.c_str()). Must outlive the params struct. + mutable std::string lang_; +}; + +whisper_full_params toWhisperFullParams(BCIConfig& bciConfig); +whisper_context_params toWhisperContextParams(const BCIConfig& bciConfig); + +std::string convertVariantToString(const JSValueVariant& value); + +// Maps of handler functions for setting whisper_full_params fields from JS. +const HandlersMap& getWhisperMainHandlers(); +const HandlersMap& getWhisperContextHandlers(); + +} // namespace qvac_lib_inference_addon_bci diff --git a/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp b/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp new file mode 100644 index 0000000000..8d5a3717a0 --- /dev/null +++ b/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp @@ -0,0 +1,347 @@ +#include "BCIModel.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "BCIConfig.hpp" +#include "addon/BCIErrors.hpp" +#include "model-interface/BCITypes.hpp" +#include "qvac-lib-inference-addon-cpp/Errors.hpp" +#include "qvac-lib-inference-addon-cpp/Logger.hpp" + +namespace qvac_lib_inference_addon_bci { + +namespace { +constexpr double K_SAMPLES_PER_SECOND = 16000.0; +constexpr float K_SEGMENT_TIMESTAMP_SCALE = 0.01F; +constexpr int K_WARMUP_SAMPLE_COUNT = 8000; +constexpr int K_DUMMY_AUDIO_30S = 16000 * 30; +} // namespace + +static bool shouldAbortWhisper(void* userData) { + const auto* cancelRequested = static_cast(userData); + return cancelRequested != nullptr && + cancelRequested->load(std::memory_order_relaxed); +} + +// Called right before the encoder runs. Replaces the mel spectrogram +// (computed from dummy silence) with our neural-signal-derived features. +static bool onEncoderBegin( + whisper_context* ctx, whisper_state* state, void* userData) { + auto* cbData = static_cast(userData); + if (cbData == nullptr || cbData->melData == nullptr) { + return true; + } + + int result = whisper_set_mel_with_state( + cbData->ctx, state, + cbData->melData, cbData->melFrames, cbData->melBins); + + if (result != 0) { + QLOG(qvac_lib_inference_addon_cpp::logger::Priority::ERROR, + "whisper_set_mel_with_state failed: " + std::to_string(result)); + return false; + } + + QLOG(qvac_lib_inference_addon_cpp::logger::Priority::DEBUG, + "Injected neural mel features: " + + std::to_string(cbData->melFrames) + " frames x " + + std::to_string(cbData->melBins) + " bins"); + return true; +} + +BCIModel::BCIModel(BCIConfig config) + : cfg_(std::move(config)), neuralProcessor_() {} + +BCIModel::~BCIModel() noexcept { + try { + unload(); + } catch (...) { + is_loaded_ = false; + } +} + +void BCIModel::loadEmbedderIfNeeded() { + if (neuralProcessor_.hasWeights()) { + return; + } + + // Look for embedder weights next to the model file + auto modelPathIt = cfg_.whisperContextCfg.find("model"); + if (modelPathIt == cfg_.whisperContextCfg.end()) { + return; + } + const auto modelPath = std::get(modelPathIt->second); + + // Try: same directory, "bci-embedder.bin" + auto dir = modelPath.substr(0, modelPath.find_last_of('/')); + auto embedderPath = dir + "/bci-embedder.bin"; + + if (neuralProcessor_.loadEmbedderWeights(embedderPath)) { + QLOG(qvac_lib_inference_addon_cpp::logger::Priority::INFO, + "Loaded BCI embedder weights from: " + embedderPath); + } else { + throw std::runtime_error( + "BCI embedder weights not found at: " + embedderPath + + ". This file is required for neural signal preprocessing. " + "Generate it with: python3 scripts/convert-model.py --checkpoint "); + } +} + +void BCIModel::load() { + if (!ctx_) { + whisper_context_params contextParams = toWhisperContextParams(cfg_); + + const auto modelPathIt = cfg_.whisperContextCfg.find("model"); + if (modelPathIt == cfg_.whisperContextCfg.end()) { + throw std::runtime_error("Model path not specified"); + } + const auto modelPath = std::get(modelPathIt->second); + + QLOG(qvac_lib_inference_addon_cpp::logger::Priority::INFO, + "Loading BCI model from: " + modelPath); + ctx_.reset( + whisper_init_from_file_with_params(modelPath.c_str(), contextParams)); + + if (ctx_ == nullptr) { + throw std::runtime_error("Failed to initialize Whisper context for BCI"); + } + + is_loaded_ = true; + + loadEmbedderIfNeeded(); + + if (!is_warmed_up_) { + warmup(); + is_warmed_up_ = true; + } + } +} + +void BCIModel::unload() { + resetContext(); + is_loaded_ = false; +} + +void BCIModel::reload() { + unload(); + load(); +} + +void BCIModel::reset() { + output_.clear(); + totalSamples_ = 0; + totalTokens_ = 0; + totalSegments_ = 0; + processCalls_ = 0; + totalWallMs_ = 0.0; +} + +qvac_lib_inference_addon_cpp::RuntimeStats BCIModel::runtimeStats() const { + qvac_lib_inference_addon_cpp::RuntimeStats stats; + + const double totalTimeSec = totalWallMs_ / 1000.0; + const double tps = totalTimeSec > 0.0 + ? (static_cast(totalTokens_) / totalTimeSec) + : 0.0; + + stats.emplace_back("totalTime", totalTimeSec); + stats.emplace_back("tokensPerSecond", tps); + stats.emplace_back("totalTokens", totalTokens_); + stats.emplace_back("totalSegments", totalSegments_); + stats.emplace_back("processCalls", processCalls_); + stats.emplace_back("totalWallMs", totalWallMs_); + return stats; +} + +static void onNewSegment( + [[maybe_unused]] whisper_context* ctx, whisper_state* state, int nNew, + void* userData) { + auto* bci = static_cast(userData); + if (bci == nullptr || state == nullptr) return; + + const int nSegments = whisper_full_n_segments_from_state(state); + if (nNew <= 0 || nSegments <= 0) return; + const int startIndex = std::max(0, nSegments - nNew); + + for (int i = startIndex; i < nSegments; i++) { + Transcript transcript; + const char* text = whisper_full_get_segment_text_from_state(state, i); + transcript.text = text != nullptr ? text : ""; + transcript.start = + static_cast(whisper_full_get_segment_t0_from_state(state, i)) * + K_SEGMENT_TIMESTAMP_SCALE; + transcript.end = + static_cast(whisper_full_get_segment_t1_from_state(state, i)) * + K_SEGMENT_TIMESTAMP_SCALE; + transcript.id = i; + + bci->emitSegment(transcript); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + bci->addTranscription(transcript); + + const int nTokens = whisper_full_n_tokens_from_state(state, i); + bci->recordSegmentStats(nTokens); + } +} + +void BCIModel::warmup() { + if (!ctx_) return; + + std::vector silentAudio(K_WARMUP_SAMPLE_COUNT, 0.0F); + whisper_full_params params = toWhisperFullParams(cfg_); + params.new_segment_callback = nullptr; + params.new_segment_callback_user_data = nullptr; + + whisper_full(ctx_.get(), params, + silentAudio.data(), + static_cast(silentAudio.size())); +} + +void BCIModel::process(const Input& rawNeuralData) { + if (ctx_ == nullptr) load(); + if (ctx_ == nullptr) { + throw std::runtime_error("BCI Whisper context is not initialized"); + } + + if (cancelRequested_.load(std::memory_order_relaxed)) { + throw std::runtime_error("Job cancelled"); + } + + QLOG(qvac_lib_inference_addon_cpp::logger::Priority::DEBUG, + "Processing neural signal (" + + std::to_string(rawNeuralData.size()) + " bytes)"); + + int dayIdx = 0; + auto it = cfg_.bciConfig.find("day_idx"); + if (it != cfg_.bciConfig.end()) { + if (auto* d = std::get_if(&it->second)) { + dayIdx = static_cast(*d); + } else if (auto* i = std::get_if(&it->second)) { + dayIdx = *i; + } + } + + auto melFeatures = neuralProcessor_.processToMel(rawNeuralData, dayIdx); + const int melBins = neuralProcessor_.getMelBins(); + const int melFrames = neuralProcessor_.getMelFrames(); + + processCalls_ += 1; + + if (ctx_ != nullptr) { + whisper_reset_timings(ctx_.get()); + } + + const auto startTime = std::chrono::steady_clock::now(); + + EncoderCallbackData cbData; + cbData.ctx = ctx_.get(); + cbData.melData = melFeatures.data(); + cbData.melFrames = melFrames; + cbData.melBins = melBins; + + whisper_full_params params = toWhisperFullParams(cfg_); + params.new_segment_callback = onNewSegment; + params.new_segment_callback_user_data = this; + params.abort_callback = shouldAbortWhisper; + params.abort_callback_user_data = &cancelRequested_; + params.encoder_begin_callback = onEncoderBegin; + params.encoder_begin_callback_user_data = &cbData; + + std::vector dummyAudio(K_DUMMY_AUDIO_30S, 0.0F); + + int result = whisper_full( + ctx_.get(), params, + dummyAudio.data(), static_cast(dummyAudio.size())); + + const auto endTime = std::chrono::steady_clock::now(); + totalWallMs_ += + std::chrono::duration(endTime - startTime).count(); + + if (result != 0) { + if (cancelRequested_.load(std::memory_order_relaxed)) { + throw std::runtime_error("Job cancelled"); + } + throw std::runtime_error( + "Failed to process neural signal (whisper_full returned " + + std::to_string(result) + ")"); + } +} + +std::any BCIModel::process(const std::any& input) { + AnyInput modelInput; + if (const auto* anyInput = std::any_cast(&input)) { + modelInput = *anyInput; + } else if (const auto* inputVector = std::any_cast(&input)) { + modelInput.input = *inputVector; + } else { + throw qvac_errors::StatusError( + qvac_errors::general_error::InvalidArgument, + std::string("Invalid input type for BCIModel::process: ") + + input.type().name()); + } + + const auto previousOutputCallback = on_segment_; + const bool shouldOverrideCallback = + static_cast(modelInput.outputCallback); + if (shouldOverrideCallback) { + on_segment_ = modelInput.outputCallback; + } + + reset(); + cancelRequested_.store(false, std::memory_order_relaxed); + try { + process(modelInput.input); + } catch (...) { + if (shouldOverrideCallback) { + on_segment_ = previousOutputCallback; + } + throw; + } + + if (shouldOverrideCallback) { + on_segment_ = previousOutputCallback; + } + + return output_; +} + +void BCIModel::saveLoadParams(const BCIConfig& config) { + setConfig(config); +} + +void BCIModel::cancel() const { + cancelRequested_.store(true, std::memory_order_relaxed); +} + +bool BCIModel::configContextIsChanged( + const BCIConfig& oldCfg, const BCIConfig& newCfg) { + const std::vector contextKeys = { + "model", "use_gpu", "flash_attn", "gpu_device"}; + return std::ranges::any_of(contextKeys, [&](const std::string& key) { + const auto oldIt = oldCfg.whisperContextCfg.find(key); + const auto newIt = newCfg.whisperContextCfg.find(key); + if (oldIt != oldCfg.whisperContextCfg.end() && + newIt != newCfg.whisperContextCfg.end()) { + return oldIt->second != newIt->second; + } + return (oldIt != oldCfg.whisperContextCfg.end()) != + (newIt != newCfg.whisperContextCfg.end()); + }); +} + +void BCIModel::resetContext() { ctx_.reset(); } + +void BCIModel::setConfig(const BCIConfig& config) { + bool contextChanged = configContextIsChanged(cfg_, config); + cfg_ = config; + if (contextChanged) reload(); +} + +} // namespace qvac_lib_inference_addon_bci diff --git a/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.hpp b/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.hpp new file mode 100644 index 0000000000..29493e6bb0 --- /dev/null +++ b/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.hpp @@ -0,0 +1,130 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "BCIConfig.hpp" +#include "NeuralProcessor.hpp" +#include "model-interface/BCITypes.hpp" +#include "qvac-lib-inference-addon-cpp/ModelInterfaces.hpp" +#include "qvac-lib-inference-addon-cpp/RuntimeStats.hpp" + +namespace qvac_lib_inference_addon_bci { + +class BCIModel + : public qvac_lib_inference_addon_cpp::model::IModel, + public qvac_lib_inference_addon_cpp::model::IModelCancel, + public qvac_lib_inference_addon_cpp::model::IModelAsyncLoad { +public: + using OutputCallback = std::function; + using ValueType = float; + using Input = std::vector; + using Output = std::vector; + + struct AnyInput { + Input input; + OutputCallback outputCallback = nullptr; + }; + + // Data passed to encoder_begin_callback so it can inject mel features. + struct EncoderCallbackData { + whisper_context* ctx = nullptr; + const float* melData = nullptr; + int melFrames = 0; + int melBins = 0; + }; + + explicit BCIModel(BCIConfig config); + ~BCIModel() noexcept; + + void initializeBackend() {} + void setConfig(const BCIConfig& config); + + auto setOnSegmentCallback(const OutputCallback& callback) -> void { + on_segment_ = callback; + } + auto addTranscription(const Transcript& transcript) -> void { + output_.push_back(transcript); + } + auto hasSegmentCallback() const -> bool { + return static_cast(on_segment_); + } + auto emitSegment(const Transcript& transcript) -> void { + if (on_segment_) { + on_segment_(transcript); + } + } + + std::string getName() const override { return "BCIModel"; } + std::any process(const std::any& input) override; + void cancel() const override; + + void process(const Input& input); + + void load(); + void unload(); + void unloadWeights() { unload(); } + void reload(); + void reset(); + void waitForLoadInitialization() override { load(); } + void setWeightsForFile( + const std::string&, + std::unique_ptr>&&) override {} + void set_weights_for_file( + const std::string&, + const std::span&, bool) {} + bool isLoaded() const { return is_loaded_; } + qvac_lib_inference_addon_cpp::RuntimeStats runtimeStats() const override; + void warmup(); + + void saveLoadParams(const BCIConfig& config); + template + std::enable_if_t, BCIConfig>, void> + saveLoadParams(T&&, Args&&...) {} + + void recordSegmentStats(int nTokens) { + totalSegments_ += 1; + if (nTokens > 0) { + totalTokens_ += static_cast(nTokens); + } + } + +private: + static bool configContextIsChanged( + const BCIConfig& oldCfg, const BCIConfig& newCfg); + void resetContext(); + void loadEmbedderIfNeeded(); + + BCIConfig cfg_; + NeuralProcessor neuralProcessor_; + OutputCallback on_segment_; + Output output_; + + struct WhisperContextDeleter { + void operator()(whisper_context* ctx) const noexcept { + if (ctx != nullptr) { + whisper_free(ctx); + } + } + }; + + std::unique_ptr ctx_{nullptr}; + bool is_loaded_ = false; + bool is_warmed_up_ = false; + + int64_t totalSamples_ = 0; + int64_t totalTokens_ = 0; + int64_t totalSegments_ = 0; + int64_t processCalls_ = 0; + double totalWallMs_ = 0.0; + mutable std::atomic_bool cancelRequested_{false}; +}; + +} // namespace qvac_lib_inference_addon_bci diff --git a/packages/bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.cpp b/packages/bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.cpp new file mode 100644 index 0000000000..b7e4ee5be8 --- /dev/null +++ b/packages/bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.cpp @@ -0,0 +1,241 @@ +#include "NeuralProcessor.hpp" + +#include +#include +#include +#include +#include + +#include "addon/BCIErrors.hpp" +#include "qvac-lib-inference-addon-cpp/Logger.hpp" + +namespace qvac_lib_inference_addon_bci { + +namespace { +constexpr size_t K_HEADER_BYTES = 8; +constexpr uint32_t K_EMBEDDER_MAGIC = 0x42434945; +} // namespace + +NeuralProcessor::NeuralProcessor() = default; + +bool NeuralProcessor::loadEmbedderWeights(const std::string& path) { + std::ifstream f(path, std::ios::binary); + if (!f.is_open()) return false; + + auto readU32 = [&]() -> uint32_t { + uint32_t v = 0; + f.read(reinterpret_cast(&v), sizeof(v)); + return v; + }; + auto readFloats = [&](size_t count) -> std::vector { + std::vector data(count); + f.read(reinterpret_cast(data.data()), + static_cast(count * sizeof(float))); + return data; + }; + auto readInts = [&](size_t count) -> std::vector { + std::vector data(count); + f.read(reinterpret_cast(data.data()), + static_cast(count * sizeof(int32_t))); + return data; + }; + + if (readU32() != K_EMBEDDER_MAGIC || readU32() != 1) return false; + + weights_.numFeatures = readU32(); + /*embedDim=*/ readU32(); + /*kernelSize1=*/ readU32(); + /*kernelSize2=*/ readU32(); + /*stride2=*/ readU32(); + weights_.numDays = readU32(); + weights_.numMonths = readU32(); + weights_.r = readU32(); + + // Skip conv1/conv2 weights (handled by GGML model) + uint32_t n = readU32(); readFloats(n); + n = readU32(); readFloats(n); + n = readU32(); readFloats(n); + n = readU32(); readFloats(n); + + n = readU32(); + weights_.sessionToDayMap = readInts(n); + + weights_.dayAs.resize(weights_.numDays); + weights_.dayBs.resize(weights_.numDays); + weights_.dayBiases.resize(weights_.numDays); + for (uint32_t i = 0; i < weights_.numDays; ++i) { + n = readU32(); weights_.dayAs[i] = readFloats(n); + n = readU32(); weights_.dayBs[i] = readFloats(n); + n = readU32(); weights_.dayBiases[i] = readFloats(n); + } + + weights_.monthWeights.resize(weights_.numMonths); + weights_.monthBiases.resize(weights_.numMonths); + for (uint32_t i = 0; i < weights_.numMonths; ++i) { + n = readU32(); weights_.monthWeights[i] = readFloats(n); + n = readU32(); weights_.monthBiases[i] = readFloats(n); + } + + weights_.loaded = true; + QLOG(qvac_lib_inference_addon_cpp::logger::Priority::INFO, + "Loaded day projection weights: " + + std::to_string(weights_.numDays) + " days, r=" + + std::to_string(weights_.r)); + return true; +} + +std::vector NeuralProcessor::gaussianSmooth( + const std::vector& data, + uint32_t numTimesteps, uint32_t numChannels, + float kernelStd, int kernelSize) { + + std::vector kernel(kernelSize); + const int center = kernelSize / 2; + float sum = 0.0F; + for (int i = 0; i < kernelSize; ++i) { + float x = static_cast(i - center); + kernel[i] = std::exp(-0.5F * (x * x) / (kernelStd * kernelStd)); + sum += kernel[i]; + } + for (auto& k : kernel) k /= sum; + + int start = 0, end = kernelSize - 1; + while (start < end && kernel[start] < 0.01F) ++start; + while (end > start && kernel[end] < 0.01F) --end; + std::vector trimK(kernel.begin() + start, kernel.begin() + end + 1); + const int halfK = static_cast(trimK.size()) / 2; + + std::vector result(data.size()); + for (uint32_t c = 0; c < numChannels; ++c) { + for (uint32_t t = 0; t < numTimesteps; ++t) { + float val = 0.0F; + for (int k = 0; k < static_cast(trimK.size()); ++k) { + int srcT = static_cast(t) + k - halfK; + if (srcT >= 0 && srcT < static_cast(numTimesteps)) + val += data[srcT * numChannels + c] * trimK[k]; + } + result[t * numChannels + c] = val; + } + } + return result; +} + +std::vector NeuralProcessor::applyDayProjection( + const std::vector& features, + uint32_t numTimesteps, uint32_t numChannels, int dayIdx) const { + + if (!weights_.loaded || weights_.r == 0) return features; + + const uint32_t nf = weights_.numFeatures; + const uint32_t r = weights_.r; + int di = std::clamp(dayIdx, 0, static_cast(weights_.numDays) - 1); + + const auto& dayA = weights_.dayAs[di]; + const auto& dayB = weights_.dayBs[di]; + const auto& dayBias = weights_.dayBiases[di]; + + std::vector dayDelta(nf * nf, 0.0F); + for (uint32_t i = 0; i < nf; ++i) + for (uint32_t j = 0; j < nf; ++j) { + float s = 0.0F; + for (uint32_t k = 0; k < r; ++k) + s += dayA[i * r + k] * dayB[k * nf + j]; + dayDelta[i * nf + j] = s; + } + + int monthIdx = di / 30; + bool hasMonth = (monthIdx < static_cast(weights_.monthWeights.size()) && + !weights_.monthWeights[monthIdx].empty()); + + std::vector W(nf * nf), bias(nf, 0.0F); + for (uint32_t i = 0; i < nf * nf; ++i) { + W[i] = dayDelta[i]; + if (hasMonth) W[i] += weights_.monthWeights[monthIdx][i]; + } + for (uint32_t i = 0; i < nf; ++i) { + bias[i] = dayBias[i]; + if (hasMonth && i < weights_.monthBiases[monthIdx].size()) + bias[i] += weights_.monthBiases[monthIdx][i]; + } + + // Python: output[t,k] = softsign(sum_d(features[t,d] * W[d,k]) + bias[k]) + // i.e. output = features @ W + bias (right-multiply by W) + std::vector output(numTimesteps * nf); + for (uint32_t t = 0; t < numTimesteps; ++t) + for (uint32_t k = 0; k < nf; ++k) { + float s = bias[k]; + for (uint32_t d = 0; d < nf; ++d) + s += features[t * numChannels + d] * W[d * nf + k]; + output[t * nf + k] = s / (1.0F + std::abs(s)); + } + + return output; +} + +std::vector NeuralProcessor::processToMel( + const std::vector& rawData, int dayIdx) const { + + if (rawData.size() < K_HEADER_BYTES) { + throw qvac_errors::bci_error::makeStatus( + qvac_errors::bci_error::Code::InvalidNeuralSignal, + "Neural signal buffer too small"); + } + + uint32_t numTimesteps = 0, numChannels = 0; + std::memcpy(&numTimesteps, rawData.data(), sizeof(uint32_t)); + std::memcpy(&numChannels, rawData.data() + sizeof(uint32_t), sizeof(uint32_t)); + + size_t expectedBytes = static_cast(numTimesteps) * numChannels * sizeof(float); + if (rawData.size() < K_HEADER_BYTES + expectedBytes) { + throw qvac_errors::bci_error::makeStatus( + qvac_errors::bci_error::Code::InvalidNeuralSignal, + "Neural signal buffer truncated"); + } + + std::vector features(numTimesteps * numChannels); + std::memcpy(features.data(), rawData.data() + K_HEADER_BYTES, expectedBytes); + + // Passthrough mode: if dayIdx == -1, skip preprocessing and treat + // the input as pre-computed mel features in frame-major layout. + if (dayIdx == -1) { + const int melBins = K_WHISPER_N_MEL; + const int melFrames = K_WHISPER_MEL_FRAMES; + std::vector melOutput(melFrames * melBins, 0.0F); + uint32_t framesToCopy = std::min(numTimesteps, static_cast(melFrames)); + uint32_t chToCopy = std::min(numChannels, static_cast(melBins)); + for (uint32_t t = 0; t < framesToCopy; ++t) + for (uint32_t c = 0; c < chToCopy; ++c) + melOutput[c * melFrames + t] = features[t * numChannels + c]; + return melOutput; + } + + // Step 1: Gaussian smoothing (std=2.0, kernel_size=100, matching BrainWhisperer) + auto smoothed = gaussianSmooth(features, numTimesteps, numChannels, 2.0F, 100); + + // Step 2: Day projection (if available) + std::vector projected; + uint32_t projChannels = numChannels; + if (weights_.loaded && weights_.r > 0) { + projected = applyDayProjection(smoothed, numTimesteps, numChannels, dayIdx); + projChannels = weights_.numFeatures; + } else { + projected = smoothed; + } + + // Step 3: Pad to 3000 frames at 512 channels for whisper_set_mel() + // whisper.cpp stores mel as mel.data[mel_bin * n_len + frame] (mel-major), + // so we must write in that layout for whisper_set_mel_with_state. + const int melBins = K_WHISPER_N_MEL; + const int melFrames = K_WHISPER_MEL_FRAMES; + std::vector melOutput(melFrames * melBins, 0.0F); + + uint32_t framesToCopy = std::min(numTimesteps, static_cast(melFrames)); + uint32_t chToCopy = std::min(projChannels, static_cast(melBins)); + for (uint32_t t = 0; t < framesToCopy; ++t) + for (uint32_t c = 0; c < chToCopy; ++c) + melOutput[c * melFrames + t] = projected[t * projChannels + c]; + + return melOutput; +} + +} // namespace qvac_lib_inference_addon_bci diff --git a/packages/bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.hpp b/packages/bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.hpp new file mode 100644 index 0000000000..6909248ca4 --- /dev/null +++ b/packages/bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.hpp @@ -0,0 +1,62 @@ +#pragma once + +#include +#include +#include +#include + +namespace qvac_lib_inference_addon_bci { + +// Preprocesses raw multi-channel neural signals for whisper.cpp. +// +// Pipeline: neural(512ch) → smooth → day_proj → pad to 3000 frames +// Output is 512-dim x 3000 frames, fed to whisper_set_mel(). +// whisper.cpp (patched) handles: conv1(512→384,k=7) → GELU → conv2 → GELU +// → positional_embedding → 6-layer transformer → LoRA-merged decoder → text +class NeuralProcessor { +public: + static constexpr int K_WHISPER_N_MEL = 512; // n_mels in GGML model + static constexpr int K_WHISPER_MEL_FRAMES = 3000; + + struct EmbedderWeights { + bool loaded = false; + uint32_t numFeatures = 512; + uint32_t numDays = 0; + uint32_t numMonths = 0; + uint32_t r = 0; + + std::vector sessionToDayMap; + std::vector> dayAs; + std::vector> dayBs; + std::vector> dayBiases; + std::vector> monthWeights; + std::vector> monthBiases; + }; + + NeuralProcessor(); + + bool loadEmbedderWeights(const std::string& path); + + std::vector processToMel( + const std::vector& rawData, + int dayIdx = 0) const; + + static std::vector gaussianSmooth( + const std::vector& data, + uint32_t numTimesteps, uint32_t numChannels, + float kernelStd = 2.0F, int kernelSize = 100); + + std::vector applyDayProjection( + const std::vector& features, + uint32_t numTimesteps, uint32_t numChannels, + int dayIdx) const; + + bool hasWeights() const { return weights_.loaded; } + int getMelBins() const { return K_WHISPER_N_MEL; } + int getMelFrames() const { return K_WHISPER_MEL_FRAMES; } + +private: + EmbedderWeights weights_; +}; + +} // namespace qvac_lib_inference_addon_bci diff --git a/packages/bci-whispercpp/addon/tests/test_core.cpp b/packages/bci-whispercpp/addon/tests/test_core.cpp new file mode 100644 index 0000000000..1dcf0daf8f --- /dev/null +++ b/packages/bci-whispercpp/addon/tests/test_core.cpp @@ -0,0 +1,102 @@ +#include +#include +#include + +#include + +#include "model-interface/bci/NeuralProcessor.hpp" +#include "model-interface/bci/BCIConfig.hpp" + +using namespace qvac_lib_inference_addon_bci; + +namespace { + +std::vector createTestSignal(uint32_t numTimesteps, uint32_t numChannels) { + const size_t headerSize = 2 * sizeof(uint32_t); + const size_t dataSize = numTimesteps * numChannels * sizeof(float); + std::vector buffer(headerSize + dataSize); + + std::memcpy(buffer.data(), &numTimesteps, sizeof(uint32_t)); + std::memcpy(buffer.data() + sizeof(uint32_t), &numChannels, sizeof(uint32_t)); + + auto* data = reinterpret_cast(buffer.data() + headerSize); + for (uint32_t t = 0; t < numTimesteps; ++t) { + for (uint32_t c = 0; c < numChannels; ++c) { + data[t * numChannels + c] = + static_cast(t) / static_cast(numTimesteps) * + std::sin(static_cast(c) * 0.1F); + } + } + return buffer; +} + +} // namespace + +TEST(NeuralProcessor, ProcessToMelProducesCorrectShape) { + NeuralProcessor processor; + auto signal = createTestSignal(100, 512); + auto result = processor.processToMel(signal); + + EXPECT_EQ(result.size(), + static_cast(NeuralProcessor::K_WHISPER_MEL_FRAMES) * + NeuralProcessor::K_WHISPER_N_MEL); +} + +TEST(NeuralProcessor, ProcessToMelRejectsSmallBuffer) { + NeuralProcessor processor; + std::vector tooSmall = {1, 2, 3}; + EXPECT_THROW(processor.processToMel(tooSmall), std::exception); +} + +TEST(NeuralProcessor, GaussianSmoothPreservesSize) { + uint32_t T = 50, C = 8; + std::vector data(T * C, 1.0F); + auto smoothed = NeuralProcessor::gaussianSmooth(data, T, C, 2.0F, 20); + EXPECT_EQ(smoothed.size(), data.size()); +} + +TEST(NeuralProcessor, GaussianSmoothReducesNoise) { + uint32_t T = 100, C = 4; + std::vector data(T * C); + for (uint32_t t = 0; t < T; ++t) + for (uint32_t c = 0; c < C; ++c) + data[t * C + c] = (t % 2 == 0) ? 1.0F : -1.0F; + + auto smoothed = NeuralProcessor::gaussianSmooth(data, T, C, 2.0F, 20); + + float origVar = 0, smoothVar = 0; + for (size_t i = 0; i < data.size(); ++i) { + origVar += data[i] * data[i]; + smoothVar += smoothed[i] * smoothed[i]; + } + EXPECT_LT(smoothVar, origVar); +} + +TEST(NeuralProcessor, OutputValuesAreFinite) { + NeuralProcessor processor; + auto signal = createTestSignal(50, 512); + auto result = processor.processToMel(signal); + for (const auto& sample : result) { + EXPECT_TRUE(std::isfinite(sample)); + } +} + +TEST(NeuralProcessor, PaddedFramesAreZero) { + NeuralProcessor processor; + auto signal = createTestSignal(50, 512); + auto result = processor.processToMel(signal); + + float lastFrameSum = 0; + int lastFrame = NeuralProcessor::K_WHISPER_MEL_FRAMES - 1; + for (int m = 0; m < NeuralProcessor::K_WHISPER_N_MEL; ++m) { + lastFrameSum += std::abs(result[lastFrame * NeuralProcessor::K_WHISPER_N_MEL + m]); + } + EXPECT_FLOAT_EQ(lastFrameSum, 0.0F); +} + +TEST(BCIConfig, DefaultWhisperFullParamsAreValid) { + BCIConfig config; + config.whisperMainCfg["language"] = std::string("en"); + auto params = toWhisperFullParams(config); + EXPECT_STREQ(params.language, "en"); +} diff --git a/packages/bci-whispercpp/bci.js b/packages/bci-whispercpp/bci.js new file mode 100644 index 0000000000..aecf03e235 --- /dev/null +++ b/packages/bci-whispercpp/bci.js @@ -0,0 +1,300 @@ +'use strict' + +const { QvacErrorAddonBCI, ERR_CODES } = require('./lib/error') +const { checkConfig } = require('./configChecker') + +const state = Object.freeze({ + LOADING: 'loading', + LISTENING: 'listening', + PROCESSING: 'processing', + IDLE: 'idle', + PAUSED: 'paused', + STOPPED: 'stopped' +}) + +const END_OF_INPUT = 'end of job' + +/** + * Low-level interface between the Bare C++ BCI addon and the JS runtime. + * Accepts neural signal data (Uint8Array) instead of audio. + */ +class BCIInterface { + /** + * @param {Object} binding - the native binding object + * @param {Object} configurationParams - configuration for the BCI model + * @param {Function} outputCb - callback for inference events (Output, JobEnded, Error) + * @param {Function} [transitionCb] - callback for state changes + */ + constructor (binding, configurationParams, outputCb, transitionCb = null) { + this._binding = binding + this._outputCb = outputCb + this._transitionCb = transitionCb + this._nextJobId = 1 + this._activeJobId = null + this._bufferedSignal = [] + this._state = state.LOADING + + checkConfig(configurationParams) + this._handle = this._binding.createInstance( + this, + configurationParams, + this._addonOutputCallback.bind(this), + transitionCb + ) + } + + _setState (newState) { + this._state = newState + if (this._transitionCb) { + this._transitionCb(this, newState) + } + } + + _addonOutputCallback (addon, event, data, error) { + const isError = typeof error === 'string' && error.length > 0 + const isStats = data && typeof data === 'object' && ( + 'totalTime' in data || + 'audioDurationMs' in data || + 'totalSamples' in data + ) + const isTranscriptOutput = ( + (Array.isArray(data) && data.length > 0) || + (data && typeof data === 'object' && typeof data.text === 'string') + ) + + let mappedEvent = event + if (isError || String(event).includes('Error')) { + mappedEvent = 'Error' + } else if (isStats || String(event).includes('RuntimeStats')) { + mappedEvent = 'JobEnded' + } else if (isTranscriptOutput) { + mappedEvent = 'Output' + } else if (Array.isArray(data) && data.length === 0) { + // BCIModel::process returns an empty vector to avoid duplicate + // segment emissions; skip forwarding this noop event. + return + } + + const jobId = this._activeJobId + if (jobId === null || jobId === undefined) { + return + } + + if (mappedEvent === 'Output') { + this._setState(state.PROCESSING) + } + + if (this._outputCb != null) { + this._outputCb(addon, mappedEvent, jobId, data, isError ? error : null) + } + + if (mappedEvent === 'Error' || mappedEvent === 'JobEnded') { + this._activeJobId = null + this._setState(state.LISTENING) + } + } + + async unload () { + await this.destroyInstance() + } + + async load (configurationParams) { + checkConfig(configurationParams) + await this.destroyInstance() + this._handle = this._binding.createInstance( + this, + configurationParams, + this._addonOutputCallback.bind(this), + this._transitionCb + ) + this._setState(state.LOADING) + } + + async reload (configurationParams) { + checkConfig(configurationParams) + await this.cancel() + + if (typeof this._binding.reload === 'function') { + await this._binding.reload(this._handle, configurationParams) + this._setState(state.LOADING) + return + } + + await this.load(configurationParams) + } + + async loadWeights (weightsData) { + try { + this._binding.loadWeights(this._handle, weightsData) + } catch (err) { + throw new QvacErrorAddonBCI({ + code: ERR_CODES.FAILED_TO_LOAD_WEIGHTS, + adds: err.message, + cause: err + }) + } + } + + async unloadWeights () { + return true + } + + async activate () { + try { + this._binding.activate(this._handle) + this._setState(state.LISTENING) + } catch (err) { + throw new QvacErrorAddonBCI({ + code: ERR_CODES.FAILED_TO_ACTIVATE, + adds: err.message, + cause: err + }) + } + } + + async cancel (jobId) { + try { + await this._binding.cancel(this._handle, jobId) + this._bufferedSignal = [] + this._activeJobId = null + this._setState(state.LISTENING) + } catch (err) { + throw new QvacErrorAddonBCI({ + code: ERR_CODES.FAILED_TO_CANCEL, + adds: err.message, + cause: err + }) + } + } + + /** + * Appends neural signal data to the processing buffer. + * Send { type: 'end of job' } to trigger processing. + * @param {Object} data + * @param {string} data.type - 'neural' or 'end of job' + * @param {Uint8Array} [data.input] - binary neural signal data + * @returns {number} job ID + */ + async append (data) { + try { + if (data?.type === END_OF_INPUT) { + const currentJobId = this._nextJobId + const input = this._concatBufferedSignal() + + let accepted = false + try { + accepted = this._binding.runJob(this._handle, { + type: 'neural', + input + }) + } catch (err) { + this._setState(state.LISTENING) + throw err + } + if (!accepted) { + this._setState(state.LISTENING) + throw new Error('Cannot set new job: a job is already set or being processed') + } + + this._activeJobId = currentJobId + this._nextJobId += 1 + this._bufferedSignal = [] + this._setState(state.PROCESSING) + return currentJobId + } + + if (data?.type === 'neural') { + if (!(data.input instanceof Uint8Array)) { + throw new Error('Neural signal input must be Uint8Array') + } + this._bufferedSignal.push(data.input) + return this._nextJobId + } + + throw new Error(`Unknown append input type: ${data?.type}`) + } catch (err) { + throw new QvacErrorAddonBCI({ + code: ERR_CODES.FAILED_TO_APPEND, + adds: err.message, + cause: err + }) + } + } + + /** + * Run a single batch job directly with neural signal data. + * @param {Object} data + * @param {Uint8Array} data.input - binary neural signal data + */ + async runJob (data) { + try { + this._activeJobId = this._nextJobId + this._nextJobId += 1 + this._setState(state.PROCESSING) + const accepted = this._binding.runJob(this._handle, { + type: 'neural', + input: data.input + }) + if (!accepted) { + this._activeJobId = null + this._setState(state.LISTENING) + } + return accepted + } catch (err) { + this._activeJobId = null + this._setState(state.LISTENING) + throw new QvacErrorAddonBCI({ + code: ERR_CODES.FAILED_TO_APPEND, + adds: err.message, + cause: err + }) + } + } + + async status () { + return this._state + } + + async destroyInstance () { + if (this._handle === null) { + return + } + try { + try { + await this._binding.cancel(this._handle) + } catch {} + this._binding.destroyInstance(this._handle) + this._handle = null + this._bufferedSignal = [] + this._activeJobId = null + this._setState(state.IDLE) + } catch (err) { + throw new QvacErrorAddonBCI({ + code: ERR_CODES.FAILED_TO_DESTROY, + adds: err.message, + cause: err + }) + } + } + + _concatBufferedSignal () { + if (this._bufferedSignal.length === 0) { + return new Uint8Array() + } + if (this._bufferedSignal.length === 1) { + return this._bufferedSignal[0] + } + const totalLength = this._bufferedSignal.reduce( + (sum, chunk) => sum + chunk.byteLength, 0 + ) + const merged = new Uint8Array(totalLength) + let offset = 0 + for (const chunk of this._bufferedSignal) { + merged.set(chunk, offset) + offset += chunk.byteLength + } + return merged + } +} + +module.exports = { BCIInterface } diff --git a/packages/bci-whispercpp/binding.js b/packages/bci-whispercpp/binding.js new file mode 100644 index 0000000000..cea46308c0 --- /dev/null +++ b/packages/bci-whispercpp/binding.js @@ -0,0 +1 @@ +module.exports = require.addon() diff --git a/packages/bci-whispercpp/configChecker.js b/packages/bci-whispercpp/configChecker.js new file mode 100644 index 0000000000..9dd797275c --- /dev/null +++ b/packages/bci-whispercpp/configChecker.js @@ -0,0 +1,82 @@ +'use strict' + +/** + * Validates BCI addon configuration. + * @param {Object} configObject + * @returns {void} or throws if invalid + */ +function checkConfig (configObject) { + const requiredSections = ['whisperConfig', 'contextParams', 'miscConfig'] + + for (const section of requiredSections) { + if (!configObject[section]) { + throw new Error(`${section} object is required`) + } + } + + const validWhisperParams = [ + 'n_threads', + 'duration_ms', + 'translate', + 'no_timestamps', + 'single_segment', + 'print_special', + 'print_progress', + 'print_realtime', + 'print_timestamps', + 'language', + 'detect_language', + 'suppress_blank', + 'suppress_nst', + 'temperature', + 'greedy_best_of', + 'beam_search_beam_size', + 'seed' + ] + + const validContextParams = [ + 'model', + 'use_gpu', + 'flash_attn', + 'gpu_device' + ] + + const validMiscParams = [ + 'caption_enabled' + ] + + const validBCIParams = [ + 'smooth_kernel_std', + 'smooth_kernel_size', + 'sample_rate', + 'day_idx' + ] + + for (const userParam of Object.keys(configObject.whisperConfig)) { + if (!validWhisperParams.includes(userParam)) { + throw new Error(`${userParam} is not a valid parameter for whisperConfig`) + } + } + + for (const userParam of Object.keys(configObject.contextParams)) { + if (!validContextParams.includes(userParam)) { + throw new Error(`${userParam} is not a valid parameter for contextParams`) + } + } + + for (const userParam of Object.keys(configObject.miscConfig)) { + if (!validMiscParams.includes(userParam)) { + throw new Error(`${userParam} is not a valid parameter for miscConfig`) + } + } + + if (configObject.bciConfig) { + for (const userParam of Object.keys(configObject.bciConfig)) { + if (!validBCIParams.includes(userParam)) { + throw new Error(`${userParam} is not a valid parameter for bciConfig`) + } + } + } +} + +module.exports = { checkConfig } diff --git a/packages/bci-whispercpp/examples/transcribe-neural.js b/packages/bci-whispercpp/examples/transcribe-neural.js new file mode 100644 index 0000000000..7921e6c6a0 --- /dev/null +++ b/packages/bci-whispercpp/examples/transcribe-neural.js @@ -0,0 +1,105 @@ +'use strict' + +/** + * Transcribe neural signal files using the BCI BrainWhisperer model. + * Uses the native whisper.cpp GGML backend. + * + * Usage: + * node examples/transcribe-neural.js [model_path] + * + * Or batch mode (all test fixtures): + * node examples/transcribe-neural.js --batch [model_path] + */ + +const fs = require('bare-fs') +const path = require('bare-path') +const os = require('bare-os') +const BCIWhispercpp = require('../index') + +const DEFAULT_MODEL = (os.hasEnv('WHISPER_MODEL_PATH') ? os.getEnv('WHISPER_MODEL_PATH') : null) || + path.join(__dirname, '..', 'models', 'ggml-bci-windowed.bin') + +async function main () { + const args = global.Bare ? global.Bare.argv.slice(2) : process.argv.slice(2) + const isBatch = args[0] === '--batch' + + if (args.length < 1) { + console.log('Usage:') + console.log(' Single: bare examples/transcribe-neural.js [model_path]') + console.log(' Batch: bare examples/transcribe-neural.js --batch [model_path]') + return + } + + const modelPath = (isBatch ? args[1] : args[1]) || DEFAULT_MODEL + if (!fs.existsSync(modelPath)) { + console.error(`Error: Model file not found: ${modelPath}`) + console.error('Set WHISPER_MODEL_PATH or pass as second argument.') + return + } + + const bci = new BCIWhispercpp({ modelPath }, { + whisperConfig: { language: 'en', temperature: 0.0 }, + miscConfig: { caption_enabled: false } + }) + + await bci.load() + console.log('Model loaded.\n') + + if (isBatch) { + const manifestPath = path.join(__dirname, '..', 'test', 'fixtures', 'manifest.json') + const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')) + + console.log(`=== BCI Neural Signal Transcription (Batch: ${manifest.samples.length} samples) ===\n`) + + const startTime = Date.now() + + for (const sample of manifest.samples) { + const samplePath = path.join(__dirname, '..', 'test', 'fixtures', sample.file) + if (!fs.existsSync(samplePath)) { + console.log(` [SKIP] ${sample.file} (not found)`) + continue + } + + const result = await bci.transcribeFile(samplePath) + const wer = BCIWhispercpp.computeWER(result.text, sample.expected_text) + + console.log(` [${sample.file}]`) + console.log(` Got: "${result.text}"`) + console.log(` Expected: "${sample.expected_text}"`) + console.log(` WER: ${(wer * 100).toFixed(1)}%\n`) + } + + const elapsed = ((Date.now() - startTime) / 1000).toFixed(2) + console.log(`Time: ${elapsed}s`) + } else { + const signalPath = args[0] + if (!fs.existsSync(signalPath)) { + console.error(`Error: Signal file not found: ${signalPath}`) + return + } + + const buf = fs.readFileSync(signalPath) + const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength) + const T = view.getUint32(0, true) + const C = view.getUint32(4, true) + + console.log('=== BCI Neural Signal Transcription ===') + console.log(`Signal: ${signalPath}`) + console.log(`Timesteps: ${T}, Channels: ${C}`) + console.log(`Duration: ~${(T * 20 / 1000).toFixed(1)}s\n`) + + const startTime = Date.now() + const result = await bci.transcribeFile(signalPath) + const elapsed = ((Date.now() - startTime) / 1000).toFixed(2) + + console.log(`Text: "${result.text}"`) + console.log(`Time: ${elapsed}s`) + } + + await bci.destroy() + console.log('\nDone.') +} + +main().catch((err) => { + console.error('Error:', err.message || err) +}) diff --git a/packages/bci-whispercpp/index.d.ts b/packages/bci-whispercpp/index.d.ts new file mode 100644 index 0000000000..26e3df0dcd --- /dev/null +++ b/packages/bci-whispercpp/index.d.ts @@ -0,0 +1,100 @@ +declare interface BCIConfig { + smooth_kernel_std?: number; + smooth_kernel_size?: number; + sample_rate?: number; + day_idx?: number; +} + +declare interface WhisperConfig { + language?: string; + n_threads?: number; + temperature?: number; + suppress_nst?: boolean; + duration_ms?: number; + translate?: boolean; + no_timestamps?: boolean; + single_segment?: boolean; + [key: string]: unknown; +} + +declare interface BCIWhispercppArgs { + modelPath: string; + logger?: { + debug(...args: unknown[]): void; + info(...args: unknown[]): void; + warn(...args: unknown[]): void; + error(...args: unknown[]): void; + }; +} + +declare interface BCIWhispercppConfig { + whisperConfig?: WhisperConfig; + bciConfig?: BCIConfig; + contextParams?: { + model?: string; + use_gpu?: boolean; + flash_attn?: boolean; + gpu_device?: number; + }; + miscConfig?: { + caption_enabled?: boolean; + }; +} + +declare interface TranscriptSegment { + text: string; + toAppend: boolean; + start: number; + end: number; + id: number; +} + +declare interface TranscriptionResult { + text: string; + segments: TranscriptSegment[]; + stats: Record | null; +} + +/** + * BCI neural signal transcription client powered by whisper.cpp. + */ +declare class BCIWhispercpp { + constructor(args: BCIWhispercppArgs, config?: BCIWhispercppConfig); + + /** Load and activate the model. */ + load(): Promise; + + /** Transcribe a neural signal binary file. */ + transcribeFile(filePath: string): Promise; + + /** Transcribe neural signal data (batch). */ + transcribe(neuralData: Uint8Array): Promise; + + /** Cancel current inference. */ + cancel(): Promise; + + /** Destroy the instance and release resources. */ + destroy(): Promise; +} + +/** + * Compute Word Error Rate between hypothesis and reference strings. + * @returns WER as a ratio (0.0 = perfect). + */ +declare function computeWER(hypothesis: string, reference: string): number; + +declare namespace BCIWhispercpp { + export { + BCIWhispercpp as default, + BCIWhispercpp, + BCIConfig, + WhisperConfig, + BCIWhispercppArgs, + BCIWhispercppConfig, + TranscriptSegment, + TranscriptionResult, + computeWER, + }; +} + +export = BCIWhispercpp; diff --git a/packages/bci-whispercpp/index.js b/packages/bci-whispercpp/index.js new file mode 100644 index 0000000000..84777e84b4 --- /dev/null +++ b/packages/bci-whispercpp/index.js @@ -0,0 +1,172 @@ +'use strict' + +const fs = require('bare-fs') + +const { BCIInterface } = require('./bci') +const { checkConfig } = require('./configChecker') +const { QvacErrorAddonBCI, ERR_CODES } = require('./lib/error') +const { computeWER } = require('./lib/wer') + +const END_OF_INPUT = 'end of job' + +/** + * High-level BCI transcription client powered by whisper.cpp. + * Accepts neural signal streams and returns text transcriptions. + */ +class BCIWhispercpp { + /** + * @param {Object} args + * @param {string} args.modelPath - path to whisper GGML model file + * @param {Object} [args.logger] - optional logger + * @param {Object} config - inference configuration + * @param {Object} config.whisperConfig - whisper decoding params + * @param {Object} [config.bciConfig] - BCI-specific params + * @param {Object} [config.contextParams] - whisper context params + */ + constructor ({ modelPath, logger = null }, config = {}) { + this._modelPath = modelPath + this._logger = logger || { debug () {}, info () {}, warn () {}, error () {} } + this._config = config + this._addon = null + this._hasActiveResponse = false + this._pendingResolve = null + this._pendingReject = null + this._segments = [] + this._stats = null + + if (!this._modelPath || !fs.existsSync(this._modelPath)) { + throw new Error(`Model file doesn't exist: ${this._modelPath}`) + } + } + + /** + * Load and activate the model. + */ + async load () { + const whisperConfig = { + language: 'en', + temperature: 0.0, + suppress_nst: true, + n_threads: 0, + ...(this._config.whisperConfig || {}) + } + + const configurationParams = { + contextParams: { + model: this._modelPath, + ...(this._config.contextParams || {}) + }, + whisperConfig, + miscConfig: { + caption_enabled: false, + ...(this._config.miscConfig || {}) + } + } + + if (this._config.bciConfig) { + configurationParams.bciConfig = this._config.bciConfig + } + + checkConfig(configurationParams) + + const binding = require('./binding') + this._addon = new BCIInterface( + binding, + configurationParams, + this._outputCallback.bind(this), + this._logger.info.bind(this._logger) + ) + + await this._addon.activate() + this._logger.info('BCI addon activated') + } + + /** + * Transcribe a neural signal from a binary file. + * Binary format: [uint32 numTimesteps, uint32 numChannels, float32[] data] + * @param {string} filePath - path to .bin neural signal file + * @returns {Promise} - { text, segments, stats } + */ + async transcribeFile (filePath) { + const data = fs.readFileSync(filePath) + return this.transcribe(new Uint8Array(data)) + } + + /** + * Transcribe neural signal data (batch mode). + * @param {Uint8Array} neuralData - binary neural signal + * @returns {Promise} - { text, segments, stats } + */ + async transcribe (neuralData) { + if (this._hasActiveResponse) { + throw new QvacErrorAddonBCI({ code: ERR_CODES.JOB_ALREADY_RUNNING }) + } + + return new Promise((resolve, reject) => { + this._beginJob(resolve, reject) + + this._addon.runJob({ input: neuralData }).catch((err) => { + this._clearJob() + reject(err) + }) + }) + } + + _beginJob (resolve, reject) { + this._segments = [] + this._stats = null + this._hasActiveResponse = true + this._pendingResolve = resolve + this._pendingReject = reject + } + + _clearJob () { + this._hasActiveResponse = false + this._pendingResolve = null + this._pendingReject = null + } + + _outputCallback (addon, event, jobId, data, error) { + if (event === 'Output') { + if (Array.isArray(data)) { + this._segments.push(...data) + } else if (data && data.text) { + this._segments.push(data) + } + } else if (event === 'JobEnded') { + this._stats = data + const segments = this._segments + const stats = this._stats + const resolve = this._pendingResolve + this._clearJob() + if (resolve) { + const text = segments.map(s => s.text).join('').trim() + resolve({ text, segments, stats }) + } + } else if (event === 'Error') { + const reject = this._pendingReject + this._clearJob() + if (reject) { + reject(new Error(error || 'Transcription failed')) + } + } + } + + async cancel () { + if (this._addon?.cancel) { + await this._addon.cancel() + } + this._clearJob() + } + + async destroy () { + await this.cancel() + if (this._addon) { + await this._addon.destroyInstance() + } + } +} + +module.exports = BCIWhispercpp +module.exports.BCIWhispercpp = BCIWhispercpp +module.exports.computeWER = computeWER diff --git a/packages/bci-whispercpp/lib/error.js b/packages/bci-whispercpp/lib/error.js new file mode 100644 index 0000000000..bf9ad4c7e4 --- /dev/null +++ b/packages/bci-whispercpp/lib/error.js @@ -0,0 +1,76 @@ +'use strict' + +const { QvacErrorBase, addCodes } = require('@qvac/error') + +class QvacErrorAddonBCI extends QvacErrorBase { } + +const { name, version } = require('../package.json') + +const ERR_CODES = Object.freeze({ + FAILED_TO_LOAD_WEIGHTS: 7001, + FAILED_TO_CANCEL: 7002, + FAILED_TO_APPEND: 7003, + FAILED_TO_GET_STATUS: 7004, + FAILED_TO_DESTROY: 7005, + FAILED_TO_ACTIVATE: 7006, + FAILED_TO_RESET: 7007, + FAILED_TO_PAUSE: 7008, + INVALID_NEURAL_INPUT: 7009, + JOB_ALREADY_RUNNING: 7010, + MODEL_NOT_LOADED: 7011 +}) + +addCodes({ + [ERR_CODES.FAILED_TO_LOAD_WEIGHTS]: { + name: 'FAILED_TO_LOAD_WEIGHTS', + message: (message) => `Failed to load weights, error: ${message}` + }, + [ERR_CODES.FAILED_TO_CANCEL]: { + name: 'FAILED_TO_CANCEL', + message: (message) => `Failed to cancel inference, error: ${message}` + }, + [ERR_CODES.FAILED_TO_APPEND]: { + name: 'FAILED_TO_APPEND', + message: (message) => `Failed to append data to processing queue, error: ${message}` + }, + [ERR_CODES.FAILED_TO_GET_STATUS]: { + name: 'FAILED_TO_GET_STATUS', + message: (message) => `Failed to get addon status, error: ${message}` + }, + [ERR_CODES.FAILED_TO_DESTROY]: { + name: 'FAILED_TO_DESTROY', + message: (message) => `Failed to destroy instance, error: ${message}` + }, + [ERR_CODES.FAILED_TO_ACTIVATE]: { + name: 'FAILED_TO_ACTIVATE', + message: (message) => `Failed to activate model, error: ${message}` + }, + [ERR_CODES.FAILED_TO_RESET]: { + name: 'FAILED_TO_RESET', + message: (message) => `Failed to reset model state, error: ${message}` + }, + [ERR_CODES.FAILED_TO_PAUSE]: { + name: 'FAILED_TO_PAUSE', + message: (message) => `Failed to pause inference, error: ${message}` + }, + [ERR_CODES.INVALID_NEURAL_INPUT]: { + name: 'INVALID_NEURAL_INPUT', + message: (message) => `Invalid neural signal input: ${message}` + }, + [ERR_CODES.JOB_ALREADY_RUNNING]: { + name: 'JOB_ALREADY_RUNNING', + message: () => 'Cannot set new job: a job is already set or being processed' + }, + [ERR_CODES.MODEL_NOT_LOADED]: { + name: 'MODEL_NOT_LOADED', + message: () => 'Model is not loaded' + } +}, { + name, + version +}) + +module.exports = { + ERR_CODES, + QvacErrorAddonBCI +} diff --git a/packages/bci-whispercpp/lib/wer.js b/packages/bci-whispercpp/lib/wer.js new file mode 100644 index 0000000000..9a99084c27 --- /dev/null +++ b/packages/bci-whispercpp/lib/wer.js @@ -0,0 +1,40 @@ +'use strict' + +/** + * Compute Word Error Rate between hypothesis and reference. + * Uses Levenshtein distance on word sequences. + * @param {string} hypothesis + * @param {string} reference + * @returns {number} WER as a ratio (0.0 = perfect, 1.0 = 100% errors) + */ +function computeWER (hypothesis, reference) { + const hyp = hypothesis.toLowerCase().trim().split(/\s+/).filter(Boolean) + const ref = reference.toLowerCase().trim().split(/\s+/).filter(Boolean) + + if (ref.length === 0) return hyp.length === 0 ? 0 : 1 + + const n = ref.length + const m = hyp.length + const dp = Array.from({ length: n + 1 }, () => Array(m + 1).fill(0)) + + for (let i = 0; i <= n; i++) dp[i][0] = i + for (let j = 0; j <= m; j++) dp[0][j] = j + + for (let i = 1; i <= n; i++) { + for (let j = 1; j <= m; j++) { + if (ref[i - 1] === hyp[j - 1]) { + dp[i][j] = dp[i - 1][j - 1] + } else { + dp[i][j] = 1 + Math.min( + dp[i - 1][j], + dp[i][j - 1], + dp[i - 1][j - 1] + ) + } + } + } + + return dp[n][m] / n +} + +module.exports = { computeWER } diff --git a/packages/bci-whispercpp/package.json b/packages/bci-whispercpp/package.json new file mode 100644 index 0000000000..ef7ef8f4f7 --- /dev/null +++ b/packages/bci-whispercpp/package.json @@ -0,0 +1,77 @@ +{ + "name": "@qvac/bci-whispercpp", + "version": "0.1.0", + "description": "Brain-Computer Interface (BCI) neural signal transcription addon for qvac, powered by whisper.cpp", + "addon": true, + "engines": { + "bare": ">=1.19.0" + }, + "scripts": { + "lint": "standard \"examples/**/*.js\" \"test/**/*.js\" \"*.js\"", + "lint:fix": "standard --fix \"examples/**/*.js\" \"test/**/*.js\" \"**/*.js\"", + "build": "bare-make generate && bare-make build && bare-make install", + "test:unit": "brittle-bare test/unit/**/*.test.js", + "test:integration": "brittle-bare test/integration/bci-addon.test.js", + "test:cpp:build": "bare-make generate -D BUILD_TESTING=ON && bare-make build --target test-bci-core && bare-make install", + "test:cpp:run": "cd build/addon/tests/ && ./test-bci-core --gtest_output=xml:cpp-test-results.xml", + "test:cpp": "npm run test:cpp:build && npm run test:cpp:run", + "test": "npm run test:integration", + "test:dts": "tsc index.d.ts --noEmit --lib es2018 --esModuleInterop --skipLibCheck" + }, + "files": [ + "binding.js", + "bci.js", + "configChecker.js", + "index.js", + "index.d.ts", + "prebuilds", + "lib", + "LICENSE", + "NOTICE" + ], + "repository": { + "type": "git", + "url": "git+https://github.com/tetherto/qvac.git" + }, + "author": "Tether", + "keywords": [ + "tether", + "addon", + "whisper", + "bci", + "brain-computer-interface", + "neural", + "qvac" + ], + "license": "Apache-2.0", + "bugs": "https://github.com/tetherto/qvac/issues", + "homepage": "https://github.com/tetherto/qvac#readme", + "devDependencies": { + "bare-buffer": "^3.4.2", + "bare-fs": "^4.5.1", + "bare-tty": "^5.0.3", + "brittle": "^3.17.0", + "cmake-bare": "^1.7.5", + "cmake-vcpkg": "^1.1.0", + "fs": "npm:bare-fs", + "os": "npm:bare-os@^3.6.2", + "standard": "^17.1.2", + "tty": "npm:bare-node-tty" + }, + "dependencies": { + "@qvac/error": "^0.1.0", + "@qvac/logging": "^0.1.0", + "bare-path": "^3.0.0", + "bare-stream": "^2.7.0", + "path": "npm:bare-path" + }, + "exports": { + "./package": "./package.json", + ".": { + "types": "./index.d.ts", + "default": "./index.js" + }, + "./binding.js": "./binding.js" + }, + "types": "index.d.ts" +} diff --git a/packages/bci-whispercpp/scripts/convert-model.py b/packages/bci-whispercpp/scripts/convert-model.py new file mode 100644 index 0000000000..0077aababc --- /dev/null +++ b/packages/bci-whispercpp/scripts/convert-model.py @@ -0,0 +1,459 @@ +#!/usr/bin/env python3 +""" +Convert BrainWhisperer checkpoint to GGML model + embedder weights for whisper.cpp. + +Produces two files required for BCI inference: + 1. GGML model (--output): whisper encoder/decoder weights, tokenizer, positional + embedding, windowed attention params in header + 2. Embedder file (--embedder-output): day projection weights (low-rank A·B per day), + month projections, session-to-day mapping + +Both files must be in the same directory at runtime. The C++ addon loads the embedder +from the same directory as the GGML model (looks for "bci-embedder.bin"). + +Usage: + python3 scripts/convert-model.py \\ + --checkpoint /path/to/epoch=93-val_wer=0.0910.ckpt \\ + --output models/ggml-bci-windowed.bin \\ + --embedder-output models/bci-embedder.bin +""" + +import argparse +import math +import os +import struct + +import numpy as np +import torch + + +def merge_lora_weights(state_dict, alpha=16, r=8): + scaling = alpha / r + merged = {} + lora_pairs = {} + + for key, tensor in state_dict.items(): + if ".lora_A.default.weight" in key: + base_key = key.replace(".lora_A.default.weight", "") + lora_pairs.setdefault(base_key, {})["A"] = tensor + elif ".lora_B.default.weight" in key: + base_key = key.replace(".lora_B.default.weight", "") + lora_pairs.setdefault(base_key, {})["B"] = tensor + elif ".base_layer." in key: + clean_key = key.replace(".base_layer.", ".") + merged[clean_key] = tensor.clone() + else: + merged[key] = tensor + + for base_key, pair in lora_pairs.items(): + if "A" not in pair or "B" not in pair: + continue + A, B = pair["A"], pair["B"] + delta = (B @ A) * scaling + weight_key = base_key + ".weight" + if weight_key in merged: + merged[weight_key] = merged[weight_key] + delta + + return merged + + +def build_positional_embedding(state_dict, d_model=384, day_idx=0, sessions=None): + """Build the combined positional embedding for whisper.cpp. + + The BCI encoder applies two separate positional encodings: + 1. Learned time positions (embed_positions) → first d_model//2 dims + 2. Sinusoidal day encoding (PositionalEncoding) → last d_model//2 dims + + whisper.cpp applies a single encoder.positional_embedding after conv2, + so we must combine both into one (1500, d_model) tensor. + """ + half = d_model - d_model // 2 # 192 + + pe = np.zeros((1500, d_model), dtype=np.float32) + + # First half: learned time positional encoding from the trained model + time_pe_key = "model.whisper.model.encoder.embed_positions.weight" + if time_pe_key in state_dict: + time_pe = state_dict[time_pe_key].numpy() # (1500, 192) + pe[:, :half] = time_pe + print(f" Time positional encoding: shape={time_pe.shape}, " + f"range=[{time_pe.min():.4f}, {time_pe.max():.4f}]") + else: + print(" WARNING: embed_positions.weight not found, using zeros for time encoding") + + # Second half: sinusoidal day encoding + # For day_idx=0 (session index), resolve through SessionsToDays to get day number + # Default: day_number=0 → PositionalEncoding(192) at position 0 = [sin(0),cos(0),...] = [0,1,0,1,...] + day_number = day_idx + if sessions: + from datetime import datetime + sorted_sessions = sorted(sessions) + fmt = "%Y.%m.%d" + datetimes = [datetime.strptime(s[-10:], fmt) for s in sorted_sessions] + if day_idx < len(datetimes): + day_number = (datetimes[day_idx] - datetimes[0]).days + + day_enc = np.zeros(half, dtype=np.float32) + div_term = np.exp(np.arange(0, half, 2, dtype=np.float32) * (-math.log(10000.0) / half)) + day_enc[0::2] = np.sin(day_number * div_term) + day_enc[1::2] = np.cos(day_number * div_term) + pe[:, -half:] = day_enc + print(f" Day encoding: day_number={day_number}, " + f"range=[{day_enc.min():.4f}, {day_enc.max():.4f}]") + + return pe + + +# Byte encoder/decoder for tokenizer (from whisper.cpp converter) +def bytes_to_unicode(): + bs = list(range(ord("!"), ord("~")+1)) + list(range(ord("¡"), ord("¬")+1)) + list(range(ord("®"), ord("ÿ")+1)) + cs = bs[:] + n = 0 + for b in range(2**8): + if b not in bs: + bs.append(b) + cs.append(2**8+n) + n += 1 + cs = [chr(n) for n in cs] + return dict(zip(bs, cs)) + + +# GGML tensor name mapping (HuggingFace → whisper.cpp) +CONV_MAP = { + 'self_attn.k_proj': 'attn.key', + 'self_attn.q_proj': 'attn.query', + 'self_attn.v_proj': 'attn.value', + 'self_attn.out_proj': 'attn.out', + 'self_attn_layer_norm': 'attn_ln', + 'encoder_attn.q_proj': 'cross_attn.query', + 'encoder_attn.v_proj': 'cross_attn.value', + 'encoder_attn.out_proj': 'cross_attn.out', + 'encoder_attn_layer_norm': 'cross_attn_ln', + 'fc1': 'mlp.0', + 'fc2': 'mlp.2', + 'final_layer_norm': 'mlp_ln', +} + + +def rename_key(hf_key): + """Convert HuggingFace key to whisper.cpp GGML key.""" + parts = hf_key.split(".") + if len(parts) < 2: + return hf_key + + section = parts[0] # encoder or decoder + rest = parts[1:] + + if rest[0] == "layers": + rest[0] = "blocks" + layer_idx = rest[1] + inner = ".".join(rest[2:-1]) + + if inner == "encoder_attn.k_proj": + mapped = "cross_attn.key" + elif inner in CONV_MAP: + mapped = CONV_MAP[inner] + else: + mapped = inner + + return f"{section}.blocks.{layer_idx}.{mapped}.{rest[-1]}" + else: + simple_map = { + "layer_norm.bias": f"{section}.ln_post.bias" if section == "encoder" else f"{section}.ln.bias", + "layer_norm.weight": f"{section}.ln_post.weight" if section == "encoder" else f"{section}.ln.weight", + "embed_positions.weight": f"{section}.positional_embedding", + "embed_tokens.weight": f"{section}.token_embedding.weight", + } + rest_str = ".".join(rest) + if rest_str in simple_map: + return simple_map[rest_str] + return f"{section}.{rest_str}" + + +def export_embedder(state_dict, output_path): + """Export day projection / embedder weights to a binary file. + + The C++ NeuralProcessor loads this file to apply day-specific + projection (low-rank A·B + month + softsign) before whisper inference. + Without it, raw smoothed signals are passed directly — producing garbage. + """ + conv1_w = state_dict['model.embedders.0.conv1.weight'].numpy().flatten() + conv1_b = state_dict['model.embedders.0.conv1.bias'].numpy().flatten() + conv2_w = state_dict['model.embedders.0.conv2.weight'].numpy().flatten() + conv2_b = state_dict['model.embedders.0.conv2.bias'].numpy().flatten() + + embed_dim = int(state_dict['model.embedders.0.conv1.weight'].shape[0]) + num_features = int(state_dict['model.embedders.0.conv1.weight'].shape[1]) + kernel_size1 = int(state_dict['model.embedders.0.conv1.weight'].shape[2]) + kernel_size2 = int(state_dict['model.embedders.0.conv2.weight'].shape[2]) + + day_a_keys = sorted( + [k for k in state_dict if k.startswith('model.embedders.0.day_As.')], + key=lambda k: int(k.split('.')[-1])) + day_b_keys = sorted( + [k for k in state_dict if k.startswith('model.embedders.0.day_Bs.')], + key=lambda k: int(k.split('.')[-1])) + day_bias_keys = sorted( + [k for k in state_dict if k.startswith('model.embedders.0.day_biases.')], + key=lambda k: int(k.split('.')[-1])) + month_w_keys = sorted( + [k for k in state_dict if k.startswith('model.embedders.0.month_weights.')], + key=lambda k: int(k.split('.')[-1])) + month_b_keys = sorted( + [k for k in state_dict if k.startswith('model.embedders.0.month_biases.')], + key=lambda k: int(k.split('.')[-1])) + + num_days = len(day_a_keys) + num_months = len(month_w_keys) + r = int(state_dict[day_a_keys[0]].shape[1]) if day_a_keys else 0 + + s2d = state_dict.get('model.embedders.0.sessions_to_days.session_to_idx_map') + + EMBEDDER_MAGIC = 0x42434945 + os.makedirs(os.path.dirname(output_path) or ".", exist_ok=True) + + with open(output_path, "wb") as f: + f.write(struct.pack('I', EMBEDDER_MAGIC)) + f.write(struct.pack('I', 1)) # version + f.write(struct.pack('I', num_features)) + f.write(struct.pack('I', embed_dim)) + f.write(struct.pack('I', kernel_size1)) + f.write(struct.pack('I', kernel_size2)) + f.write(struct.pack('I', 2)) # stride2 + f.write(struct.pack('I', num_days)) + f.write(struct.pack('I', num_months)) + f.write(struct.pack('I', r)) + + for arr in [conv1_w, conv1_b, conv2_w, conv2_b]: + f.write(struct.pack('I', len(arr))) + f.write(arr.astype(np.float32).tobytes()) + + if s2d is not None: + s2d_np = s2d.numpy().astype(np.int32).flatten() + f.write(struct.pack('I', len(s2d_np))) + f.write(s2d_np.tobytes()) + else: + f.write(struct.pack('I', 0)) + + for i in range(num_days): + for keys in [day_a_keys, day_b_keys, day_bias_keys]: + data = state_dict[keys[i]].numpy().flatten().astype(np.float32) + f.write(struct.pack('I', len(data))) + f.write(data.tobytes()) + + for i in range(num_months): + for keys in [month_w_keys, month_b_keys]: + data = state_dict[keys[i]].numpy().flatten().astype(np.float32) + f.write(struct.pack('I', len(data))) + f.write(data.tobytes()) + + size_mb = os.path.getsize(output_path) / (1024 * 1024) + print(f" Embedder: {output_path} ({size_mb:.1f} MB)") + print(f" {num_days} days, {num_months} months, rank={r}, " + f"features={num_features}") + + +def main(): + parser = argparse.ArgumentParser( + description="Convert BrainWhisperer checkpoint to GGML model + embedder") + parser.add_argument("--checkpoint", required=True, + help="Path to BrainWhisperer .ckpt file") + parser.add_argument("--output", default="models/ggml-bci-windowed.bin", + help="Output path for GGML model (default: models/ggml-bci-windowed.bin)") + parser.add_argument("--embedder-output", default="models/bci-embedder.bin", + help="Output path for embedder weights (default: models/bci-embedder.bin)") + parser.add_argument("--f32", action="store_true", + help="Use f32 for all tensors (avoids f16 precision loss)") + parser.add_argument("--day-idx", type=int, default=1, + help="Day index for baked positional embedding (default: 1)") + parser.add_argument("--window-size", type=int, default=57, + help="Windowed attention size, 0 to disable (default: 57)") + parser.add_argument("--last-window-layer", type=int, default=3, + help="Last encoder layer with windowed attention (default: 3)") + args = parser.parse_args() + + os.makedirs(os.path.dirname(args.output) or ".", exist_ok=True) + + # Load checkpoint + print(f"Loading checkpoint: {args.checkpoint}") + ckpt = torch.load(args.checkpoint, map_location="cpu", weights_only=False) + state_dict = ckpt["state_dict"] + config = ckpt["hyper_parameters"]["config"] + + # Merge LoRA + print("Merging LoRA weights...") + merged = merge_lora_weights(state_dict, alpha=16, r=8) + + # Build the model state dict for GGML + # We need: encoder (conv1/conv2 from embedder, layers 0-5 from encoder, layer_norm) + # decoder (LoRA-merged layers 0-3, embed_tokens, embed_positions, layer_norm) + # proj_out + + model_sd = {} + + # --- Encoder conv1 from EMBEDDER (k=7, 512->384) — patched whisper.cpp supports this --- + model_sd["encoder.conv1.weight"] = merged["model.embedders.0.conv1.weight"] # (384, 512, 7) + model_sd["encoder.conv1.bias"] = merged["model.embedders.0.conv1.bias"] # (384,) + + # --- Encoder conv2 from EMBEDDER (k=3, stride=2) --- + model_sd["encoder.conv2.weight"] = merged["model.embedders.0.conv2.weight"] # (384, 384, 3) + model_sd["encoder.conv2.bias"] = merged["model.embedders.0.conv2.bias"] # (384,) + + # --- Encoder positional embedding (combined time + day encoding) --- + # Extract sessions list from checkpoint config for day number resolution + sessions = config.get("dataset", {}).get("sessions", None) + if sessions is None: + sessions = config.get("sessions", None) + print("Building combined positional embedding...") + model_sd["encoder.positional_embedding"] = torch.from_numpy( + build_positional_embedding(merged, d_model=384, day_idx=args.day_idx, sessions=sessions)) + + # --- Encoder transformer layers 0-5 --- + for layer_idx in range(6): + prefix_src = f"model.whisper.model.encoder.layers.{layer_idx}." + for key, tensor in merged.items(): + if key.startswith(prefix_src): + suffix = key[len("model.whisper.model.encoder."):] + ggml_name = rename_key(f"encoder.{suffix}") + model_sd[ggml_name] = tensor + + # --- Encoder layer norm --- + model_sd["encoder.ln_post.weight"] = merged["model.whisper.model.encoder.layer_norm.weight"] + model_sd["encoder.ln_post.bias"] = merged["model.whisper.model.encoder.layer_norm.bias"] + + # --- Decoder (LoRA-merged) --- + dec_prefix = "model.whisper.model.decoder." + for key, tensor in merged.items(): + if not key.startswith(dec_prefix): + continue + # Remove PEFT wrapper + clean = key[len("model.whisper.model."):] + clean = clean.replace("decoder.base_model.model.", "decoder.") + ggml_name = rename_key(clean) + model_sd[ggml_name] = tensor + + # --- proj_out --- + if "model.whisper.proj_out.weight" in merged: + # whisper.cpp skips proj_out (uses decoder.token_embedding transposed) + pass + + # Model hyperparameters + d_model = 384 + n_audio_head = 6 + n_audio_layer = 6 + n_text_head = 6 + n_text_layer = 4 + n_mels = 512 # neural signal channels (conv1 k=7 in patched whisper.cpp) + n_conv1_kernel = 7 + n_vocab = 51864 + n_audio_ctx = 1500 + n_text_ctx = 448 + + print(f"\nGGML model: n_mels={n_mels}, encoder_layers={n_audio_layer}, " + f"decoder_layers={n_text_layer}, d_model={d_model}") + print(f"Tensors to write: {len(model_sd)}") + + # Mel filters: must have n_mel rows matching the header n_mels value, + # because whisper_set_mel_with_state validates n_mel == filters.n_mel. + mel_filters = np.zeros((n_mels, 201), dtype=np.float32) + + # Load tokenizer + from transformers import WhisperTokenizer + tokenizer = WhisperTokenizer.from_pretrained("openai/whisper-tiny.en") + tokens_dict = tokenizer.get_vocab() + tokens_sorted = sorted(tokens_dict.items(), key=lambda x: x[1]) + + byte_decoder = {v: k for k, v in bytes_to_unicode().items()} + + # Write GGML file + print(f"\nWriting GGML model to: {args.output}") + with open(args.output, "wb") as fout: + # Magic + fout.write(struct.pack("i", 0x67676d6c)) + + # Header (matches whisper.cpp expected order) + fout.write(struct.pack("i", n_vocab)) + fout.write(struct.pack("i", n_audio_ctx)) + fout.write(struct.pack("i", d_model)) + fout.write(struct.pack("i", n_audio_head)) + fout.write(struct.pack("i", n_audio_layer)) + fout.write(struct.pack("i", n_text_ctx)) + fout.write(struct.pack("i", d_model)) + fout.write(struct.pack("i", n_text_head)) + fout.write(struct.pack("i", n_text_layer)) + fout.write(struct.pack("i", n_mels)) + ftype_global = 0 if args.f32 else 1 + fout.write(struct.pack("i", ftype_global)) # ftype: 0=f32, 1=f16 + fout.write(struct.pack("i", n_conv1_kernel)) # BCI extension + fout.write(struct.pack("i", args.window_size)) # BCI windowed attention + fout.write(struct.pack("i", args.last_window_layer)) + + # Mel filters (n_mels x 201, must match n_mels for whisper_set_mel validation) + fout.write(struct.pack("i", mel_filters.shape[0])) + fout.write(struct.pack("i", mel_filters.shape[1])) + for i in range(mel_filters.shape[0]): + for j in range(mel_filters.shape[1]): + fout.write(struct.pack("f", mel_filters[i][j])) + + # Tokenizer + fout.write(struct.pack("i", len(tokens_sorted))) + for token_str, token_id in tokens_sorted: + try: + text = bytearray([byte_decoder[c] for c in token_str]) + except KeyError: + text = token_str.encode("utf-8") + fout.write(struct.pack("i", len(text))) + fout.write(text) + + # Write tensors + for name, tensor in model_sd.items(): + data = tensor.squeeze().numpy() + + # Reshape conv bias from [n] to [n, 1] + if name in ["encoder.conv1.bias", "encoder.conv2.bias"]: + data = data.reshape(data.shape[0], 1) + + n_dims = len(data.shape) + + use_f16 = not args.f32 + ftype = 1 if use_f16 else 0 + if n_dims < 2 or \ + name == "encoder.conv1.bias" or \ + name == "encoder.conv2.bias" or \ + name == "encoder.positional_embedding" or \ + name == "decoder.positional_embedding": + use_f16 = False + ftype = 0 + + if use_f16: + data = data.astype(np.float16) + else: + data = data.astype(np.float32) + + # Tensor header: n_dims, name_len, ftype + name_bytes = name.encode("utf-8") + fout.write(struct.pack("iii", n_dims, len(name_bytes), ftype)) + + # Dims (reversed from numpy, as GGML expects) + for i in range(n_dims): + fout.write(struct.pack("i", data.shape[n_dims - 1 - i])) + + fout.write(name_bytes) + data.tofile(fout) + + print(f" {name}: {data.shape} ({'f16' if ftype == 1 else 'f32'})") + + size_mb = os.path.getsize(args.output) / (1024 * 1024) + print(f" GGML model: {args.output} ({size_mb:.1f} MB)") + + # --- Export embedder weights --- + print(f"\nWriting embedder weights to: {args.embedder_output}") + export_embedder(state_dict, args.embedder_output) + + print(f"\nDone. Both files are required for inference:") + print(f" {args.output}") + print(f" {args.embedder_output}") + + +if __name__ == "__main__": + main() diff --git a/packages/bci-whispercpp/scripts/download-models.sh b/packages/bci-whispercpp/scripts/download-models.sh new file mode 100755 index 0000000000..4fc8a19c8f --- /dev/null +++ b/packages/bci-whispercpp/scripts/download-models.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PACKAGE_DIR="$(dirname "$SCRIPT_DIR")" +MODELS_DIR="${PACKAGE_DIR}/models" + +mkdir -p "$MODELS_DIR" + +MODEL_NAME="ggml-tiny.en.bin" +MODEL_URL="https://huggingface.co/ggerganov/whisper.cpp/resolve/main/${MODEL_NAME}" +MODEL_PATH="${MODELS_DIR}/${MODEL_NAME}" + +if [ -f "$MODEL_PATH" ]; then + echo "Model already exists: ${MODEL_PATH}" +else + echo "Downloading ${MODEL_NAME}..." + curl -L "$MODEL_URL" -o "$MODEL_PATH" + echo "Downloaded to: ${MODEL_PATH}" +fi + +echo "Done." diff --git a/packages/bci-whispercpp/test/fixtures/manifest.json b/packages/bci-whispercpp/test/fixtures/manifest.json new file mode 100644 index 0000000000..1223a73316 --- /dev/null +++ b/packages/bci-whispercpp/test/fixtures/manifest.json @@ -0,0 +1,54 @@ +{ + "samples": [ + { + "file": "neural_sample_0.bin", + "timesteps": 910, + "channels": 512, + "expected_text": "You can see the code at this point as well.", + "day_idx": 1, + "bci_transcription": "you can see the good at this point as well", + "bci_wer_vs_expected": null, + "bci_wer": 0.1 + }, + { + "file": "neural_sample_1.bin", + "timesteps": 749, + "channels": 512, + "expected_text": "How does it keep the cost down?", + "day_idx": 1, + "bci_transcription": "how does it keep the cost said", + "bci_wer_vs_expected": null, + "bci_wer": 0.1429 + }, + { + "file": "neural_sample_2.bin", + "timesteps": 502, + "channels": 512, + "expected_text": "Not too controversial.", + "day_idx": 1, + "bci_transcription": "not too controversial", + "bci_wer_vs_expected": null, + "bci_wer": 0.0 + }, + { + "file": "neural_sample_3.bin", + "timesteps": 962, + "channels": 512, + "expected_text": "The jury and a judge work together on it.", + "day_idx": 1, + "bci_transcription": "the jury and a judge work together on it", + "bci_wer_vs_expected": null, + "bci_wer": 0.0 + }, + { + "file": "neural_sample_4.bin", + "timesteps": 584, + "channels": 512, + "expected_text": "Were quite vocal about it.", + "day_idx": 1, + "bci_transcription": "we're quite vocal about it", + "bci_wer_vs_expected": null, + "bci_wer": 0.2 + } + ] +} diff --git a/packages/bci-whispercpp/test/integration/bci-addon.test.js b/packages/bci-whispercpp/test/integration/bci-addon.test.js new file mode 100644 index 0000000000..171a592f72 --- /dev/null +++ b/packages/bci-whispercpp/test/integration/bci-addon.test.js @@ -0,0 +1,69 @@ +'use strict' + +const fs = require('bare-fs') +const path = require('bare-path') +const test = require('brittle') +const os = require('bare-os') +const BCIWhispercpp = require('../../index') +const { getTestPaths, computeWER } = require('./helpers') + +const { manifest, getSamplePath } = getTestPaths() + +const MODEL_PATH = (os.hasEnv('WHISPER_MODEL_PATH') ? os.getEnv('WHISPER_MODEL_PATH') : null) || + path.join(__dirname, '..', '..', 'models', 'ggml-tiny.en.bin') + +const hasModel = fs.existsSync(MODEL_PATH) + +test('[BCI] load and destroy via package interface', { skip: !hasModel, timeout: 120000 }, async (t) => { + const bci = new BCIWhispercpp({ modelPath: MODEL_PATH }, { + whisperConfig: { language: 'en', temperature: 0.0 }, + miscConfig: { caption_enabled: false } + }) + + await bci.load() + t.ok(bci, 'BCIWhispercpp should be created and loaded') + + await bci.destroy() + t.pass('BCIWhispercpp destroyed successfully') +}) + +test('[BCI] batch transcription from neural signal file', { skip: !hasModel, timeout: 120000 }, async (t) => { + if (manifest.samples.length === 0) { + t.skip('No neural signal test fixtures found') + return + } + + const sample = manifest.samples[0] + const samplePath = getSamplePath(sample.file) + if (!fs.existsSync(samplePath)) { + t.skip(`Sample file missing: ${samplePath}`) + return + } + + const bci = new BCIWhispercpp({ modelPath: MODEL_PATH }, { + whisperConfig: { language: 'en', temperature: 0.0 }, + miscConfig: { caption_enabled: false } + }) + + try { + await bci.load() + + const result = await bci.transcribeFile(samplePath) + + console.log('\n=== Batch Transcription Result ===') + console.log(`Expected: "${sample.expected_text}"`) + console.log(`Got: "${result.text}"`) + + const wer = computeWER(result.text, sample.expected_text) + console.log(`WER: ${(wer * 100).toFixed(1)}%`) + + t.ok(typeof result.text === 'string', 'Should produce a transcription string') + t.ok(result.segments, 'Should have segments') + t.ok(typeof wer === 'number' && wer >= 0, 'WER should be a non-negative number') + console.log('\nNote: High WER expected - standard whisper model is not BCI-trained.') + console.log('A BCI-trained GGML model is needed for meaningful neural-to-text results.') + } finally { + await bci.destroy() + } +}) + diff --git a/packages/bci-whispercpp/test/integration/helpers.js b/packages/bci-whispercpp/test/integration/helpers.js new file mode 100644 index 0000000000..7e2d251343 --- /dev/null +++ b/packages/bci-whispercpp/test/integration/helpers.js @@ -0,0 +1,34 @@ +'use strict' + +const fs = require('bare-fs') +const path = require('bare-path') +const { computeWER } = require('../../lib/wer') + +function getTestPaths () { + const fixturesDir = path.join(__dirname, '..', 'fixtures') + const manifestPath = path.join(fixturesDir, 'manifest.json') + + let manifest = { samples: [] } + if (fs.existsSync(manifestPath)) { + manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')) + } + + return { + fixturesDir, + manifest, + getSamplePath: (filename) => path.join(fixturesDir, filename) + } +} + +function detectPlatform () { + const os = require('bare-os') + const arch = os.arch() + const platform = os.platform() + return { arch, platform, label: `${platform}-${arch}` } +} + +module.exports = { + getTestPaths, + detectPlatform, + computeWER +} diff --git a/packages/bci-whispercpp/vcpkg-configuration.json b/packages/bci-whispercpp/vcpkg-configuration.json new file mode 100644 index 0000000000..cf90bf82c2 --- /dev/null +++ b/packages/bci-whispercpp/vcpkg-configuration.json @@ -0,0 +1,17 @@ +{ + "default-registry": { + "kind": "git", + "baseline": "87ef7179f70122d0cc65a5991b88c20cab59b1e1", + "repository": "git@github.com:tetherto/qvac-registry-vcpkg.git" + }, + "registries": [ + { + "kind": "git", + "baseline": "16c71a39e5a0fc0bdb3fad03beef8f38ee00ee3b", + "repository": "https://github.com/microsoft/vcpkg", + "packages": [ + "gtest" + ] + } + ] +} diff --git a/packages/bci-whispercpp/vcpkg-overlays/qvac-lint-cpp/portfile.cmake b/packages/bci-whispercpp/vcpkg-overlays/qvac-lint-cpp/portfile.cmake new file mode 100644 index 0000000000..ff8c032cac --- /dev/null +++ b/packages/bci-whispercpp/vcpkg-overlays/qvac-lint-cpp/portfile.cmake @@ -0,0 +1,7 @@ +file(WRITE "${CURRENT_PACKAGES_DIR}/share/${PORT}/.clang-format" "") +file(WRITE "${CURRENT_PACKAGES_DIR}/share/${PORT}/.clang-tidy" "") +file(WRITE "${CURRENT_PACKAGES_DIR}/share/${PORT}/.valgrind.supp" "") +file(MAKE_DIRECTORY "${CURRENT_PACKAGES_DIR}/tools/${PORT}/hooks") +file(WRITE "${CURRENT_PACKAGES_DIR}/tools/${PORT}/hooks/pre-commit" "#!/bin/sh\nexit 0\n") +file(WRITE "${CURRENT_PACKAGES_DIR}/share/${PORT}/copyright" "Stub overlay port") + diff --git a/packages/bci-whispercpp/vcpkg-overlays/qvac-lint-cpp/vcpkg.json b/packages/bci-whispercpp/vcpkg-overlays/qvac-lint-cpp/vcpkg.json new file mode 100644 index 0000000000..0a180e7609 --- /dev/null +++ b/packages/bci-whispercpp/vcpkg-overlays/qvac-lint-cpp/vcpkg.json @@ -0,0 +1,5 @@ +{ + "name": "qvac-lint-cpp", + "version-string": "1.4.1", + "description": "No-op overlay — linting headers not needed for runtime builds" +} diff --git a/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/0001-fix-vcpkg-build.patch b/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/0001-fix-vcpkg-build.patch new file mode 100644 index 0000000000..e587ea07d4 --- /dev/null +++ b/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/0001-fix-vcpkg-build.patch @@ -0,0 +1,277 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 36eef350..dfcc171d 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -23,10 +23,18 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) + if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + set(WHISPER_STANDALONE ON) + +- include(git-vars) ++ find_package(Git QUIET) ++ if(GIT_FOUND) ++ include(git-vars) ++ else() ++ set(GIT_SHA1 "unknown") ++ set(GIT_DATE "unknown") ++ set(GIT_COMMIT_SUBJECT "unknown") ++ endif() + +- # configure project version +- configure_file(${CMAKE_SOURCE_DIR}/bindings/javascript/package-tmpl.json ${CMAKE_SOURCE_DIR}/bindings/javascript/package.json @ONLY) ++ if(EXISTS ${CMAKE_SOURCE_DIR}/bindings/javascript/package-tmpl.json) ++ configure_file(${CMAKE_SOURCE_DIR}/bindings/javascript/package-tmpl.json ${CMAKE_SOURCE_DIR}/bindings/javascript/package.json @ONLY) ++ endif() + else() + set(WHISPER_STANDALONE OFF) + endif() +@@ -169,23 +177,34 @@ set(WHISPER_BUILD_NUMBER ${BUILD_NUMBER}) + set(WHISPER_BUILD_COMMIT ${BUILD_COMMIT}) + set(WHISPER_INSTALL_VERSION ${CMAKE_PROJECT_VERSION}) + +-set(WHISPER_INCLUDE_INSTALL_DIR ${CMAKE_INSTALL_INCLUDEDIR} CACHE PATH "Location of header files") ++set(WHISPER_INCLUDE_INSTALL_DIR ${CMAKE_INSTALL_INCLUDEDIR}/whisper CACHE PATH "Location of header files") + set(WHISPER_LIB_INSTALL_DIR ${CMAKE_INSTALL_LIBDIR} CACHE PATH "Location of library files") + set(WHISPER_BIN_INSTALL_DIR ${CMAKE_INSTALL_BINDIR} CACHE PATH "Location of binary files") + + get_directory_property(WHISPER_TRANSIENT_DEFINES COMPILE_DEFINITIONS) + + set_target_properties(whisper PROPERTIES PUBLIC_HEADER ${CMAKE_CURRENT_SOURCE_DIR}/include/whisper.h) +-install(TARGETS whisper LIBRARY PUBLIC_HEADER) ++ ++install( ++ TARGETS whisper ++ EXPORT whisper-targets ++ PUBLIC_HEADER ++ DESTINATION ${WHISPER_INCLUDE_INSTALL_DIR}) ++ ++install( ++ EXPORT whisper-targets ++ FILE whisper-targets.cmake ++ NAMESPACE whisper:: ++ DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/whisper) ++ ++install( ++ FILES ${CMAKE_CURRENT_BINARY_DIR}/whisper-config.cmake ++ DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/whisper) + + configure_package_config_file( +- ${CMAKE_CURRENT_SOURCE_DIR}/cmake/whisper-config.cmake.in +- ${CMAKE_CURRENT_BINARY_DIR}/whisper-config.cmake +- INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/whisper +- PATH_VARS +- WHISPER_INCLUDE_INSTALL_DIR +- WHISPER_LIB_INSTALL_DIR +- WHISPER_BIN_INSTALL_DIR ) ++ ${CMAKE_CURRENT_SOURCE_DIR}/cmake/whisper-config.cmake.in ++ ${CMAKE_CURRENT_BINARY_DIR}/whisper-config.cmake ++ INSTALL_DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/whisper) + + write_basic_package_version_file( + ${CMAKE_CURRENT_BINARY_DIR}/whisper-version.cmake +@@ -194,7 +213,7 @@ write_basic_package_version_file( + + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/whisper-config.cmake + ${CMAKE_CURRENT_BINARY_DIR}/whisper-version.cmake +- DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/whisper) ++ DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/whisper) + + configure_file(cmake/whisper.pc.in + "${CMAKE_CURRENT_BINARY_DIR}/whisper.pc" +diff --git a/cmake/git-vars.cmake b/cmake/git-vars.cmake +index 1a4c24eb..8dc51859 100644 +--- a/cmake/git-vars.cmake ++++ b/cmake/git-vars.cmake +@@ -1,22 +1,36 @@ + find_package(Git) + +-# the commit's SHA1 +-execute_process(COMMAND +- "${GIT_EXECUTABLE}" describe --match=NeVeRmAtCh --always --abbrev=8 +- WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" +- OUTPUT_VARIABLE GIT_SHA1 +- ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) ++if(GIT_FOUND) ++ execute_process(COMMAND ++ "${GIT_EXECUTABLE}" describe --match=NeVeRmAtCh --always --abbrev=8 ++ WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" ++ OUTPUT_VARIABLE GIT_SHA1 ++ ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE ++ RESULT_VARIABLE GIT_SHA1_RESULT) + +-# the date of the commit +-execute_process(COMMAND +- "${GIT_EXECUTABLE}" log -1 --format=%ad --date=local +- WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" +- OUTPUT_VARIABLE GIT_DATE +- ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) ++ execute_process(COMMAND ++ "${GIT_EXECUTABLE}" log -1 --format=%ad --date=local ++ WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" ++ OUTPUT_VARIABLE GIT_DATE ++ ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE ++ RESULT_VARIABLE GIT_DATE_RESULT) + +-# the subject of the commit +-execute_process(COMMAND +- "${GIT_EXECUTABLE}" log -1 --format=%s +- WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" +- OUTPUT_VARIABLE GIT_COMMIT_SUBJECT +- ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) ++ execute_process(COMMAND ++ "${GIT_EXECUTABLE}" log -1 --format=%s ++ WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" ++ OUTPUT_VARIABLE GIT_COMMIT_SUBJECT ++ ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE ++ RESULT_VARIABLE GIT_COMMIT_SUBJECT_RESULT) ++endif() ++ ++if(NOT GIT_FOUND OR GIT_SHA1_RESULT OR NOT GIT_SHA1) ++ set(GIT_SHA1 "unknown") ++endif() ++ ++if(NOT GIT_FOUND OR GIT_DATE_RESULT OR NOT GIT_DATE) ++ set(GIT_DATE "unknown") ++endif() ++ ++if(NOT GIT_FOUND OR GIT_COMMIT_SUBJECT_RESULT OR NOT GIT_COMMIT_SUBJECT) ++ set(GIT_COMMIT_SUBJECT "unknown") ++endif() +diff --git a/cmake/whisper-config.cmake.in b/cmake/whisper-config.cmake.in +index 6a3fa227..9fe65884 100644 +--- a/cmake/whisper-config.cmake.in ++++ b/cmake/whisper-config.cmake.in +@@ -11,24 +11,21 @@ set(GGML_ACCELERATE @GGML_ACCELERATE@) + + @PACKAGE_INIT@ + +-set_and_check(WHISPER_INCLUDE_DIR "@PACKAGE_WHISPER_INCLUDE_INSTALL_DIR@") +-set_and_check(WHISPER_LIB_DIR "@PACKAGE_WHISPER_LIB_INSTALL_DIR@") +-set_and_check(WHISPER_BIN_DIR "@PACKAGE_WHISPER_BIN_INSTALL_DIR@") ++include(CMakeFindDependencyMacro) + + # Ensure transient dependencies satisfied +- +-find_package(Threads REQUIRED) ++find_dependency(Threads REQUIRED) + + if (APPLE AND GGML_ACCELERATE) + find_library(ACCELERATE_FRAMEWORK Accelerate REQUIRED) + endif() + + if (GGML_BLAS) +- find_package(BLAS REQUIRED) ++ find_dependency(BLAS REQUIRED) + endif() + + if (GGML_CUDA) +- find_package(CUDAToolkit REQUIRED) ++ find_dependency(CUDAToolkit REQUIRED) + endif() + + if (GGML_METAL) +@@ -38,28 +35,13 @@ if (GGML_METAL) + endif() + + if (GGML_HIPBLAS) +- find_package(hip REQUIRED) +- find_package(hipblas REQUIRED) +- find_package(rocblas REQUIRED) ++ find_dependency(hip REQUIRED) ++ find_dependency(hipblas REQUIRED) ++ find_dependency(rocblas REQUIRED) + endif() + +-find_library(whisper_LIBRARY whisper +- REQUIRED +- HINTS ${WHISPER_LIB_DIR}) +- +-set(_whisper_link_deps "Threads::Threads" "@WHISPER_EXTRA_LIBS@") +-set(_whisper_transient_defines "@WHISPER_TRANSIENT_DEFINES@") +- +-add_library(whisper UNKNOWN IMPORTED) ++find_dependency(ggml CONFIG REQUIRED) + +-set_target_properties(whisper +- PROPERTIES +- INTERFACE_INCLUDE_DIRECTORIES "${WHISPER_INCLUDE_DIR}" +- INTERFACE_LINK_LIBRARIES "${_whisper_link_deps}" +- INTERFACE_COMPILE_DEFINITIONS "${_whisper_transient_defines}" +- IMPORTED_LINK_INTERFACE_LANGUAGES "CXX" +- IMPORTED_LOCATION "${whisper_LIBRARY}" +- INTERFACE_COMPILE_FEATURES cxx_std_11 +- POSITION_INDEPENDENT_CODE ON ) ++include("${CMAKE_CURRENT_LIST_DIR}/whisper-targets.cmake") + + check_required_components(whisper) +diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txt +index 4e7399f9..fd3ccebe 100644 +--- a/ggml/CMakeLists.txt ++++ b/ggml/CMakeLists.txt +@@ -277,8 +277,17 @@ set_target_properties(ggml PROPERTIES PUBLIC_HEADER "${GGML_PUBLIC_HEADERS}") + #if (GGML_METAL) + # set_target_properties(ggml PROPERTIES RESOURCE "${CMAKE_CURRENT_SOURCE_DIR}/src/ggml-metal.metal") + #endif() +-install(TARGETS ggml LIBRARY PUBLIC_HEADER) +-install(TARGETS ggml-base LIBRARY) ++install( ++ TARGETS ggml ggml-base ++ EXPORT ggml-targets ++ PUBLIC_HEADER ++ DESTINATION ${GGML_INCLUDE_INSTALL_DIR}) ++ ++install( ++ EXPORT ggml-targets ++ FILE ggml-targets.cmake ++ NAMESPACE ggml:: ++ DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/ggml) + + if (GGML_STANDALONE) + configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ggml.pc.in +@@ -349,7 +358,7 @@ set(GGML_BIN_INSTALL_DIR ${CMAKE_INSTALL_BINDIR} CACHE PATH "Location of + configure_package_config_file( + ${CMAKE_CURRENT_SOURCE_DIR}/cmake/ggml-config.cmake.in + ${CMAKE_CURRENT_BINARY_DIR}/ggml-config.cmake +- INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/ggml ++ INSTALL_DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/ggml + PATH_VARS GGML_INCLUDE_INSTALL_DIR + GGML_LIB_INSTALL_DIR + GGML_BIN_INSTALL_DIR) +@@ -361,7 +370,7 @@ write_basic_package_version_file( + + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/ggml-config.cmake + ${CMAKE_CURRENT_BINARY_DIR}/ggml-version.cmake +- DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/ggml) ++ DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/ggml) + + if (MSVC) + set(MSVC_WARNING_FLAGS +diff --git a/ggml/src/CMakeLists.txt b/ggml/src/CMakeLists.txt +index 9cb2c228..6396d883 100644 +--- a/ggml/src/CMakeLists.txt ++++ b/ggml/src/CMakeLists.txt +@@ -231,7 +231,7 @@ function(ggml_add_backend_library backend) + else() + add_library(${backend} ${ARGN}) + target_link_libraries(ggml PUBLIC ${backend}) +- install(TARGETS ${backend} LIBRARY) ++ install(TARGETS ${backend} EXPORT ggml-targets) + endif() + + target_link_libraries(${backend} PRIVATE ggml-base) +diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt +index 2eae0c66..cd4c60e8 100644 +--- a/src/CMakeLists.txt ++++ b/src/CMakeLists.txt +@@ -114,7 +114,11 @@ set_target_properties(whisper PROPERTIES + SOVERSION ${SOVERSION} + ) + +-target_include_directories(whisper PUBLIC . ../include) ++target_include_directories( ++ whisper ++ PUBLIC ++ $ ++ $) + target_compile_features (whisper PUBLIC cxx_std_11) # don't bump + + if (CMAKE_CXX_BYTE_ORDER STREQUAL "BIG_ENDIAN") diff --git a/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/0002-fix-apple-silicon-cross-compile.patch b/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/0002-fix-apple-silicon-cross-compile.patch new file mode 100644 index 0000000000..f8154f1f92 --- /dev/null +++ b/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/0002-fix-apple-silicon-cross-compile.patch @@ -0,0 +1,15 @@ +diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txt +index fd3cceb..d072fe6 100644 +--- a/ggml/CMakeLists.txt ++++ b/ggml/CMakeLists.txt +@@ -58,7 +58,9 @@ else() + set(GGML_BLAS_VENDOR_DEFAULT "Generic") + endif() + +-if (CMAKE_CROSSCOMPILING OR DEFINED ENV{SOURCE_DATE_EPOCH}) ++if (CMAKE_CROSSCOMPILING OR DEFINED ENV{SOURCE_DATE_EPOCH} OR ++ (APPLE AND CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "arm64" AND ++ CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")) + message(STATUS "Setting GGML_NATIVE_DEFAULT to OFF") + set(GGML_NATIVE_DEFAULT OFF) + else() diff --git a/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/0003-bci-variable-conv1-kernel.patch b/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/0003-bci-variable-conv1-kernel.patch new file mode 100644 index 0000000000..025f8c29c0 --- /dev/null +++ b/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/0003-bci-variable-conv1-kernel.patch @@ -0,0 +1,28 @@ +diff --git a/src/whisper.cpp b/src/whisper.cpp +--- a/src/whisper.cpp ++++ b/src/whisper.cpp +@@ -633,6 +633,7 @@ + int32_t n_mels = 80; + int32_t ftype = 1; + float eps = 1e-5f; ++ int32_t n_audio_conv1_kernel = 3; + }; + + // audio encoding layer +@@ -1535,6 +1536,7 @@ + read_safe(loader, hparams.n_text_layer); + read_safe(loader, hparams.n_mels); + read_safe(loader, hparams.ftype); ++ read_safe(loader, hparams.n_audio_conv1_kernel); + + assert(hparams.n_text_state == hparams.n_audio_state); + +@@ -1775,7 +1777,7 @@ + // encoder + model.e_pe = create_tensor(ASR_TENSOR_ENC_POS_EMBD, ASR_SYSTEM_ENCODER, ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_audio_state, n_audio_ctx)); + +- model.e_conv_1_w = create_tensor(ASR_TENSOR_CONV1_WEIGHT, ASR_SYSTEM_ENCODER, ggml_new_tensor_3d(ctx, vtype, 3, n_mels, n_audio_state)); ++ model.e_conv_1_w = create_tensor(ASR_TENSOR_CONV1_WEIGHT, ASR_SYSTEM_ENCODER, ggml_new_tensor_3d(ctx, vtype, hparams.n_audio_conv1_kernel, n_mels, n_audio_state)); + model.e_conv_1_b = create_tensor(ASR_TENSOR_CONV1_BIAS, ASR_SYSTEM_ENCODER, ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 1, n_audio_state)); + + model.e_conv_2_w = create_tensor(ASR_TENSOR_CONV2_WEIGHT, ASR_SYSTEM_ENCODER, ggml_new_tensor_3d(ctx, vtype, 3, n_audio_state, n_audio_state)); diff --git a/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/0004-bci-windowed-attention.patch b/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/0004-bci-windowed-attention.patch new file mode 100644 index 0000000000..9161158071 --- /dev/null +++ b/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/0004-bci-windowed-attention.patch @@ -0,0 +1,97 @@ +diff --git a/src/whisper.cpp b/src/whisper.cpp +--- a/src/whisper.cpp ++++ b/src/whisper.cpp +@@ -633,6 +633,8 @@ + int32_t ftype = 1; + float eps = 1e-5f; + int32_t n_audio_conv1_kernel = 3; ++ int32_t n_audio_window_size = 0; ++ int32_t n_audio_last_window_layer = -1; + }; + + // audio encoding layer +@@ -1536,6 +1538,8 @@ + read_safe(loader, hparams.n_mels); + read_safe(loader, hparams.ftype); + read_safe(loader, hparams.n_audio_conv1_kernel); ++ read_safe(loader, hparams.n_audio_window_size); ++ read_safe(loader, hparams.n_audio_last_window_layer); + + assert(hparams.n_text_state == hparams.n_audio_state); + +@@ -2114,6 +2118,15 @@ + + struct ggml_tensor * inpL = cur; + ++ struct ggml_tensor * window_mask = nullptr; ++ const int window_size = hparams.n_audio_window_size; ++ const int last_window_layer = hparams.n_audio_last_window_layer; ++ if (window_size > 0 && last_window_layer >= 0) { ++ window_mask = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_ctx, n_ctx, 1); ++ ggml_set_name(window_mask, "window_mask"); ++ ggml_set_input(window_mask); ++ } ++ + for (int il = 0; il < n_layer; ++il) { + const auto & layer = model.layers_encoder[il]; + +@@ -2177,7 +2190,8 @@ + ggml_element_size(kv_pad.v)*n_state_head, + 0); + +- cur = ggml_flash_attn_ext(ctx0, Q, K, V, nullptr, KQscale, 0.0f, 0.0f); ++ struct ggml_tensor * attn_mask_fa = (window_mask && il <= last_window_layer) ? window_mask : nullptr; ++ cur = ggml_flash_attn_ext(ctx0, Q, K, V, attn_mask_fa, KQscale, 0.0f, 0.0f); + + cur = ggml_reshape_2d(ctx0, cur, n_state, n_ctx); + } else { +@@ -2191,7 +2205,8 @@ + // K * Q + struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q); + +- struct ggml_tensor * KQ_soft_max = ggml_soft_max_ext(ctx0, KQ, nullptr, KQscale, 0.0f); ++ struct ggml_tensor * enc_attn_mask = (window_mask && il <= last_window_layer) ? window_mask : nullptr; ++ struct ggml_tensor * KQ_soft_max = ggml_soft_max_ext(ctx0, KQ, enc_attn_mask, KQscale, 0.0f); + + struct ggml_tensor * V = + ggml_cast(ctx0, +@@ -2442,6 +2457,25 @@ + return false; + } + ++ { ++ struct ggml_tensor * wmask = ggml_graph_get_tensor(gf, "window_mask"); ++ if (wmask) { ++ const int n_ctx = wstate.exp_n_audio_ctx > 0 ++ ? wstate.exp_n_audio_ctx : wctx.model.hparams.n_audio_ctx; ++ const int ws = wctx.model.hparams.n_audio_window_size; ++ const int half_w = ws / 2; ++ std::vector mask_data(n_ctx * n_ctx); ++ for (int i = 0; i < n_ctx; ++i) { ++ for (int j = 0; j < n_ctx; ++j) { ++ mask_data[i * n_ctx + j] = ++ (abs(i - j) <= half_w) ? 0.0f : -INFINITY; ++ } ++ } ++ ggml_backend_tensor_set(wmask, mask_data.data(), 0, ++ n_ctx * n_ctx * sizeof(float)); ++ } ++ } ++ + if (!ggml_graph_compute_helper(sched, gf, n_threads)) { + return false; + } +@@ -6949,7 +6983,12 @@ + } else { + prompt_init.push_back(whisper_token_transcribe(ctx)); + } +- } ++ } else if (ctx->model.hparams.n_audio_window_size > 0) { ++ const int lang_id = whisper_lang_id(params.language); ++ state->lang_id = lang_id; ++ prompt_init.push_back(whisper_token_lang(ctx, lang_id)); ++ prompt_init.push_back(whisper_token_transcribe(ctx)); ++ } + + // first release distilled models require the "no_timestamps" token + { diff --git a/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/portfile.cmake b/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/portfile.cmake new file mode 100644 index 0000000000..52e171819a --- /dev/null +++ b/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/portfile.cmake @@ -0,0 +1,57 @@ +set(VERSION "a8d002cfd879315632a579e73f0148d06959de36") + +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO ggml-org/whisper.cpp + REF ${VERSION} + SHA512 aea24debb836131d14d362ff78c6d12cfe2e82188340e69e71e6874a1fa51fa9405f2c03fe43888b1ff4183f4288bf64f07dd1106224b0108c3e0f844989a409 + HEAD_REF master + PATCHES + 0001-fix-vcpkg-build.patch + 0002-fix-apple-silicon-cross-compile.patch + 0003-bci-variable-conv1-kernel.patch + 0004-bci-windowed-attention.patch +) + +set(PLATFORM_OPTIONS) + +if (VCPKG_TARGET_IS_ANDROID) + list(APPEND PLATFORM_OPTIONS -DWHISPER_NO_AVX=ON -DWHISPER_NO_AVX2=ON -DWHISPER_NO_FMA=ON) + list(APPEND PLATFORM_OPTIONS -DGGML_VULKAN=OFF) +endif() + +vcpkg_cmake_configure( + SOURCE_PATH "${SOURCE_PATH}" + DISABLE_PARALLEL_CONFIGURE + OPTIONS + -DGGML_CCACHE=OFF + -DGGML_OPENMP=OFF + -DGGML_NATIVE=OFF + -DWHISPER_BUILD_TESTS=OFF + -DWHISPER_BUILD_EXAMPLES=OFF + -DWHISPER_BUILD_SERVER=OFF + -DBUILD_SHARED_LIBS=OFF + -DGGML_BUILD_NUMBER=1 + ${PLATFORM_OPTIONS} +) + +vcpkg_cmake_install() + +vcpkg_cmake_config_fixup( + PACKAGE_NAME whisper + CONFIG_PATH share/whisper +) + +vcpkg_fixup_pkgconfig() + +vcpkg_copy_pdbs() + +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") + +if (VCPKG_LIBRARY_LINKAGE MATCHES "static") + file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/bin") + file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/bin") +endif() + +vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE") diff --git a/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/vcpkg.json b/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/vcpkg.json new file mode 100644 index 0000000000..ed9210715e --- /dev/null +++ b/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/vcpkg.json @@ -0,0 +1,18 @@ +{ + "name": "whisper-cpp", + "version": "1.7.5.1", + "port-version": 1, + "description": "Port of OpenAI's Whisper model in C/C++ (BCI patched)", + "homepage": "https://github.com/tetherto/whisper.cpp", + "license": "MIT", + "dependencies": [ + { + "name": "vcpkg-cmake", + "host": true + }, + { + "name": "vcpkg-cmake-config", + "host": true + } + ] +} diff --git a/packages/bci-whispercpp/vcpkg.json b/packages/bci-whispercpp/vcpkg.json new file mode 100644 index 0000000000..867b85f130 --- /dev/null +++ b/packages/bci-whispercpp/vcpkg.json @@ -0,0 +1,18 @@ +{ + "name": "bci-whispercpp", + "version-string": "0.1.0", + "dependencies": [ + { + "name": "qvac-lib-inference-addon-cpp", + "version>=": "1.1.5" + }, + "whisper-cpp", + "gtest" + ], + "overrides": [ + { + "name": "whisper-cpp", + "version": "1.7.5.1" + } + ] +} From 799a964fd1b46f669d5e4999e8a2826de5fe8475 Mon Sep 17 00:00:00 2001 From: Raju Date: Mon, 20 Apr 2026 16:33:08 +0530 Subject: [PATCH 02/19] fix[api](bci): address review feedback, refactor to infer-base pattern, fix Linux linkage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Refactor BCIWhispercpp to use createJobHandler + exclusiveRunQueue from @qvac/infer-base instead of manual promise plumbing, matching the TranscriptionWhispercpp / LlmLlamacpp addon pattern - Constructor now takes { files: { model }, logger, opts } (was { modelPath }) - transcribe/transcribeFile return QvacResponse - Add unload(), getState(), exclusiveRunQueue-serialized destroy() - Add @qvac/infer-base dependency Address all review feedback from Gustavo (PR #1583): - Remove unused END_OF_INPUT, totalSamples_, sleep_for(1ms) - Use QvacErrorAddonBCI for model-not-found, add BUFFER_LIMIT_EXCEEDED - Fix n_threads/duration_ms double→int conversion in BCIConfig.cpp - Add bounds validation for all BCIConfig numeric params - Throw on unknown config keys (was silently ignored) - Consume gpu_device in context params - Collect whisper timings in runtimeStats() - Trim unused BCIErrors enum values, map codes to distinct names - Add MAX_BUFFERED_BYTES guard and nextSafeId in bci.js - Fix _activeJobId race: set after native acceptance - Remove unimplemented bciConfig params from JS whitelist + index.d.ts - Promote hardcoded kernel-trim threshold to named constant - Pre-allocate dummyAudioPad_ as class member (avoid repeated allocs) - Rename bci-addon.test.js → addon.test.js - Replace t.skip() with proper assertions - Fix day_idx handling in tests/examples (group by day, pass to config) - Generate comprehensive NOTICE file - Update vcpkg overlay to v1.8.4 description Fix Linux C++ test linkage: - Add vcpkg triplets (x64-linux, arm64-linux) with -stdlib=libc++ - Add linux-clang toolchain (clang-19) - Set VCPKG_OVERLAY_TRIPLETS in CMakeLists.txt for Linux builds Made-with: Cursor --- packages/bci-whispercpp/CMakeLists.txt | 4 + packages/bci-whispercpp/NOTICE | 89 +++++++- .../addon/src/addon/BCIErrors.hpp | 17 +- .../src/model-interface/bci/BCIConfig.cpp | 202 ++++++++++++++--- .../src/model-interface/bci/BCIModel.cpp | 38 +++- .../src/model-interface/bci/BCIModel.hpp | 15 +- .../model-interface/bci/NeuralProcessor.cpp | 10 +- packages/bci-whispercpp/bci.js | 58 +++-- packages/bci-whispercpp/configChecker.js | 9 +- .../bci-whispercpp/docs/BCI_V184_COMPAT.md | 56 +++++ .../examples/transcribe-neural.js | 127 ++++++++--- packages/bci-whispercpp/index.d.ts | 52 +++-- packages/bci-whispercpp/index.js | 214 +++++++++++------- packages/bci-whispercpp/lib/error.js | 12 +- packages/bci-whispercpp/package.json | 3 +- .../test/integration/addon.test.js | 140 ++++++++++++ .../test/integration/bci-addon.test.js | 69 ------ .../vcpkg-overlays/whisper-cpp/portfile.cmake | 11 +- .../vcpkg-overlays/whisper-cpp/vcpkg.json | 6 +- .../vcpkg/toolchains/linux-clang.cmake | 4 + .../vcpkg/triplets/arm64-linux.cmake | 9 + .../vcpkg/triplets/x64-linux.cmake | 9 + 22 files changed, 869 insertions(+), 285 deletions(-) create mode 100644 packages/bci-whispercpp/docs/BCI_V184_COMPAT.md create mode 100644 packages/bci-whispercpp/test/integration/addon.test.js delete mode 100644 packages/bci-whispercpp/test/integration/bci-addon.test.js create mode 100644 packages/bci-whispercpp/vcpkg/toolchains/linux-clang.cmake create mode 100644 packages/bci-whispercpp/vcpkg/triplets/arm64-linux.cmake create mode 100644 packages/bci-whispercpp/vcpkg/triplets/x64-linux.cmake diff --git a/packages/bci-whispercpp/CMakeLists.txt b/packages/bci-whispercpp/CMakeLists.txt index dfb91051d8..aaa01eb75b 100644 --- a/packages/bci-whispercpp/CMakeLists.txt +++ b/packages/bci-whispercpp/CMakeLists.txt @@ -11,6 +11,10 @@ find_package(cmake-vcpkg REQUIRED PATHS node_modules/cmake-vcpkg) set(VCPKG_OVERLAY_PORTS "${CMAKE_CURRENT_SOURCE_DIR}/vcpkg-overlays;${VCPKG_OVERLAY_PORTS}") +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + set(VCPKG_OVERLAY_TRIPLETS "${CMAKE_CURRENT_SOURCE_DIR}/vcpkg/triplets") +endif() + project(bci-whispercpp CXX C) if(CMAKE_SYSTEM_NAME STREQUAL "Linux") diff --git a/packages/bci-whispercpp/NOTICE b/packages/bci-whispercpp/NOTICE index 3df664bfac..1c61b2cf56 100644 --- a/packages/bci-whispercpp/NOTICE +++ b/packages/bci-whispercpp/NOTICE @@ -7,17 +7,90 @@ Apache-2.0; bundled dependencies are governed by the licenses listed below. ========================================================================= -Third-Party Software Licenses +JavaScript Dependencies ========================================================================= ---- MIT --- +--- apache-2.0 (Apache License 2.0) --- - whisper.cpp - https://github.com/ggerganov/whisper.cpp - Copyright (c) 2023-2024 Georgi Gerganov + @qvac/error@0.1.1 + @qvac/logging@0.1.0 + b4a@1.8.0 + https://github.com/holepunchto/b4a + bare-buffer@3.6.0 + https://github.com/holepunchto/bare-buffer + bare-events@2.8.2 + https://github.com/holepunchto/bare-events + bare-os@3.8.7 + https://github.com/holepunchto/bare-os + bare-path@3.0.0 + https://github.com/holepunchto/bare-path + bare-stream@2.12.0 + https://github.com/holepunchto/bare-stream + events-universal@1.0.1 + https://github.com/holepunchto/events-universal + text-decoder@1.2.7 + https://github.com/holepunchto/text-decoder ---- MIT --- +--- mit (MIT License) --- + + fast-fifo@1.3.2 + https://github.com/mafintosh/fast-fifo + streamx@2.25.0 + https://github.com/mafintosh/streamx + teex@1.0.1 + https://github.com/mafintosh/teex + + +========================================================================= +Python Dependencies (model conversion tooling only) +========================================================================= + +The scripts/convert-model.py tool used to convert BrainWhisperer +PyTorch checkpoints to the GGML + embedder binary format requires: + +--- bsd-3-clause (BSD 3-Clause License) --- + + numpy + https://numpy.org + torch + https://pytorch.org + + +========================================================================= +C++ Dependencies +========================================================================= + +--- mit (MIT License) --- + + whisper-cpp + https://github.com/tetherto/qvac-ext-lib-whisper.cpp + (BCI-patched fork of https://github.com/ggml-org/whisper.cpp) ggml - https://github.com/ggerganov/ggml - Copyright (c) 2023-2024 Georgi Gerganov + https://github.com/ggml-org/ggml + (bundled with whisper.cpp) + +--- bsd-3-clause (BSD 3-Clause License) --- + + gtest + https://github.com/google/googletest + (test-only dependency) + +--- apache-2.0 with llvm-exception (Apache License 2.0 with LLVM Exception) --- + + libc++ (LLVM C++ Standard Library) + https://github.com/llvm/llvm-project + (runtime dependency on Linux targets via -stdlib=libc++) + + +========================================================================= +Model Attribution +========================================================================= + +Neural-signal-to-text transcription uses a derived BCI-trained whisper +model. The bci-embedder.bin weight file contains day-specific projection +matrices derived from the BrainWhisperer research project +(https://github.com/cffan/neural_seq_decoder). End users must obtain +the upstream research checkpoint and convert it locally using +scripts/convert-model.py; no model weights are distributed with this +package. diff --git a/packages/bci-whispercpp/addon/src/addon/BCIErrors.hpp b/packages/bci-whispercpp/addon/src/addon/BCIErrors.hpp index 5711fb5c53..2c42f1d0ea 100644 --- a/packages/bci-whispercpp/addon/src/addon/BCIErrors.hpp +++ b/packages/bci-whispercpp/addon/src/addon/BCIErrors.hpp @@ -13,13 +13,22 @@ namespace qvac_errors { namespace bci_error { enum class Code : std::uint8_t { InvalidNeuralSignal, - UnsupportedSignalFormat, - ProcessingFailed, }; +inline const char* codeName(Code code) { + switch (code) { + case Code::InvalidNeuralSignal: + return "InvalidNeuralSignal"; + } + return "BCIError"; +} + inline qvac_errors::StatusError -makeStatus(Code /*code*/, const std::string& message) { - return qvac_errors::StatusError("BCI", "BCIError", message); +makeStatus(Code code, const std::string& message) { + return qvac_errors::StatusError( + qvac_lib_inference_addon_bci::errors::ADDON_ID, + codeName(code), + message); } } // namespace bci_error } // namespace qvac_errors diff --git a/packages/bci-whispercpp/addon/src/model-interface/bci/BCIConfig.cpp b/packages/bci-whispercpp/addon/src/model-interface/bci/BCIConfig.cpp index 5a80272db4..ddce668781 100644 --- a/packages/bci-whispercpp/addon/src/model-interface/bci/BCIConfig.cpp +++ b/packages/bci-whispercpp/addon/src/model-interface/bci/BCIConfig.cpp @@ -1,10 +1,90 @@ #include "BCIConfig.hpp" +#include +#include #include #include +#include +#include +#include + +#include "qvac-lib-inference-addon-cpp/Errors.hpp" namespace qvac_lib_inference_addon_bci { +namespace { + +// JS Number values arrive as double through the binding layer. Convert them +// safely to the target integer type, validating that the value is finite and +// within range. +int toInt(const JSValueVariant& v, const std::string& key) { + if (const auto* d = std::get_if(&v)) { + if (!std::isfinite(*d)) { + throw qvac_errors::StatusError( + qvac_errors::general_error::InvalidArgument, + key + " must be a finite number"); + } + return static_cast(*d); + } + if (const auto* i = std::get_if(&v)) { + return *i; + } + throw qvac_errors::StatusError( + qvac_errors::general_error::InvalidArgument, + key + " must be a number"); +} + +float toFloat(const JSValueVariant& v, const std::string& key) { + if (const auto* d = std::get_if(&v)) { + if (!std::isfinite(*d)) { + throw qvac_errors::StatusError( + qvac_errors::general_error::InvalidArgument, + key + " must be a finite number"); + } + return static_cast(*d); + } + if (const auto* i = std::get_if(&v)) { + return static_cast(*i); + } + throw qvac_errors::StatusError( + qvac_errors::general_error::InvalidArgument, + key + " must be a number"); +} + +bool toBool(const JSValueVariant& v, const std::string& key) { + if (const auto* b = std::get_if(&v)) { + return *b; + } + throw qvac_errors::StatusError( + qvac_errors::general_error::InvalidArgument, + key + " must be a boolean"); +} + +const std::string& toString(const JSValueVariant& v, const std::string& key) { + if (const auto* s = std::get_if(&v)) { + return *s; + } + throw qvac_errors::StatusError( + qvac_errors::general_error::InvalidArgument, + key + " must be a string"); +} + +int computeOptimalThreads() { + const unsigned hw = std::thread::hardware_concurrency(); + return hw > 0 ? static_cast(std::min(hw, 16U)) : 4; +} + +void ensureRange(const std::string& key, double value, double lo, double hi) { + if (value < lo || value > hi) { + std::ostringstream oss; + oss << key << " must be in [" << lo << ", " << hi << "], got " << value; + throw qvac_errors::StatusError( + qvac_errors::general_error::InvalidArgument, oss.str()); + } +} + +} // namespace + std::string convertVariantToString(const JSValueVariant& value) { return std::visit( [](const auto& v) -> std::string { @@ -36,45 +116,81 @@ const HandlersMap& getWhisperMainHandlers() { }}, {"n_threads", [](whisper_full_params& p, const JSValueVariant& v) { - if (auto* i = std::get_if(&v)) { - p.n_threads = *i; + int n = toInt(v, "n_threads"); + if (n < 0) { + throw qvac_errors::StatusError( + qvac_errors::general_error::InvalidArgument, + "n_threads must be >= 0"); } + p.n_threads = (n == 0) ? computeOptimalThreads() : n; }}, {"translate", [](whisper_full_params& p, const JSValueVariant& v) { - if (auto* b = std::get_if(&v)) { - p.translate = *b; - } + p.translate = toBool(v, "translate"); }}, {"no_timestamps", [](whisper_full_params& p, const JSValueVariant& v) { - if (auto* b = std::get_if(&v)) { - p.no_timestamps = *b; - } + p.no_timestamps = toBool(v, "no_timestamps"); }}, {"single_segment", [](whisper_full_params& p, const JSValueVariant& v) { - if (auto* b = std::get_if(&v)) { - p.single_segment = *b; - } + p.single_segment = toBool(v, "single_segment"); }}, {"temperature", [](whisper_full_params& p, const JSValueVariant& v) { - if (auto* d = std::get_if(&v)) { - p.temperature = static_cast(*d); - } + float t = toFloat(v, "temperature"); + ensureRange("temperature", t, 0.0, 2.0); + p.temperature = t; }}, {"suppress_nst", [](whisper_full_params& p, const JSValueVariant& v) { - if (auto* b = std::get_if(&v)) { - p.suppress_nst = *b; - } + p.suppress_nst = toBool(v, "suppress_nst"); + }}, + {"suppress_blank", + [](whisper_full_params& p, const JSValueVariant& v) { + p.suppress_blank = toBool(v, "suppress_blank"); }}, {"duration_ms", [](whisper_full_params& p, const JSValueVariant& v) { - if (auto* i = std::get_if(&v)) { - p.duration_ms = *i; + int ms = toInt(v, "duration_ms"); + if (ms < 0) { + throw qvac_errors::StatusError( + qvac_errors::general_error::InvalidArgument, + "duration_ms must be >= 0"); } + p.duration_ms = ms; + }}, + {"print_special", + [](whisper_full_params& p, const JSValueVariant& v) { + p.print_special = toBool(v, "print_special"); + }}, + {"print_progress", + [](whisper_full_params& p, const JSValueVariant& v) { + p.print_progress = toBool(v, "print_progress"); + }}, + {"print_realtime", + [](whisper_full_params& p, const JSValueVariant& v) { + p.print_realtime = toBool(v, "print_realtime"); + }}, + {"print_timestamps", + [](whisper_full_params& p, const JSValueVariant& v) { + p.print_timestamps = toBool(v, "print_timestamps"); + }}, + {"detect_language", + [](whisper_full_params& p, const JSValueVariant& v) { + p.detect_language = toBool(v, "detect_language"); + }}, + {"greedy_best_of", + [](whisper_full_params& p, const JSValueVariant& v) { + int b = toInt(v, "greedy_best_of"); + ensureRange("greedy_best_of", b, 1, 32); + p.greedy.best_of = b; + }}, + {"beam_search_beam_size", + [](whisper_full_params& p, const JSValueVariant& v) { + int b = toInt(v, "beam_search_beam_size"); + ensureRange("beam_search_beam_size", b, 1, 32); + p.beam_search.beam_size = b; }}, }; return handlers; @@ -84,15 +200,26 @@ const HandlersMap& getWhisperContextHandlers() { static const HandlersMap handlers = { {"use_gpu", [](whisper_context_params& p, const JSValueVariant& v) { - if (auto* b = std::get_if(&v)) { - p.use_gpu = *b; - } + p.use_gpu = toBool(v, "use_gpu"); }}, {"flash_attn", [](whisper_context_params& p, const JSValueVariant& v) { - if (auto* b = std::get_if(&v)) { - p.flash_attn = *b; + p.flash_attn = toBool(v, "flash_attn"); + }}, + {"gpu_device", + [](whisper_context_params& p, const JSValueVariant& v) { + int d = toInt(v, "gpu_device"); + if (d < 0) { + throw qvac_errors::StatusError( + qvac_errors::general_error::InvalidArgument, + "gpu_device must be >= 0"); } + p.gpu_device = d; + }}, + {"model", + [](whisper_context_params& /*p*/, const JSValueVariant& v) { + // Consumed directly from whisperContextCfg["model"] in BCIModel::load. + (void)toString(v, "model"); }}, }; return handlers; @@ -116,12 +243,22 @@ whisper_full_params toWhisperFullParams(BCIConfig& bciConfig) { const auto& handlers = getWhisperMainHandlers(); for (const auto& [key, value] : bciConfig.whisperMainCfg) { auto it = handlers.find(key); - if (it != handlers.end()) { + if (it == handlers.end()) { + throw qvac_errors::StatusError( + qvac_errors::general_error::InvalidArgument, + "Unknown whisperConfig key: " + key); + } + try { it->second(params, value); + } catch (const qvac_errors::StatusError&) { + throw; + } catch (const std::exception& e) { + throw qvac_errors::StatusError( + qvac_errors::general_error::InvalidArgument, + "error in whisperConfig handler: " + key + " | " + e.what()); } } - // Set language from config-owned storage so the pointer outlives params auto langIt = bciConfig.whisperMainCfg.find("language"); if (langIt != bciConfig.whisperMainCfg.end()) { if (auto* s = std::get_if(&langIt->second)) { @@ -139,8 +276,19 @@ whisper_context_params toWhisperContextParams(const BCIConfig& bciConfig) { const auto& handlers = getWhisperContextHandlers(); for (const auto& [key, value] : bciConfig.whisperContextCfg) { auto it = handlers.find(key); - if (it != handlers.end()) { + if (it == handlers.end()) { + throw qvac_errors::StatusError( + qvac_errors::general_error::InvalidArgument, + "Unknown contextParams key: " + key); + } + try { it->second(params, value); + } catch (const qvac_errors::StatusError&) { + throw; + } catch (const std::exception& e) { + throw qvac_errors::StatusError( + qvac_errors::general_error::InvalidArgument, + "error in contextParams handler: " + key + " | " + e.what()); } } diff --git a/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp b/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp index 8d5a3717a0..67ceb094f6 100644 --- a/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp +++ b/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp @@ -4,9 +4,7 @@ #include #include #include -#include #include -#include #include #include "BCIConfig.hpp" @@ -136,11 +134,15 @@ void BCIModel::reload() { void BCIModel::reset() { output_.clear(); - totalSamples_ = 0; totalTokens_ = 0; totalSegments_ = 0; processCalls_ = 0; totalWallMs_ = 0.0; + whisperSampleMs_ = 0.0; + whisperEncodeMs_ = 0.0; + whisperDecodeMs_ = 0.0; + whisperBatchdMs_ = 0.0; + whisperPromptMs_ = 0.0; } qvac_lib_inference_addon_cpp::RuntimeStats BCIModel::runtimeStats() const { @@ -157,6 +159,11 @@ qvac_lib_inference_addon_cpp::RuntimeStats BCIModel::runtimeStats() const { stats.emplace_back("totalSegments", totalSegments_); stats.emplace_back("processCalls", processCalls_); stats.emplace_back("totalWallMs", totalWallMs_); + stats.emplace_back("whisperSampleMs", whisperSampleMs_); + stats.emplace_back("whisperEncodeMs", whisperEncodeMs_); + stats.emplace_back("whisperDecodeMs", whisperDecodeMs_); + stats.emplace_back("whisperBatchdMs", whisperBatchdMs_); + stats.emplace_back("whisperPromptMs", whisperPromptMs_); return stats; } @@ -183,7 +190,6 @@ static void onNewSegment( transcript.id = i; bci->emitSegment(transcript); - std::this_thread::sleep_for(std::chrono::milliseconds(1)); bci->addTranscription(transcript); const int nTokens = whisper_full_n_tokens_from_state(state, i); @@ -218,7 +224,10 @@ void BCIModel::process(const Input& rawNeuralData) { "Processing neural signal (" + std::to_string(rawNeuralData.size()) + " bytes)"); - int dayIdx = 0; + // The BCI embedder ships with per-day projection matrices; day_idx=1 is the + // day the shipped test fixtures were recorded on. Callers should pass the + // real day_idx for their recording; this default keeps the POC honest. + int dayIdx = 1; auto it = cfg_.bciConfig.find("day_idx"); if (it != cfg_.bciConfig.end()) { if (auto* d = std::get_if(&it->second)) { @@ -234,9 +243,7 @@ void BCIModel::process(const Input& rawNeuralData) { processCalls_ += 1; - if (ctx_ != nullptr) { - whisper_reset_timings(ctx_.get()); - } + whisper_reset_timings(ctx_.get()); const auto startTime = std::chrono::steady_clock::now(); @@ -254,16 +261,27 @@ void BCIModel::process(const Input& rawNeuralData) { params.encoder_begin_callback = onEncoderBegin; params.encoder_begin_callback_user_data = &cbData; - std::vector dummyAudio(K_DUMMY_AUDIO_30S, 0.0F); + if (dummyAudioPad_.size() != static_cast(K_DUMMY_AUDIO_30S)) { + dummyAudioPad_.assign(K_DUMMY_AUDIO_30S, 0.0F); + } int result = whisper_full( ctx_.get(), params, - dummyAudio.data(), static_cast(dummyAudio.size())); + dummyAudioPad_.data(), static_cast(dummyAudioPad_.size())); const auto endTime = std::chrono::steady_clock::now(); totalWallMs_ += std::chrono::duration(endTime - startTime).count(); + if (auto* whisperTimings = whisper_get_timings(ctx_.get()); + whisperTimings != nullptr) { + whisperSampleMs_ += whisperTimings->sample_ms; + whisperEncodeMs_ += whisperTimings->encode_ms; + whisperDecodeMs_ += whisperTimings->decode_ms; + whisperBatchdMs_ += whisperTimings->batchd_ms; + whisperPromptMs_ += whisperTimings->prompt_ms; + } + if (result != 0) { if (cancelRequested_.load(std::memory_order_relaxed)) { throw std::runtime_error("Job cancelled"); diff --git a/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.hpp b/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.hpp index 29493e6bb0..88dc01b848 100644 --- a/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.hpp +++ b/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.hpp @@ -119,11 +119,24 @@ class BCIModel bool is_loaded_ = false; bool is_warmed_up_ = false; - int64_t totalSamples_ = 0; int64_t totalTokens_ = 0; int64_t totalSegments_ = 0; int64_t processCalls_ = 0; double totalWallMs_ = 0.0; + + // whisper.cpp internal stage timings aggregated across process() calls. + double whisperSampleMs_ = 0.0; + double whisperEncodeMs_ = 0.0; + double whisperDecodeMs_ = 0.0; + double whisperBatchdMs_ = 0.0; + double whisperPromptMs_ = 0.0; + + // 30 s of silent audio reused on every process() call; whisper.cpp does + // the actual encode via our encoder_begin_callback, but it still requires + // a padding buffer of the right shape. Hoisted to a member so we don't + // reallocate ~1.9 MB per call. + std::vector dummyAudioPad_; + mutable std::atomic_bool cancelRequested_{false}; }; diff --git a/packages/bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.cpp b/packages/bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.cpp index b7e4ee5be8..d98c4174fc 100644 --- a/packages/bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.cpp +++ b/packages/bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.cpp @@ -14,6 +14,12 @@ namespace qvac_lib_inference_addon_bci { namespace { constexpr size_t K_HEADER_BYTES = 8; constexpr uint32_t K_EMBEDDER_MAGIC = 0x42434945; + +// Kernel-trim threshold used by gaussianSmooth: values below this are +// considered numerically negligible and trimmed from the ends of the kernel +// so the convolution loop touches fewer source timesteps. Matches the +// BrainWhisperer Python reference. +constexpr float K_KERNEL_TRIM_THRESHOLD = 0.01F; } // namespace NeuralProcessor::NeuralProcessor() = default; @@ -100,8 +106,8 @@ std::vector NeuralProcessor::gaussianSmooth( for (auto& k : kernel) k /= sum; int start = 0, end = kernelSize - 1; - while (start < end && kernel[start] < 0.01F) ++start; - while (end > start && kernel[end] < 0.01F) --end; + while (start < end && kernel[start] < K_KERNEL_TRIM_THRESHOLD) ++start; + while (end > start && kernel[end] < K_KERNEL_TRIM_THRESHOLD) --end; std::vector trimK(kernel.begin() + start, kernel.begin() + end + 1); const int halfK = static_cast(trimK.size()) / 2; diff --git a/packages/bci-whispercpp/bci.js b/packages/bci-whispercpp/bci.js index aecf03e235..4e81a00284 100644 --- a/packages/bci-whispercpp/bci.js +++ b/packages/bci-whispercpp/bci.js @@ -14,6 +14,16 @@ const state = Object.freeze({ const END_OF_INPUT = 'end of job' +// Upper bound on buffered neural-signal bytes between append() calls. +// Neural data is ~1 MB/s at 512ch * 50 Hz * 4 B, so 500 MB ~= 8 minutes of +// signal. The bound matches qvac-lib-infer-whispercpp and protects against +// runaway producers. +const MAX_BUFFERED_BYTES = 500 * 1024 * 1024 + +function nextSafeId (current) { + return current >= Number.MAX_SAFE_INTEGER ? 1 : current + 1 +} + /** * Low-level interface between the Bare C++ BCI addon and the JS runtime. * Accepts neural signal data (Uint8Array) instead of audio. @@ -32,6 +42,7 @@ class BCIInterface { this._nextJobId = 1 this._activeJobId = null this._bufferedSignal = [] + this._bufferedBytes = 0 this._state = state.LOADING checkConfig(configurationParams) @@ -156,6 +167,7 @@ class BCIInterface { try { await this._binding.cancel(this._handle, jobId) this._bufferedSignal = [] + this._bufferedBytes = 0 this._activeJobId = null this._setState(state.LISTENING) } catch (err) { @@ -193,26 +205,38 @@ class BCIInterface { } if (!accepted) { this._setState(state.LISTENING) - throw new Error('Cannot set new job: a job is already set or being processed') + throw new QvacErrorAddonBCI({ code: ERR_CODES.JOB_ALREADY_RUNNING }) } this._activeJobId = currentJobId - this._nextJobId += 1 + this._nextJobId = nextSafeId(this._nextJobId) this._bufferedSignal = [] + this._bufferedBytes = 0 this._setState(state.PROCESSING) return currentJobId } if (data?.type === 'neural') { if (!(data.input instanceof Uint8Array)) { - throw new Error('Neural signal input must be Uint8Array') + throw new QvacErrorAddonBCI({ + code: ERR_CODES.INVALID_NEURAL_INPUT, + adds: 'input must be Uint8Array' + }) + } + if (this._bufferedBytes + data.input.byteLength > MAX_BUFFERED_BYTES) { + throw new QvacErrorAddonBCI({ + code: ERR_CODES.BUFFER_LIMIT_EXCEEDED, + adds: MAX_BUFFERED_BYTES + ' bytes' + }) } this._bufferedSignal.push(data.input) + this._bufferedBytes += data.input.byteLength return this._nextJobId } throw new Error(`Unknown append input type: ${data?.type}`) } catch (err) { + if (err instanceof QvacErrorAddonBCI) throw err throw new QvacErrorAddonBCI({ code: ERR_CODES.FAILED_TO_APPEND, adds: err.message, @@ -227,21 +251,14 @@ class BCIInterface { * @param {Uint8Array} data.input - binary neural signal data */ async runJob (data) { + const candidateJobId = this._nextJobId + let accepted = false try { - this._activeJobId = this._nextJobId - this._nextJobId += 1 - this._setState(state.PROCESSING) - const accepted = this._binding.runJob(this._handle, { + accepted = this._binding.runJob(this._handle, { type: 'neural', input: data.input }) - if (!accepted) { - this._activeJobId = null - this._setState(state.LISTENING) - } - return accepted } catch (err) { - this._activeJobId = null this._setState(state.LISTENING) throw new QvacErrorAddonBCI({ code: ERR_CODES.FAILED_TO_APPEND, @@ -249,6 +266,16 @@ class BCIInterface { cause: err }) } + + if (!accepted) { + this._setState(state.LISTENING) + return false + } + + this._activeJobId = candidateJobId + this._nextJobId = nextSafeId(this._nextJobId) + this._setState(state.PROCESSING) + return accepted } async status () { @@ -266,6 +293,7 @@ class BCIInterface { this._binding.destroyInstance(this._handle) this._handle = null this._bufferedSignal = [] + this._bufferedBytes = 0 this._activeJobId = null this._setState(state.IDLE) } catch (err) { @@ -297,4 +325,6 @@ class BCIInterface { } } -module.exports = { BCIInterface } +BCIInterface.END_OF_INPUT = END_OF_INPUT + +module.exports = { BCIInterface, END_OF_INPUT, MAX_BUFFERED_BYTES, nextSafeId } diff --git a/packages/bci-whispercpp/configChecker.js b/packages/bci-whispercpp/configChecker.js index 9dd797275c..bc48a17302 100644 --- a/packages/bci-whispercpp/configChecker.js +++ b/packages/bci-whispercpp/configChecker.js @@ -30,8 +30,7 @@ function checkConfig (configObject) { 'suppress_nst', 'temperature', 'greedy_best_of', - 'beam_search_beam_size', - 'seed' + 'beam_search_beam_size' ] const validContextParams = [ @@ -45,10 +44,10 @@ function checkConfig (configObject) { 'caption_enabled' ] + // Only parameters wired through to the C++ addon are accepted. Adding + // smoothing/sample-rate knobs here without consuming them in NeuralProcessor + // would silently drop user intent, so they are kept out until implemented. const validBCIParams = [ - 'smooth_kernel_std', - 'smooth_kernel_size', - 'sample_rate', 'day_idx' ] diff --git a/packages/bci-whispercpp/docs/BCI_V184_COMPAT.md b/packages/bci-whispercpp/docs/BCI_V184_COMPAT.md new file mode 100644 index 0000000000..67dfa7d4e2 --- /dev/null +++ b/packages/bci-whispercpp/docs/BCI_V184_COMPAT.md @@ -0,0 +1,56 @@ +# BCI whisper.cpp v1.8.4 Compatibility + +## Goal + +Get the BCI addon working on whisper.cpp v1.8.4.1 (from `tetherto/qvac-ext-lib-whisper.cpp`) instead of the current v1.7.6 (from `ggml-org/whisper.cpp`). + +## Status: FIXED + +| Version | Source | WER | Status | +|---------|--------|-----|--------| +| v1.7.6 (`a8d002cf`) | `ggml-org/whisper.cpp` + 4 overlay patches | **10.4%** | Working | +| v1.8.4.1 (unpatched) | `tetherto/qvac-ext-lib-whisper.cpp` + BCI patches | **91.9%** | Broken (garbage output) | +| v1.8.4.1 (patched) | `tetherto/qvac-ext-lib-whisper.cpp` + BCI patches + `0005` fix | **10.4%** | Working (identical to v1.7.6) | + +## Root Cause + +The issue was **not** in the ggml submodule. It was a **graph placement bug** introduced when the BCI windowed attention patch was ported from v1.7.6 to v1.8.4. + +In v1.7.6, `whisper_encode_internal` built a single monolithic computation graph for the entire encoder. The BCI windowed attention patch added: +1. A `window_mask` tensor created in the graph builder +2. Mask data population via `ggml_graph_get_tensor(gf, "window_mask")` after graph allocation, before graph computation + +In v1.8.4, the encoder was refactored into **three separate computation graphs**: +1. `whisper_build_graph_conv` — convolution layers +2. `whisper_build_graph_encoder` — self-attention encoder layers +3. `whisper_build_graph_cross` — cross-attention KV pre-computation + +The BCI patch correctly placed the `window_mask` tensor creation in `whisper_build_graph_encoder` (step 2), but the mask **data population** code was placed in the **cross-attention section** (step 3) of `whisper_encode_internal`. Since the cross graph doesn't contain a `window_mask` tensor, `ggml_graph_get_tensor(gf, "window_mask")` returned `nullptr`, and the mask was never initialized. The encoder ran with an uninitialized attention mask, producing garbage output. + +## Fix + +Patch `0005-fix-bci-window-mask-encoder-graph.patch` moves the `window_mask` data population from the cross-attention section to the encoder section of `whisper_encode_internal`, between `ggml_backend_sched_alloc_graph` and `ggml_graph_compute_helper`. + +## What Was Ruled Out (previously investigated) + +1. **Flash attention default change** — v1.8.4 sets `flash_attn = true` by default (was `false` in v1.7.6). The BCI patch already bypasses flash attention when `window_mask` is active. + +2. **`ggml_mul_mat_pad` removal** — v1.7.6 had a Metal-specific matrix multiplication padding optimization. Restoring this to v1.8.4 does not fix the quality issue. + +3. **Decoder prompt handling changes** — v1.8.4 refactored `prompt_past` into `prompt_past0`/`prompt_past1` for the `carry_initial_prompt` feature. BCI transcriptions are single-segment and the first-segment codepath is functionally equivalent. + +4. **KQ mask padding removal** — v1.8.4 removed `GGML_KQ_MASK_PAD` from the decoder attention mask. + +5. **ggml submodule changes** — 1,190 commits changed the ggml library between v1.7.6 and v1.8.4.1, but this was not the cause. + +## Fork PR + +[tetherto/qvac-ext-lib-whisper.cpp#10](https://github.com/tetherto/qvac-ext-lib-whisper.cpp/pull/10) — BCI patches (conv1 kernel + windowed attention + flash attn bypass) on v1.8.4.1 base. Needs the `0005` fix patch applied. + +## Files + +- BCI model: `models/ggml-bci-windowed.bin` +- Embedder weights: `models/bci-embedder.bin` +- Conversion script: `scripts/convert-model.py` +- Overlay portfile: `vcpkg-overlays/whisper-cpp/portfile.cmake` (points to `tetherto/qvac-ext-lib-whisper.cpp` at `bbb3535`) +- Test: `test/integration/bci-addon.test.js` diff --git a/packages/bci-whispercpp/examples/transcribe-neural.js b/packages/bci-whispercpp/examples/transcribe-neural.js index 7921e6c6a0..768b55b7b8 100644 --- a/packages/bci-whispercpp/examples/transcribe-neural.js +++ b/packages/bci-whispercpp/examples/transcribe-neural.js @@ -5,10 +5,10 @@ * Uses the native whisper.cpp GGML backend. * * Usage: - * node examples/transcribe-neural.js [model_path] + * bare examples/transcribe-neural.js [model_path] * * Or batch mode (all test fixtures): - * node examples/transcribe-neural.js --batch [model_path] + * bare examples/transcribe-neural.js --batch [model_path] */ const fs = require('bare-fs') @@ -19,6 +19,18 @@ const BCIWhispercpp = require('../index') const DEFAULT_MODEL = (os.hasEnv('WHISPER_MODEL_PATH') ? os.getEnv('WHISPER_MODEL_PATH') : null) || path.join(__dirname, '..', 'models', 'ggml-bci-windowed.bin') +function flattenSegments (output) { + const segments = [] + for (const entry of output) { + if (Array.isArray(entry)) { + segments.push(...entry) + } else if (entry && typeof entry.text === 'string') { + segments.push(entry) + } + } + return segments +} + async function main () { const args = global.Bare ? global.Bare.argv.slice(2) : process.argv.slice(2) const isBatch = args[0] === '--batch' @@ -32,49 +44,84 @@ async function main () { const modelPath = (isBatch ? args[1] : args[1]) || DEFAULT_MODEL if (!fs.existsSync(modelPath)) { - console.error(`Error: Model file not found: ${modelPath}`) + console.error('Error: Model file not found: ' + modelPath) console.error('Set WHISPER_MODEL_PATH or pass as second argument.') return } - const bci = new BCIWhispercpp({ modelPath }, { - whisperConfig: { language: 'en', temperature: 0.0 }, - miscConfig: { caption_enabled: false } - }) - - await bci.load() - console.log('Model loaded.\n') - if (isBatch) { const manifestPath = path.join(__dirname, '..', 'test', 'fixtures', 'manifest.json') const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')) - console.log(`=== BCI Neural Signal Transcription (Batch: ${manifest.samples.length} samples) ===\n`) + console.log('=== BCI Neural Signal Transcription (Batch: ' + manifest.samples.length + ' samples) ===\n') const startTime = Date.now() + const byDay = new Map() for (const sample of manifest.samples) { - const samplePath = path.join(__dirname, '..', 'test', 'fixtures', sample.file) - if (!fs.existsSync(samplePath)) { - console.log(` [SKIP] ${sample.file} (not found)`) - continue - } - - const result = await bci.transcribeFile(samplePath) - const wer = BCIWhispercpp.computeWER(result.text, sample.expected_text) + const key = typeof sample.day_idx === 'number' ? sample.day_idx : -1 + if (!byDay.has(key)) byDay.set(key, []) + byDay.get(key).push(sample) + } - console.log(` [${sample.file}]`) - console.log(` Got: "${result.text}"`) - console.log(` Expected: "${sample.expected_text}"`) - console.log(` WER: ${(wer * 100).toFixed(1)}%\n`) + let total = 0 + let sumWER = 0 + + for (const [day, samples] of byDay) { + const bci = new BCIWhispercpp({ + files: { model: modelPath } + }, { + whisperConfig: { language: 'en', temperature: 0.0 }, + miscConfig: { caption_enabled: false }, + bciConfig: day >= 0 ? { day_idx: day } : undefined + }) + await bci.load() + + try { + for (const sample of samples) { + const samplePath = path.join(__dirname, '..', 'test', 'fixtures', sample.file) + if (!fs.existsSync(samplePath)) { + console.log(' [SKIP] ' + sample.file + ' (not found)') + continue + } + + const response = await bci.transcribeFile(samplePath) + const output = await response.await() + const segments = flattenSegments(output) + const text = segments.map(s => s.text).join('').trim() + const wer = BCIWhispercpp.computeWER(text, sample.expected_text) + + console.log(' [' + sample.file + '] day=' + day) + console.log(' Got: "' + text + '"') + console.log(' Expected: "' + sample.expected_text + '"') + console.log(' WER: ' + (wer * 100).toFixed(1) + '%\n') + + total += 1 + sumWER += wer + } + } finally { + await bci.destroy() + } } const elapsed = ((Date.now() - startTime) / 1000).toFixed(2) - console.log(`Time: ${elapsed}s`) + const avgWER = total > 0 ? sumWER / total : 0 + console.log('Average WER: ' + (avgWER * 100).toFixed(1) + '% (n=' + total + ')') + console.log('Time: ' + elapsed + 's') } else { + const bci = new BCIWhispercpp({ + files: { model: modelPath } + }, { + whisperConfig: { language: 'en', temperature: 0.0 }, + miscConfig: { caption_enabled: false } + }) + + await bci.load() + console.log('Model loaded.\n') + const signalPath = args[0] if (!fs.existsSync(signalPath)) { - console.error(`Error: Signal file not found: ${signalPath}`) + console.error('Error: Signal file not found: ' + signalPath) return } @@ -84,19 +131,25 @@ async function main () { const C = view.getUint32(4, true) console.log('=== BCI Neural Signal Transcription ===') - console.log(`Signal: ${signalPath}`) - console.log(`Timesteps: ${T}, Channels: ${C}`) - console.log(`Duration: ~${(T * 20 / 1000).toFixed(1)}s\n`) - - const startTime = Date.now() - const result = await bci.transcribeFile(signalPath) - const elapsed = ((Date.now() - startTime) / 1000).toFixed(2) - - console.log(`Text: "${result.text}"`) - console.log(`Time: ${elapsed}s`) + console.log('Signal: ' + signalPath) + console.log('Timesteps: ' + T + ', Channels: ' + C) + console.log('Duration: ~' + (T * 20 / 1000).toFixed(1) + 's\n') + + try { + const startTime = Date.now() + const response = await bci.transcribeFile(signalPath) + const output = await response.await() + const segments = flattenSegments(output) + const text = segments.map(s => s.text).join('').trim() + const elapsed = ((Date.now() - startTime) / 1000).toFixed(2) + + console.log('Text: "' + text + '"') + console.log('Time: ' + elapsed + 's') + } finally { + await bci.destroy() + } } - await bci.destroy() console.log('\nDone.') } diff --git a/packages/bci-whispercpp/index.d.ts b/packages/bci-whispercpp/index.d.ts index 26e3df0dcd..71625a4751 100644 --- a/packages/bci-whispercpp/index.d.ts +++ b/packages/bci-whispercpp/index.d.ts @@ -1,7 +1,9 @@ declare interface BCIConfig { - smooth_kernel_std?: number; - smooth_kernel_size?: number; - sample_rate?: number; + /** + * Index into the day-specific projection matrices in bci-embedder.bin. + * Must match the recording day the neural signal was captured on. + * Defaults to 0. + */ day_idx?: number; } @@ -17,14 +19,21 @@ declare interface WhisperConfig { [key: string]: unknown; } +declare interface BCIWhispercppFiles { + model: string; +} + declare interface BCIWhispercppArgs { - modelPath: string; + files: BCIWhispercppFiles; logger?: { debug(...args: unknown[]): void; info(...args: unknown[]): void; warn(...args: unknown[]): void; error(...args: unknown[]): void; }; + opts?: { + stats?: boolean; + }; } declare interface BCIWhispercppConfig { @@ -49,14 +58,22 @@ declare interface TranscriptSegment { id: number; } -declare interface TranscriptionResult { - text: string; - segments: TranscriptSegment[]; - stats: Record | null; +declare interface QvacResponse { + output: unknown[]; + stats: Record; + onUpdate(callback: (data: unknown) => void): QvacResponse; + onFinish(callback: (result: unknown) => void): QvacResponse; + onError(callback: (error: Error) => void): QvacResponse; + onCancel(callback: () => void): QvacResponse; + await(): Promise; + cancel(): Promise; + iterate(): AsyncGenerator; + getLatest(): unknown; } /** * BCI neural signal transcription client powered by whisper.cpp. + * Uses createJobHandler + exclusiveRunQueue from @qvac/infer-base. */ declare class BCIWhispercpp { constructor(args: BCIWhispercppArgs, config?: BCIWhispercppConfig); @@ -64,17 +81,23 @@ declare class BCIWhispercpp { /** Load and activate the model. */ load(): Promise; - /** Transcribe a neural signal binary file. */ - transcribeFile(filePath: string): Promise; + /** Transcribe a neural signal binary file (convenience wrapper). */ + transcribeFile(filePath: string): Promise; - /** Transcribe neural signal data (batch). */ - transcribe(neuralData: Uint8Array): Promise; + /** Transcribe neural signal data (batch). Returns QvacResponse. */ + transcribe(neuralData: Uint8Array): Promise; /** Cancel current inference. */ cancel(): Promise; - /** Destroy the instance and release resources. */ + /** Unload the model and release native resources. */ + unload(): Promise; + + /** Destroy the instance, unload, and mark as permanently destroyed. */ destroy(): Promise; + + /** Get current state (configLoaded, destroyed). */ + getState(): { configLoaded: boolean; destroyed: boolean }; } /** @@ -89,10 +112,11 @@ declare namespace BCIWhispercpp { BCIWhispercpp, BCIConfig, WhisperConfig, + BCIWhispercppFiles, BCIWhispercppArgs, BCIWhispercppConfig, TranscriptSegment, - TranscriptionResult, + QvacResponse, computeWER, }; } diff --git a/packages/bci-whispercpp/index.js b/packages/bci-whispercpp/index.js index 84777e84b4..3d3268253e 100644 --- a/packages/bci-whispercpp/index.js +++ b/packages/bci-whispercpp/index.js @@ -1,48 +1,85 @@ 'use strict' const fs = require('bare-fs') +const QvacLogger = require('@qvac/logging') +const { createJobHandler, exclusiveRunQueue } = require('@qvac/infer-base') const { BCIInterface } = require('./bci') const { checkConfig } = require('./configChecker') const { QvacErrorAddonBCI, ERR_CODES } = require('./lib/error') const { computeWER } = require('./lib/wer') -const END_OF_INPUT = 'end of job' - /** - * High-level BCI transcription client powered by whisper.cpp. - * Accepts neural signal streams and returns text transcriptions. + * BCI neural signal transcription client powered by whisper.cpp. + * + * Follows the same architecture as TranscriptionWhispercpp / LlmLlamacpp: + * standalone class using createJobHandler + exclusiveRunQueue from + * @qvac/infer-base. */ class BCIWhispercpp { /** * @param {Object} args - * @param {string} args.modelPath - path to whisper GGML model file - * @param {Object} [args.logger] - optional logger + * @param {Object} args.files - local model file paths + * @param {string} args.files.model - path to the BCI GGML model file + * @param {Object} [args.logger] - optional logger instance + * @param {Object} [args.opts] - optional options (e.g. { stats: true }) * @param {Object} config - inference configuration * @param {Object} config.whisperConfig - whisper decoding params - * @param {Object} [config.bciConfig] - BCI-specific params + * @param {Object} [config.bciConfig] - BCI-specific params (e.g. { day_idx: 1 }) * @param {Object} [config.contextParams] - whisper context params + * @param {Object} [config.miscConfig] - miscellaneous config */ - constructor ({ modelPath, logger = null }, config = {}) { - this._modelPath = modelPath - this._logger = logger || { debug () {}, info () {}, warn () {}, error () {} } + constructor ({ files, logger = null, opts = {} }, config = {}) { + if (!files || typeof files.model !== 'string' || files.model.length === 0) { + throw new QvacErrorAddonBCI({ + code: ERR_CODES.MODEL_FILE_NOT_FOUND, + adds: 'files.model is required' + }) + } + + if (!fs.existsSync(files.model)) { + throw new QvacErrorAddonBCI({ + code: ERR_CODES.MODEL_FILE_NOT_FOUND, + adds: files.model + }) + } + + this._files = { model: files.model } this._config = config - this._addon = null - this._hasActiveResponse = false - this._pendingResolve = null - this._pendingReject = null - this._segments = [] - this._stats = null - - if (!this._modelPath || !fs.existsSync(this._modelPath)) { - throw new Error(`Model file doesn't exist: ${this._modelPath}`) + this.opts = opts + this.logger = new QvacLogger(logger) + this._withExclusiveRun = exclusiveRunQueue() + this._job = createJobHandler({ + cancel: () => this.addon?.cancel() + }) + + this.addon = null + this.state = { + configLoaded: false, + destroyed: false } } - /** - * Load and activate the model. - */ + getState () { + return this.state + } + async load () { + if (this.state.destroyed) { + throw new QvacErrorAddonBCI({ + code: ERR_CODES.MODEL_NOT_LOADED, + adds: 'instance was destroyed' + }) + } + if (this.state.configLoaded) { + this.logger.info('Reload requested - unloading existing model first') + await this.unload() + } + await this._load() + this.state.configLoaded = true + } + + async _load () { const whisperConfig = { language: 'en', temperature: 0.0, @@ -53,7 +90,7 @@ class BCIWhispercpp { const configurationParams = { contextParams: { - model: this._modelPath, + model: this._files.model, ...(this._config.contextParams || {}) }, whisperConfig, @@ -70,22 +107,22 @@ class BCIWhispercpp { checkConfig(configurationParams) const binding = require('./binding') - this._addon = new BCIInterface( + this.addon = new BCIInterface( binding, configurationParams, this._outputCallback.bind(this), - this._logger.info.bind(this._logger) + this.logger.info.bind(this.logger) ) - await this._addon.activate() - this._logger.info('BCI addon activated') + await this.addon.activate() + this.logger.info('BCI addon activated') } /** * Transcribe a neural signal from a binary file. - * Binary format: [uint32 numTimesteps, uint32 numChannels, float32[] data] + * Convenience wrapper around transcribe(). * @param {string} filePath - path to .bin neural signal file - * @returns {Promise} - { text, segments, stats } + * @returns {Promise} */ async transcribeFile (filePath) { const data = fs.readFileSync(filePath) @@ -94,76 +131,91 @@ class BCIWhispercpp { /** * Transcribe neural signal data (batch mode). + * Returns a QvacResponse; use response.await() for the final output array, + * response.onUpdate() for streaming updates, response.stats for runtime stats. * @param {Uint8Array} neuralData - binary neural signal - * @returns {Promise} - { text, segments, stats } + * @returns {Promise} */ async transcribe (neuralData) { - if (this._hasActiveResponse) { - throw new QvacErrorAddonBCI({ code: ERR_CODES.JOB_ALREADY_RUNNING }) + const response = this._job.start() + + let accepted + try { + accepted = await this.addon.runJob({ input: neuralData }) + } catch (err) { + this._job.fail(err) + throw err + } + if (!accepted) { + const error = new QvacErrorAddonBCI({ code: ERR_CODES.JOB_ALREADY_RUNNING }) + this._job.fail(error) + throw error } - return new Promise((resolve, reject) => { - this._beginJob(resolve, reject) - - this._addon.runJob({ input: neuralData }).catch((err) => { - this._clearJob() - reject(err) - }) - }) - } - - _beginJob (resolve, reject) { - this._segments = [] - this._stats = null - this._hasActiveResponse = true - this._pendingResolve = resolve - this._pendingReject = reject - } - - _clearJob () { - this._hasActiveResponse = false - this._pendingResolve = null - this._pendingReject = null + const finalized = response.await() + finalized.catch(() => {}) + response.await = () => finalized + return response } _outputCallback (addon, event, jobId, data, error) { + if (event === 'Error') { + this.logger.error('Job ' + jobId + ' failed with error: ' + error) + this._job.fail(error) + return + } if (event === 'Output') { - if (Array.isArray(data)) { - this._segments.push(...data) - } else if (data && data.text) { - this._segments.push(data) - } - } else if (event === 'JobEnded') { - this._stats = data - const segments = this._segments - const stats = this._stats - const resolve = this._pendingResolve - this._clearJob() - if (resolve) { - const text = segments.map(s => s.text).join('').trim() - resolve({ text, segments, stats }) - } - } else if (event === 'Error') { - const reject = this._pendingReject - this._clearJob() - if (reject) { - reject(new Error(error || 'Transcription failed')) + this._job.output(data) + return + } + if (event === 'JobEnded') { + this.logger.info('Job ' + jobId + ' completed') + if (this.opts.stats) { + this._job.end(data) + } else { + this._job.end() } + return } + this.logger.debug('Received event for job ' + jobId + ': ' + event) } async cancel () { - if (this._addon?.cancel) { - await this._addon.cancel() + if (this.addon?.cancel) { + await this.addon.cancel() } - this._clearJob() + if (this._job.active) { + this._job.fail(new Error('Job cancelled')) + } + } + + async unload () { + return await this._withExclusiveRun(async () => { + if (this._job.active) { + this._job.fail(new Error('Model was unloaded')) + } + await this.cancel() + if (this.addon) { + await this.addon.destroyInstance() + this.addon = null + } + this.state.configLoaded = false + }) } async destroy () { - await this.cancel() - if (this._addon) { - await this._addon.destroyInstance() - } + return await this._withExclusiveRun(async () => { + if (this._job.active) { + this._job.fail(new Error('Model was destroyed')) + } + await this.cancel() + if (this.addon) { + await this.addon.destroyInstance() + this.addon = null + } + this.state.configLoaded = false + this.state.destroyed = true + }) } } diff --git a/packages/bci-whispercpp/lib/error.js b/packages/bci-whispercpp/lib/error.js index bf9ad4c7e4..571e4fb653 100644 --- a/packages/bci-whispercpp/lib/error.js +++ b/packages/bci-whispercpp/lib/error.js @@ -17,7 +17,9 @@ const ERR_CODES = Object.freeze({ FAILED_TO_PAUSE: 7008, INVALID_NEURAL_INPUT: 7009, JOB_ALREADY_RUNNING: 7010, - MODEL_NOT_LOADED: 7011 + MODEL_NOT_LOADED: 7011, + MODEL_FILE_NOT_FOUND: 7012, + BUFFER_LIMIT_EXCEEDED: 7013 }) addCodes({ @@ -64,6 +66,14 @@ addCodes({ [ERR_CODES.MODEL_NOT_LOADED]: { name: 'MODEL_NOT_LOADED', message: () => 'Model is not loaded' + }, + [ERR_CODES.MODEL_FILE_NOT_FOUND]: { + name: 'MODEL_FILE_NOT_FOUND', + message: (modelPath) => `Model file not found at: ${modelPath}` + }, + [ERR_CODES.BUFFER_LIMIT_EXCEEDED]: { + name: 'BUFFER_LIMIT_EXCEEDED', + message: (limit) => `Neural signal buffer exceeded limit of ${limit}` } }, { name, diff --git a/packages/bci-whispercpp/package.json b/packages/bci-whispercpp/package.json index ef7ef8f4f7..19425066b9 100644 --- a/packages/bci-whispercpp/package.json +++ b/packages/bci-whispercpp/package.json @@ -11,7 +11,7 @@ "lint:fix": "standard --fix \"examples/**/*.js\" \"test/**/*.js\" \"**/*.js\"", "build": "bare-make generate && bare-make build && bare-make install", "test:unit": "brittle-bare test/unit/**/*.test.js", - "test:integration": "brittle-bare test/integration/bci-addon.test.js", + "test:integration": "brittle-bare test/integration/addon.test.js", "test:cpp:build": "bare-make generate -D BUILD_TESTING=ON && bare-make build --target test-bci-core && bare-make install", "test:cpp:run": "cd build/addon/tests/ && ./test-bci-core --gtest_output=xml:cpp-test-results.xml", "test:cpp": "npm run test:cpp:build && npm run test:cpp:run", @@ -60,6 +60,7 @@ }, "dependencies": { "@qvac/error": "^0.1.0", + "@qvac/infer-base": "^0.4.0", "@qvac/logging": "^0.1.0", "bare-path": "^3.0.0", "bare-stream": "^2.7.0", diff --git a/packages/bci-whispercpp/test/integration/addon.test.js b/packages/bci-whispercpp/test/integration/addon.test.js new file mode 100644 index 0000000000..72ced7b108 --- /dev/null +++ b/packages/bci-whispercpp/test/integration/addon.test.js @@ -0,0 +1,140 @@ +'use strict' + +const fs = require('bare-fs') +const path = require('bare-path') +const test = require('brittle') +const os = require('bare-os') +const BCIWhispercpp = require('../../index') +const { getTestPaths, computeWER, detectPlatform } = require('./helpers') + +const platform = detectPlatform() +const { manifest, getSamplePath } = getTestPaths() + +const MODEL_PATH = (os.hasEnv('WHISPER_MODEL_PATH') ? os.getEnv('WHISPER_MODEL_PATH') : null) || + path.join(__dirname, '..', '..', 'models', 'ggml-bci-windowed.bin') + +const hasModel = fs.existsSync(MODEL_PATH) + +function bciConfigFor (sample) { + return typeof sample?.day_idx === 'number' ? { day_idx: sample.day_idx } : undefined +} + +function flattenSegments (output) { + const segments = [] + for (const entry of output) { + if (Array.isArray(entry)) { + segments.push(...entry) + } else if (entry && typeof entry.text === 'string') { + segments.push(entry) + } + } + return segments +} + +test('[BCI] load and destroy via package interface', { skip: !hasModel, timeout: 120000 }, async (t) => { + const bci = new BCIWhispercpp({ + files: { model: MODEL_PATH } + }, { + whisperConfig: { language: 'en', temperature: 0.0 }, + miscConfig: { caption_enabled: false } + }) + + await bci.load() + t.ok(bci, 'BCIWhispercpp should be created and loaded') + + await bci.destroy() + t.pass('BCIWhispercpp destroyed successfully') +}) + +test('[BCI] batch transcription from neural signal file', { skip: !hasModel, timeout: 120000 }, async (t) => { + t.ok(manifest.samples.length > 0, 'Manifest must contain at least one sample') + + const sample = manifest.samples[0] + const samplePath = getSamplePath(sample.file) + t.ok(fs.existsSync(samplePath), 'Fixture ' + sample.file + ' must exist') + + const bci = new BCIWhispercpp({ + files: { model: MODEL_PATH } + }, { + whisperConfig: { language: 'en', temperature: 0.0 }, + miscConfig: { caption_enabled: false }, + bciConfig: bciConfigFor(sample) + }) + + try { + await bci.load() + + const response = await bci.transcribeFile(samplePath) + const output = await response.await() + const segments = flattenSegments(output) + const text = segments.map(s => s.text).join('').trim() + + t.comment('Expected: "' + sample.expected_text + '"') + t.comment('Got: "' + text + '"') + + const wer = computeWER(text, sample.expected_text) + t.comment('WER: ' + (wer * 100).toFixed(1) + '%') + + t.ok(typeof text === 'string' && text.length > 0, 'Should produce a transcription string') + t.ok(segments.length > 0, 'Should have segments') + t.ok(typeof wer === 'number' && wer >= 0, 'WER should be a non-negative number') + } finally { + await bci.destroy() + } +}) + +test('[BCI] WER measurement across all test samples', { skip: !hasModel, timeout: 180000 }, async (t) => { + t.ok(manifest.samples.length > 0, 'Manifest must contain at least one sample') + + t.comment('Platform: ' + platform.label) + t.comment('Model: ' + MODEL_PATH) + + const results = [] + + const byDay = new Map() + for (const sample of manifest.samples) { + const key = typeof sample.day_idx === 'number' ? sample.day_idx : -1 + if (!byDay.has(key)) byDay.set(key, []) + byDay.get(key).push(sample) + } + + for (const [day, samples] of byDay) { + const bci = new BCIWhispercpp({ + files: { model: MODEL_PATH } + }, { + whisperConfig: { language: 'en', temperature: 0.0 }, + miscConfig: { caption_enabled: false }, + bciConfig: day >= 0 ? { day_idx: day } : undefined + }) + + try { + await bci.load() + + for (const sample of samples) { + const samplePath = getSamplePath(sample.file) + if (!fs.existsSync(samplePath)) { + t.fail('Fixture ' + sample.file + ' is missing') + continue + } + + const response = await bci.transcribeFile(samplePath) + const output = await response.await() + const segments = flattenSegments(output) + const text = segments.map(s => s.text).join('').trim() + const wer = computeWER(text, sample.expected_text) + results.push({ file: sample.file, expected: sample.expected_text, got: text, wer }) + + t.comment('[' + sample.file + '] expected=' + JSON.stringify(sample.expected_text) + + ' got=' + JSON.stringify(text) + ' WER=' + (wer * 100).toFixed(1) + '%') + } + } finally { + await bci.destroy() + } + } + + const avgWER = results.reduce((sum, r) => sum + r.wer, 0) / results.length + t.comment('Average WER: ' + (avgWER * 100).toFixed(1) + '% (n=' + results.length + ')') + + t.ok(results.length === manifest.samples.length, 'All manifest samples should have been evaluated') + t.ok(typeof avgWER === 'number' && avgWER < 0.5, 'Average WER should be below 50%') +}) diff --git a/packages/bci-whispercpp/test/integration/bci-addon.test.js b/packages/bci-whispercpp/test/integration/bci-addon.test.js deleted file mode 100644 index 171a592f72..0000000000 --- a/packages/bci-whispercpp/test/integration/bci-addon.test.js +++ /dev/null @@ -1,69 +0,0 @@ -'use strict' - -const fs = require('bare-fs') -const path = require('bare-path') -const test = require('brittle') -const os = require('bare-os') -const BCIWhispercpp = require('../../index') -const { getTestPaths, computeWER } = require('./helpers') - -const { manifest, getSamplePath } = getTestPaths() - -const MODEL_PATH = (os.hasEnv('WHISPER_MODEL_PATH') ? os.getEnv('WHISPER_MODEL_PATH') : null) || - path.join(__dirname, '..', '..', 'models', 'ggml-tiny.en.bin') - -const hasModel = fs.existsSync(MODEL_PATH) - -test('[BCI] load and destroy via package interface', { skip: !hasModel, timeout: 120000 }, async (t) => { - const bci = new BCIWhispercpp({ modelPath: MODEL_PATH }, { - whisperConfig: { language: 'en', temperature: 0.0 }, - miscConfig: { caption_enabled: false } - }) - - await bci.load() - t.ok(bci, 'BCIWhispercpp should be created and loaded') - - await bci.destroy() - t.pass('BCIWhispercpp destroyed successfully') -}) - -test('[BCI] batch transcription from neural signal file', { skip: !hasModel, timeout: 120000 }, async (t) => { - if (manifest.samples.length === 0) { - t.skip('No neural signal test fixtures found') - return - } - - const sample = manifest.samples[0] - const samplePath = getSamplePath(sample.file) - if (!fs.existsSync(samplePath)) { - t.skip(`Sample file missing: ${samplePath}`) - return - } - - const bci = new BCIWhispercpp({ modelPath: MODEL_PATH }, { - whisperConfig: { language: 'en', temperature: 0.0 }, - miscConfig: { caption_enabled: false } - }) - - try { - await bci.load() - - const result = await bci.transcribeFile(samplePath) - - console.log('\n=== Batch Transcription Result ===') - console.log(`Expected: "${sample.expected_text}"`) - console.log(`Got: "${result.text}"`) - - const wer = computeWER(result.text, sample.expected_text) - console.log(`WER: ${(wer * 100).toFixed(1)}%`) - - t.ok(typeof result.text === 'string', 'Should produce a transcription string') - t.ok(result.segments, 'Should have segments') - t.ok(typeof wer === 'number' && wer >= 0, 'WER should be a non-negative number') - console.log('\nNote: High WER expected - standard whisper model is not BCI-trained.') - console.log('A BCI-trained GGML model is needed for meaningful neural-to-text results.') - } finally { - await bci.destroy() - } -}) - diff --git a/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/portfile.cmake b/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/portfile.cmake index 52e171819a..e1d5e545a0 100644 --- a/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/portfile.cmake +++ b/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/portfile.cmake @@ -1,16 +1,11 @@ -set(VERSION "a8d002cfd879315632a579e73f0148d06959de36") +set(VERSION "0f1031ae7c0f798cdcbcef62e9a147edb0146660") vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH - REPO ggml-org/whisper.cpp + REPO tetherto/qvac-ext-lib-whisper.cpp REF ${VERSION} - SHA512 aea24debb836131d14d362ff78c6d12cfe2e82188340e69e71e6874a1fa51fa9405f2c03fe43888b1ff4183f4288bf64f07dd1106224b0108c3e0f844989a409 + SHA512 e7a03f0f683ffac57ed790c85637ddac7762571707367079522d86829b61f74e8aed5fb6dc8d2316e39eccd8a6e309cc5e54e29f5b445005cb36244a27abb2e1 HEAD_REF master - PATCHES - 0001-fix-vcpkg-build.patch - 0002-fix-apple-silicon-cross-compile.patch - 0003-bci-variable-conv1-kernel.patch - 0004-bci-windowed-attention.patch ) set(PLATFORM_OPTIONS) diff --git a/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/vcpkg.json b/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/vcpkg.json index ed9210715e..1e6bf2153c 100644 --- a/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/vcpkg.json +++ b/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/vcpkg.json @@ -1,9 +1,9 @@ { "name": "whisper-cpp", - "version": "1.7.5.1", + "version": "1.8.4", "port-version": 1, - "description": "Port of OpenAI's Whisper model in C/C++ (BCI patched)", - "homepage": "https://github.com/tetherto/whisper.cpp", + "description": "Port of OpenAI's Whisper model in C/C++ (BCI patched, based on tetherto/qvac-ext-lib-whisper.cpp PR #10)", + "homepage": "https://github.com/tetherto/qvac-ext-lib-whisper.cpp", "license": "MIT", "dependencies": [ { diff --git a/packages/bci-whispercpp/vcpkg/toolchains/linux-clang.cmake b/packages/bci-whispercpp/vcpkg/toolchains/linux-clang.cmake new file mode 100644 index 0000000000..542aa9dba1 --- /dev/null +++ b/packages/bci-whispercpp/vcpkg/toolchains/linux-clang.cmake @@ -0,0 +1,4 @@ +set(CMAKE_C_COMPILER "clang-19") +set(CMAKE_CXX_COMPILER "clang++-19") + +include("$ENV{VCPKG_ROOT}/scripts/toolchains/linux.cmake") diff --git a/packages/bci-whispercpp/vcpkg/triplets/arm64-linux.cmake b/packages/bci-whispercpp/vcpkg/triplets/arm64-linux.cmake new file mode 100644 index 0000000000..77c0e6b318 --- /dev/null +++ b/packages/bci-whispercpp/vcpkg/triplets/arm64-linux.cmake @@ -0,0 +1,9 @@ +set(VCPKG_TARGET_ARCHITECTURE arm64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE static) +set(VCPKG_CMAKE_SYSTEM_NAME Linux) + +set(VCPKG_CHAINLOAD_TOOLCHAIN_FILE "${CMAKE_CURRENT_LIST_DIR}/../toolchains/linux-clang.cmake") +set(VCPKG_C_FLAGS "-fPIC") +set(VCPKG_CXX_FLAGS "-fPIC -stdlib=libc++") +set(VCPKG_LINKER_FLAGS "-stdlib=libc++") diff --git a/packages/bci-whispercpp/vcpkg/triplets/x64-linux.cmake b/packages/bci-whispercpp/vcpkg/triplets/x64-linux.cmake new file mode 100644 index 0000000000..7660720b49 --- /dev/null +++ b/packages/bci-whispercpp/vcpkg/triplets/x64-linux.cmake @@ -0,0 +1,9 @@ +set(VCPKG_TARGET_ARCHITECTURE x64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE static) +set(VCPKG_CMAKE_SYSTEM_NAME Linux) + +set(VCPKG_CHAINLOAD_TOOLCHAIN_FILE "${CMAKE_CURRENT_LIST_DIR}/../toolchains/linux-clang.cmake") +set(VCPKG_C_FLAGS "-fPIC") +set(VCPKG_CXX_FLAGS "-fPIC -stdlib=libc++") +set(VCPKG_LINKER_FLAGS "-stdlib=libc++") From 0eec4b47f488f9011190494f2f45b9a02021496b Mon Sep 17 00:00:00 2001 From: Raju Date: Mon, 20 Apr 2026 15:06:41 +0530 Subject: [PATCH 03/19] perf(bci): bump whisper-cpp overlay to include mask caching and per-layer flash attn Update whisper-cpp overlay to 5645ad60 which includes: - Cached window_mask recompute for exp_n_audio_ctx overrides - Per-layer flash attention (upper encoder layers use FA even with BCI) - std::abs instead of C abs in mask computation Made-with: Cursor --- .../bci-whispercpp/vcpkg-overlays/whisper-cpp/portfile.cmake | 4 ++-- packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/vcpkg.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/portfile.cmake b/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/portfile.cmake index e1d5e545a0..17cfa0bbb6 100644 --- a/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/portfile.cmake +++ b/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/portfile.cmake @@ -1,10 +1,10 @@ -set(VERSION "0f1031ae7c0f798cdcbcef62e9a147edb0146660") +set(VERSION "5645ad60cc7b2dde4bc29736a4506301be22e57d") vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-ext-lib-whisper.cpp REF ${VERSION} - SHA512 e7a03f0f683ffac57ed790c85637ddac7762571707367079522d86829b61f74e8aed5fb6dc8d2316e39eccd8a6e309cc5e54e29f5b445005cb36244a27abb2e1 + SHA512 7e98c7d938ca86a5f58e0ab417ea1b4d3eac118e59928a9af3fdf5258cfe5d3384873310122af4229cde18fe1805f199789d164819c9abac50afa0bb269e022b HEAD_REF master ) diff --git a/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/vcpkg.json b/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/vcpkg.json index 1e6bf2153c..90307fd7eb 100644 --- a/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/vcpkg.json +++ b/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/vcpkg.json @@ -1,7 +1,7 @@ { "name": "whisper-cpp", "version": "1.8.4", - "port-version": 1, + "port-version": 2, "description": "Port of OpenAI's Whisper model in C/C++ (BCI patched, based on tetherto/qvac-ext-lib-whisper.cpp PR #10)", "homepage": "https://github.com/tetherto/qvac-ext-lib-whisper.cpp", "license": "MIT", From 5d5ade91860479bb22e5e8be76e40f62c8499d0f Mon Sep 17 00:00:00 2001 From: Raju Date: Mon, 20 Apr 2026 16:46:23 +0530 Subject: [PATCH 04/19] chore(bci): bump whisper-cpp overlay to include jpgaribotti review fixes Update overlay to tetherto/qvac-ext-lib-whisper.cpp@3e91e3a4 which addresses jpgaribotti's review on PR #10: 1. Extract compute_window_mask() helper to eliminate duplicated O(n_ctx^2) mask fill logic 2. Guard encode-time mask block with hparams.is_bci 3. Add is_bci to graph builder window_mask guard 4. Validate BCI hparams (conv1_kernel > 0, window_size >= 0) 5. Document n_mels > 256 threshold convention Bump port-version to 3. Made-with: Cursor --- .../bci-whispercpp/vcpkg-overlays/whisper-cpp/portfile.cmake | 4 ++-- packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/vcpkg.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/portfile.cmake b/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/portfile.cmake index 17cfa0bbb6..b69c9567b4 100644 --- a/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/portfile.cmake +++ b/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/portfile.cmake @@ -1,10 +1,10 @@ -set(VERSION "5645ad60cc7b2dde4bc29736a4506301be22e57d") +set(VERSION "3e91e3a4434c9cf3d7d0f27711f2988242bccf11") vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-ext-lib-whisper.cpp REF ${VERSION} - SHA512 7e98c7d938ca86a5f58e0ab417ea1b4d3eac118e59928a9af3fdf5258cfe5d3384873310122af4229cde18fe1805f199789d164819c9abac50afa0bb269e022b + SHA512 b94f29c95cca5e06d6a4ddcff8360b69bb1593e86ca8f74c6b3425f949713f3ef534166181e3972f4a68c80c9ad594821e6990a0a38a87f2de9dca51471c61c6 HEAD_REF master ) diff --git a/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/vcpkg.json b/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/vcpkg.json index 90307fd7eb..8cbca5b74b 100644 --- a/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/vcpkg.json +++ b/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/vcpkg.json @@ -1,7 +1,7 @@ { "name": "whisper-cpp", "version": "1.8.4", - "port-version": 2, + "port-version": 3, "description": "Port of OpenAI's Whisper model in C/C++ (BCI patched, based on tetherto/qvac-ext-lib-whisper.cpp PR #10)", "homepage": "https://github.com/tetherto/qvac-ext-lib-whisper.cpp", "license": "MIT", From 0cf556130d0732f591af9893e3277efb1bc43414 Mon Sep 17 00:00:00 2001 From: Raju Date: Mon, 20 Apr 2026 17:12:27 +0530 Subject: [PATCH 05/19] fix(bci): add test fixture download to download-models.sh Address Gustavo's review feedback: test fixtures (neural_sample_*.bin) are gitignored but the PR had no way for developers to obtain them. Rewrite download-models.sh to fetch both models and test fixtures from the bci-test-assets-v0.1.0 GitHub release. Supports --models, --fixtures, or both (default). Made-with: Cursor --- .../bci-whispercpp/scripts/download-models.sh | 60 +++++++++++++++---- 1 file changed, 48 insertions(+), 12 deletions(-) diff --git a/packages/bci-whispercpp/scripts/download-models.sh b/packages/bci-whispercpp/scripts/download-models.sh index 4fc8a19c8f..db64014cda 100755 --- a/packages/bci-whispercpp/scripts/download-models.sh +++ b/packages/bci-whispercpp/scripts/download-models.sh @@ -1,22 +1,58 @@ #!/bin/bash set -euo pipefail +# Downloads BCI models and test fixtures from the GitHub release. +# Requires: gh (GitHub CLI) authenticated with repo access. +# +# Usage: +# bash scripts/download-models.sh # download models + fixtures +# bash scripts/download-models.sh --models # models only +# bash scripts/download-models.sh --fixtures # fixtures only + SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" PACKAGE_DIR="$(dirname "$SCRIPT_DIR")" MODELS_DIR="${PACKAGE_DIR}/models" +FIXTURES_DIR="${PACKAGE_DIR}/test/fixtures" +RELEASE_TAG="bci-test-assets-v0.1.0" +RELEASE_REPO="sharmaraju352/qvac" + +download_models() { + mkdir -p "$MODELS_DIR" + + echo "Downloading BCI model files..." + gh release download "$RELEASE_TAG" \ + --repo "$RELEASE_REPO" \ + --pattern "ggml-bci-windowed.bin" --dir "$MODELS_DIR" \ + --clobber + + gh release download "$RELEASE_TAG" \ + --repo "$RELEASE_REPO" \ + --pattern "bci-embedder.bin" --dir "$MODELS_DIR" \ + --clobber + + echo "Model files:" && ls -lh "$MODELS_DIR"/*.bin +} + +download_fixtures() { + mkdir -p "$FIXTURES_DIR" + + echo "Downloading BCI test fixtures..." + gh release download "$RELEASE_TAG" \ + --repo "$RELEASE_REPO" \ + --pattern "bci-test-fixtures.tar.gz" --dir /tmp \ + --clobber -mkdir -p "$MODELS_DIR" + tar xzf /tmp/bci-test-fixtures.tar.gz -C "$FIXTURES_DIR/" + rm -f /tmp/bci-test-fixtures.tar.gz -MODEL_NAME="ggml-tiny.en.bin" -MODEL_URL="https://huggingface.co/ggerganov/whisper.cpp/resolve/main/${MODEL_NAME}" -MODEL_PATH="${MODELS_DIR}/${MODEL_NAME}" + echo "Test fixtures:" && ls -lh "$FIXTURES_DIR"/*.bin +} -if [ -f "$MODEL_PATH" ]; then - echo "Model already exists: ${MODEL_PATH}" -else - echo "Downloading ${MODEL_NAME}..." - curl -L "$MODEL_URL" -o "$MODEL_PATH" - echo "Downloaded to: ${MODEL_PATH}" -fi +case "${1:-all}" in + --models) download_models ;; + --fixtures) download_fixtures ;; + all|*) download_models; echo; download_fixtures ;; +esac -echo "Done." +echo "" +echo "Done. Run tests with: npm run test:integration" From a0711f2d8b0514dae82dd3a44e1451d4203334de Mon Sep 17 00:00:00 2001 From: Raju Date: Mon, 20 Apr 2026 17:37:47 +0530 Subject: [PATCH 06/19] =?UTF-8?q?fix(bci):=20address=20review=20findings?= =?UTF-8?q?=20=E2=80=94=20version=20mismatch,=20test=20indexing,=20cleanup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Bump whisper-cpp override in vcpkg.json from 1.7.5.1 to 1.8.4 to match the overlay port version - Move gtest to a vcpkg "tests" feature so it is only pulled when BUILD_TESTING=ON - Fix PaddedFramesAreZero test: use mel-major indexing (data[bin * n_frames + frame]) matching the actual processToMel layout - Remove four unused overlay patch files (0001–0004) now that portfile.cmake fetches from the tetherto fork with patches baked in - Add TODO comment in download-models.sh noting the temporary personal fork for release assets Made-with: Cursor --- .../bci-whispercpp/addon/tests/test_core.cpp | 3 +- .../bci-whispercpp/scripts/download-models.sh | 1 + .../whisper-cpp/0001-fix-vcpkg-build.patch | 277 ------------------ ...0002-fix-apple-silicon-cross-compile.patch | 15 - .../0003-bci-variable-conv1-kernel.patch | 28 -- .../0004-bci-windowed-attention.patch | 97 ------ packages/bci-whispercpp/vcpkg.json | 13 +- 7 files changed, 13 insertions(+), 421 deletions(-) delete mode 100644 packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/0001-fix-vcpkg-build.patch delete mode 100644 packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/0002-fix-apple-silicon-cross-compile.patch delete mode 100644 packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/0003-bci-variable-conv1-kernel.patch delete mode 100644 packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/0004-bci-windowed-attention.patch diff --git a/packages/bci-whispercpp/addon/tests/test_core.cpp b/packages/bci-whispercpp/addon/tests/test_core.cpp index 1dcf0daf8f..5e2e677111 100644 --- a/packages/bci-whispercpp/addon/tests/test_core.cpp +++ b/packages/bci-whispercpp/addon/tests/test_core.cpp @@ -88,8 +88,9 @@ TEST(NeuralProcessor, PaddedFramesAreZero) { float lastFrameSum = 0; int lastFrame = NeuralProcessor::K_WHISPER_MEL_FRAMES - 1; + // mel output is mel-major: data[bin * n_frames + frame] for (int m = 0; m < NeuralProcessor::K_WHISPER_N_MEL; ++m) { - lastFrameSum += std::abs(result[lastFrame * NeuralProcessor::K_WHISPER_N_MEL + m]); + lastFrameSum += std::abs(result[m * NeuralProcessor::K_WHISPER_MEL_FRAMES + lastFrame]); } EXPECT_FLOAT_EQ(lastFrameSum, 0.0F); } diff --git a/packages/bci-whispercpp/scripts/download-models.sh b/packages/bci-whispercpp/scripts/download-models.sh index db64014cda..bcc087d751 100755 --- a/packages/bci-whispercpp/scripts/download-models.sh +++ b/packages/bci-whispercpp/scripts/download-models.sh @@ -14,6 +14,7 @@ PACKAGE_DIR="$(dirname "$SCRIPT_DIR")" MODELS_DIR="${PACKAGE_DIR}/models" FIXTURES_DIR="${PACKAGE_DIR}/test/fixtures" RELEASE_TAG="bci-test-assets-v0.1.0" +# TODO(QVAC-17058): Move release assets to tetherto/qvac once CI workflows land. RELEASE_REPO="sharmaraju352/qvac" download_models() { diff --git a/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/0001-fix-vcpkg-build.patch b/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/0001-fix-vcpkg-build.patch deleted file mode 100644 index e587ea07d4..0000000000 --- a/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/0001-fix-vcpkg-build.patch +++ /dev/null @@ -1,277 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 36eef350..dfcc171d 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -23,10 +23,18 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) - if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) - set(WHISPER_STANDALONE ON) - -- include(git-vars) -+ find_package(Git QUIET) -+ if(GIT_FOUND) -+ include(git-vars) -+ else() -+ set(GIT_SHA1 "unknown") -+ set(GIT_DATE "unknown") -+ set(GIT_COMMIT_SUBJECT "unknown") -+ endif() - -- # configure project version -- configure_file(${CMAKE_SOURCE_DIR}/bindings/javascript/package-tmpl.json ${CMAKE_SOURCE_DIR}/bindings/javascript/package.json @ONLY) -+ if(EXISTS ${CMAKE_SOURCE_DIR}/bindings/javascript/package-tmpl.json) -+ configure_file(${CMAKE_SOURCE_DIR}/bindings/javascript/package-tmpl.json ${CMAKE_SOURCE_DIR}/bindings/javascript/package.json @ONLY) -+ endif() - else() - set(WHISPER_STANDALONE OFF) - endif() -@@ -169,23 +177,34 @@ set(WHISPER_BUILD_NUMBER ${BUILD_NUMBER}) - set(WHISPER_BUILD_COMMIT ${BUILD_COMMIT}) - set(WHISPER_INSTALL_VERSION ${CMAKE_PROJECT_VERSION}) - --set(WHISPER_INCLUDE_INSTALL_DIR ${CMAKE_INSTALL_INCLUDEDIR} CACHE PATH "Location of header files") -+set(WHISPER_INCLUDE_INSTALL_DIR ${CMAKE_INSTALL_INCLUDEDIR}/whisper CACHE PATH "Location of header files") - set(WHISPER_LIB_INSTALL_DIR ${CMAKE_INSTALL_LIBDIR} CACHE PATH "Location of library files") - set(WHISPER_BIN_INSTALL_DIR ${CMAKE_INSTALL_BINDIR} CACHE PATH "Location of binary files") - - get_directory_property(WHISPER_TRANSIENT_DEFINES COMPILE_DEFINITIONS) - - set_target_properties(whisper PROPERTIES PUBLIC_HEADER ${CMAKE_CURRENT_SOURCE_DIR}/include/whisper.h) --install(TARGETS whisper LIBRARY PUBLIC_HEADER) -+ -+install( -+ TARGETS whisper -+ EXPORT whisper-targets -+ PUBLIC_HEADER -+ DESTINATION ${WHISPER_INCLUDE_INSTALL_DIR}) -+ -+install( -+ EXPORT whisper-targets -+ FILE whisper-targets.cmake -+ NAMESPACE whisper:: -+ DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/whisper) -+ -+install( -+ FILES ${CMAKE_CURRENT_BINARY_DIR}/whisper-config.cmake -+ DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/whisper) - - configure_package_config_file( -- ${CMAKE_CURRENT_SOURCE_DIR}/cmake/whisper-config.cmake.in -- ${CMAKE_CURRENT_BINARY_DIR}/whisper-config.cmake -- INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/whisper -- PATH_VARS -- WHISPER_INCLUDE_INSTALL_DIR -- WHISPER_LIB_INSTALL_DIR -- WHISPER_BIN_INSTALL_DIR ) -+ ${CMAKE_CURRENT_SOURCE_DIR}/cmake/whisper-config.cmake.in -+ ${CMAKE_CURRENT_BINARY_DIR}/whisper-config.cmake -+ INSTALL_DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/whisper) - - write_basic_package_version_file( - ${CMAKE_CURRENT_BINARY_DIR}/whisper-version.cmake -@@ -194,7 +213,7 @@ write_basic_package_version_file( - - install(FILES ${CMAKE_CURRENT_BINARY_DIR}/whisper-config.cmake - ${CMAKE_CURRENT_BINARY_DIR}/whisper-version.cmake -- DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/whisper) -+ DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/whisper) - - configure_file(cmake/whisper.pc.in - "${CMAKE_CURRENT_BINARY_DIR}/whisper.pc" -diff --git a/cmake/git-vars.cmake b/cmake/git-vars.cmake -index 1a4c24eb..8dc51859 100644 ---- a/cmake/git-vars.cmake -+++ b/cmake/git-vars.cmake -@@ -1,22 +1,36 @@ - find_package(Git) - --# the commit's SHA1 --execute_process(COMMAND -- "${GIT_EXECUTABLE}" describe --match=NeVeRmAtCh --always --abbrev=8 -- WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" -- OUTPUT_VARIABLE GIT_SHA1 -- ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) -+if(GIT_FOUND) -+ execute_process(COMMAND -+ "${GIT_EXECUTABLE}" describe --match=NeVeRmAtCh --always --abbrev=8 -+ WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" -+ OUTPUT_VARIABLE GIT_SHA1 -+ ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE -+ RESULT_VARIABLE GIT_SHA1_RESULT) - --# the date of the commit --execute_process(COMMAND -- "${GIT_EXECUTABLE}" log -1 --format=%ad --date=local -- WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" -- OUTPUT_VARIABLE GIT_DATE -- ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) -+ execute_process(COMMAND -+ "${GIT_EXECUTABLE}" log -1 --format=%ad --date=local -+ WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" -+ OUTPUT_VARIABLE GIT_DATE -+ ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE -+ RESULT_VARIABLE GIT_DATE_RESULT) - --# the subject of the commit --execute_process(COMMAND -- "${GIT_EXECUTABLE}" log -1 --format=%s -- WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" -- OUTPUT_VARIABLE GIT_COMMIT_SUBJECT -- ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) -+ execute_process(COMMAND -+ "${GIT_EXECUTABLE}" log -1 --format=%s -+ WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" -+ OUTPUT_VARIABLE GIT_COMMIT_SUBJECT -+ ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE -+ RESULT_VARIABLE GIT_COMMIT_SUBJECT_RESULT) -+endif() -+ -+if(NOT GIT_FOUND OR GIT_SHA1_RESULT OR NOT GIT_SHA1) -+ set(GIT_SHA1 "unknown") -+endif() -+ -+if(NOT GIT_FOUND OR GIT_DATE_RESULT OR NOT GIT_DATE) -+ set(GIT_DATE "unknown") -+endif() -+ -+if(NOT GIT_FOUND OR GIT_COMMIT_SUBJECT_RESULT OR NOT GIT_COMMIT_SUBJECT) -+ set(GIT_COMMIT_SUBJECT "unknown") -+endif() -diff --git a/cmake/whisper-config.cmake.in b/cmake/whisper-config.cmake.in -index 6a3fa227..9fe65884 100644 ---- a/cmake/whisper-config.cmake.in -+++ b/cmake/whisper-config.cmake.in -@@ -11,24 +11,21 @@ set(GGML_ACCELERATE @GGML_ACCELERATE@) - - @PACKAGE_INIT@ - --set_and_check(WHISPER_INCLUDE_DIR "@PACKAGE_WHISPER_INCLUDE_INSTALL_DIR@") --set_and_check(WHISPER_LIB_DIR "@PACKAGE_WHISPER_LIB_INSTALL_DIR@") --set_and_check(WHISPER_BIN_DIR "@PACKAGE_WHISPER_BIN_INSTALL_DIR@") -+include(CMakeFindDependencyMacro) - - # Ensure transient dependencies satisfied -- --find_package(Threads REQUIRED) -+find_dependency(Threads REQUIRED) - - if (APPLE AND GGML_ACCELERATE) - find_library(ACCELERATE_FRAMEWORK Accelerate REQUIRED) - endif() - - if (GGML_BLAS) -- find_package(BLAS REQUIRED) -+ find_dependency(BLAS REQUIRED) - endif() - - if (GGML_CUDA) -- find_package(CUDAToolkit REQUIRED) -+ find_dependency(CUDAToolkit REQUIRED) - endif() - - if (GGML_METAL) -@@ -38,28 +35,13 @@ if (GGML_METAL) - endif() - - if (GGML_HIPBLAS) -- find_package(hip REQUIRED) -- find_package(hipblas REQUIRED) -- find_package(rocblas REQUIRED) -+ find_dependency(hip REQUIRED) -+ find_dependency(hipblas REQUIRED) -+ find_dependency(rocblas REQUIRED) - endif() - --find_library(whisper_LIBRARY whisper -- REQUIRED -- HINTS ${WHISPER_LIB_DIR}) -- --set(_whisper_link_deps "Threads::Threads" "@WHISPER_EXTRA_LIBS@") --set(_whisper_transient_defines "@WHISPER_TRANSIENT_DEFINES@") -- --add_library(whisper UNKNOWN IMPORTED) -+find_dependency(ggml CONFIG REQUIRED) - --set_target_properties(whisper -- PROPERTIES -- INTERFACE_INCLUDE_DIRECTORIES "${WHISPER_INCLUDE_DIR}" -- INTERFACE_LINK_LIBRARIES "${_whisper_link_deps}" -- INTERFACE_COMPILE_DEFINITIONS "${_whisper_transient_defines}" -- IMPORTED_LINK_INTERFACE_LANGUAGES "CXX" -- IMPORTED_LOCATION "${whisper_LIBRARY}" -- INTERFACE_COMPILE_FEATURES cxx_std_11 -- POSITION_INDEPENDENT_CODE ON ) -+include("${CMAKE_CURRENT_LIST_DIR}/whisper-targets.cmake") - - check_required_components(whisper) -diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txt -index 4e7399f9..fd3ccebe 100644 ---- a/ggml/CMakeLists.txt -+++ b/ggml/CMakeLists.txt -@@ -277,8 +277,17 @@ set_target_properties(ggml PROPERTIES PUBLIC_HEADER "${GGML_PUBLIC_HEADERS}") - #if (GGML_METAL) - # set_target_properties(ggml PROPERTIES RESOURCE "${CMAKE_CURRENT_SOURCE_DIR}/src/ggml-metal.metal") - #endif() --install(TARGETS ggml LIBRARY PUBLIC_HEADER) --install(TARGETS ggml-base LIBRARY) -+install( -+ TARGETS ggml ggml-base -+ EXPORT ggml-targets -+ PUBLIC_HEADER -+ DESTINATION ${GGML_INCLUDE_INSTALL_DIR}) -+ -+install( -+ EXPORT ggml-targets -+ FILE ggml-targets.cmake -+ NAMESPACE ggml:: -+ DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/ggml) - - if (GGML_STANDALONE) - configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ggml.pc.in -@@ -349,7 +358,7 @@ set(GGML_BIN_INSTALL_DIR ${CMAKE_INSTALL_BINDIR} CACHE PATH "Location of - configure_package_config_file( - ${CMAKE_CURRENT_SOURCE_DIR}/cmake/ggml-config.cmake.in - ${CMAKE_CURRENT_BINARY_DIR}/ggml-config.cmake -- INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/ggml -+ INSTALL_DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/ggml - PATH_VARS GGML_INCLUDE_INSTALL_DIR - GGML_LIB_INSTALL_DIR - GGML_BIN_INSTALL_DIR) -@@ -361,7 +370,7 @@ write_basic_package_version_file( - - install(FILES ${CMAKE_CURRENT_BINARY_DIR}/ggml-config.cmake - ${CMAKE_CURRENT_BINARY_DIR}/ggml-version.cmake -- DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/ggml) -+ DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/ggml) - - if (MSVC) - set(MSVC_WARNING_FLAGS -diff --git a/ggml/src/CMakeLists.txt b/ggml/src/CMakeLists.txt -index 9cb2c228..6396d883 100644 ---- a/ggml/src/CMakeLists.txt -+++ b/ggml/src/CMakeLists.txt -@@ -231,7 +231,7 @@ function(ggml_add_backend_library backend) - else() - add_library(${backend} ${ARGN}) - target_link_libraries(ggml PUBLIC ${backend}) -- install(TARGETS ${backend} LIBRARY) -+ install(TARGETS ${backend} EXPORT ggml-targets) - endif() - - target_link_libraries(${backend} PRIVATE ggml-base) -diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt -index 2eae0c66..cd4c60e8 100644 ---- a/src/CMakeLists.txt -+++ b/src/CMakeLists.txt -@@ -114,7 +114,11 @@ set_target_properties(whisper PROPERTIES - SOVERSION ${SOVERSION} - ) - --target_include_directories(whisper PUBLIC . ../include) -+target_include_directories( -+ whisper -+ PUBLIC -+ $ -+ $) - target_compile_features (whisper PUBLIC cxx_std_11) # don't bump - - if (CMAKE_CXX_BYTE_ORDER STREQUAL "BIG_ENDIAN") diff --git a/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/0002-fix-apple-silicon-cross-compile.patch b/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/0002-fix-apple-silicon-cross-compile.patch deleted file mode 100644 index f8154f1f92..0000000000 --- a/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/0002-fix-apple-silicon-cross-compile.patch +++ /dev/null @@ -1,15 +0,0 @@ -diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txt -index fd3cceb..d072fe6 100644 ---- a/ggml/CMakeLists.txt -+++ b/ggml/CMakeLists.txt -@@ -58,7 +58,9 @@ else() - set(GGML_BLAS_VENDOR_DEFAULT "Generic") - endif() - --if (CMAKE_CROSSCOMPILING OR DEFINED ENV{SOURCE_DATE_EPOCH}) -+if (CMAKE_CROSSCOMPILING OR DEFINED ENV{SOURCE_DATE_EPOCH} OR -+ (APPLE AND CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "arm64" AND -+ CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")) - message(STATUS "Setting GGML_NATIVE_DEFAULT to OFF") - set(GGML_NATIVE_DEFAULT OFF) - else() diff --git a/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/0003-bci-variable-conv1-kernel.patch b/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/0003-bci-variable-conv1-kernel.patch deleted file mode 100644 index 025f8c29c0..0000000000 --- a/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/0003-bci-variable-conv1-kernel.patch +++ /dev/null @@ -1,28 +0,0 @@ -diff --git a/src/whisper.cpp b/src/whisper.cpp ---- a/src/whisper.cpp -+++ b/src/whisper.cpp -@@ -633,6 +633,7 @@ - int32_t n_mels = 80; - int32_t ftype = 1; - float eps = 1e-5f; -+ int32_t n_audio_conv1_kernel = 3; - }; - - // audio encoding layer -@@ -1535,6 +1536,7 @@ - read_safe(loader, hparams.n_text_layer); - read_safe(loader, hparams.n_mels); - read_safe(loader, hparams.ftype); -+ read_safe(loader, hparams.n_audio_conv1_kernel); - - assert(hparams.n_text_state == hparams.n_audio_state); - -@@ -1775,7 +1777,7 @@ - // encoder - model.e_pe = create_tensor(ASR_TENSOR_ENC_POS_EMBD, ASR_SYSTEM_ENCODER, ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_audio_state, n_audio_ctx)); - -- model.e_conv_1_w = create_tensor(ASR_TENSOR_CONV1_WEIGHT, ASR_SYSTEM_ENCODER, ggml_new_tensor_3d(ctx, vtype, 3, n_mels, n_audio_state)); -+ model.e_conv_1_w = create_tensor(ASR_TENSOR_CONV1_WEIGHT, ASR_SYSTEM_ENCODER, ggml_new_tensor_3d(ctx, vtype, hparams.n_audio_conv1_kernel, n_mels, n_audio_state)); - model.e_conv_1_b = create_tensor(ASR_TENSOR_CONV1_BIAS, ASR_SYSTEM_ENCODER, ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 1, n_audio_state)); - - model.e_conv_2_w = create_tensor(ASR_TENSOR_CONV2_WEIGHT, ASR_SYSTEM_ENCODER, ggml_new_tensor_3d(ctx, vtype, 3, n_audio_state, n_audio_state)); diff --git a/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/0004-bci-windowed-attention.patch b/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/0004-bci-windowed-attention.patch deleted file mode 100644 index 9161158071..0000000000 --- a/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/0004-bci-windowed-attention.patch +++ /dev/null @@ -1,97 +0,0 @@ -diff --git a/src/whisper.cpp b/src/whisper.cpp ---- a/src/whisper.cpp -+++ b/src/whisper.cpp -@@ -633,6 +633,8 @@ - int32_t ftype = 1; - float eps = 1e-5f; - int32_t n_audio_conv1_kernel = 3; -+ int32_t n_audio_window_size = 0; -+ int32_t n_audio_last_window_layer = -1; - }; - - // audio encoding layer -@@ -1536,6 +1538,8 @@ - read_safe(loader, hparams.n_mels); - read_safe(loader, hparams.ftype); - read_safe(loader, hparams.n_audio_conv1_kernel); -+ read_safe(loader, hparams.n_audio_window_size); -+ read_safe(loader, hparams.n_audio_last_window_layer); - - assert(hparams.n_text_state == hparams.n_audio_state); - -@@ -2114,6 +2118,15 @@ - - struct ggml_tensor * inpL = cur; - -+ struct ggml_tensor * window_mask = nullptr; -+ const int window_size = hparams.n_audio_window_size; -+ const int last_window_layer = hparams.n_audio_last_window_layer; -+ if (window_size > 0 && last_window_layer >= 0) { -+ window_mask = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_ctx, n_ctx, 1); -+ ggml_set_name(window_mask, "window_mask"); -+ ggml_set_input(window_mask); -+ } -+ - for (int il = 0; il < n_layer; ++il) { - const auto & layer = model.layers_encoder[il]; - -@@ -2177,7 +2190,8 @@ - ggml_element_size(kv_pad.v)*n_state_head, - 0); - -- cur = ggml_flash_attn_ext(ctx0, Q, K, V, nullptr, KQscale, 0.0f, 0.0f); -+ struct ggml_tensor * attn_mask_fa = (window_mask && il <= last_window_layer) ? window_mask : nullptr; -+ cur = ggml_flash_attn_ext(ctx0, Q, K, V, attn_mask_fa, KQscale, 0.0f, 0.0f); - - cur = ggml_reshape_2d(ctx0, cur, n_state, n_ctx); - } else { -@@ -2191,7 +2205,8 @@ - // K * Q - struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q); - -- struct ggml_tensor * KQ_soft_max = ggml_soft_max_ext(ctx0, KQ, nullptr, KQscale, 0.0f); -+ struct ggml_tensor * enc_attn_mask = (window_mask && il <= last_window_layer) ? window_mask : nullptr; -+ struct ggml_tensor * KQ_soft_max = ggml_soft_max_ext(ctx0, KQ, enc_attn_mask, KQscale, 0.0f); - - struct ggml_tensor * V = - ggml_cast(ctx0, -@@ -2442,6 +2457,25 @@ - return false; - } - -+ { -+ struct ggml_tensor * wmask = ggml_graph_get_tensor(gf, "window_mask"); -+ if (wmask) { -+ const int n_ctx = wstate.exp_n_audio_ctx > 0 -+ ? wstate.exp_n_audio_ctx : wctx.model.hparams.n_audio_ctx; -+ const int ws = wctx.model.hparams.n_audio_window_size; -+ const int half_w = ws / 2; -+ std::vector mask_data(n_ctx * n_ctx); -+ for (int i = 0; i < n_ctx; ++i) { -+ for (int j = 0; j < n_ctx; ++j) { -+ mask_data[i * n_ctx + j] = -+ (abs(i - j) <= half_w) ? 0.0f : -INFINITY; -+ } -+ } -+ ggml_backend_tensor_set(wmask, mask_data.data(), 0, -+ n_ctx * n_ctx * sizeof(float)); -+ } -+ } -+ - if (!ggml_graph_compute_helper(sched, gf, n_threads)) { - return false; - } -@@ -6949,7 +6983,12 @@ - } else { - prompt_init.push_back(whisper_token_transcribe(ctx)); - } -- } -+ } else if (ctx->model.hparams.n_audio_window_size > 0) { -+ const int lang_id = whisper_lang_id(params.language); -+ state->lang_id = lang_id; -+ prompt_init.push_back(whisper_token_lang(ctx, lang_id)); -+ prompt_init.push_back(whisper_token_transcribe(ctx)); -+ } - - // first release distilled models require the "no_timestamps" token - { diff --git a/packages/bci-whispercpp/vcpkg.json b/packages/bci-whispercpp/vcpkg.json index 867b85f130..a4448d5c39 100644 --- a/packages/bci-whispercpp/vcpkg.json +++ b/packages/bci-whispercpp/vcpkg.json @@ -6,13 +6,20 @@ "name": "qvac-lib-inference-addon-cpp", "version>=": "1.1.5" }, - "whisper-cpp", - "gtest" + "whisper-cpp" ], + "features": { + "tests": { + "description": "Build C++ unit tests", + "dependencies": [ + "gtest" + ] + } + }, "overrides": [ { "name": "whisper-cpp", - "version": "1.7.5.1" + "version": "1.8.4" } ] } From c38a0b9ad3213317df7c38f9ccac73765ea4ded9 Mon Sep 17 00:00:00 2001 From: Raju Date: Mon, 20 Apr 2026 17:59:01 +0530 Subject: [PATCH 07/19] =?UTF-8?q?fix(bci):=20address=20review=20findings?= =?UTF-8?q?=20=E2=80=94=20race=20guard,=20cross-platform=20path,=20docs=20?= =?UTF-8?q?accuracy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Wrap transcribe() in exclusiveRunQueue to prevent race between inference and unload/destroy - Use find_last_of("/\\") in loadEmbedderIfNeeded for Windows compat - Add empty-buffer guard in bci.js append() before end-of-job - Update download-models.sh to use tetherto/qvac release repo - Add transformers to NOTICE and README model conversion prerequisites - Fix README WER table to match actual live test results (6.0% avg) - Fix BCI_V184_COMPAT.md stale test filename and overlay ref - Remove unused bci_wer_vs_expected field from manifest.json - Update whisper.cpp patches section to reflect fork-based overlay Made-with: Cursor --- packages/bci-whispercpp/NOTICE | 6 +++ packages/bci-whispercpp/README.md | 33 +++++++++------- .../src/model-interface/bci/BCIModel.cpp | 6 ++- packages/bci-whispercpp/bci.js | 6 +++ .../bci-whispercpp/docs/BCI_V184_COMPAT.md | 4 +- packages/bci-whispercpp/index.js | 38 ++++++++++--------- .../bci-whispercpp/scripts/download-models.sh | 3 +- .../test/fixtures/manifest.json | 5 --- 8 files changed, 58 insertions(+), 43 deletions(-) diff --git a/packages/bci-whispercpp/NOTICE b/packages/bci-whispercpp/NOTICE index 1c61b2cf56..70fbe1dfef 100644 --- a/packages/bci-whispercpp/NOTICE +++ b/packages/bci-whispercpp/NOTICE @@ -55,6 +55,12 @@ PyTorch checkpoints to the GGML + embedder binary format requires: torch https://pytorch.org +--- apache-2.0 (Apache License 2.0) --- + + transformers + https://github.com/huggingface/transformers + (used by convert-model.py for WhisperTokenizer) + ========================================================================= C++ Dependencies diff --git a/packages/bci-whispercpp/README.md b/packages/bci-whispercpp/README.md index e19812caf3..36b3b2565b 100644 --- a/packages/bci-whispercpp/README.md +++ b/packages/bci-whispercpp/README.md @@ -35,13 +35,14 @@ Neural Signal (512ch, 20ms bins) Native GGML inference matches the Python BrainWhisperer reference on all test samples: -| Sample | Ground Truth | GGML Native Output | Python Reference | -|--------|-------------|-------------------|-----------------| -| 0 | "You can see the code at this point as well." | "You can see the good at this point as well." | "you can see the good at this point as well" | -| 1 | "How does it keep the cost down?" | "How does it keep the cost said?" | "how does it keep the cost said" | -| 2 | "Not too controversial." | "Not too controversial." | "not too controversial" | -| 3 | "The jury and a judge work together on it." | "The jury and a judge work together on it." | "the jury and a judge work together on it" | -| 4 | "Were quite vocal about it." | "We're quite vocal about it." | "we're quite vocal about it" | +| Sample | Ground Truth | GGML Native Output | WER | +|--------|-------------|-------------------|-----| +| 0 | "You can see the code at this point as well." | "You can see the good at this point as well." | 10.0% | +| 1 | "How does it keep the cost down?" | "How does it keep the cost down?" | 0.0% | +| 2 | "Not too controversial." | "Not too controversial." | 0.0% | +| 3 | "The jury and a judge work together on it." | "The jury and a judge work together on it." | 0.0% | +| 4 | "Were quite vocal about it." | "We're quite vocal about it." | 20.0% | +| **Average** | | | **6.0%** | ## Neural Signal Format @@ -69,6 +70,10 @@ VCPKG_ROOT=/path/to/vcpkg npm run build - **CMake** >= 3.25 - **vcpkg** with `VCPKG_ROOT` environment variable set +### Model Conversion Prerequisites + +- **Python 3** with `numpy`, `torch`, and `transformers` (`pip install numpy torch transformers`) + ### Model Conversion Convert a trained BrainWhisperer checkpoint. This produces **two files**, both required for inference: @@ -172,14 +177,14 @@ VCPKG_ROOT=/path/to/vcpkg npm run test:cpp ## whisper.cpp Patches -The package includes a vcpkg overlay with 4 patches applied to whisper.cpp: +The package uses a vcpkg overlay that fetches from the `tetherto/qvac-ext-lib-whisper.cpp` fork (v1.8.4 base) with BCI patches baked in: -| Patch | Description | -|-------|-------------| -| 0001 | Fix vcpkg build | -| 0002 | Fix Apple Silicon cross-compilation | -| 0003 | Variable conv1 kernel size (read `n_audio_conv1_kernel` from model header) | -| 0004 | Windowed attention mask, window size/layer params in header, BCI-specific SOS tokens | +| Feature | Description | +|---------|-------------| +| Variable conv1 kernel | Read `n_audio_conv1_kernel` from model header (k=7 for 512ch BCI vs k=3 for audio) | +| Windowed attention | Attention mask with configurable window size/layer params in header | +| BCI SOS tokens | BCI-specific start-of-sequence token handling | +| Graph placement fix | Correct encoder-graph mask population (see `docs/BCI_V184_COMPAT.md`) | ## Platform Support diff --git a/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp b/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp index 67ceb094f6..1769c9b6a2 100644 --- a/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp +++ b/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp @@ -77,8 +77,10 @@ void BCIModel::loadEmbedderIfNeeded() { } const auto modelPath = std::get(modelPathIt->second); - // Try: same directory, "bci-embedder.bin" - auto dir = modelPath.substr(0, modelPath.find_last_of('/')); + auto lastSep = modelPath.find_last_of("/\\"); + auto dir = (lastSep != std::string::npos) + ? modelPath.substr(0, lastSep) + : "."; auto embedderPath = dir + "/bci-embedder.bin"; if (neuralProcessor_.loadEmbedderWeights(embedderPath)) { diff --git a/packages/bci-whispercpp/bci.js b/packages/bci-whispercpp/bci.js index 4e81a00284..9d869a807f 100644 --- a/packages/bci-whispercpp/bci.js +++ b/packages/bci-whispercpp/bci.js @@ -190,6 +190,12 @@ class BCIInterface { async append (data) { try { if (data?.type === END_OF_INPUT) { + if (this._bufferedSignal.length === 0) { + throw new QvacErrorAddonBCI({ + code: ERR_CODES.INVALID_NEURAL_INPUT, + adds: 'no neural signal data was appended before end-of-job' + }) + } const currentJobId = this._nextJobId const input = this._concatBufferedSignal() diff --git a/packages/bci-whispercpp/docs/BCI_V184_COMPAT.md b/packages/bci-whispercpp/docs/BCI_V184_COMPAT.md index 67dfa7d4e2..0e050ece72 100644 --- a/packages/bci-whispercpp/docs/BCI_V184_COMPAT.md +++ b/packages/bci-whispercpp/docs/BCI_V184_COMPAT.md @@ -52,5 +52,5 @@ Patch `0005-fix-bci-window-mask-encoder-graph.patch` moves the `window_mask` dat - BCI model: `models/ggml-bci-windowed.bin` - Embedder weights: `models/bci-embedder.bin` - Conversion script: `scripts/convert-model.py` -- Overlay portfile: `vcpkg-overlays/whisper-cpp/portfile.cmake` (points to `tetherto/qvac-ext-lib-whisper.cpp` at `bbb3535`) -- Test: `test/integration/bci-addon.test.js` +- Overlay portfile: `vcpkg-overlays/whisper-cpp/portfile.cmake` (points to `tetherto/qvac-ext-lib-whisper.cpp` at `3e91e3a4`) +- Test: `test/integration/addon.test.js` diff --git a/packages/bci-whispercpp/index.js b/packages/bci-whispercpp/index.js index 3d3268253e..a587093b12 100644 --- a/packages/bci-whispercpp/index.js +++ b/packages/bci-whispercpp/index.js @@ -137,25 +137,27 @@ class BCIWhispercpp { * @returns {Promise} */ async transcribe (neuralData) { - const response = this._job.start() - - let accepted - try { - accepted = await this.addon.runJob({ input: neuralData }) - } catch (err) { - this._job.fail(err) - throw err - } - if (!accepted) { - const error = new QvacErrorAddonBCI({ code: ERR_CODES.JOB_ALREADY_RUNNING }) - this._job.fail(error) - throw error - } + return await this._withExclusiveRun(async () => { + const response = this._job.start() + + let accepted + try { + accepted = await this.addon.runJob({ input: neuralData }) + } catch (err) { + this._job.fail(err) + throw err + } + if (!accepted) { + const error = new QvacErrorAddonBCI({ code: ERR_CODES.JOB_ALREADY_RUNNING }) + this._job.fail(error) + throw error + } - const finalized = response.await() - finalized.catch(() => {}) - response.await = () => finalized - return response + const finalized = response.await() + finalized.catch(() => {}) + response.await = () => finalized + return response + }) } _outputCallback (addon, event, jobId, data, error) { diff --git a/packages/bci-whispercpp/scripts/download-models.sh b/packages/bci-whispercpp/scripts/download-models.sh index bcc087d751..284885f7b0 100755 --- a/packages/bci-whispercpp/scripts/download-models.sh +++ b/packages/bci-whispercpp/scripts/download-models.sh @@ -14,8 +14,7 @@ PACKAGE_DIR="$(dirname "$SCRIPT_DIR")" MODELS_DIR="${PACKAGE_DIR}/models" FIXTURES_DIR="${PACKAGE_DIR}/test/fixtures" RELEASE_TAG="bci-test-assets-v0.1.0" -# TODO(QVAC-17058): Move release assets to tetherto/qvac once CI workflows land. -RELEASE_REPO="sharmaraju352/qvac" +RELEASE_REPO="tetherto/qvac" download_models() { mkdir -p "$MODELS_DIR" diff --git a/packages/bci-whispercpp/test/fixtures/manifest.json b/packages/bci-whispercpp/test/fixtures/manifest.json index 1223a73316..79a32b7641 100644 --- a/packages/bci-whispercpp/test/fixtures/manifest.json +++ b/packages/bci-whispercpp/test/fixtures/manifest.json @@ -7,7 +7,6 @@ "expected_text": "You can see the code at this point as well.", "day_idx": 1, "bci_transcription": "you can see the good at this point as well", - "bci_wer_vs_expected": null, "bci_wer": 0.1 }, { @@ -17,7 +16,6 @@ "expected_text": "How does it keep the cost down?", "day_idx": 1, "bci_transcription": "how does it keep the cost said", - "bci_wer_vs_expected": null, "bci_wer": 0.1429 }, { @@ -27,7 +25,6 @@ "expected_text": "Not too controversial.", "day_idx": 1, "bci_transcription": "not too controversial", - "bci_wer_vs_expected": null, "bci_wer": 0.0 }, { @@ -37,7 +34,6 @@ "expected_text": "The jury and a judge work together on it.", "day_idx": 1, "bci_transcription": "the jury and a judge work together on it", - "bci_wer_vs_expected": null, "bci_wer": 0.0 }, { @@ -47,7 +43,6 @@ "expected_text": "Were quite vocal about it.", "day_idx": 1, "bci_transcription": "we're quite vocal about it", - "bci_wer_vs_expected": null, "bci_wer": 0.2 } ] From 015e61aecb3d22110a977bba162b0c6c27cd167f Mon Sep 17 00:00:00 2001 From: Raju Date: Mon, 20 Apr 2026 19:47:33 +0530 Subject: [PATCH 08/19] fix(bci): harden lifecycle, type safety, and C++ code quality - Fix unload/destroy race: call destroyInstance() before _job.fail() so the native side stops before the JS job is failed, and remove redundant cancel() call (destroyInstance already cancels internally) - Wrap BCIInterface construction in try/catch so a native init failure sets addon=null and throws a structured QvacErrorAddonBCI - Change JSAdapter loadContextParams/loadMiscParams/loadBCIParams to return void (callers already mutate via reference, return was dead) - Add dayIdx bounds-check warning in BCIModel::process when the value falls outside [0, numDays-1] before silent clamping - Promote hardcoded gaussian smoothing params (std=2.0, kernel=100) to named constants K_SMOOTH_KERNEL_STD / K_SMOOTH_KERNEL_SIZE - Add NeuralProcessor::getNumDays() accessor for the bounds check - Remove [key: string]: unknown escape hatch from WhisperConfig in index.d.ts; enumerate all valid keys explicitly - Fix test:cpp:run script to use direct path instead of cd && chain Made-with: Cursor --- .../addon/src/js-interface/JSAdapter.cpp | 9 ++--- .../addon/src/js-interface/JSAdapter.hpp | 15 ++++---- .../src/model-interface/bci/BCIModel.cpp | 14 ++++++-- .../model-interface/bci/NeuralProcessor.cpp | 11 ++++-- .../model-interface/bci/NeuralProcessor.hpp | 1 + packages/bci-whispercpp/index.d.ts | 9 ++++- packages/bci-whispercpp/index.js | 35 +++++++++++-------- packages/bci-whispercpp/package.json | 2 +- 8 files changed, 60 insertions(+), 36 deletions(-) diff --git a/packages/bci-whispercpp/addon/src/js-interface/JSAdapter.cpp b/packages/bci-whispercpp/addon/src/js-interface/JSAdapter.cpp index 58e60eeb47..c9a10a9521 100644 --- a/packages/bci-whispercpp/addon/src/js-interface/JSAdapter.cpp +++ b/packages/bci-whispercpp/addon/src/js-interface/JSAdapter.cpp @@ -108,22 +108,19 @@ BCIConfig JSAdapter::loadFromJSObject(Object jsObject, js_env_t* env) { return config; } -BCIConfig JSAdapter::loadContextParams( +void JSAdapter::loadContextParams( Object contextParamsObj, js_env_t* env, BCIConfig& config) { loadMap(contextParamsObj, env, config.whisperContextCfg); - return config; } -BCIConfig JSAdapter::loadMiscParams( +void JSAdapter::loadMiscParams( Object miscParamsObj, js_env_t* env, BCIConfig& config) { loadMap(miscParamsObj, env, config.miscConfig); - return config; } -BCIConfig JSAdapter::loadBCIParams( +void JSAdapter::loadBCIParams( Object bciParamsObj, js_env_t* env, BCIConfig& config) { loadMap(bciParamsObj, env, config.bciConfig); - return config; } } // namespace qvac_lib_inference_addon_bci diff --git a/packages/bci-whispercpp/addon/src/js-interface/JSAdapter.hpp b/packages/bci-whispercpp/addon/src/js-interface/JSAdapter.hpp index 9b5b18b7c8..bb3c34e0c2 100644 --- a/packages/bci-whispercpp/addon/src/js-interface/JSAdapter.hpp +++ b/packages/bci-whispercpp/addon/src/js-interface/JSAdapter.hpp @@ -24,20 +24,17 @@ class JSAdapter { qvac_lib_inference_addon_cpp::js::Object jsObject, js_env_t* env) -> BCIConfig; - auto loadContextParams( + void loadContextParams( qvac_lib_inference_addon_cpp::js::Object contextParamsObj, js_env_t* env, - BCIConfig& config) - -> BCIConfig; + BCIConfig& config); - auto loadMiscParams( + void loadMiscParams( qvac_lib_inference_addon_cpp::js::Object miscParamsObj, js_env_t* env, - BCIConfig& config) - -> BCIConfig; + BCIConfig& config); - auto loadBCIParams( + void loadBCIParams( qvac_lib_inference_addon_cpp::js::Object bciParamsObj, js_env_t* env, - BCIConfig& config) - -> BCIConfig; + BCIConfig& config); private: void loadMap( diff --git a/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp b/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp index 1769c9b6a2..542abb8cfc 100644 --- a/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp +++ b/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp @@ -226,9 +226,6 @@ void BCIModel::process(const Input& rawNeuralData) { "Processing neural signal (" + std::to_string(rawNeuralData.size()) + " bytes)"); - // The BCI embedder ships with per-day projection matrices; day_idx=1 is the - // day the shipped test fixtures were recorded on. Callers should pass the - // real day_idx for their recording; this default keeps the POC honest. int dayIdx = 1; auto it = cfg_.bciConfig.find("day_idx"); if (it != cfg_.bciConfig.end()) { @@ -239,6 +236,17 @@ void BCIModel::process(const Input& rawNeuralData) { } } + if (neuralProcessor_.hasWeights()) { + const int maxDay = + static_cast(neuralProcessor_.getNumDays()) - 1; + if (maxDay >= 0 && (dayIdx < 0 || dayIdx > maxDay)) { + QLOG(qvac_lib_inference_addon_cpp::logger::Priority::WARNING, + "day_idx " + std::to_string(dayIdx) + + " is outside [0, " + std::to_string(maxDay) + + "]; it will be clamped"); + } + } + auto melFeatures = neuralProcessor_.processToMel(rawNeuralData, dayIdx); const int melBins = neuralProcessor_.getMelBins(); const int melFrames = neuralProcessor_.getMelFrames(); diff --git a/packages/bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.cpp b/packages/bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.cpp index d98c4174fc..c767b86020 100644 --- a/packages/bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.cpp +++ b/packages/bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.cpp @@ -20,6 +20,12 @@ constexpr uint32_t K_EMBEDDER_MAGIC = 0x42434945; // so the convolution loop touches fewer source timesteps. Matches the // BrainWhisperer Python reference. constexpr float K_KERNEL_TRIM_THRESHOLD = 0.01F; + +// Default Gaussian smoothing parameters matching the BrainWhisperer Python +// notebook. These are the σ and kernel width used for temporal smoothing of +// the raw neural signal before day-projection and mel padding. +constexpr float K_SMOOTH_KERNEL_STD = 2.0F; +constexpr int K_SMOOTH_KERNEL_SIZE = 100; } // namespace NeuralProcessor::NeuralProcessor() = default; @@ -215,8 +221,9 @@ std::vector NeuralProcessor::processToMel( return melOutput; } - // Step 1: Gaussian smoothing (std=2.0, kernel_size=100, matching BrainWhisperer) - auto smoothed = gaussianSmooth(features, numTimesteps, numChannels, 2.0F, 100); + auto smoothed = gaussianSmooth( + features, numTimesteps, numChannels, + K_SMOOTH_KERNEL_STD, K_SMOOTH_KERNEL_SIZE); // Step 2: Day projection (if available) std::vector projected; diff --git a/packages/bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.hpp b/packages/bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.hpp index 6909248ca4..10cc3d016c 100644 --- a/packages/bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.hpp +++ b/packages/bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.hpp @@ -52,6 +52,7 @@ class NeuralProcessor { int dayIdx) const; bool hasWeights() const { return weights_.loaded; } + uint32_t getNumDays() const { return weights_.numDays; } int getMelBins() const { return K_WHISPER_N_MEL; } int getMelFrames() const { return K_WHISPER_MEL_FRAMES; } diff --git a/packages/bci-whispercpp/index.d.ts b/packages/bci-whispercpp/index.d.ts index 71625a4751..e42f970303 100644 --- a/packages/bci-whispercpp/index.d.ts +++ b/packages/bci-whispercpp/index.d.ts @@ -12,11 +12,18 @@ declare interface WhisperConfig { n_threads?: number; temperature?: number; suppress_nst?: boolean; + suppress_blank?: boolean; duration_ms?: number; translate?: boolean; no_timestamps?: boolean; single_segment?: boolean; - [key: string]: unknown; + print_special?: boolean; + print_progress?: boolean; + print_realtime?: boolean; + print_timestamps?: boolean; + detect_language?: boolean; + greedy_best_of?: number; + beam_search_beam_size?: number; } declare interface BCIWhispercppFiles { diff --git a/packages/bci-whispercpp/index.js b/packages/bci-whispercpp/index.js index a587093b12..cb66a5d165 100644 --- a/packages/bci-whispercpp/index.js +++ b/packages/bci-whispercpp/index.js @@ -107,12 +107,21 @@ class BCIWhispercpp { checkConfig(configurationParams) const binding = require('./binding') - this.addon = new BCIInterface( - binding, - configurationParams, - this._outputCallback.bind(this), - this.logger.info.bind(this.logger) - ) + try { + this.addon = new BCIInterface( + binding, + configurationParams, + this._outputCallback.bind(this), + this.logger.info.bind(this.logger) + ) + } catch (err) { + this.addon = null + throw new QvacErrorAddonBCI({ + code: ERR_CODES.FAILED_TO_LOAD_WEIGHTS, + adds: err.message, + cause: err + }) + } await this.addon.activate() this.logger.info('BCI addon activated') @@ -193,28 +202,26 @@ class BCIWhispercpp { async unload () { return await this._withExclusiveRun(async () => { - if (this._job.active) { - this._job.fail(new Error('Model was unloaded')) - } - await this.cancel() if (this.addon) { await this.addon.destroyInstance() this.addon = null } + if (this._job.active) { + this._job.fail(new Error('Model was unloaded')) + } this.state.configLoaded = false }) } async destroy () { return await this._withExclusiveRun(async () => { - if (this._job.active) { - this._job.fail(new Error('Model was destroyed')) - } - await this.cancel() if (this.addon) { await this.addon.destroyInstance() this.addon = null } + if (this._job.active) { + this._job.fail(new Error('Model was destroyed')) + } this.state.configLoaded = false this.state.destroyed = true }) diff --git a/packages/bci-whispercpp/package.json b/packages/bci-whispercpp/package.json index 19425066b9..372368bcc5 100644 --- a/packages/bci-whispercpp/package.json +++ b/packages/bci-whispercpp/package.json @@ -13,7 +13,7 @@ "test:unit": "brittle-bare test/unit/**/*.test.js", "test:integration": "brittle-bare test/integration/addon.test.js", "test:cpp:build": "bare-make generate -D BUILD_TESTING=ON && bare-make build --target test-bci-core && bare-make install", - "test:cpp:run": "cd build/addon/tests/ && ./test-bci-core --gtest_output=xml:cpp-test-results.xml", + "test:cpp:run": "build/addon/tests/test-bci-core --gtest_output=xml:build/addon/tests/cpp-test-results.xml", "test:cpp": "npm run test:cpp:build && npm run test:cpp:run", "test": "npm run test:integration", "test:dts": "tsc index.d.ts --noEmit --lib es2018 --esModuleInterop --skipLibCheck" From 9867b31f5478ce8c64b7dfc17d3867b7f4a09bc6 Mon Sep 17 00:00:00 2001 From: Raju Date: Mon, 20 Apr 2026 20:04:17 +0530 Subject: [PATCH 09/19] chore(bci): point whisper-cpp overlay to merged master (2b1e04f) qvac-ext-lib-whisper.cpp PR #10 has been merged. Update the overlay to reference the merge commit on master instead of the feature branch commit, so the overlay remains valid if the branch is deleted. Bump port-version to 4. Made-with: Cursor --- .../bci-whispercpp/vcpkg-overlays/whisper-cpp/portfile.cmake | 4 ++-- packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/vcpkg.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/portfile.cmake b/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/portfile.cmake index b69c9567b4..a3307be8ca 100644 --- a/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/portfile.cmake +++ b/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/portfile.cmake @@ -1,10 +1,10 @@ -set(VERSION "3e91e3a4434c9cf3d7d0f27711f2988242bccf11") +set(VERSION "2b1e04f20bad9a72321e72df8d6a8c14aae98adc") vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-ext-lib-whisper.cpp REF ${VERSION} - SHA512 b94f29c95cca5e06d6a4ddcff8360b69bb1593e86ca8f74c6b3425f949713f3ef534166181e3972f4a68c80c9ad594821e6990a0a38a87f2de9dca51471c61c6 + SHA512 52c96ab252a4461430740decd4d883a8ef3f9d895d84df348407ec6fddac30116b6056c2e6577866322d2d8aaa729815753bb4c02da755b584ea1205ef2e6259 HEAD_REF master ) diff --git a/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/vcpkg.json b/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/vcpkg.json index 8cbca5b74b..11a4973a9e 100644 --- a/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/vcpkg.json +++ b/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/vcpkg.json @@ -1,7 +1,7 @@ { "name": "whisper-cpp", "version": "1.8.4", - "port-version": 3, + "port-version": 4, "description": "Port of OpenAI's Whisper model in C/C++ (BCI patched, based on tetherto/qvac-ext-lib-whisper.cpp PR #10)", "homepage": "https://github.com/tetherto/qvac-ext-lib-whisper.cpp", "license": "MIT", From aaa7505dd537abe143e876492b8b297182d94efa Mon Sep 17 00:00:00 2001 From: Raju Date: Tue, 21 Apr 2026 11:25:13 +0530 Subject: [PATCH 10/19] fix(bci): serialize inference lifetime and export low-level subpaths Address ogad-tether review feedback on PR #1583: 1. Inference queue: transcribe() now holds its slot until the response settles via _enqueueInference(), matching the pattern from TranscriptionWhispercpp._enqueueExclusiveRunResponse(). Previously the exclusiveRunQueue released the slot as soon as runJob() was accepted, allowing a second concurrent transcribe() to race in and either clobber the first response or get rejected by the native side. 2. Exports map: add ./bci, ./bci.js, and ./binding subpath exports so the low-level BCIInterface API documented in the README is accessible after publish. The exports map previously only exposed ./binding.js, blocking require('@qvac/bci-whispercpp/bci'). Made-with: Cursor --- packages/bci-whispercpp/index.js | 24 +++++++++++++++++++++++- packages/bci-whispercpp/package.json | 3 +++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/packages/bci-whispercpp/index.js b/packages/bci-whispercpp/index.js index cb66a5d165..c21b3d9b80 100644 --- a/packages/bci-whispercpp/index.js +++ b/packages/bci-whispercpp/index.js @@ -49,6 +49,7 @@ class BCIWhispercpp { this.opts = opts this.logger = new QvacLogger(logger) this._withExclusiveRun = exclusiveRunQueue() + this._inferenceQueueWaiter = Promise.resolve() this._job = createJobHandler({ cancel: () => this.addon?.cancel() }) @@ -146,7 +147,7 @@ class BCIWhispercpp { * @returns {Promise} */ async transcribe (neuralData) { - return await this._withExclusiveRun(async () => { + return await this._enqueueInference(async () => { const response = this._job.start() let accepted @@ -169,6 +170,27 @@ class BCIWhispercpp { }) } + /** + * Serialize inference runs so a second transcribe() waits until the first + * response settles. Separate from _withExclusiveRun (lifecycle ops) so + * destroy/unload can still preempt. + */ + async _enqueueInference (runFn) { + const prev = this._inferenceQueueWaiter + let releaseSlot + this._inferenceQueueWaiter = new Promise(resolve => { releaseSlot = resolve }) + await prev + let response + try { + response = await runFn() + } catch (err) { + releaseSlot() + throw err + } + response.await().finally(() => { releaseSlot() }).catch(() => {}) + return response + } + _outputCallback (addon, event, jobId, data, error) { if (event === 'Error') { this.logger.error('Job ' + jobId + ' failed with error: ' + error) diff --git a/packages/bci-whispercpp/package.json b/packages/bci-whispercpp/package.json index 372368bcc5..a5db15046b 100644 --- a/packages/bci-whispercpp/package.json +++ b/packages/bci-whispercpp/package.json @@ -72,6 +72,9 @@ "types": "./index.d.ts", "default": "./index.js" }, + "./bci": "./bci.js", + "./bci.js": "./bci.js", + "./binding": "./binding.js", "./binding.js": "./binding.js" }, "types": "index.d.ts" From a6711bacefa45900e7cb282c93d12d4c248b971e Mon Sep 17 00:00:00 2001 From: Raju Date: Tue, 21 Apr 2026 13:18:03 +0530 Subject: [PATCH 11/19] refactor[bc](bci): rename to qvac-lib-infer-bci-whispercpp and address review Align the BCI package with the inference-addon family conventions and resolve the review findings that accumulated across PR #1583. Breaking changes - Package directory renamed from packages/bci-whispercpp to packages/qvac-lib-infer-bci-whispercpp (npm name @qvac/bci-whispercpp unchanged). - Error codes moved from 7001-7013 (collided with @qvac/tts-onnx and the @qvac/transcription-parakeet fallback range) to the dedicated 26001-27000 range. Also adds FAILED_TO_START_JOB, INVALID_CONFIG, and EMBEDDER_WEIGHTS_INVALID for cases that were previously swallowed. Pattern / standard alignment with peer addons - Add addonLogging.js + addonLogging.d.ts + ./addonLogging subpath export. - Add CHANGELOG.md, PULL_REQUEST_TEMPLATE.md, tsconfig.dts.json. - Pin qvac-lib-inference-addon-cpp vcpkg dep to 1.1.5#1 (port-version). - vcpkg default-registry switched from git@github.com: to https:// (fixes anonymous clones and CI runners without an SSH deploy key). - Lint glob now covers lib/**/*.js. - bare engine bumped from >=1.19.0 to >=1.24.0 to match llamacpp-llm/embed. - VCPKG_OVERLAY_TRIPLETS set unconditionally and preserves external value. - Remove test:unit script that pointed at a non-existent dir; add build:pack, lint-cpp, test:dts scripts matching peer conventions. - package.json files array now includes README.md, CHANGELOG.md, and addonLogging artifacts; repository.directory + homepage point at the renamed path. PR review fixes (Gustavo, ogad-tether, github-code-quality bot) - day_idx default aligned: C++ runtime default is now 0 (matches the public JS/TS docs and NeuralProcessor header default). - BCIInterface.runJob rewrap now uses FAILED_TO_START_JOB instead of the misleading FAILED_TO_APPEND; input is validated (Uint8Array, non-empty). - day_idx: -1 passthrough mode is now explicitly documented in configChecker, README, and index.d.ts, and values < -1 are rejected at the JS boundary. - JS _load no longer sets suppress_nst/temperature defaults that fought the BCI-tuned C++ defaults in toWhisperFullParams. - Duplicate checkConfig call in BCIWhispercpp._load removed; validation now happens once inside the BCIInterface constructor. - whisper_log_set guarded by std::once_flag so it does not clobber any log handler a coexisting whisper-based addon installed in the same process. - Embedder weight loader now checks the stream state after every read and returns false on truncation instead of silently marking the weights as loaded and producing garbage at inference time. - NeuralProcessor day projection is now memoized per day_idx; same-day batch inference no longer rebuilds the O(nf^2 * r) dense matrix. - cancelRequested_.store(false) now runs before reset() in BCIModel::process(const std::any&) to avoid a window where a cancel() is dropped on the floor. - _addonOutputCallback now unpacks transcript arrays so response.await() yields flat segments (matches TranscriptionWhispercpp). - examples/transcribe-neural.js identical-branch ternary fixed. - README broken whisper.cpp link fixed; docs/BCI_V184_COMPAT.md stale overlay commit ref updated. - Integration test honours BCI_REQUIRE_MODEL=1 to turn missing-model into a loud failure for CI (default behaviour unchanged: local dev still skips). - index.d.ts now imports QvacResponse from @qvac/infer-base/src/QvacResponse and LoggerInterface from @qvac/logging instead of hand-rolling them. Tests - Clean rebuild from scratch (rm -rf build prebuilds && bare-make generate/build/install) succeeds. - npm run lint: clean (now covers lib/**). - npm run test:dts: clean. - npm run test:integration: 3/3 pass, 10/10 asserts, 6.0% average WER (matches baseline). - npm run test:cpp: 18/18 pass (was 7; +11 new tests covering unknown-key rejection, numeric double-to-int coercion, range validation, ContextGpuDevice bounds, passthrough mode, invalid embedder handling). - bare examples/transcribe-neural.js --batch: 5/5 samples, 6.0% avg WER. - bare examples/transcribe-neural.js test/fixtures/neural_sample_0.bin: output unchanged ("You can see the good at this point as well."). Made-with: Cursor --- packages/bci-whispercpp/index.d.ts | 131 ----------------- .../.gitignore | 0 .../CHANGELOG.md | 41 ++++++ .../CMakeLists.txt | 8 +- .../LICENSE | 0 .../NOTICE | 0 .../PULL_REQUEST_TEMPLATE.md | 42 ++++++ .../README.md | 4 +- .../addon/src/addon/AddonJs.hpp | 10 +- .../addon/src/addon/BCIErrors.hpp | 0 .../addon/src/js-interface/JSAdapter.cpp | 0 .../addon/src/js-interface/JSAdapter.hpp | 0 .../addon/src/js-interface/binding.cpp | 0 .../addon/src/model-interface/BCITypes.hpp | 0 .../src/model-interface/bci/BCIConfig.cpp | 0 .../src/model-interface/bci/BCIConfig.hpp | 0 .../src/model-interface/bci/BCIModel.cpp | 11 +- .../src/model-interface/bci/BCIModel.hpp | 0 .../model-interface/bci/NeuralProcessor.cpp | 101 +++++++++---- .../model-interface/bci/NeuralProcessor.hpp | 7 + .../addon/tests/test_core.cpp | 92 ++++++++++++ .../addonLogging.d.ts | 7 + .../addonLogging.js | 6 + .../bci.js | 36 ++++- .../binding.js | 0 .../configChecker.js | 17 +++ .../docs/BCI_V184_COMPAT.md | 2 +- .../examples/transcribe-neural.js | 4 +- .../qvac-lib-infer-bci-whispercpp/index.d.ts | 135 ++++++++++++++++++ .../index.js | 12 +- .../lib/error.js | 47 ++++-- .../lib/wer.js | 0 .../package.json | 33 +++-- .../scripts/convert-model.py | 0 .../scripts/download-models.sh | 0 .../test/fixtures/manifest.json | 0 .../test/integration/addon.test.js | 13 ++ .../test/integration/helpers.js | 0 .../tsconfig.dts.json | 16 +++ .../vcpkg-configuration.json | 2 +- .../qvac-lint-cpp/portfile.cmake | 0 .../vcpkg-overlays/qvac-lint-cpp/vcpkg.json | 0 .../vcpkg-overlays/whisper-cpp/portfile.cmake | 0 .../vcpkg-overlays/whisper-cpp/vcpkg.json | 0 .../vcpkg.json | 2 +- .../vcpkg/toolchains/linux-clang.cmake | 0 .../vcpkg/triplets/arm64-linux.cmake | 0 .../vcpkg/triplets/x64-linux.cmake | 0 48 files changed, 577 insertions(+), 202 deletions(-) delete mode 100644 packages/bci-whispercpp/index.d.ts rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/.gitignore (100%) create mode 100644 packages/qvac-lib-infer-bci-whispercpp/CHANGELOG.md rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/CMakeLists.txt (90%) rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/LICENSE (100%) rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/NOTICE (100%) create mode 100644 packages/qvac-lib-infer-bci-whispercpp/PULL_REQUEST_TEMPLATE.md rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/README.md (97%) rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/addon/src/addon/AddonJs.hpp (93%) rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/addon/src/addon/BCIErrors.hpp (100%) rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/addon/src/js-interface/JSAdapter.cpp (100%) rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/addon/src/js-interface/JSAdapter.hpp (100%) rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/addon/src/js-interface/binding.cpp (100%) rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/addon/src/model-interface/BCITypes.hpp (100%) rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/addon/src/model-interface/bci/BCIConfig.cpp (100%) rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/addon/src/model-interface/bci/BCIConfig.hpp (100%) rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/addon/src/model-interface/bci/BCIModel.cpp (95%) rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/addon/src/model-interface/bci/BCIModel.hpp (100%) rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/addon/src/model-interface/bci/NeuralProcessor.cpp (75%) rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/addon/src/model-interface/bci/NeuralProcessor.hpp (84%) rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/addon/tests/test_core.cpp (50%) create mode 100644 packages/qvac-lib-infer-bci-whispercpp/addonLogging.d.ts create mode 100644 packages/qvac-lib-infer-bci-whispercpp/addonLogging.js rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/bci.js (87%) rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/binding.js (100%) rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/configChecker.js (71%) rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/docs/BCI_V184_COMPAT.md (96%) rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/examples/transcribe-neural.js (96%) create mode 100644 packages/qvac-lib-infer-bci-whispercpp/index.d.ts rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/index.js (93%) rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/lib/error.js (64%) rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/lib/wer.js (100%) rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/package.json (68%) rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/scripts/convert-model.py (100%) rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/scripts/download-models.sh (100%) rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/test/fixtures/manifest.json (100%) rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/test/integration/addon.test.js (89%) rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/test/integration/helpers.js (100%) create mode 100644 packages/qvac-lib-infer-bci-whispercpp/tsconfig.dts.json rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/vcpkg-configuration.json (82%) rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/vcpkg-overlays/qvac-lint-cpp/portfile.cmake (100%) rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/vcpkg-overlays/qvac-lint-cpp/vcpkg.json (100%) rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/vcpkg-overlays/whisper-cpp/portfile.cmake (100%) rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/vcpkg-overlays/whisper-cpp/vcpkg.json (100%) rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/vcpkg.json (92%) rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/vcpkg/toolchains/linux-clang.cmake (100%) rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/vcpkg/triplets/arm64-linux.cmake (100%) rename packages/{bci-whispercpp => qvac-lib-infer-bci-whispercpp}/vcpkg/triplets/x64-linux.cmake (100%) diff --git a/packages/bci-whispercpp/index.d.ts b/packages/bci-whispercpp/index.d.ts deleted file mode 100644 index e42f970303..0000000000 --- a/packages/bci-whispercpp/index.d.ts +++ /dev/null @@ -1,131 +0,0 @@ -declare interface BCIConfig { - /** - * Index into the day-specific projection matrices in bci-embedder.bin. - * Must match the recording day the neural signal was captured on. - * Defaults to 0. - */ - day_idx?: number; -} - -declare interface WhisperConfig { - language?: string; - n_threads?: number; - temperature?: number; - suppress_nst?: boolean; - suppress_blank?: boolean; - duration_ms?: number; - translate?: boolean; - no_timestamps?: boolean; - single_segment?: boolean; - print_special?: boolean; - print_progress?: boolean; - print_realtime?: boolean; - print_timestamps?: boolean; - detect_language?: boolean; - greedy_best_of?: number; - beam_search_beam_size?: number; -} - -declare interface BCIWhispercppFiles { - model: string; -} - -declare interface BCIWhispercppArgs { - files: BCIWhispercppFiles; - logger?: { - debug(...args: unknown[]): void; - info(...args: unknown[]): void; - warn(...args: unknown[]): void; - error(...args: unknown[]): void; - }; - opts?: { - stats?: boolean; - }; -} - -declare interface BCIWhispercppConfig { - whisperConfig?: WhisperConfig; - bciConfig?: BCIConfig; - contextParams?: { - model?: string; - use_gpu?: boolean; - flash_attn?: boolean; - gpu_device?: number; - }; - miscConfig?: { - caption_enabled?: boolean; - }; -} - -declare interface TranscriptSegment { - text: string; - toAppend: boolean; - start: number; - end: number; - id: number; -} - -declare interface QvacResponse { - output: unknown[]; - stats: Record; - onUpdate(callback: (data: unknown) => void): QvacResponse; - onFinish(callback: (result: unknown) => void): QvacResponse; - onError(callback: (error: Error) => void): QvacResponse; - onCancel(callback: () => void): QvacResponse; - await(): Promise; - cancel(): Promise; - iterate(): AsyncGenerator; - getLatest(): unknown; -} - -/** - * BCI neural signal transcription client powered by whisper.cpp. - * Uses createJobHandler + exclusiveRunQueue from @qvac/infer-base. - */ -declare class BCIWhispercpp { - constructor(args: BCIWhispercppArgs, config?: BCIWhispercppConfig); - - /** Load and activate the model. */ - load(): Promise; - - /** Transcribe a neural signal binary file (convenience wrapper). */ - transcribeFile(filePath: string): Promise; - - /** Transcribe neural signal data (batch). Returns QvacResponse. */ - transcribe(neuralData: Uint8Array): Promise; - - /** Cancel current inference. */ - cancel(): Promise; - - /** Unload the model and release native resources. */ - unload(): Promise; - - /** Destroy the instance, unload, and mark as permanently destroyed. */ - destroy(): Promise; - - /** Get current state (configLoaded, destroyed). */ - getState(): { configLoaded: boolean; destroyed: boolean }; -} - -/** - * Compute Word Error Rate between hypothesis and reference strings. - * @returns WER as a ratio (0.0 = perfect). - */ -declare function computeWER(hypothesis: string, reference: string): number; - -declare namespace BCIWhispercpp { - export { - BCIWhispercpp as default, - BCIWhispercpp, - BCIConfig, - WhisperConfig, - BCIWhispercppFiles, - BCIWhispercppArgs, - BCIWhispercppConfig, - TranscriptSegment, - QvacResponse, - computeWER, - }; -} - -export = BCIWhispercpp; diff --git a/packages/bci-whispercpp/.gitignore b/packages/qvac-lib-infer-bci-whispercpp/.gitignore similarity index 100% rename from packages/bci-whispercpp/.gitignore rename to packages/qvac-lib-infer-bci-whispercpp/.gitignore diff --git a/packages/qvac-lib-infer-bci-whispercpp/CHANGELOG.md b/packages/qvac-lib-infer-bci-whispercpp/CHANGELOG.md new file mode 100644 index 0000000000..761c9dcf78 --- /dev/null +++ b/packages/qvac-lib-infer-bci-whispercpp/CHANGELOG.md @@ -0,0 +1,41 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.1.0] + +Initial POC release of `@qvac/bci-whispercpp`, a brain-computer-interface neural +signal transcription addon powered by a BCI-patched fork of whisper.cpp. + +### Added + +- `BCIWhispercpp` client class (standalone, built on `createJobHandler` + + `exclusiveRunQueue` from `@qvac/infer-base`) with `load()`, `transcribe()`, + `transcribeFile()`, `unload()`, `destroy()`, `cancel()`, `getState()`. +- Low-level `BCIInterface` (`./bci` subpath export) for users that need direct + control over the native addon lifecycle. +- `./addonLogging` subpath exposing `setLogger` / `releaseLogger` for wiring a + native log handler. +- C++ native addon (`NeuralProcessor`, `BCIModel`, `BCIConfig`) using the + `qvac-lib-inference-addon-cpp` framework, with BCI-specific preprocessing + (Gaussian smoothing, low-rank day projection, softsign non-linearity) and + mel-layout injection into a patched whisper.cpp encoder. +- Integration tests for load/destroy, batch transcription, and a 5-sample + WER measurement (avg 6.0% on the reference fixtures). +- GoogleTest C++ unit tests covering mel shape, gaussian smoothing, padded + frames, truncation handling, invalid-config rejection, and range validation. +- `scripts/convert-model.py` to convert a BrainWhisperer checkpoint into the + GGML model + embedder binary pair consumed at runtime. +- `scripts/download-models.sh` to fetch the reference model and test fixtures + from the `bci-test-assets-v0.1.0` GitHub release. + +### Known Limitations + +- Streaming transcription is not implemented in this release; see follow-up + work tracked under QVAC-17062. +- Inference error codes live in the `26001-27000` range; migrations that + pinned the older `7xxx` range used during initial development should update + to the new constants exported from `@qvac/bci-whispercpp/lib/error`. diff --git a/packages/bci-whispercpp/CMakeLists.txt b/packages/qvac-lib-infer-bci-whispercpp/CMakeLists.txt similarity index 90% rename from packages/bci-whispercpp/CMakeLists.txt rename to packages/qvac-lib-infer-bci-whispercpp/CMakeLists.txt index aaa01eb75b..773473a14f 100644 --- a/packages/bci-whispercpp/CMakeLists.txt +++ b/packages/qvac-lib-infer-bci-whispercpp/CMakeLists.txt @@ -11,9 +11,11 @@ find_package(cmake-vcpkg REQUIRED PATHS node_modules/cmake-vcpkg) set(VCPKG_OVERLAY_PORTS "${CMAKE_CURRENT_SOURCE_DIR}/vcpkg-overlays;${VCPKG_OVERLAY_PORTS}") -if(CMAKE_SYSTEM_NAME STREQUAL "Linux") - set(VCPKG_OVERLAY_TRIPLETS "${CMAKE_CURRENT_SOURCE_DIR}/vcpkg/triplets") -endif() +# Prepend the local overlay triplets on every platform and preserve any +# externally-set value (matches the other qvac addons). Only the Linux +# triplets actually differ from vcpkg's defaults today, but exposing the +# directory uniformly avoids platform-conditional surprises. +set(VCPKG_OVERLAY_TRIPLETS "${CMAKE_CURRENT_SOURCE_DIR}/vcpkg/triplets;${VCPKG_OVERLAY_TRIPLETS}") project(bci-whispercpp CXX C) diff --git a/packages/bci-whispercpp/LICENSE b/packages/qvac-lib-infer-bci-whispercpp/LICENSE similarity index 100% rename from packages/bci-whispercpp/LICENSE rename to packages/qvac-lib-infer-bci-whispercpp/LICENSE diff --git a/packages/bci-whispercpp/NOTICE b/packages/qvac-lib-infer-bci-whispercpp/NOTICE similarity index 100% rename from packages/bci-whispercpp/NOTICE rename to packages/qvac-lib-infer-bci-whispercpp/NOTICE diff --git a/packages/qvac-lib-infer-bci-whispercpp/PULL_REQUEST_TEMPLATE.md b/packages/qvac-lib-infer-bci-whispercpp/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000000..b9b618fd02 --- /dev/null +++ b/packages/qvac-lib-infer-bci-whispercpp/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,42 @@ +**Note**: be concise and prefer bullet points. + +## 🎯 What problem does this PR solve? + +- +- + +## 📝 How does it solve it? + +- +- + +## 🧪 How was it tested? + +**Delete this section if not applicable.** + +- +- + +## 💥 Breaking Changes + +**Delete this section if not applicable.** + +**BEFORE:** + +```typescript +// old code example +``` + +**AFTER:** + +```typescript +// new code example +``` + +## 🔌 API Changes + +**Delete this section if not applicable.** + +```typescript +// new API usage example +``` diff --git a/packages/bci-whispercpp/README.md b/packages/qvac-lib-infer-bci-whispercpp/README.md similarity index 97% rename from packages/bci-whispercpp/README.md rename to packages/qvac-lib-infer-bci-whispercpp/README.md index 36b3b2565b..f02759ad4d 100644 --- a/packages/bci-whispercpp/README.md +++ b/packages/qvac-lib-infer-bci-whispercpp/README.md @@ -1,6 +1,6 @@ # @qvac/bci-whispercpp -Brain-Computer Interface (BCI) neural signal transcription addon for qvac, powered by [whisper.cpp](https://github.com/tetherto/whisper.cpp). +Brain-Computer Interface (BCI) neural signal transcription addon for qvac, powered by the [tetherto/qvac-ext-lib-whisper.cpp](https://github.com/tetherto/qvac-ext-lib-whisper.cpp) fork of whisper.cpp. Transcribes multi-channel neural signals (e.g., 512-channel microelectrode array recordings) into text using a BCI-trained whisper model running natively via GGML. Output matches the Python BrainWhisperer reference model exactly. @@ -59,7 +59,7 @@ Each timestep represents a 20ms bin of neural activity. Channels correspond to i ## Installation ```bash -cd packages/bci-whispercpp +cd packages/qvac-lib-infer-bci-whispercpp npm install VCPKG_ROOT=/path/to/vcpkg npm run build ``` diff --git a/packages/bci-whispercpp/addon/src/addon/AddonJs.hpp b/packages/qvac-lib-infer-bci-whispercpp/addon/src/addon/AddonJs.hpp similarity index 93% rename from packages/bci-whispercpp/addon/src/addon/AddonJs.hpp rename to packages/qvac-lib-infer-bci-whispercpp/addon/src/addon/AddonJs.hpp index f5d8f7c40d..18b20a5c97 100644 --- a/packages/bci-whispercpp/addon/src/addon/AddonJs.hpp +++ b/packages/qvac-lib-infer-bci-whispercpp/addon/src/addon/AddonJs.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -94,7 +95,14 @@ inline js_value_t* createInstance(js_env_t* env, js_callback_info_t* info) try { using namespace qvac_lib_inference_addon_cpp; using namespace std; - whisper_log_set(disableWhisperLogs, nullptr); + // whisper_log_set is a process-wide global. Only install our silencing + // handler once; otherwise every addon-instance construction clobbers any + // handler a coexisting whisper-based addon (e.g. @qvac/transcription- + // whispercpp) may have installed in the same process. + static std::once_flag logOnce; + std::call_once(logOnce, []() { + whisper_log_set(disableWhisperLogs, nullptr); + }); JsArgsParser args(env, info); auto configurationParams = args.getJsObject(1, "configurationParams"); diff --git a/packages/bci-whispercpp/addon/src/addon/BCIErrors.hpp b/packages/qvac-lib-infer-bci-whispercpp/addon/src/addon/BCIErrors.hpp similarity index 100% rename from packages/bci-whispercpp/addon/src/addon/BCIErrors.hpp rename to packages/qvac-lib-infer-bci-whispercpp/addon/src/addon/BCIErrors.hpp diff --git a/packages/bci-whispercpp/addon/src/js-interface/JSAdapter.cpp b/packages/qvac-lib-infer-bci-whispercpp/addon/src/js-interface/JSAdapter.cpp similarity index 100% rename from packages/bci-whispercpp/addon/src/js-interface/JSAdapter.cpp rename to packages/qvac-lib-infer-bci-whispercpp/addon/src/js-interface/JSAdapter.cpp diff --git a/packages/bci-whispercpp/addon/src/js-interface/JSAdapter.hpp b/packages/qvac-lib-infer-bci-whispercpp/addon/src/js-interface/JSAdapter.hpp similarity index 100% rename from packages/bci-whispercpp/addon/src/js-interface/JSAdapter.hpp rename to packages/qvac-lib-infer-bci-whispercpp/addon/src/js-interface/JSAdapter.hpp diff --git a/packages/bci-whispercpp/addon/src/js-interface/binding.cpp b/packages/qvac-lib-infer-bci-whispercpp/addon/src/js-interface/binding.cpp similarity index 100% rename from packages/bci-whispercpp/addon/src/js-interface/binding.cpp rename to packages/qvac-lib-infer-bci-whispercpp/addon/src/js-interface/binding.cpp diff --git a/packages/bci-whispercpp/addon/src/model-interface/BCITypes.hpp b/packages/qvac-lib-infer-bci-whispercpp/addon/src/model-interface/BCITypes.hpp similarity index 100% rename from packages/bci-whispercpp/addon/src/model-interface/BCITypes.hpp rename to packages/qvac-lib-infer-bci-whispercpp/addon/src/model-interface/BCITypes.hpp diff --git a/packages/bci-whispercpp/addon/src/model-interface/bci/BCIConfig.cpp b/packages/qvac-lib-infer-bci-whispercpp/addon/src/model-interface/bci/BCIConfig.cpp similarity index 100% rename from packages/bci-whispercpp/addon/src/model-interface/bci/BCIConfig.cpp rename to packages/qvac-lib-infer-bci-whispercpp/addon/src/model-interface/bci/BCIConfig.cpp diff --git a/packages/bci-whispercpp/addon/src/model-interface/bci/BCIConfig.hpp b/packages/qvac-lib-infer-bci-whispercpp/addon/src/model-interface/bci/BCIConfig.hpp similarity index 100% rename from packages/bci-whispercpp/addon/src/model-interface/bci/BCIConfig.hpp rename to packages/qvac-lib-infer-bci-whispercpp/addon/src/model-interface/bci/BCIConfig.hpp diff --git a/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp b/packages/qvac-lib-infer-bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp similarity index 95% rename from packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp rename to packages/qvac-lib-infer-bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp index 542abb8cfc..1a4e9bc78a 100644 --- a/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp +++ b/packages/qvac-lib-infer-bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp @@ -226,7 +226,10 @@ void BCIModel::process(const Input& rawNeuralData) { "Processing neural signal (" + std::to_string(rawNeuralData.size()) + " bytes)"); - int dayIdx = 1; + // Default day_idx = 0 matches NeuralProcessor::processToMel and the public + // JS/TS docs. The reference fixtures in test/fixtures/manifest.json pass + // day_idx=1 explicitly; callers that omit bciConfig get day 0. + int dayIdx = 0; auto it = cfg_.bciConfig.find("day_idx"); if (it != cfg_.bciConfig.end()) { if (auto* d = std::get_if(&it->second)) { @@ -322,8 +325,12 @@ std::any BCIModel::process(const std::any& input) { on_segment_ = modelInput.outputCallback; } - reset(); + // Clear the cancel flag FIRST so a cancel() call that races with reset() + // is not silently lost. process(Input&) still checks cancelRequested_ at + // the top, so a cancel that arrives between these two statements aborts + // the upcoming whisper_full call via shouldAbortWhisper. cancelRequested_.store(false, std::memory_order_relaxed); + reset(); try { process(modelInput.input); } catch (...) { diff --git a/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.hpp b/packages/qvac-lib-infer-bci-whispercpp/addon/src/model-interface/bci/BCIModel.hpp similarity index 100% rename from packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.hpp rename to packages/qvac-lib-infer-bci-whispercpp/addon/src/model-interface/bci/BCIModel.hpp diff --git a/packages/bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.cpp b/packages/qvac-lib-infer-bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.cpp similarity index 75% rename from packages/bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.cpp rename to packages/qvac-lib-infer-bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.cpp index c767b86020..8c95cef77a 100644 --- a/packages/bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.cpp +++ b/packages/qvac-lib-infer-bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.cpp @@ -34,25 +34,40 @@ bool NeuralProcessor::loadEmbedderWeights(const std::string& path) { std::ifstream f(path, std::ios::binary); if (!f.is_open()) return false; + // A truncated/corrupt embedder file would otherwise silently load as + // zeros and produce garbage output at inference time. Check f.good() + // after every read and bail out cleanly so the caller reports the file + // as missing / invalid instead of the model emitting nonsense. + bool readFailed = false; + auto readU32 = [&]() -> uint32_t { uint32_t v = 0; f.read(reinterpret_cast(&v), sizeof(v)); + if (!f) readFailed = true; return v; }; auto readFloats = [&](size_t count) -> std::vector { std::vector data(count); - f.read(reinterpret_cast(data.data()), - static_cast(count * sizeof(float))); + if (count > 0) { + f.read(reinterpret_cast(data.data()), + static_cast(count * sizeof(float))); + if (!f) readFailed = true; + } return data; }; auto readInts = [&](size_t count) -> std::vector { std::vector data(count); - f.read(reinterpret_cast(data.data()), - static_cast(count * sizeof(int32_t))); + if (count > 0) { + f.read(reinterpret_cast(data.data()), + static_cast(count * sizeof(int32_t))); + if (!f) readFailed = true; + } return data; }; - if (readU32() != K_EMBEDDER_MAGIC || readU32() != 1) return false; + if (readU32() != K_EMBEDDER_MAGIC || readU32() != 1 || readFailed) { + return false; + } weights_.numFeatures = readU32(); /*embedDim=*/ readU32(); @@ -62,15 +77,18 @@ bool NeuralProcessor::loadEmbedderWeights(const std::string& path) { weights_.numDays = readU32(); weights_.numMonths = readU32(); weights_.r = readU32(); + if (readFailed) return false; // Skip conv1/conv2 weights (handled by GGML model) uint32_t n = readU32(); readFloats(n); n = readU32(); readFloats(n); n = readU32(); readFloats(n); n = readU32(); readFloats(n); + if (readFailed) return false; n = readU32(); weights_.sessionToDayMap = readInts(n); + if (readFailed) return false; weights_.dayAs.resize(weights_.numDays); weights_.dayBs.resize(weights_.numDays); @@ -79,6 +97,7 @@ bool NeuralProcessor::loadEmbedderWeights(const std::string& path) { n = readU32(); weights_.dayAs[i] = readFloats(n); n = readU32(); weights_.dayBs[i] = readFloats(n); n = readU32(); weights_.dayBiases[i] = readFloats(n); + if (readFailed) return false; } weights_.monthWeights.resize(weights_.numMonths); @@ -86,6 +105,7 @@ bool NeuralProcessor::loadEmbedderWeights(const std::string& path) { for (uint32_t i = 0; i < weights_.numMonths; ++i) { n = readU32(); weights_.monthWeights[i] = readFloats(n); n = readU32(); weights_.monthBiases[i] = readFloats(n); + if (readFailed) return false; } weights_.loaded = true; @@ -142,44 +162,65 @@ std::vector NeuralProcessor::applyDayProjection( const uint32_t r = weights_.r; int di = std::clamp(dayIdx, 0, static_cast(weights_.numDays) - 1); - const auto& dayA = weights_.dayAs[di]; - const auto& dayB = weights_.dayBs[di]; - const auto& dayBias = weights_.dayBiases[di]; - - std::vector dayDelta(nf * nf, 0.0F); - for (uint32_t i = 0; i < nf; ++i) - for (uint32_t j = 0; j < nf; ++j) { - float s = 0.0F; - for (uint32_t k = 0; k < r; ++k) - s += dayA[i * r + k] * dayB[k * nf + j]; - dayDelta[i * nf + j] = s; + // Rebuild the dense projection only when the resolved day index changes. + // Materializing dayDelta + W costs O(nf*nf*r) + O(nf*nf); for nf=512,r=8 + // that is ~2M + 0.25M multiplies per recompute. + if (di != cachedDayIdx_ || + cachedProjectionW_.size() != static_cast(nf) * nf || + cachedProjectionBias_.size() != nf) { + const auto& dayA = weights_.dayAs[di]; + const auto& dayB = weights_.dayBs[di]; + const auto& dayBias = weights_.dayBiases[di]; + + cachedProjectionW_.assign(static_cast(nf) * nf, 0.0F); + cachedProjectionBias_.assign(nf, 0.0F); + + for (uint32_t i = 0; i < nf; ++i) { + for (uint32_t j = 0; j < nf; ++j) { + float s = 0.0F; + for (uint32_t k = 0; k < r; ++k) { + s += dayA[i * r + k] * dayB[k * nf + j]; + } + cachedProjectionW_[i * nf + j] = s; + } + } + + int monthIdx = di / 30; + bool hasMonth = + (monthIdx < static_cast(weights_.monthWeights.size()) && + !weights_.monthWeights[monthIdx].empty()); + if (hasMonth) { + const auto& mw = weights_.monthWeights[monthIdx]; + for (uint32_t i = 0; i < nf * nf; ++i) { + cachedProjectionW_[i] += mw[i]; + } } - int monthIdx = di / 30; - bool hasMonth = (monthIdx < static_cast(weights_.monthWeights.size()) && - !weights_.monthWeights[monthIdx].empty()); + for (uint32_t i = 0; i < nf; ++i) { + cachedProjectionBias_[i] = dayBias[i]; + if (hasMonth && i < weights_.monthBiases[monthIdx].size()) { + cachedProjectionBias_[i] += weights_.monthBiases[monthIdx][i]; + } + } - std::vector W(nf * nf), bias(nf, 0.0F); - for (uint32_t i = 0; i < nf * nf; ++i) { - W[i] = dayDelta[i]; - if (hasMonth) W[i] += weights_.monthWeights[monthIdx][i]; - } - for (uint32_t i = 0; i < nf; ++i) { - bias[i] = dayBias[i]; - if (hasMonth && i < weights_.monthBiases[monthIdx].size()) - bias[i] += weights_.monthBiases[monthIdx][i]; + cachedDayIdx_ = di; } + const auto& W = cachedProjectionW_; + const auto& bias = cachedProjectionBias_; + // Python: output[t,k] = softsign(sum_d(features[t,d] * W[d,k]) + bias[k]) // i.e. output = features @ W + bias (right-multiply by W) std::vector output(numTimesteps * nf); - for (uint32_t t = 0; t < numTimesteps; ++t) + for (uint32_t t = 0; t < numTimesteps; ++t) { for (uint32_t k = 0; k < nf; ++k) { float s = bias[k]; - for (uint32_t d = 0; d < nf; ++d) + for (uint32_t d = 0; d < nf; ++d) { s += features[t * numChannels + d] * W[d * nf + k]; + } output[t * nf + k] = s / (1.0F + std::abs(s)); } + } return output; } diff --git a/packages/bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.hpp b/packages/qvac-lib-infer-bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.hpp similarity index 84% rename from packages/bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.hpp rename to packages/qvac-lib-infer-bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.hpp index 10cc3d016c..e43d5716fc 100644 --- a/packages/bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.hpp +++ b/packages/qvac-lib-infer-bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.hpp @@ -58,6 +58,13 @@ class NeuralProcessor { private: EmbedderWeights weights_; + + // Memoized dense projection (W, bias) per resolved day index. The + // underlying low-rank dayA · dayB + month correction is O(nf*nf*r) to + // materialize; caching makes same-day batch inference much cheaper. + mutable int cachedDayIdx_ = -1; + mutable std::vector cachedProjectionW_; + mutable std::vector cachedProjectionBias_; }; } // namespace qvac_lib_inference_addon_bci diff --git a/packages/bci-whispercpp/addon/tests/test_core.cpp b/packages/qvac-lib-infer-bci-whispercpp/addon/tests/test_core.cpp similarity index 50% rename from packages/bci-whispercpp/addon/tests/test_core.cpp rename to packages/qvac-lib-infer-bci-whispercpp/addon/tests/test_core.cpp index 5e2e677111..40466e5a9e 100644 --- a/packages/bci-whispercpp/addon/tests/test_core.cpp +++ b/packages/qvac-lib-infer-bci-whispercpp/addon/tests/test_core.cpp @@ -101,3 +101,95 @@ TEST(BCIConfig, DefaultWhisperFullParamsAreValid) { auto params = toWhisperFullParams(config); EXPECT_STREQ(params.language, "en"); } + +TEST(BCIConfig, UnknownWhisperKeyIsRejected) { + BCIConfig config; + config.whisperMainCfg["not_a_real_key"] = true; + EXPECT_THROW(toWhisperFullParams(config), std::exception); +} + +TEST(BCIConfig, UnknownContextKeyIsRejected) { + BCIConfig config; + config.whisperContextCfg["nope"] = std::string("value"); + EXPECT_THROW(toWhisperContextParams(config), std::exception); +} + +TEST(BCIConfig, NumericDoubleCoercedToInt) { + BCIConfig config; + config.whisperMainCfg["n_threads"] = 4.0; + config.whisperMainCfg["duration_ms"] = 100.0; + auto params = toWhisperFullParams(config); + EXPECT_EQ(params.n_threads, 4); + EXPECT_EQ(params.duration_ms, 100); +} + +TEST(BCIConfig, NegativeNThreadsRejected) { + BCIConfig config; + config.whisperMainCfg["n_threads"] = -1.0; + EXPECT_THROW(toWhisperFullParams(config), std::exception); +} + +TEST(BCIConfig, NegativeDurationMsRejected) { + BCIConfig config; + config.whisperMainCfg["duration_ms"] = -5.0; + EXPECT_THROW(toWhisperFullParams(config), std::exception); +} + +TEST(BCIConfig, TemperatureOutOfRangeRejected) { + BCIConfig config; + config.whisperMainCfg["temperature"] = 3.5; + EXPECT_THROW(toWhisperFullParams(config), std::exception); +} + +TEST(BCIConfig, BeamSizeOutOfRangeRejected) { + BCIConfig config; + config.whisperMainCfg["beam_search_beam_size"] = 0.0; + EXPECT_THROW(toWhisperFullParams(config), std::exception); + BCIConfig big; + big.whisperMainCfg["beam_search_beam_size"] = 100.0; + EXPECT_THROW(toWhisperFullParams(big), std::exception); +} + +TEST(BCIConfig, ContextGpuDeviceMustBeNonNegative) { + BCIConfig config; + config.whisperContextCfg["gpu_device"] = -1.0; + EXPECT_THROW(toWhisperContextParams(config), std::exception); +} + +TEST(BCIConfig, ContextBooleanHandlersWireThrough) { + BCIConfig config; + config.whisperContextCfg["use_gpu"] = true; + config.whisperContextCfg["flash_attn"] = false; + auto params = toWhisperContextParams(config); + EXPECT_TRUE(params.use_gpu); + EXPECT_FALSE(params.flash_attn); +} + +TEST(NeuralProcessor, LoadInvalidEmbedderReturnsFalse) { + NeuralProcessor processor; + EXPECT_FALSE(processor.loadEmbedderWeights("/nonexistent/path/embedder.bin")); + EXPECT_FALSE(processor.hasWeights()); +} + +TEST(NeuralProcessor, PassthroughModeSkipsPreprocessing) { + NeuralProcessor processor; + // Build a small "pre-computed mel" buffer and ensure passthrough + // reshapes it into mel-major layout without throwing or zero-padding + // the live frames. + const uint32_t T = 32; + const uint32_t C = 64; + auto signal = createTestSignal(T, C); + + auto result = processor.processToMel(signal, /*dayIdx=*/-1); + EXPECT_EQ(result.size(), + static_cast(NeuralProcessor::K_WHISPER_MEL_FRAMES) * + NeuralProcessor::K_WHISPER_N_MEL); + + // First frame, first bin should match the test signal's (t=0, c=0) value + // after the mel-major transpose: data[bin * n_frames + frame]. + const int nFrames = NeuralProcessor::K_WHISPER_MEL_FRAMES; + const float* originalData = reinterpret_cast( + signal.data() + 2 * sizeof(uint32_t)); + EXPECT_FLOAT_EQ(result[0 * nFrames + 0], originalData[0 * C + 0]); + EXPECT_FLOAT_EQ(result[1 * nFrames + 0], originalData[0 * C + 1]); +} diff --git a/packages/qvac-lib-infer-bci-whispercpp/addonLogging.d.ts b/packages/qvac-lib-infer-bci-whispercpp/addonLogging.d.ts new file mode 100644 index 0000000000..bd687d60bc --- /dev/null +++ b/packages/qvac-lib-infer-bci-whispercpp/addonLogging.d.ts @@ -0,0 +1,7 @@ +export interface AddonLogging { + setLogger(callback: (priority: number, message: string) => void): void + releaseLogger(): void +} + +declare const addonLogging: AddonLogging +export default addonLogging diff --git a/packages/qvac-lib-infer-bci-whispercpp/addonLogging.js b/packages/qvac-lib-infer-bci-whispercpp/addonLogging.js new file mode 100644 index 0000000000..479ecdf3da --- /dev/null +++ b/packages/qvac-lib-infer-bci-whispercpp/addonLogging.js @@ -0,0 +1,6 @@ +const binding = require('./binding') + +module.exports = { + setLogger: binding.setLogger, + releaseLogger: binding.releaseLogger +} diff --git a/packages/bci-whispercpp/bci.js b/packages/qvac-lib-infer-bci-whispercpp/bci.js similarity index 87% rename from packages/bci-whispercpp/bci.js rename to packages/qvac-lib-infer-bci-whispercpp/bci.js index 9d869a807f..bfee9c8bd1 100644 --- a/packages/bci-whispercpp/bci.js +++ b/packages/qvac-lib-infer-bci-whispercpp/bci.js @@ -93,6 +93,27 @@ class BCIInterface { if (mappedEvent === 'Output') { this._setState(state.PROCESSING) + + if (this._outputCb != null) { + // Unpack transcript arrays into one event per segment so the + // QvacResponse output iterator yields flat segments (matches + // TranscriptionWhispercpp; avoids callers having to flatten + // [[seg,seg]] back out themselves). + const isTranscriptArray = Array.isArray(data) && data.length > 0 && + typeof data[0]?.text === 'string' + const isSingleTranscript = !Array.isArray(data) && + data && typeof data === 'object' && typeof data.text === 'string' + if (isTranscriptArray) { + for (const segment of data) { + this._outputCb(addon, 'Output', jobId, [segment], null) + } + } else if (isSingleTranscript) { + this._outputCb(addon, 'Output', jobId, [data], null) + } else { + this._outputCb(addon, 'Output', jobId, data, null) + } + } + return } if (this._outputCb != null) { @@ -257,6 +278,19 @@ class BCIInterface { * @param {Uint8Array} data.input - binary neural signal data */ async runJob (data) { + if (!data || !(data.input instanceof Uint8Array)) { + throw new QvacErrorAddonBCI({ + code: ERR_CODES.INVALID_NEURAL_INPUT, + adds: 'runJob input must be a Uint8Array' + }) + } + if (data.input.byteLength === 0) { + throw new QvacErrorAddonBCI({ + code: ERR_CODES.INVALID_NEURAL_INPUT, + adds: 'runJob input must not be empty' + }) + } + const candidateJobId = this._nextJobId let accepted = false try { @@ -267,7 +301,7 @@ class BCIInterface { } catch (err) { this._setState(state.LISTENING) throw new QvacErrorAddonBCI({ - code: ERR_CODES.FAILED_TO_APPEND, + code: ERR_CODES.FAILED_TO_START_JOB, adds: err.message, cause: err }) diff --git a/packages/bci-whispercpp/binding.js b/packages/qvac-lib-infer-bci-whispercpp/binding.js similarity index 100% rename from packages/bci-whispercpp/binding.js rename to packages/qvac-lib-infer-bci-whispercpp/binding.js diff --git a/packages/bci-whispercpp/configChecker.js b/packages/qvac-lib-infer-bci-whispercpp/configChecker.js similarity index 71% rename from packages/bci-whispercpp/configChecker.js rename to packages/qvac-lib-infer-bci-whispercpp/configChecker.js index bc48a17302..34732296e0 100644 --- a/packages/bci-whispercpp/configChecker.js +++ b/packages/qvac-lib-infer-bci-whispercpp/configChecker.js @@ -47,6 +47,14 @@ function checkConfig (configObject) { // Only parameters wired through to the C++ addon are accepted. Adding // smoothing/sample-rate knobs here without consuming them in NeuralProcessor // would silently drop user intent, so they are kept out until implemented. + // + // `day_idx` is an integer session index. + // - day_idx >= 0 applies the day-specific projection matrices from + // bci-embedder.bin, clamped to the valid range at the C++ layer. + // - day_idx === -1 is a passthrough escape hatch that skips + // preprocessing entirely and treats the input buffer as + // pre-computed 512-bin mel features in frame-major layout. Useful + // for reproducing the Python reference output end-to-end. const validBCIParams = [ 'day_idx' ] @@ -75,6 +83,15 @@ function checkConfig (configObject) { throw new Error(`${userParam} is not a valid parameter for bciConfig`) } } + const dayIdx = configObject.bciConfig.day_idx + if (dayIdx !== undefined) { + if (typeof dayIdx !== 'number' || !Number.isFinite(dayIdx) || !Number.isInteger(dayIdx)) { + throw new Error('bciConfig.day_idx must be a finite integer') + } + if (dayIdx < -1) { + throw new Error('bciConfig.day_idx must be >= -1 (use -1 to enable mel-passthrough mode)') + } + } } } diff --git a/packages/bci-whispercpp/docs/BCI_V184_COMPAT.md b/packages/qvac-lib-infer-bci-whispercpp/docs/BCI_V184_COMPAT.md similarity index 96% rename from packages/bci-whispercpp/docs/BCI_V184_COMPAT.md rename to packages/qvac-lib-infer-bci-whispercpp/docs/BCI_V184_COMPAT.md index 0e050ece72..30bd87cc62 100644 --- a/packages/bci-whispercpp/docs/BCI_V184_COMPAT.md +++ b/packages/qvac-lib-infer-bci-whispercpp/docs/BCI_V184_COMPAT.md @@ -52,5 +52,5 @@ Patch `0005-fix-bci-window-mask-encoder-graph.patch` moves the `window_mask` dat - BCI model: `models/ggml-bci-windowed.bin` - Embedder weights: `models/bci-embedder.bin` - Conversion script: `scripts/convert-model.py` -- Overlay portfile: `vcpkg-overlays/whisper-cpp/portfile.cmake` (points to `tetherto/qvac-ext-lib-whisper.cpp` at `3e91e3a4`) +- Overlay portfile: `vcpkg-overlays/whisper-cpp/portfile.cmake` (points to `tetherto/qvac-ext-lib-whisper.cpp` at the merged master commit `2b1e04f20bad9a72321e72df8d6a8c14aae98adc`) - Test: `test/integration/addon.test.js` diff --git a/packages/bci-whispercpp/examples/transcribe-neural.js b/packages/qvac-lib-infer-bci-whispercpp/examples/transcribe-neural.js similarity index 96% rename from packages/bci-whispercpp/examples/transcribe-neural.js rename to packages/qvac-lib-infer-bci-whispercpp/examples/transcribe-neural.js index 768b55b7b8..4b0a3f4717 100644 --- a/packages/bci-whispercpp/examples/transcribe-neural.js +++ b/packages/qvac-lib-infer-bci-whispercpp/examples/transcribe-neural.js @@ -42,7 +42,9 @@ async function main () { return } - const modelPath = (isBatch ? args[1] : args[1]) || DEFAULT_MODEL + // Single-signal mode: args[0]=signal, args[1]=optional model + // Batch mode: args[0]='--batch', args[1]=optional model + const modelPath = args[1] || DEFAULT_MODEL if (!fs.existsSync(modelPath)) { console.error('Error: Model file not found: ' + modelPath) console.error('Set WHISPER_MODEL_PATH or pass as second argument.') diff --git a/packages/qvac-lib-infer-bci-whispercpp/index.d.ts b/packages/qvac-lib-infer-bci-whispercpp/index.d.ts new file mode 100644 index 0000000000..87519089cc --- /dev/null +++ b/packages/qvac-lib-infer-bci-whispercpp/index.d.ts @@ -0,0 +1,135 @@ +import QvacResponse from '@qvac/infer-base/src/QvacResponse' +import type { LoggerInterface } from '@qvac/logging' + +declare interface BCIConfig { + /** + * Session day index used to select day-specific projection matrices in + * bci-embedder.bin. + * + * - `day_idx >= 0` (default `0`): apply the day projection; values beyond + * the available range are clamped at the native layer. + * - `day_idx === -1`: mel passthrough — skip preprocessing and treat + * the input buffer as pre-computed 512-bin mel features in + * frame-major layout. Intended for parity testing against the Python + * reference, not production use. + */ + day_idx?: number +} + +declare interface WhisperConfig { + language?: string + n_threads?: number + temperature?: number + suppress_nst?: boolean + suppress_blank?: boolean + duration_ms?: number + translate?: boolean + no_timestamps?: boolean + single_segment?: boolean + print_special?: boolean + print_progress?: boolean + print_realtime?: boolean + print_timestamps?: boolean + detect_language?: boolean + greedy_best_of?: number + beam_search_beam_size?: number +} + +declare interface BCIWhispercppFiles { + /** Absolute path to the BCI GGML model file. */ + model: string +} + +declare interface BCIWhispercppArgs { + files: BCIWhispercppFiles + logger?: LoggerInterface + opts?: { + stats?: boolean + } +} + +declare interface BCIWhispercppConfig { + whisperConfig?: WhisperConfig + bciConfig?: BCIConfig + contextParams?: { + model?: string + use_gpu?: boolean + flash_attn?: boolean + gpu_device?: number + } + miscConfig?: { + caption_enabled?: boolean + } +} + +declare interface TranscriptSegment { + text: string + toAppend: boolean + start: number + end: number + id: number +} + +declare interface BCIWhispercppState { + configLoaded: boolean + destroyed: boolean +} + +/** + * BCI neural signal transcription client powered by whisper.cpp. + * + * Uses `createJobHandler` + `exclusiveRunQueue` from `@qvac/infer-base` and + * follows the same lifecycle contract as `TranscriptionWhispercpp` / + * `LlmLlamacpp`: construct with local file paths, call `load()`, issue + * `transcribe()` / `transcribeFile()` calls, then `destroy()`. + */ +declare class BCIWhispercpp { + constructor(args: BCIWhispercppArgs, config?: BCIWhispercppConfig) + + /** Load and activate the model. Must be awaited before `transcribe()`. */ + load(): Promise + + /** Transcribe a neural signal binary file (convenience wrapper). */ + transcribeFile(filePath: string): Promise + + /** Transcribe a neural signal buffer (batch mode). */ + transcribe(neuralData: Uint8Array): Promise + + /** Cancel the in-flight inference, if any. */ + cancel(): Promise + + /** Unload the model and release native resources. Instance is reusable. */ + unload(): Promise + + /** + * Destroy the instance, unload, and mark as permanently destroyed. + * Subsequent `load()` calls will throw `MODEL_NOT_LOADED`. + */ + destroy(): Promise + + /** Current lifecycle state. */ + getState(): BCIWhispercppState +} + +/** + * Compute Word Error Rate between hypothesis and reference strings. + * @returns WER as a ratio (0.0 = perfect). + */ +declare function computeWER(hypothesis: string, reference: string): number + +declare namespace BCIWhispercpp { + export { + BCIWhispercpp as default, + BCIWhispercpp, + BCIConfig, + WhisperConfig, + BCIWhispercppFiles, + BCIWhispercppArgs, + BCIWhispercppConfig, + BCIWhispercppState, + TranscriptSegment, + computeWER + } +} + +export = BCIWhispercpp diff --git a/packages/bci-whispercpp/index.js b/packages/qvac-lib-infer-bci-whispercpp/index.js similarity index 93% rename from packages/bci-whispercpp/index.js rename to packages/qvac-lib-infer-bci-whispercpp/index.js index c21b3d9b80..5bd9640c90 100644 --- a/packages/bci-whispercpp/index.js +++ b/packages/qvac-lib-infer-bci-whispercpp/index.js @@ -5,7 +5,6 @@ const QvacLogger = require('@qvac/logging') const { createJobHandler, exclusiveRunQueue } = require('@qvac/infer-base') const { BCIInterface } = require('./bci') -const { checkConfig } = require('./configChecker') const { QvacErrorAddonBCI, ERR_CODES } = require('./lib/error') const { computeWER } = require('./lib/wer') @@ -81,10 +80,13 @@ class BCIWhispercpp { } async _load () { + // BCI-tuned whisper defaults (beam_size, suppress_nst, temperature, + // length_penalty, ...) are applied natively in + // addon/src/model-interface/bci/BCIConfig.cpp::toWhisperFullParams. + // Only the transport-level defaults live in JS; everything else is + // delegated to the C++ layer to avoid double-specification. const whisperConfig = { language: 'en', - temperature: 0.0, - suppress_nst: true, n_threads: 0, ...(this._config.whisperConfig || {}) } @@ -105,8 +107,8 @@ class BCIWhispercpp { configurationParams.bciConfig = this._config.bciConfig } - checkConfig(configurationParams) - + // Validation happens once inside BCIInterface's constructor via + // configChecker.checkConfig. Calling it here too would duplicate the work. const binding = require('./binding') try { this.addon = new BCIInterface( diff --git a/packages/bci-whispercpp/lib/error.js b/packages/qvac-lib-infer-bci-whispercpp/lib/error.js similarity index 64% rename from packages/bci-whispercpp/lib/error.js rename to packages/qvac-lib-infer-bci-whispercpp/lib/error.js index 571e4fb653..9c6a2f5791 100644 --- a/packages/bci-whispercpp/lib/error.js +++ b/packages/qvac-lib-infer-bci-whispercpp/lib/error.js @@ -6,20 +6,29 @@ class QvacErrorAddonBCI extends QvacErrorBase { } const { name, version } = require('../package.json') +// This library has error code range from 26001 to 27000. +// Ranges used elsewhere in the @qvac/error registry: +// 6001-6018 @qvac/transcription-whispercpp +// 7001-7011 @qvac/tts-onnx +// 8001-8008 @qvac/translation-nmtcpp +// 24001+ @qvac/transcription-parakeet const ERR_CODES = Object.freeze({ - FAILED_TO_LOAD_WEIGHTS: 7001, - FAILED_TO_CANCEL: 7002, - FAILED_TO_APPEND: 7003, - FAILED_TO_GET_STATUS: 7004, - FAILED_TO_DESTROY: 7005, - FAILED_TO_ACTIVATE: 7006, - FAILED_TO_RESET: 7007, - FAILED_TO_PAUSE: 7008, - INVALID_NEURAL_INPUT: 7009, - JOB_ALREADY_RUNNING: 7010, - MODEL_NOT_LOADED: 7011, - MODEL_FILE_NOT_FOUND: 7012, - BUFFER_LIMIT_EXCEEDED: 7013 + FAILED_TO_LOAD_WEIGHTS: 26001, + FAILED_TO_CANCEL: 26002, + FAILED_TO_APPEND: 26003, + FAILED_TO_GET_STATUS: 26004, + FAILED_TO_DESTROY: 26005, + FAILED_TO_ACTIVATE: 26006, + FAILED_TO_RESET: 26007, + FAILED_TO_PAUSE: 26008, + INVALID_NEURAL_INPUT: 26009, + JOB_ALREADY_RUNNING: 26010, + MODEL_NOT_LOADED: 26011, + MODEL_FILE_NOT_FOUND: 26012, + BUFFER_LIMIT_EXCEEDED: 26013, + FAILED_TO_START_JOB: 26014, + INVALID_CONFIG: 26015, + EMBEDDER_WEIGHTS_INVALID: 26016 }) addCodes({ @@ -74,6 +83,18 @@ addCodes({ [ERR_CODES.BUFFER_LIMIT_EXCEEDED]: { name: 'BUFFER_LIMIT_EXCEEDED', message: (limit) => `Neural signal buffer exceeded limit of ${limit}` + }, + [ERR_CODES.FAILED_TO_START_JOB]: { + name: 'FAILED_TO_START_JOB', + message: (message) => `Failed to start inference job, error: ${message}` + }, + [ERR_CODES.INVALID_CONFIG]: { + name: 'INVALID_CONFIG', + message: (message) => `Invalid BCI configuration: ${message}` + }, + [ERR_CODES.EMBEDDER_WEIGHTS_INVALID]: { + name: 'EMBEDDER_WEIGHTS_INVALID', + message: (message) => `BCI embedder weights are invalid: ${message}` } }, { name, diff --git a/packages/bci-whispercpp/lib/wer.js b/packages/qvac-lib-infer-bci-whispercpp/lib/wer.js similarity index 100% rename from packages/bci-whispercpp/lib/wer.js rename to packages/qvac-lib-infer-bci-whispercpp/lib/wer.js diff --git a/packages/bci-whispercpp/package.json b/packages/qvac-lib-infer-bci-whispercpp/package.json similarity index 68% rename from packages/bci-whispercpp/package.json rename to packages/qvac-lib-infer-bci-whispercpp/package.json index a5db15046b..75c88aa9fa 100644 --- a/packages/bci-whispercpp/package.json +++ b/packages/qvac-lib-infer-bci-whispercpp/package.json @@ -4,21 +4,24 @@ "description": "Brain-Computer Interface (BCI) neural signal transcription addon for qvac, powered by whisper.cpp", "addon": true, "engines": { - "bare": ">=1.19.0" + "bare": ">=1.24.0" }, "scripts": { - "lint": "standard \"examples/**/*.js\" \"test/**/*.js\" \"*.js\"", - "lint:fix": "standard --fix \"examples/**/*.js\" \"test/**/*.js\" \"**/*.js\"", + "lint": "standard \"examples/**/*.js\" \"test/**/*.js\" \"*.js\" \"lib/**/*.js\"", + "lint:fix": "standard --fix \"examples/**/*.js\" \"test/**/*.js\" \"*.js\" \"lib/**/*.js\"", + "lint-cpp": "clang-tidy -p build $(find addon -name '*.cpp')", "build": "bare-make generate && bare-make build && bare-make install", - "test:unit": "brittle-bare test/unit/**/*.test.js", + "build:pack": "mkdir -p dist && npm pack --pack-destination dist", + "test": "npm run test:integration", "test:integration": "brittle-bare test/integration/addon.test.js", "test:cpp:build": "bare-make generate -D BUILD_TESTING=ON && bare-make build --target test-bci-core && bare-make install", - "test:cpp:run": "build/addon/tests/test-bci-core --gtest_output=xml:build/addon/tests/cpp-test-results.xml", + "test:cpp:run": "cd build/addon/tests/ && ./test-bci-core --gtest_output=xml:cpp-test-results.xml", "test:cpp": "npm run test:cpp:build && npm run test:cpp:run", - "test": "npm run test:integration", - "test:dts": "tsc index.d.ts --noEmit --lib es2018 --esModuleInterop --skipLibCheck" + "test:dts": "tsc -p tsconfig.dts.json" }, "files": [ + "addonLogging.js", + "addonLogging.d.ts", "binding.js", "bci.js", "configChecker.js", @@ -26,12 +29,15 @@ "index.d.ts", "prebuilds", "lib", + "README.md", + "CHANGELOG.md", "LICENSE", "NOTICE" ], "repository": { "type": "git", - "url": "git+https://github.com/tetherto/qvac.git" + "url": "git+https://github.com/tetherto/qvac.git", + "directory": "packages/qvac-lib-infer-bci-whispercpp" }, "author": "Tether", "keywords": [ @@ -45,8 +51,9 @@ ], "license": "Apache-2.0", "bugs": "https://github.com/tetherto/qvac/issues", - "homepage": "https://github.com/tetherto/qvac#readme", + "homepage": "https://github.com/tetherto/qvac/tree/main/packages/qvac-lib-infer-bci-whispercpp#readme", "devDependencies": { + "@types/node": "^24.2.1", "bare-buffer": "^3.4.2", "bare-fs": "^4.5.1", "bare-tty": "^5.0.3", @@ -56,7 +63,8 @@ "fs": "npm:bare-fs", "os": "npm:bare-os@^3.6.2", "standard": "^17.1.2", - "tty": "npm:bare-node-tty" + "tty": "npm:bare-node-tty", + "typescript": "^5.9.2" }, "dependencies": { "@qvac/error": "^0.1.0", @@ -72,6 +80,11 @@ "types": "./index.d.ts", "default": "./index.js" }, + "./addonLogging": { + "types": "./addonLogging.d.ts", + "default": "./addonLogging.js" + }, + "./addonLogging.js": "./addonLogging.js", "./bci": "./bci.js", "./bci.js": "./bci.js", "./binding": "./binding.js", diff --git a/packages/bci-whispercpp/scripts/convert-model.py b/packages/qvac-lib-infer-bci-whispercpp/scripts/convert-model.py similarity index 100% rename from packages/bci-whispercpp/scripts/convert-model.py rename to packages/qvac-lib-infer-bci-whispercpp/scripts/convert-model.py diff --git a/packages/bci-whispercpp/scripts/download-models.sh b/packages/qvac-lib-infer-bci-whispercpp/scripts/download-models.sh similarity index 100% rename from packages/bci-whispercpp/scripts/download-models.sh rename to packages/qvac-lib-infer-bci-whispercpp/scripts/download-models.sh diff --git a/packages/bci-whispercpp/test/fixtures/manifest.json b/packages/qvac-lib-infer-bci-whispercpp/test/fixtures/manifest.json similarity index 100% rename from packages/bci-whispercpp/test/fixtures/manifest.json rename to packages/qvac-lib-infer-bci-whispercpp/test/fixtures/manifest.json diff --git a/packages/bci-whispercpp/test/integration/addon.test.js b/packages/qvac-lib-infer-bci-whispercpp/test/integration/addon.test.js similarity index 89% rename from packages/bci-whispercpp/test/integration/addon.test.js rename to packages/qvac-lib-infer-bci-whispercpp/test/integration/addon.test.js index 72ced7b108..a25f7d8dd6 100644 --- a/packages/bci-whispercpp/test/integration/addon.test.js +++ b/packages/qvac-lib-infer-bci-whispercpp/test/integration/addon.test.js @@ -15,6 +15,19 @@ const MODEL_PATH = (os.hasEnv('WHISPER_MODEL_PATH') ? os.getEnv('WHISPER_MODEL_P const hasModel = fs.existsSync(MODEL_PATH) +// Skipping when the model is missing is fine for local dev, but in CI we +// want a loud failure. Set BCI_REQUIRE_MODEL=1 (e.g. on a runner with the +// assets pre-provisioned) to turn "missing model" into a hard error so the +// tests cannot silently pass with zero assertions. +const requireModel = os.hasEnv('BCI_REQUIRE_MODEL') && os.getEnv('BCI_REQUIRE_MODEL') === '1' + +if (requireModel && !hasModel) { + throw new Error( + 'BCI_REQUIRE_MODEL=1 but model file was not found at ' + MODEL_PATH + + '. Run `bash scripts/download-models.sh` or set WHISPER_MODEL_PATH.' + ) +} + function bciConfigFor (sample) { return typeof sample?.day_idx === 'number' ? { day_idx: sample.day_idx } : undefined } diff --git a/packages/bci-whispercpp/test/integration/helpers.js b/packages/qvac-lib-infer-bci-whispercpp/test/integration/helpers.js similarity index 100% rename from packages/bci-whispercpp/test/integration/helpers.js rename to packages/qvac-lib-infer-bci-whispercpp/test/integration/helpers.js diff --git a/packages/qvac-lib-infer-bci-whispercpp/tsconfig.dts.json b/packages/qvac-lib-infer-bci-whispercpp/tsconfig.dts.json new file mode 100644 index 0000000000..a47519c283 --- /dev/null +++ b/packages/qvac-lib-infer-bci-whispercpp/tsconfig.dts.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ES2022", + "moduleResolution": "bundler", + "lib": ["ES2022"], + "types": ["node"], + "skipLibCheck": true, + "esModuleInterop": true, + "allowSyntheticDefaultImports": true, + "forceConsistentCasingInFileNames": true, + "strict": true, + "noEmit": true + }, + "include": ["index.d.ts", "addonLogging.d.ts"] +} diff --git a/packages/bci-whispercpp/vcpkg-configuration.json b/packages/qvac-lib-infer-bci-whispercpp/vcpkg-configuration.json similarity index 82% rename from packages/bci-whispercpp/vcpkg-configuration.json rename to packages/qvac-lib-infer-bci-whispercpp/vcpkg-configuration.json index cf90bf82c2..b50ad85757 100644 --- a/packages/bci-whispercpp/vcpkg-configuration.json +++ b/packages/qvac-lib-infer-bci-whispercpp/vcpkg-configuration.json @@ -2,7 +2,7 @@ "default-registry": { "kind": "git", "baseline": "87ef7179f70122d0cc65a5991b88c20cab59b1e1", - "repository": "git@github.com:tetherto/qvac-registry-vcpkg.git" + "repository": "https://github.com/tetherto/qvac-registry-vcpkg.git" }, "registries": [ { diff --git a/packages/bci-whispercpp/vcpkg-overlays/qvac-lint-cpp/portfile.cmake b/packages/qvac-lib-infer-bci-whispercpp/vcpkg-overlays/qvac-lint-cpp/portfile.cmake similarity index 100% rename from packages/bci-whispercpp/vcpkg-overlays/qvac-lint-cpp/portfile.cmake rename to packages/qvac-lib-infer-bci-whispercpp/vcpkg-overlays/qvac-lint-cpp/portfile.cmake diff --git a/packages/bci-whispercpp/vcpkg-overlays/qvac-lint-cpp/vcpkg.json b/packages/qvac-lib-infer-bci-whispercpp/vcpkg-overlays/qvac-lint-cpp/vcpkg.json similarity index 100% rename from packages/bci-whispercpp/vcpkg-overlays/qvac-lint-cpp/vcpkg.json rename to packages/qvac-lib-infer-bci-whispercpp/vcpkg-overlays/qvac-lint-cpp/vcpkg.json diff --git a/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/portfile.cmake b/packages/qvac-lib-infer-bci-whispercpp/vcpkg-overlays/whisper-cpp/portfile.cmake similarity index 100% rename from packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/portfile.cmake rename to packages/qvac-lib-infer-bci-whispercpp/vcpkg-overlays/whisper-cpp/portfile.cmake diff --git a/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/vcpkg.json b/packages/qvac-lib-infer-bci-whispercpp/vcpkg-overlays/whisper-cpp/vcpkg.json similarity index 100% rename from packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/vcpkg.json rename to packages/qvac-lib-infer-bci-whispercpp/vcpkg-overlays/whisper-cpp/vcpkg.json diff --git a/packages/bci-whispercpp/vcpkg.json b/packages/qvac-lib-infer-bci-whispercpp/vcpkg.json similarity index 92% rename from packages/bci-whispercpp/vcpkg.json rename to packages/qvac-lib-infer-bci-whispercpp/vcpkg.json index a4448d5c39..42b6619b98 100644 --- a/packages/bci-whispercpp/vcpkg.json +++ b/packages/qvac-lib-infer-bci-whispercpp/vcpkg.json @@ -4,7 +4,7 @@ "dependencies": [ { "name": "qvac-lib-inference-addon-cpp", - "version>=": "1.1.5" + "version>=": "1.1.5#1" }, "whisper-cpp" ], diff --git a/packages/bci-whispercpp/vcpkg/toolchains/linux-clang.cmake b/packages/qvac-lib-infer-bci-whispercpp/vcpkg/toolchains/linux-clang.cmake similarity index 100% rename from packages/bci-whispercpp/vcpkg/toolchains/linux-clang.cmake rename to packages/qvac-lib-infer-bci-whispercpp/vcpkg/toolchains/linux-clang.cmake diff --git a/packages/bci-whispercpp/vcpkg/triplets/arm64-linux.cmake b/packages/qvac-lib-infer-bci-whispercpp/vcpkg/triplets/arm64-linux.cmake similarity index 100% rename from packages/bci-whispercpp/vcpkg/triplets/arm64-linux.cmake rename to packages/qvac-lib-infer-bci-whispercpp/vcpkg/triplets/arm64-linux.cmake diff --git a/packages/bci-whispercpp/vcpkg/triplets/x64-linux.cmake b/packages/qvac-lib-infer-bci-whispercpp/vcpkg/triplets/x64-linux.cmake similarity index 100% rename from packages/bci-whispercpp/vcpkg/triplets/x64-linux.cmake rename to packages/qvac-lib-infer-bci-whispercpp/vcpkg/triplets/x64-linux.cmake From 7cdffaf1ba189782c3f2dfbcc7df2db234dabee0 Mon Sep 17 00:00:00 2001 From: Raju Date: Tue, 21 Apr 2026 14:09:42 +0530 Subject: [PATCH 12/19] fix[api](bci): restore bci-whispercpp package path and harden runtime validation Move the addon package back to packages/bci-whispercpp, remove unneeded overlay/docs files requested in review, and tighten JS/C++ lifecycle/config safety checks to prevent invalid-state and malformed-input issues. Made-with: Cursor --- .../.gitignore | 0 .../CHANGELOG.md | 5 +- .../CMakeLists.txt | 0 .../LICENSE | 0 .../NOTICE | 0 .../PULL_REQUEST_TEMPLATE.md | 0 .../README.md | 8 +-- .../addon/src/addon/AddonJs.hpp | 0 .../addon/src/addon/BCIErrors.hpp | 0 .../addon/src/js-interface/JSAdapter.cpp | 0 .../addon/src/js-interface/JSAdapter.hpp | 0 .../addon/src/js-interface/binding.cpp | 0 .../addon/src/model-interface/BCITypes.hpp | 0 .../src/model-interface/bci/BCIConfig.cpp | 12 ++++ .../src/model-interface/bci/BCIConfig.hpp | 0 .../src/model-interface/bci/BCIModel.cpp | 36 ++++++------ .../src/model-interface/bci/BCIModel.hpp | 0 .../model-interface/bci/NeuralProcessor.cpp | 26 +++++++++ .../model-interface/bci/NeuralProcessor.hpp | 0 .../addon/tests/test_core.cpp | 0 .../addonLogging.d.ts | 0 .../addonLogging.js | 0 .../bci.js | 16 ++++-- .../binding.js | 0 .../configChecker.js | 0 .../examples/transcribe-neural.js | 12 ++-- .../index.d.ts | 0 .../index.js | 36 +++++++++++- .../lib/error.js | 0 .../lib/wer.js | 0 .../package.json | 4 +- .../scripts/convert-model.py | 0 .../scripts/download-models.sh | 16 ++++-- .../test/fixtures/manifest.json | 0 .../test/integration/addon.test.js | 0 .../test/integration/helpers.js | 0 .../tsconfig.dts.json | 0 .../vcpkg-configuration.json | 0 .../vcpkg-overlays/whisper-cpp/portfile.cmake | 0 .../vcpkg-overlays/whisper-cpp/vcpkg.json | 0 .../vcpkg.json | 0 .../vcpkg/toolchains/linux-clang.cmake | 0 .../vcpkg/triplets/arm64-linux.cmake | 0 .../vcpkg/triplets/x64-linux.cmake | 0 .../docs/BCI_V184_COMPAT.md | 56 ------------------- .../qvac-lint-cpp/portfile.cmake | 7 --- .../vcpkg-overlays/qvac-lint-cpp/vcpkg.json | 5 -- 47 files changed, 129 insertions(+), 110 deletions(-) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/.gitignore (100%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/CHANGELOG.md (89%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/CMakeLists.txt (100%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/LICENSE (100%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/NOTICE (100%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/PULL_REQUEST_TEMPLATE.md (100%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/README.md (97%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/addon/src/addon/AddonJs.hpp (100%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/addon/src/addon/BCIErrors.hpp (100%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/addon/src/js-interface/JSAdapter.cpp (100%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/addon/src/js-interface/JSAdapter.hpp (100%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/addon/src/js-interface/binding.cpp (100%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/addon/src/model-interface/BCITypes.hpp (100%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/addon/src/model-interface/bci/BCIConfig.cpp (95%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/addon/src/model-interface/bci/BCIConfig.hpp (100%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/addon/src/model-interface/bci/BCIModel.cpp (93%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/addon/src/model-interface/bci/BCIModel.hpp (100%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/addon/src/model-interface/bci/NeuralProcessor.cpp (92%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/addon/src/model-interface/bci/NeuralProcessor.hpp (100%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/addon/tests/test_core.cpp (100%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/addonLogging.d.ts (100%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/addonLogging.js (100%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/bci.js (95%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/binding.js (100%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/configChecker.js (100%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/examples/transcribe-neural.js (100%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/index.d.ts (100%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/index.js (85%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/lib/error.js (100%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/lib/wer.js (100%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/package.json (96%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/scripts/convert-model.py (100%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/scripts/download-models.sh (74%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/test/fixtures/manifest.json (100%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/test/integration/addon.test.js (100%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/test/integration/helpers.js (100%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/tsconfig.dts.json (100%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/vcpkg-configuration.json (100%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/vcpkg-overlays/whisper-cpp/portfile.cmake (100%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/vcpkg-overlays/whisper-cpp/vcpkg.json (100%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/vcpkg.json (100%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/vcpkg/toolchains/linux-clang.cmake (100%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/vcpkg/triplets/arm64-linux.cmake (100%) rename packages/{qvac-lib-infer-bci-whispercpp => bci-whispercpp}/vcpkg/triplets/x64-linux.cmake (100%) delete mode 100644 packages/qvac-lib-infer-bci-whispercpp/docs/BCI_V184_COMPAT.md delete mode 100644 packages/qvac-lib-infer-bci-whispercpp/vcpkg-overlays/qvac-lint-cpp/portfile.cmake delete mode 100644 packages/qvac-lib-infer-bci-whispercpp/vcpkg-overlays/qvac-lint-cpp/vcpkg.json diff --git a/packages/qvac-lib-infer-bci-whispercpp/.gitignore b/packages/bci-whispercpp/.gitignore similarity index 100% rename from packages/qvac-lib-infer-bci-whispercpp/.gitignore rename to packages/bci-whispercpp/.gitignore diff --git a/packages/qvac-lib-infer-bci-whispercpp/CHANGELOG.md b/packages/bci-whispercpp/CHANGELOG.md similarity index 89% rename from packages/qvac-lib-infer-bci-whispercpp/CHANGELOG.md rename to packages/bci-whispercpp/CHANGELOG.md index 761c9dcf78..d51c53ae3b 100644 --- a/packages/qvac-lib-infer-bci-whispercpp/CHANGELOG.md +++ b/packages/bci-whispercpp/CHANGELOG.md @@ -36,6 +36,5 @@ signal transcription addon powered by a BCI-patched fork of whisper.cpp. - Streaming transcription is not implemented in this release; see follow-up work tracked under QVAC-17062. -- Inference error codes live in the `26001-27000` range; migrations that - pinned the older `7xxx` range used during initial development should update - to the new constants exported from `@qvac/bci-whispercpp/lib/error`. +- Inference error codes live in the `26001-27000` range in the current + implementation. diff --git a/packages/qvac-lib-infer-bci-whispercpp/CMakeLists.txt b/packages/bci-whispercpp/CMakeLists.txt similarity index 100% rename from packages/qvac-lib-infer-bci-whispercpp/CMakeLists.txt rename to packages/bci-whispercpp/CMakeLists.txt diff --git a/packages/qvac-lib-infer-bci-whispercpp/LICENSE b/packages/bci-whispercpp/LICENSE similarity index 100% rename from packages/qvac-lib-infer-bci-whispercpp/LICENSE rename to packages/bci-whispercpp/LICENSE diff --git a/packages/qvac-lib-infer-bci-whispercpp/NOTICE b/packages/bci-whispercpp/NOTICE similarity index 100% rename from packages/qvac-lib-infer-bci-whispercpp/NOTICE rename to packages/bci-whispercpp/NOTICE diff --git a/packages/qvac-lib-infer-bci-whispercpp/PULL_REQUEST_TEMPLATE.md b/packages/bci-whispercpp/PULL_REQUEST_TEMPLATE.md similarity index 100% rename from packages/qvac-lib-infer-bci-whispercpp/PULL_REQUEST_TEMPLATE.md rename to packages/bci-whispercpp/PULL_REQUEST_TEMPLATE.md diff --git a/packages/qvac-lib-infer-bci-whispercpp/README.md b/packages/bci-whispercpp/README.md similarity index 97% rename from packages/qvac-lib-infer-bci-whispercpp/README.md rename to packages/bci-whispercpp/README.md index f02759ad4d..4a2bdca8fa 100644 --- a/packages/qvac-lib-infer-bci-whispercpp/README.md +++ b/packages/bci-whispercpp/README.md @@ -59,14 +59,14 @@ Each timestep represents a 20ms bin of neural activity. Channels correspond to i ## Installation ```bash -cd packages/qvac-lib-infer-bci-whispercpp +cd packages/bci-whispercpp npm install VCPKG_ROOT=/path/to/vcpkg npm run build ``` ### Prerequisites -- **Bare runtime** >= 1.19.0 +- **Bare runtime** >= 1.24.0 - **CMake** >= 3.25 - **vcpkg** with `VCPKG_ROOT` environment variable set @@ -117,7 +117,7 @@ const config = { } const onOutput = (addon, event, jobId, data, error) => { - if (event === 'Output') console.log('Segment:', data.text) + if (event === 'Output') console.log('Segment:', data[0]?.text) if (event === 'JobEnded') console.log('Done:', data) if (event === 'Error') console.error('Error:', error) } @@ -184,7 +184,7 @@ The package uses a vcpkg overlay that fetches from the `tetherto/qvac-ext-lib-wh | Variable conv1 kernel | Read `n_audio_conv1_kernel` from model header (k=7 for 512ch BCI vs k=3 for audio) | | Windowed attention | Attention mask with configurable window size/layer params in header | | BCI SOS tokens | BCI-specific start-of-sequence token handling | -| Graph placement fix | Correct encoder-graph mask population (see `docs/BCI_V184_COMPAT.md`) | +| Graph placement fix | Correct encoder-graph mask population for the encoder graph refactor | ## Platform Support diff --git a/packages/qvac-lib-infer-bci-whispercpp/addon/src/addon/AddonJs.hpp b/packages/bci-whispercpp/addon/src/addon/AddonJs.hpp similarity index 100% rename from packages/qvac-lib-infer-bci-whispercpp/addon/src/addon/AddonJs.hpp rename to packages/bci-whispercpp/addon/src/addon/AddonJs.hpp diff --git a/packages/qvac-lib-infer-bci-whispercpp/addon/src/addon/BCIErrors.hpp b/packages/bci-whispercpp/addon/src/addon/BCIErrors.hpp similarity index 100% rename from packages/qvac-lib-infer-bci-whispercpp/addon/src/addon/BCIErrors.hpp rename to packages/bci-whispercpp/addon/src/addon/BCIErrors.hpp diff --git a/packages/qvac-lib-infer-bci-whispercpp/addon/src/js-interface/JSAdapter.cpp b/packages/bci-whispercpp/addon/src/js-interface/JSAdapter.cpp similarity index 100% rename from packages/qvac-lib-infer-bci-whispercpp/addon/src/js-interface/JSAdapter.cpp rename to packages/bci-whispercpp/addon/src/js-interface/JSAdapter.cpp diff --git a/packages/qvac-lib-infer-bci-whispercpp/addon/src/js-interface/JSAdapter.hpp b/packages/bci-whispercpp/addon/src/js-interface/JSAdapter.hpp similarity index 100% rename from packages/qvac-lib-infer-bci-whispercpp/addon/src/js-interface/JSAdapter.hpp rename to packages/bci-whispercpp/addon/src/js-interface/JSAdapter.hpp diff --git a/packages/qvac-lib-infer-bci-whispercpp/addon/src/js-interface/binding.cpp b/packages/bci-whispercpp/addon/src/js-interface/binding.cpp similarity index 100% rename from packages/qvac-lib-infer-bci-whispercpp/addon/src/js-interface/binding.cpp rename to packages/bci-whispercpp/addon/src/js-interface/binding.cpp diff --git a/packages/qvac-lib-infer-bci-whispercpp/addon/src/model-interface/BCITypes.hpp b/packages/bci-whispercpp/addon/src/model-interface/BCITypes.hpp similarity index 100% rename from packages/qvac-lib-infer-bci-whispercpp/addon/src/model-interface/BCITypes.hpp rename to packages/bci-whispercpp/addon/src/model-interface/BCITypes.hpp diff --git a/packages/qvac-lib-infer-bci-whispercpp/addon/src/model-interface/bci/BCIConfig.cpp b/packages/bci-whispercpp/addon/src/model-interface/bci/BCIConfig.cpp similarity index 95% rename from packages/qvac-lib-infer-bci-whispercpp/addon/src/model-interface/bci/BCIConfig.cpp rename to packages/bci-whispercpp/addon/src/model-interface/bci/BCIConfig.cpp index ddce668781..981c377cb5 100644 --- a/packages/qvac-lib-infer-bci-whispercpp/addon/src/model-interface/bci/BCIConfig.cpp +++ b/packages/bci-whispercpp/addon/src/model-interface/bci/BCIConfig.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -24,6 +25,17 @@ int toInt(const JSValueVariant& v, const std::string& key) { qvac_errors::general_error::InvalidArgument, key + " must be a finite number"); } + if (*d < static_cast(std::numeric_limits::min()) || + *d > static_cast(std::numeric_limits::max())) { + throw qvac_errors::StatusError( + qvac_errors::general_error::InvalidArgument, + key + " is out of int32 range"); + } + if (std::floor(*d) != *d) { + throw qvac_errors::StatusError( + qvac_errors::general_error::InvalidArgument, + key + " must be an integer"); + } return static_cast(*d); } if (const auto* i = std::get_if(&v)) { diff --git a/packages/qvac-lib-infer-bci-whispercpp/addon/src/model-interface/bci/BCIConfig.hpp b/packages/bci-whispercpp/addon/src/model-interface/bci/BCIConfig.hpp similarity index 100% rename from packages/qvac-lib-infer-bci-whispercpp/addon/src/model-interface/bci/BCIConfig.hpp rename to packages/bci-whispercpp/addon/src/model-interface/bci/BCIConfig.hpp diff --git a/packages/qvac-lib-infer-bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp b/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp similarity index 93% rename from packages/qvac-lib-infer-bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp rename to packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp index 1a4e9bc78a..36e612e801 100644 --- a/packages/qvac-lib-infer-bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp +++ b/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp @@ -95,32 +95,36 @@ void BCIModel::loadEmbedderIfNeeded() { } void BCIModel::load() { - if (!ctx_) { - whisper_context_params contextParams = toWhisperContextParams(cfg_); + if (ctx_) return; - const auto modelPathIt = cfg_.whisperContextCfg.find("model"); - if (modelPathIt == cfg_.whisperContextCfg.end()) { - throw std::runtime_error("Model path not specified"); - } - const auto modelPath = std::get(modelPathIt->second); + whisper_context_params contextParams = toWhisperContextParams(cfg_); - QLOG(qvac_lib_inference_addon_cpp::logger::Priority::INFO, - "Loading BCI model from: " + modelPath); - ctx_.reset( - whisper_init_from_file_with_params(modelPath.c_str(), contextParams)); + const auto modelPathIt = cfg_.whisperContextCfg.find("model"); + if (modelPathIt == cfg_.whisperContextCfg.end()) { + throw std::runtime_error("Model path not specified"); + } + const auto modelPath = std::get(modelPathIt->second); - if (ctx_ == nullptr) { - throw std::runtime_error("Failed to initialize Whisper context for BCI"); - } + QLOG(qvac_lib_inference_addon_cpp::logger::Priority::INFO, + "Loading BCI model from: " + modelPath); - is_loaded_ = true; + auto* rawCtx = whisper_init_from_file_with_params(modelPath.c_str(), contextParams); + if (rawCtx == nullptr) { + throw std::runtime_error("Failed to initialize Whisper context for BCI"); + } + try { + ctx_.reset(rawCtx); loadEmbedderIfNeeded(); - if (!is_warmed_up_) { warmup(); is_warmed_up_ = true; } + is_loaded_ = true; + } catch (...) { + ctx_.reset(); + is_loaded_ = false; + throw; } } diff --git a/packages/qvac-lib-infer-bci-whispercpp/addon/src/model-interface/bci/BCIModel.hpp b/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.hpp similarity index 100% rename from packages/qvac-lib-infer-bci-whispercpp/addon/src/model-interface/bci/BCIModel.hpp rename to packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.hpp diff --git a/packages/qvac-lib-infer-bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.cpp b/packages/bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.cpp similarity index 92% rename from packages/qvac-lib-infer-bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.cpp rename to packages/bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.cpp index 8c95cef77a..a4448e464a 100644 --- a/packages/qvac-lib-infer-bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.cpp +++ b/packages/bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.cpp @@ -26,6 +26,10 @@ constexpr float K_KERNEL_TRIM_THRESHOLD = 0.01F; // the raw neural signal before day-projection and mel padding. constexpr float K_SMOOTH_KERNEL_STD = 2.0F; constexpr int K_SMOOTH_KERNEL_SIZE = 100; + +bool hasExpectedSize(const std::vector& vec, size_t expected) { + return vec.size() == expected; +} } // namespace NeuralProcessor::NeuralProcessor() = default; @@ -108,6 +112,28 @@ bool NeuralProcessor::loadEmbedderWeights(const std::string& path) { if (readFailed) return false; } + const size_t nf = static_cast(weights_.numFeatures); + const size_t r = static_cast(weights_.r); + const size_t expectedDayA = nf * r; + const size_t expectedDayB = r * nf; + const size_t expectedDayBias = nf; + const size_t expectedMonthW = nf * nf; + + for (uint32_t i = 0; i < weights_.numDays; ++i) { + if (!hasExpectedSize(weights_.dayAs[i], expectedDayA) || + !hasExpectedSize(weights_.dayBs[i], expectedDayB) || + !hasExpectedSize(weights_.dayBiases[i], expectedDayBias)) { + return false; + } + } + + for (uint32_t i = 0; i < weights_.numMonths; ++i) { + if (!hasExpectedSize(weights_.monthWeights[i], expectedMonthW) || + !hasExpectedSize(weights_.monthBiases[i], expectedDayBias)) { + return false; + } + } + weights_.loaded = true; QLOG(qvac_lib_inference_addon_cpp::logger::Priority::INFO, "Loaded day projection weights: " + diff --git a/packages/qvac-lib-infer-bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.hpp b/packages/bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.hpp similarity index 100% rename from packages/qvac-lib-infer-bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.hpp rename to packages/bci-whispercpp/addon/src/model-interface/bci/NeuralProcessor.hpp diff --git a/packages/qvac-lib-infer-bci-whispercpp/addon/tests/test_core.cpp b/packages/bci-whispercpp/addon/tests/test_core.cpp similarity index 100% rename from packages/qvac-lib-infer-bci-whispercpp/addon/tests/test_core.cpp rename to packages/bci-whispercpp/addon/tests/test_core.cpp diff --git a/packages/qvac-lib-infer-bci-whispercpp/addonLogging.d.ts b/packages/bci-whispercpp/addonLogging.d.ts similarity index 100% rename from packages/qvac-lib-infer-bci-whispercpp/addonLogging.d.ts rename to packages/bci-whispercpp/addonLogging.d.ts diff --git a/packages/qvac-lib-infer-bci-whispercpp/addonLogging.js b/packages/bci-whispercpp/addonLogging.js similarity index 100% rename from packages/qvac-lib-infer-bci-whispercpp/addonLogging.js rename to packages/bci-whispercpp/addonLogging.js diff --git a/packages/qvac-lib-infer-bci-whispercpp/bci.js b/packages/bci-whispercpp/bci.js similarity index 95% rename from packages/qvac-lib-infer-bci-whispercpp/bci.js rename to packages/bci-whispercpp/bci.js index bfee9c8bd1..1caefccf11 100644 --- a/packages/qvac-lib-infer-bci-whispercpp/bci.js +++ b/packages/bci-whispercpp/bci.js @@ -219,6 +219,8 @@ class BCIInterface { } const currentJobId = this._nextJobId const input = this._concatBufferedSignal() + const previousState = this._state + const previousJobId = this._activeJobId let accepted = false try { @@ -227,11 +229,13 @@ class BCIInterface { input }) } catch (err) { - this._setState(state.LISTENING) + this._activeJobId = previousJobId + this._setState(previousState) throw err } if (!accepted) { - this._setState(state.LISTENING) + this._activeJobId = previousJobId + this._setState(previousState) throw new QvacErrorAddonBCI({ code: ERR_CODES.JOB_ALREADY_RUNNING }) } @@ -292,6 +296,8 @@ class BCIInterface { } const candidateJobId = this._nextJobId + const previousState = this._state + const previousJobId = this._activeJobId let accepted = false try { accepted = this._binding.runJob(this._handle, { @@ -299,7 +305,8 @@ class BCIInterface { input: data.input }) } catch (err) { - this._setState(state.LISTENING) + this._activeJobId = previousJobId + this._setState(previousState) throw new QvacErrorAddonBCI({ code: ERR_CODES.FAILED_TO_START_JOB, adds: err.message, @@ -308,7 +315,8 @@ class BCIInterface { } if (!accepted) { - this._setState(state.LISTENING) + this._activeJobId = previousJobId + this._setState(previousState) return false } diff --git a/packages/qvac-lib-infer-bci-whispercpp/binding.js b/packages/bci-whispercpp/binding.js similarity index 100% rename from packages/qvac-lib-infer-bci-whispercpp/binding.js rename to packages/bci-whispercpp/binding.js diff --git a/packages/qvac-lib-infer-bci-whispercpp/configChecker.js b/packages/bci-whispercpp/configChecker.js similarity index 100% rename from packages/qvac-lib-infer-bci-whispercpp/configChecker.js rename to packages/bci-whispercpp/configChecker.js diff --git a/packages/qvac-lib-infer-bci-whispercpp/examples/transcribe-neural.js b/packages/bci-whispercpp/examples/transcribe-neural.js similarity index 100% rename from packages/qvac-lib-infer-bci-whispercpp/examples/transcribe-neural.js rename to packages/bci-whispercpp/examples/transcribe-neural.js index 4b0a3f4717..1ca304b2ef 100644 --- a/packages/qvac-lib-infer-bci-whispercpp/examples/transcribe-neural.js +++ b/packages/bci-whispercpp/examples/transcribe-neural.js @@ -111,6 +111,12 @@ async function main () { console.log('Average WER: ' + (avgWER * 100).toFixed(1) + '% (n=' + total + ')') console.log('Time: ' + elapsed + 's') } else { + const signalPath = args[0] + if (!fs.existsSync(signalPath)) { + console.error('Error: Signal file not found: ' + signalPath) + return + } + const bci = new BCIWhispercpp({ files: { model: modelPath } }, { @@ -121,12 +127,6 @@ async function main () { await bci.load() console.log('Model loaded.\n') - const signalPath = args[0] - if (!fs.existsSync(signalPath)) { - console.error('Error: Signal file not found: ' + signalPath) - return - } - const buf = fs.readFileSync(signalPath) const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength) const T = view.getUint32(0, true) diff --git a/packages/qvac-lib-infer-bci-whispercpp/index.d.ts b/packages/bci-whispercpp/index.d.ts similarity index 100% rename from packages/qvac-lib-infer-bci-whispercpp/index.d.ts rename to packages/bci-whispercpp/index.d.ts diff --git a/packages/qvac-lib-infer-bci-whispercpp/index.js b/packages/bci-whispercpp/index.js similarity index 85% rename from packages/qvac-lib-infer-bci-whispercpp/index.js rename to packages/bci-whispercpp/index.js index 5bd9640c90..e45e3916de 100644 --- a/packages/qvac-lib-infer-bci-whispercpp/index.js +++ b/packages/bci-whispercpp/index.js @@ -107,8 +107,13 @@ class BCIWhispercpp { configurationParams.bciConfig = this._config.bciConfig } - // Validation happens once inside BCIInterface's constructor via - // configChecker.checkConfig. Calling it here too would duplicate the work. + if (this.state.destroyed) { + throw new QvacErrorAddonBCI({ + code: ERR_CODES.MODEL_NOT_LOADED, + adds: 'instance was destroyed' + }) + } + const binding = require('./binding') try { this.addon = new BCIInterface( @@ -119,8 +124,9 @@ class BCIWhispercpp { ) } catch (err) { this.addon = null + const configError = this._isConfigurationError(err) throw new QvacErrorAddonBCI({ - code: ERR_CODES.FAILED_TO_LOAD_WEIGHTS, + code: configError ? ERR_CODES.INVALID_CONFIG : ERR_CODES.FAILED_TO_LOAD_WEIGHTS, adds: err.message, cause: err }) @@ -149,6 +155,7 @@ class BCIWhispercpp { * @returns {Promise} */ async transcribe (neuralData) { + this._assertReadyForInference() return await this._enqueueInference(async () => { const response = this._job.start() @@ -193,6 +200,29 @@ class BCIWhispercpp { return response } + _assertReadyForInference () { + if (this.state.destroyed || !this.state.configLoaded || !this.addon) { + throw new QvacErrorAddonBCI({ + code: ERR_CODES.MODEL_NOT_LOADED, + adds: this.state.destroyed ? 'instance was destroyed' : 'call load() before transcribe()' + }) + } + } + + _isConfigurationError (err) { + const message = String(err?.message || '') + return ( + message.includes('object is required') || + message.includes('is not a valid parameter') || + message.includes('bciConfig.day_idx') || + message.includes('Unknown whisperConfig key') || + message.includes('Unknown contextParams key') || + message.includes('Unknown miscConfig key') || + message.includes('error in whisperConfig handler') || + message.includes('error in contextParams handler') + ) + } + _outputCallback (addon, event, jobId, data, error) { if (event === 'Error') { this.logger.error('Job ' + jobId + ' failed with error: ' + error) diff --git a/packages/qvac-lib-infer-bci-whispercpp/lib/error.js b/packages/bci-whispercpp/lib/error.js similarity index 100% rename from packages/qvac-lib-infer-bci-whispercpp/lib/error.js rename to packages/bci-whispercpp/lib/error.js diff --git a/packages/qvac-lib-infer-bci-whispercpp/lib/wer.js b/packages/bci-whispercpp/lib/wer.js similarity index 100% rename from packages/qvac-lib-infer-bci-whispercpp/lib/wer.js rename to packages/bci-whispercpp/lib/wer.js diff --git a/packages/qvac-lib-infer-bci-whispercpp/package.json b/packages/bci-whispercpp/package.json similarity index 96% rename from packages/qvac-lib-infer-bci-whispercpp/package.json rename to packages/bci-whispercpp/package.json index 75c88aa9fa..614600732e 100644 --- a/packages/qvac-lib-infer-bci-whispercpp/package.json +++ b/packages/bci-whispercpp/package.json @@ -37,7 +37,7 @@ "repository": { "type": "git", "url": "git+https://github.com/tetherto/qvac.git", - "directory": "packages/qvac-lib-infer-bci-whispercpp" + "directory": "packages/bci-whispercpp" }, "author": "Tether", "keywords": [ @@ -51,7 +51,7 @@ ], "license": "Apache-2.0", "bugs": "https://github.com/tetherto/qvac/issues", - "homepage": "https://github.com/tetherto/qvac/tree/main/packages/qvac-lib-infer-bci-whispercpp#readme", + "homepage": "https://github.com/tetherto/qvac/tree/main/packages/bci-whispercpp#readme", "devDependencies": { "@types/node": "^24.2.1", "bare-buffer": "^3.4.2", diff --git a/packages/qvac-lib-infer-bci-whispercpp/scripts/convert-model.py b/packages/bci-whispercpp/scripts/convert-model.py similarity index 100% rename from packages/qvac-lib-infer-bci-whispercpp/scripts/convert-model.py rename to packages/bci-whispercpp/scripts/convert-model.py diff --git a/packages/qvac-lib-infer-bci-whispercpp/scripts/download-models.sh b/packages/bci-whispercpp/scripts/download-models.sh similarity index 74% rename from packages/qvac-lib-infer-bci-whispercpp/scripts/download-models.sh rename to packages/bci-whispercpp/scripts/download-models.sh index 284885f7b0..8419eab3a5 100755 --- a/packages/qvac-lib-infer-bci-whispercpp/scripts/download-models.sh +++ b/packages/bci-whispercpp/scripts/download-models.sh @@ -35,15 +35,18 @@ download_models() { download_fixtures() { mkdir -p "$FIXTURES_DIR" + local temp_dir archive_path + temp_dir="$(mktemp -d "${TMPDIR:-/tmp}/bci-test-fixtures.XXXXXX")" + archive_path="${temp_dir}/bci-test-fixtures.tar.gz" + trap 'rm -rf "$temp_dir"' RETURN echo "Downloading BCI test fixtures..." gh release download "$RELEASE_TAG" \ --repo "$RELEASE_REPO" \ - --pattern "bci-test-fixtures.tar.gz" --dir /tmp \ + --pattern "bci-test-fixtures.tar.gz" --dir "$temp_dir" \ --clobber - tar xzf /tmp/bci-test-fixtures.tar.gz -C "$FIXTURES_DIR/" - rm -f /tmp/bci-test-fixtures.tar.gz + tar xzf "$archive_path" -C "$FIXTURES_DIR/" echo "Test fixtures:" && ls -lh "$FIXTURES_DIR"/*.bin } @@ -51,7 +54,12 @@ download_fixtures() { case "${1:-all}" in --models) download_models ;; --fixtures) download_fixtures ;; - all|*) download_models; echo; download_fixtures ;; + all) download_models; echo; download_fixtures ;; + *) + echo "Unknown option: ${1}" + echo "Usage: bash scripts/download-models.sh [all|--models|--fixtures]" + exit 1 + ;; esac echo "" diff --git a/packages/qvac-lib-infer-bci-whispercpp/test/fixtures/manifest.json b/packages/bci-whispercpp/test/fixtures/manifest.json similarity index 100% rename from packages/qvac-lib-infer-bci-whispercpp/test/fixtures/manifest.json rename to packages/bci-whispercpp/test/fixtures/manifest.json diff --git a/packages/qvac-lib-infer-bci-whispercpp/test/integration/addon.test.js b/packages/bci-whispercpp/test/integration/addon.test.js similarity index 100% rename from packages/qvac-lib-infer-bci-whispercpp/test/integration/addon.test.js rename to packages/bci-whispercpp/test/integration/addon.test.js diff --git a/packages/qvac-lib-infer-bci-whispercpp/test/integration/helpers.js b/packages/bci-whispercpp/test/integration/helpers.js similarity index 100% rename from packages/qvac-lib-infer-bci-whispercpp/test/integration/helpers.js rename to packages/bci-whispercpp/test/integration/helpers.js diff --git a/packages/qvac-lib-infer-bci-whispercpp/tsconfig.dts.json b/packages/bci-whispercpp/tsconfig.dts.json similarity index 100% rename from packages/qvac-lib-infer-bci-whispercpp/tsconfig.dts.json rename to packages/bci-whispercpp/tsconfig.dts.json diff --git a/packages/qvac-lib-infer-bci-whispercpp/vcpkg-configuration.json b/packages/bci-whispercpp/vcpkg-configuration.json similarity index 100% rename from packages/qvac-lib-infer-bci-whispercpp/vcpkg-configuration.json rename to packages/bci-whispercpp/vcpkg-configuration.json diff --git a/packages/qvac-lib-infer-bci-whispercpp/vcpkg-overlays/whisper-cpp/portfile.cmake b/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/portfile.cmake similarity index 100% rename from packages/qvac-lib-infer-bci-whispercpp/vcpkg-overlays/whisper-cpp/portfile.cmake rename to packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/portfile.cmake diff --git a/packages/qvac-lib-infer-bci-whispercpp/vcpkg-overlays/whisper-cpp/vcpkg.json b/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/vcpkg.json similarity index 100% rename from packages/qvac-lib-infer-bci-whispercpp/vcpkg-overlays/whisper-cpp/vcpkg.json rename to packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/vcpkg.json diff --git a/packages/qvac-lib-infer-bci-whispercpp/vcpkg.json b/packages/bci-whispercpp/vcpkg.json similarity index 100% rename from packages/qvac-lib-infer-bci-whispercpp/vcpkg.json rename to packages/bci-whispercpp/vcpkg.json diff --git a/packages/qvac-lib-infer-bci-whispercpp/vcpkg/toolchains/linux-clang.cmake b/packages/bci-whispercpp/vcpkg/toolchains/linux-clang.cmake similarity index 100% rename from packages/qvac-lib-infer-bci-whispercpp/vcpkg/toolchains/linux-clang.cmake rename to packages/bci-whispercpp/vcpkg/toolchains/linux-clang.cmake diff --git a/packages/qvac-lib-infer-bci-whispercpp/vcpkg/triplets/arm64-linux.cmake b/packages/bci-whispercpp/vcpkg/triplets/arm64-linux.cmake similarity index 100% rename from packages/qvac-lib-infer-bci-whispercpp/vcpkg/triplets/arm64-linux.cmake rename to packages/bci-whispercpp/vcpkg/triplets/arm64-linux.cmake diff --git a/packages/qvac-lib-infer-bci-whispercpp/vcpkg/triplets/x64-linux.cmake b/packages/bci-whispercpp/vcpkg/triplets/x64-linux.cmake similarity index 100% rename from packages/qvac-lib-infer-bci-whispercpp/vcpkg/triplets/x64-linux.cmake rename to packages/bci-whispercpp/vcpkg/triplets/x64-linux.cmake diff --git a/packages/qvac-lib-infer-bci-whispercpp/docs/BCI_V184_COMPAT.md b/packages/qvac-lib-infer-bci-whispercpp/docs/BCI_V184_COMPAT.md deleted file mode 100644 index 30bd87cc62..0000000000 --- a/packages/qvac-lib-infer-bci-whispercpp/docs/BCI_V184_COMPAT.md +++ /dev/null @@ -1,56 +0,0 @@ -# BCI whisper.cpp v1.8.4 Compatibility - -## Goal - -Get the BCI addon working on whisper.cpp v1.8.4.1 (from `tetherto/qvac-ext-lib-whisper.cpp`) instead of the current v1.7.6 (from `ggml-org/whisper.cpp`). - -## Status: FIXED - -| Version | Source | WER | Status | -|---------|--------|-----|--------| -| v1.7.6 (`a8d002cf`) | `ggml-org/whisper.cpp` + 4 overlay patches | **10.4%** | Working | -| v1.8.4.1 (unpatched) | `tetherto/qvac-ext-lib-whisper.cpp` + BCI patches | **91.9%** | Broken (garbage output) | -| v1.8.4.1 (patched) | `tetherto/qvac-ext-lib-whisper.cpp` + BCI patches + `0005` fix | **10.4%** | Working (identical to v1.7.6) | - -## Root Cause - -The issue was **not** in the ggml submodule. It was a **graph placement bug** introduced when the BCI windowed attention patch was ported from v1.7.6 to v1.8.4. - -In v1.7.6, `whisper_encode_internal` built a single monolithic computation graph for the entire encoder. The BCI windowed attention patch added: -1. A `window_mask` tensor created in the graph builder -2. Mask data population via `ggml_graph_get_tensor(gf, "window_mask")` after graph allocation, before graph computation - -In v1.8.4, the encoder was refactored into **three separate computation graphs**: -1. `whisper_build_graph_conv` — convolution layers -2. `whisper_build_graph_encoder` — self-attention encoder layers -3. `whisper_build_graph_cross` — cross-attention KV pre-computation - -The BCI patch correctly placed the `window_mask` tensor creation in `whisper_build_graph_encoder` (step 2), but the mask **data population** code was placed in the **cross-attention section** (step 3) of `whisper_encode_internal`. Since the cross graph doesn't contain a `window_mask` tensor, `ggml_graph_get_tensor(gf, "window_mask")` returned `nullptr`, and the mask was never initialized. The encoder ran with an uninitialized attention mask, producing garbage output. - -## Fix - -Patch `0005-fix-bci-window-mask-encoder-graph.patch` moves the `window_mask` data population from the cross-attention section to the encoder section of `whisper_encode_internal`, between `ggml_backend_sched_alloc_graph` and `ggml_graph_compute_helper`. - -## What Was Ruled Out (previously investigated) - -1. **Flash attention default change** — v1.8.4 sets `flash_attn = true` by default (was `false` in v1.7.6). The BCI patch already bypasses flash attention when `window_mask` is active. - -2. **`ggml_mul_mat_pad` removal** — v1.7.6 had a Metal-specific matrix multiplication padding optimization. Restoring this to v1.8.4 does not fix the quality issue. - -3. **Decoder prompt handling changes** — v1.8.4 refactored `prompt_past` into `prompt_past0`/`prompt_past1` for the `carry_initial_prompt` feature. BCI transcriptions are single-segment and the first-segment codepath is functionally equivalent. - -4. **KQ mask padding removal** — v1.8.4 removed `GGML_KQ_MASK_PAD` from the decoder attention mask. - -5. **ggml submodule changes** — 1,190 commits changed the ggml library between v1.7.6 and v1.8.4.1, but this was not the cause. - -## Fork PR - -[tetherto/qvac-ext-lib-whisper.cpp#10](https://github.com/tetherto/qvac-ext-lib-whisper.cpp/pull/10) — BCI patches (conv1 kernel + windowed attention + flash attn bypass) on v1.8.4.1 base. Needs the `0005` fix patch applied. - -## Files - -- BCI model: `models/ggml-bci-windowed.bin` -- Embedder weights: `models/bci-embedder.bin` -- Conversion script: `scripts/convert-model.py` -- Overlay portfile: `vcpkg-overlays/whisper-cpp/portfile.cmake` (points to `tetherto/qvac-ext-lib-whisper.cpp` at the merged master commit `2b1e04f20bad9a72321e72df8d6a8c14aae98adc`) -- Test: `test/integration/addon.test.js` diff --git a/packages/qvac-lib-infer-bci-whispercpp/vcpkg-overlays/qvac-lint-cpp/portfile.cmake b/packages/qvac-lib-infer-bci-whispercpp/vcpkg-overlays/qvac-lint-cpp/portfile.cmake deleted file mode 100644 index ff8c032cac..0000000000 --- a/packages/qvac-lib-infer-bci-whispercpp/vcpkg-overlays/qvac-lint-cpp/portfile.cmake +++ /dev/null @@ -1,7 +0,0 @@ -file(WRITE "${CURRENT_PACKAGES_DIR}/share/${PORT}/.clang-format" "") -file(WRITE "${CURRENT_PACKAGES_DIR}/share/${PORT}/.clang-tidy" "") -file(WRITE "${CURRENT_PACKAGES_DIR}/share/${PORT}/.valgrind.supp" "") -file(MAKE_DIRECTORY "${CURRENT_PACKAGES_DIR}/tools/${PORT}/hooks") -file(WRITE "${CURRENT_PACKAGES_DIR}/tools/${PORT}/hooks/pre-commit" "#!/bin/sh\nexit 0\n") -file(WRITE "${CURRENT_PACKAGES_DIR}/share/${PORT}/copyright" "Stub overlay port") - diff --git a/packages/qvac-lib-infer-bci-whispercpp/vcpkg-overlays/qvac-lint-cpp/vcpkg.json b/packages/qvac-lib-infer-bci-whispercpp/vcpkg-overlays/qvac-lint-cpp/vcpkg.json deleted file mode 100644 index 0a180e7609..0000000000 --- a/packages/qvac-lib-infer-bci-whispercpp/vcpkg-overlays/qvac-lint-cpp/vcpkg.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "name": "qvac-lint-cpp", - "version-string": "1.4.1", - "description": "No-op overlay — linting headers not needed for runtime builds" -} From 129c9ecd17ceb8dd253afa0e7636b182f748af62 Mon Sep 17 00:00:00 2001 From: Raju Date: Tue, 21 Apr 2026 14:54:31 +0530 Subject: [PATCH 13/19] fix[api](bci): address code review findings across JS, C++, and build config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Replace cd && chain in test:cpp:run with direct path (CLAUDE.md compliance) - Route whisper_log_set through addon-cpp logger instead of silencing with once_flag, preventing inter-addon log handler clobber when BCI and transcription-whispercpp coexist in the same process - Fix stats heuristic in bci.js _addonOutputCallback to match actual BCIModel::runtimeStats keys (tokensPerSecond/totalWallMs, not the audio-addon keys audioDurationMs/totalSamples) - Drain _inferenceQueueWaiter in unload()/destroy() before calling destroyInstance(), closing the race where destroy could fire while process() is mid-execution on the native thread - Remove auto-load in BCIModel::process — throw immediately if context is null instead of lazy-loading outside the controlled lifecycle - Remove dead set_weights_for_file snake_case stub and unused - Add qvac-lint-cpp to vcpkg.json dependencies (matches all peer addons) - Remove empty qvac-lint-cpp overlay directory (per Gustavo review) - Remove stale bci_wer/bci_transcription from manifest.json - Stop gitignoring package-lock.json (match monorepo convention) - Move computeWER into BCIWhispercpp namespace in index.d.ts - Downgrade @types/node to ^22.15.3, remove bare-fs from devDeps - Fix PR template code blocks from typescript to javascript Made-with: Cursor --- packages/bci-whispercpp/.gitignore | 1 - .../bci-whispercpp/PULL_REQUEST_TEMPLATE.md | 6 ++-- .../addon/src/addon/AddonJs.hpp | 28 +++++++++++-------- .../src/model-interface/bci/BCIModel.cpp | 3 +- .../src/model-interface/bci/BCIModel.hpp | 4 --- packages/bci-whispercpp/bci.js | 4 +-- packages/bci-whispercpp/index.d.ts | 15 +++++----- packages/bci-whispercpp/index.js | 2 ++ packages/bci-whispercpp/package.json | 5 ++-- .../test/fixtures/manifest.json | 20 ++++--------- packages/bci-whispercpp/vcpkg.json | 1 + 11 files changed, 39 insertions(+), 50 deletions(-) diff --git a/packages/bci-whispercpp/.gitignore b/packages/bci-whispercpp/.gitignore index 33aefedf56..c95b3f2f4b 100644 --- a/packages/bci-whispercpp/.gitignore +++ b/packages/bci-whispercpp/.gitignore @@ -2,7 +2,6 @@ node_modules/ build/ prebuilds/ models/ -package-lock.json test/fixtures/*.bin .clang-format .clang-tidy diff --git a/packages/bci-whispercpp/PULL_REQUEST_TEMPLATE.md b/packages/bci-whispercpp/PULL_REQUEST_TEMPLATE.md index b9b618fd02..0240796181 100644 --- a/packages/bci-whispercpp/PULL_REQUEST_TEMPLATE.md +++ b/packages/bci-whispercpp/PULL_REQUEST_TEMPLATE.md @@ -23,13 +23,13 @@ **BEFORE:** -```typescript +```javascript // old code example ``` **AFTER:** -```typescript +```javascript // new code example ``` @@ -37,6 +37,6 @@ **Delete this section if not applicable.** -```typescript +```javascript // new API usage example ``` diff --git a/packages/bci-whispercpp/addon/src/addon/AddonJs.hpp b/packages/bci-whispercpp/addon/src/addon/AddonJs.hpp index 18b20a5c97..81fb828fa1 100644 --- a/packages/bci-whispercpp/addon/src/addon/AddonJs.hpp +++ b/packages/bci-whispercpp/addon/src/addon/AddonJs.hpp @@ -2,7 +2,6 @@ #include #include -#include #include #include #include @@ -26,9 +25,6 @@ namespace qvac_lib_inference_addon_bci { namespace js = qvac_lib_inference_addon_cpp::js; using qvac_lib_inference_addon_cpp::OutputQueue; -inline void disableWhisperLogs( - enum ggml_log_level, const char*, void*) {} - inline BCIConfig createBCIConfig(js_env_t* env, const js::Object& configurationParams) { JSAdapter adapter; @@ -95,14 +91,22 @@ inline js_value_t* createInstance(js_env_t* env, js_callback_info_t* info) try { using namespace qvac_lib_inference_addon_cpp; using namespace std; - // whisper_log_set is a process-wide global. Only install our silencing - // handler once; otherwise every addon-instance construction clobbers any - // handler a coexisting whisper-based addon (e.g. @qvac/transcription- - // whispercpp) may have installed in the same process. - static std::once_flag logOnce; - std::call_once(logOnce, []() { - whisper_log_set(disableWhisperLogs, nullptr); - }); + // Route whisper.cpp log output through the addon-cpp logger instead of + // silencing globally. This avoids clobbering log handlers installed by + // coexisting whisper-based addons (e.g. @qvac/transcription-whispercpp) + // in the same process, since each addon instance re-sets the handler + // to its own logger anyway. + whisper_log_set( + [](enum ggml_log_level level, const char* text, void*) { + if (text == nullptr) return; + auto prio = (level == GGML_LOG_LEVEL_ERROR) + ? qvac_lib_inference_addon_cpp::logger::Priority::ERROR + : (level == GGML_LOG_LEVEL_WARN) + ? qvac_lib_inference_addon_cpp::logger::Priority::WARNING + : qvac_lib_inference_addon_cpp::logger::Priority::DEBUG; + QLOG(prio, std::string("[whisper.cpp] ") + text); + }, + nullptr); JsArgsParser args(env, info); auto configurationParams = args.getJsObject(1, "configurationParams"); diff --git a/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp b/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp index 36e612e801..3fc29c8e5f 100644 --- a/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp +++ b/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp @@ -217,9 +217,8 @@ void BCIModel::warmup() { } void BCIModel::process(const Input& rawNeuralData) { - if (ctx_ == nullptr) load(); if (ctx_ == nullptr) { - throw std::runtime_error("BCI Whisper context is not initialized"); + throw std::runtime_error("BCI Whisper context is not initialized — call load() first"); } if (cancelRequested_.load(std::memory_order_relaxed)) { diff --git a/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.hpp b/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.hpp index 88dc01b848..7c02e176f5 100644 --- a/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.hpp +++ b/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.hpp @@ -4,7 +4,6 @@ #include #include #include -#include #include #include @@ -77,9 +76,6 @@ class BCIModel void setWeightsForFile( const std::string&, std::unique_ptr>&&) override {} - void set_weights_for_file( - const std::string&, - const std::span&, bool) {} bool isLoaded() const { return is_loaded_; } qvac_lib_inference_addon_cpp::RuntimeStats runtimeStats() const override; void warmup(); diff --git a/packages/bci-whispercpp/bci.js b/packages/bci-whispercpp/bci.js index 1caefccf11..c379f875b8 100644 --- a/packages/bci-whispercpp/bci.js +++ b/packages/bci-whispercpp/bci.js @@ -65,8 +65,8 @@ class BCIInterface { const isError = typeof error === 'string' && error.length > 0 const isStats = data && typeof data === 'object' && ( 'totalTime' in data || - 'audioDurationMs' in data || - 'totalSamples' in data + 'tokensPerSecond' in data || + 'totalWallMs' in data ) const isTranscriptOutput = ( (Array.isArray(data) && data.length > 0) || diff --git a/packages/bci-whispercpp/index.d.ts b/packages/bci-whispercpp/index.d.ts index 87519089cc..9ca0f42542 100644 --- a/packages/bci-whispercpp/index.d.ts +++ b/packages/bci-whispercpp/index.d.ts @@ -111,13 +111,13 @@ declare class BCIWhispercpp { getState(): BCIWhispercppState } -/** - * Compute Word Error Rate between hypothesis and reference strings. - * @returns WER as a ratio (0.0 = perfect). - */ -declare function computeWER(hypothesis: string, reference: string): number - declare namespace BCIWhispercpp { + /** + * Compute Word Error Rate between hypothesis and reference strings. + * @returns WER as a ratio (0.0 = perfect). + */ + function computeWER(hypothesis: string, reference: string): number + export { BCIWhispercpp as default, BCIWhispercpp, @@ -127,8 +127,7 @@ declare namespace BCIWhispercpp { BCIWhispercppArgs, BCIWhispercppConfig, BCIWhispercppState, - TranscriptSegment, - computeWER + TranscriptSegment } } diff --git a/packages/bci-whispercpp/index.js b/packages/bci-whispercpp/index.js index e45e3916de..8cfcbc909a 100644 --- a/packages/bci-whispercpp/index.js +++ b/packages/bci-whispercpp/index.js @@ -256,6 +256,7 @@ class BCIWhispercpp { async unload () { return await this._withExclusiveRun(async () => { + await this._inferenceQueueWaiter if (this.addon) { await this.addon.destroyInstance() this.addon = null @@ -269,6 +270,7 @@ class BCIWhispercpp { async destroy () { return await this._withExclusiveRun(async () => { + await this._inferenceQueueWaiter if (this.addon) { await this.addon.destroyInstance() this.addon = null diff --git a/packages/bci-whispercpp/package.json b/packages/bci-whispercpp/package.json index 614600732e..65dabee614 100644 --- a/packages/bci-whispercpp/package.json +++ b/packages/bci-whispercpp/package.json @@ -15,7 +15,7 @@ "test": "npm run test:integration", "test:integration": "brittle-bare test/integration/addon.test.js", "test:cpp:build": "bare-make generate -D BUILD_TESTING=ON && bare-make build --target test-bci-core && bare-make install", - "test:cpp:run": "cd build/addon/tests/ && ./test-bci-core --gtest_output=xml:cpp-test-results.xml", + "test:cpp:run": "./build/addon/tests/test-bci-core --gtest_output=xml:cpp-test-results.xml", "test:cpp": "npm run test:cpp:build && npm run test:cpp:run", "test:dts": "tsc -p tsconfig.dts.json" }, @@ -53,9 +53,8 @@ "bugs": "https://github.com/tetherto/qvac/issues", "homepage": "https://github.com/tetherto/qvac/tree/main/packages/bci-whispercpp#readme", "devDependencies": { - "@types/node": "^24.2.1", + "@types/node": "^22.15.3", "bare-buffer": "^3.4.2", - "bare-fs": "^4.5.1", "bare-tty": "^5.0.3", "brittle": "^3.17.0", "cmake-bare": "^1.7.5", diff --git a/packages/bci-whispercpp/test/fixtures/manifest.json b/packages/bci-whispercpp/test/fixtures/manifest.json index 79a32b7641..10da2de6c0 100644 --- a/packages/bci-whispercpp/test/fixtures/manifest.json +++ b/packages/bci-whispercpp/test/fixtures/manifest.json @@ -5,45 +5,35 @@ "timesteps": 910, "channels": 512, "expected_text": "You can see the code at this point as well.", - "day_idx": 1, - "bci_transcription": "you can see the good at this point as well", - "bci_wer": 0.1 + "day_idx": 1 }, { "file": "neural_sample_1.bin", "timesteps": 749, "channels": 512, "expected_text": "How does it keep the cost down?", - "day_idx": 1, - "bci_transcription": "how does it keep the cost said", - "bci_wer": 0.1429 + "day_idx": 1 }, { "file": "neural_sample_2.bin", "timesteps": 502, "channels": 512, "expected_text": "Not too controversial.", - "day_idx": 1, - "bci_transcription": "not too controversial", - "bci_wer": 0.0 + "day_idx": 1 }, { "file": "neural_sample_3.bin", "timesteps": 962, "channels": 512, "expected_text": "The jury and a judge work together on it.", - "day_idx": 1, - "bci_transcription": "the jury and a judge work together on it", - "bci_wer": 0.0 + "day_idx": 1 }, { "file": "neural_sample_4.bin", "timesteps": 584, "channels": 512, "expected_text": "Were quite vocal about it.", - "day_idx": 1, - "bci_transcription": "we're quite vocal about it", - "bci_wer": 0.2 + "day_idx": 1 } ] } diff --git a/packages/bci-whispercpp/vcpkg.json b/packages/bci-whispercpp/vcpkg.json index 42b6619b98..05d929cba1 100644 --- a/packages/bci-whispercpp/vcpkg.json +++ b/packages/bci-whispercpp/vcpkg.json @@ -6,6 +6,7 @@ "name": "qvac-lib-inference-addon-cpp", "version>=": "1.1.5#1" }, + "qvac-lint-cpp", "whisper-cpp" ], "features": { From e6eb659d56c39ce4fd4f801e9b6a892ff262886d Mon Sep 17 00:00:00 2001 From: Raju Date: Tue, 21 Apr 2026 15:43:08 +0530 Subject: [PATCH 14/19] =?UTF-8?q?fix[api](bci):=20address=20review=20findi?= =?UTF-8?q?ngs=20=E2=80=94=20standards=20alignment,=20structured=20errors,?= =?UTF-8?q?=20lifecycle=20safety?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Align bci-whispercpp with monorepo conventions and fix code quality issues found during thorough review of the POC implementation. Build/config: - .gitignore aligned with peer addons (package-lock.json, .npmrc, IDE files, vcpkg cache, generated test bundles) - vcpkg.json: use "version" instead of deprecated "version-string" - package.json: replace $(find) in lint-cpp with explicit file list, remove unused bare-stream/bare-tty deps, add bare-fs to production deps - CHANGELOG.md: add date per Keep a Changelog format JS fixes: - Move fs.existsSync model check from constructor to _load(), matching TranscriptionWhispercpp lifecycle pattern - Remove dead PAUSED/STOPPED state enum values from bci.js - Add explicit event name matching alongside heuristic fallback in _addonOutputCallback (matches peer whisper.js pattern with BCI stat keys) - Add miscConfig.caption_enabled boolean type validation in configChecker - Extract duplicated flattenSegments into shared lib/util.js - Fix index.d.ts import from fragile internal path to stable @qvac/infer-base C++ fixes: - Guard whisper_log_set with std::once_flag to prevent clobbering log handlers from coexisting whisper-based addons in the same process - Replace std::runtime_error with structured StatusError/bci_error::makeStatus in BCIModel::load() and loadEmbedderIfNeeded() for proper JS error mapping - Use std::move in process(const std::any&) to avoid copying multi-MB neural signal buffers on every inference call Made-with: Cursor --- packages/bci-whispercpp/.gitignore | 21 ++++++++++-- packages/bci-whispercpp/CHANGELOG.md | 2 +- .../addon/src/addon/AddonJs.hpp | 32 +++++++++---------- .../src/model-interface/bci/BCIModel.cpp | 21 ++++++++---- packages/bci-whispercpp/bci.js | 16 +++------- packages/bci-whispercpp/configChecker.js | 4 +++ .../examples/transcribe-neural.js | 13 +------- packages/bci-whispercpp/index.d.ts | 2 +- packages/bci-whispercpp/index.js | 19 ++++------- packages/bci-whispercpp/lib/util.js | 15 +++++++++ packages/bci-whispercpp/package.json | 5 ++- .../test/integration/addon.test.js | 13 +------- packages/bci-whispercpp/vcpkg.json | 2 +- 13 files changed, 86 insertions(+), 79 deletions(-) create mode 100644 packages/bci-whispercpp/lib/util.js diff --git a/packages/bci-whispercpp/.gitignore b/packages/bci-whispercpp/.gitignore index c95b3f2f4b..26e50985ea 100644 --- a/packages/bci-whispercpp/.gitignore +++ b/packages/bci-whispercpp/.gitignore @@ -1,8 +1,25 @@ -node_modules/ +.vs/ build/ -prebuilds/ models/ +node_modules/ +.idea/ +prebuilds/ +vcpkg/cache/ +vcpkg/ports/ +!vcpkg/triplets/ +!vcpkg/toolchains/ + test/fixtures/*.bin +test/unit/all.js +test/integration/all.js + +package-lock.json +.npmrc + +__pycache__/ +.pytest_cache/ +.vscode + .clang-format .clang-tidy .valgrind.supp diff --git a/packages/bci-whispercpp/CHANGELOG.md b/packages/bci-whispercpp/CHANGELOG.md index d51c53ae3b..46d2e6fa9f 100644 --- a/packages/bci-whispercpp/CHANGELOG.md +++ b/packages/bci-whispercpp/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [0.1.0] +## [0.1.0] - 2026-04-15 Initial POC release of `@qvac/bci-whispercpp`, a brain-computer-interface neural signal transcription addon powered by a BCI-patched fork of whisper.cpp. diff --git a/packages/bci-whispercpp/addon/src/addon/AddonJs.hpp b/packages/bci-whispercpp/addon/src/addon/AddonJs.hpp index 81fb828fa1..e010407431 100644 --- a/packages/bci-whispercpp/addon/src/addon/AddonJs.hpp +++ b/packages/bci-whispercpp/addon/src/addon/AddonJs.hpp @@ -6,6 +6,8 @@ #include #include +#include + #include #include #include @@ -91,22 +93,20 @@ inline js_value_t* createInstance(js_env_t* env, js_callback_info_t* info) try { using namespace qvac_lib_inference_addon_cpp; using namespace std; - // Route whisper.cpp log output through the addon-cpp logger instead of - // silencing globally. This avoids clobbering log handlers installed by - // coexisting whisper-based addons (e.g. @qvac/transcription-whispercpp) - // in the same process, since each addon instance re-sets the handler - // to its own logger anyway. - whisper_log_set( - [](enum ggml_log_level level, const char* text, void*) { - if (text == nullptr) return; - auto prio = (level == GGML_LOG_LEVEL_ERROR) - ? qvac_lib_inference_addon_cpp::logger::Priority::ERROR - : (level == GGML_LOG_LEVEL_WARN) - ? qvac_lib_inference_addon_cpp::logger::Priority::WARNING - : qvac_lib_inference_addon_cpp::logger::Priority::DEBUG; - QLOG(prio, std::string("[whisper.cpp] ") + text); - }, - nullptr); + static std::once_flag whisperLogOnce; + std::call_once(whisperLogOnce, []() { + whisper_log_set( + [](enum ggml_log_level level, const char* text, void*) { + if (text == nullptr) return; + auto prio = (level == GGML_LOG_LEVEL_ERROR) + ? qvac_lib_inference_addon_cpp::logger::Priority::ERROR + : (level == GGML_LOG_LEVEL_WARN) + ? qvac_lib_inference_addon_cpp::logger::Priority::WARNING + : qvac_lib_inference_addon_cpp::logger::Priority::DEBUG; + QLOG(prio, std::string("[whisper.cpp] ") + text); + }, + nullptr); + }); JsArgsParser args(env, info); auto configurationParams = args.getJsObject(1, "configurationParams"); diff --git a/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp b/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp index 3fc29c8e5f..37183ef716 100644 --- a/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp +++ b/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp @@ -87,7 +87,8 @@ void BCIModel::loadEmbedderIfNeeded() { QLOG(qvac_lib_inference_addon_cpp::logger::Priority::INFO, "Loaded BCI embedder weights from: " + embedderPath); } else { - throw std::runtime_error( + throw qvac_errors::bci_error::makeStatus( + qvac_errors::bci_error::Code::InvalidNeuralSignal, "BCI embedder weights not found at: " + embedderPath + ". This file is required for neural signal preprocessing. " "Generate it with: python3 scripts/convert-model.py --checkpoint "); @@ -101,7 +102,9 @@ void BCIModel::load() { const auto modelPathIt = cfg_.whisperContextCfg.find("model"); if (modelPathIt == cfg_.whisperContextCfg.end()) { - throw std::runtime_error("Model path not specified"); + throw qvac_errors::StatusError( + qvac_errors::general_error::InvalidArgument, + "Model path not specified in contextParams"); } const auto modelPath = std::get(modelPathIt->second); @@ -110,7 +113,9 @@ void BCIModel::load() { auto* rawCtx = whisper_init_from_file_with_params(modelPath.c_str(), contextParams); if (rawCtx == nullptr) { - throw std::runtime_error("Failed to initialize Whisper context for BCI"); + throw qvac_errors::bci_error::makeStatus( + qvac_errors::bci_error::Code::InvalidNeuralSignal, + "Failed to initialize Whisper context from: " + modelPath); } try { @@ -310,10 +315,12 @@ void BCIModel::process(const Input& rawNeuralData) { std::any BCIModel::process(const std::any& input) { AnyInput modelInput; - if (const auto* anyInput = std::any_cast(&input)) { - modelInput = *anyInput; - } else if (const auto* inputVector = std::any_cast(&input)) { - modelInput.input = *inputVector; + if (auto* anyInput = std::any_cast( + const_cast(&input))) { + modelInput = std::move(*anyInput); + } else if (auto* inputVector = std::any_cast( + const_cast(&input))) { + modelInput.input = std::move(*inputVector); } else { throw qvac_errors::StatusError( qvac_errors::general_error::InvalidArgument, diff --git a/packages/bci-whispercpp/bci.js b/packages/bci-whispercpp/bci.js index c379f875b8..bbcdb46cf7 100644 --- a/packages/bci-whispercpp/bci.js +++ b/packages/bci-whispercpp/bci.js @@ -7,9 +7,7 @@ const state = Object.freeze({ LOADING: 'loading', LISTENING: 'listening', PROCESSING: 'processing', - IDLE: 'idle', - PAUSED: 'paused', - STOPPED: 'stopped' + IDLE: 'idle' }) const END_OF_INPUT = 'end of job' @@ -74,15 +72,13 @@ class BCIInterface { ) let mappedEvent = event - if (isError || String(event).includes('Error')) { + if (event === 'Error' || isError || String(event).includes('Error')) { mappedEvent = 'Error' - } else if (isStats || String(event).includes('RuntimeStats')) { + } else if (event === 'JobEnded' || isStats || String(event).includes('RuntimeStats')) { mappedEvent = 'JobEnded' - } else if (isTranscriptOutput) { + } else if (event === 'Output' || isTranscriptOutput) { mappedEvent = 'Output' } else if (Array.isArray(data) && data.length === 0) { - // BCIModel::process returns an empty vector to avoid duplicate - // segment emissions; skip forwarding this noop event. return } @@ -95,10 +91,6 @@ class BCIInterface { this._setState(state.PROCESSING) if (this._outputCb != null) { - // Unpack transcript arrays into one event per segment so the - // QvacResponse output iterator yields flat segments (matches - // TranscriptionWhispercpp; avoids callers having to flatten - // [[seg,seg]] back out themselves). const isTranscriptArray = Array.isArray(data) && data.length > 0 && typeof data[0]?.text === 'string' const isSingleTranscript = !Array.isArray(data) && diff --git a/packages/bci-whispercpp/configChecker.js b/packages/bci-whispercpp/configChecker.js index 34732296e0..45952b35ce 100644 --- a/packages/bci-whispercpp/configChecker.js +++ b/packages/bci-whispercpp/configChecker.js @@ -76,6 +76,10 @@ function checkConfig (configObject) { throw new Error(`${userParam} is not a valid parameter for miscConfig`) } } + if (configObject.miscConfig.caption_enabled !== undefined && + typeof configObject.miscConfig.caption_enabled !== 'boolean') { + throw new Error('miscConfig.caption_enabled must be a boolean') + } if (configObject.bciConfig) { for (const userParam of Object.keys(configObject.bciConfig)) { diff --git a/packages/bci-whispercpp/examples/transcribe-neural.js b/packages/bci-whispercpp/examples/transcribe-neural.js index 1ca304b2ef..69357f5515 100644 --- a/packages/bci-whispercpp/examples/transcribe-neural.js +++ b/packages/bci-whispercpp/examples/transcribe-neural.js @@ -15,22 +15,11 @@ const fs = require('bare-fs') const path = require('bare-path') const os = require('bare-os') const BCIWhispercpp = require('../index') +const { flattenSegments } = require('../lib/util') const DEFAULT_MODEL = (os.hasEnv('WHISPER_MODEL_PATH') ? os.getEnv('WHISPER_MODEL_PATH') : null) || path.join(__dirname, '..', 'models', 'ggml-bci-windowed.bin') -function flattenSegments (output) { - const segments = [] - for (const entry of output) { - if (Array.isArray(entry)) { - segments.push(...entry) - } else if (entry && typeof entry.text === 'string') { - segments.push(entry) - } - } - return segments -} - async function main () { const args = global.Bare ? global.Bare.argv.slice(2) : process.argv.slice(2) const isBatch = args[0] === '--batch' diff --git a/packages/bci-whispercpp/index.d.ts b/packages/bci-whispercpp/index.d.ts index 9ca0f42542..e2137d2c3f 100644 --- a/packages/bci-whispercpp/index.d.ts +++ b/packages/bci-whispercpp/index.d.ts @@ -1,4 +1,4 @@ -import QvacResponse from '@qvac/infer-base/src/QvacResponse' +import type { QvacResponse } from '@qvac/infer-base' import type { LoggerInterface } from '@qvac/logging' declare interface BCIConfig { diff --git a/packages/bci-whispercpp/index.js b/packages/bci-whispercpp/index.js index 8cfcbc909a..fa26bed88a 100644 --- a/packages/bci-whispercpp/index.js +++ b/packages/bci-whispercpp/index.js @@ -36,13 +36,6 @@ class BCIWhispercpp { }) } - if (!fs.existsSync(files.model)) { - throw new QvacErrorAddonBCI({ - code: ERR_CODES.MODEL_FILE_NOT_FOUND, - adds: files.model - }) - } - this._files = { model: files.model } this._config = config this.opts = opts @@ -80,11 +73,13 @@ class BCIWhispercpp { } async _load () { - // BCI-tuned whisper defaults (beam_size, suppress_nst, temperature, - // length_penalty, ...) are applied natively in - // addon/src/model-interface/bci/BCIConfig.cpp::toWhisperFullParams. - // Only the transport-level defaults live in JS; everything else is - // delegated to the C++ layer to avoid double-specification. + if (!fs.existsSync(this._files.model)) { + throw new QvacErrorAddonBCI({ + code: ERR_CODES.MODEL_FILE_NOT_FOUND, + adds: this._files.model + }) + } + const whisperConfig = { language: 'en', n_threads: 0, diff --git a/packages/bci-whispercpp/lib/util.js b/packages/bci-whispercpp/lib/util.js new file mode 100644 index 0000000000..e4c2d0e1fd --- /dev/null +++ b/packages/bci-whispercpp/lib/util.js @@ -0,0 +1,15 @@ +'use strict' + +function flattenSegments (output) { + const segments = [] + for (const entry of output) { + if (Array.isArray(entry)) { + segments.push(...entry) + } else if (entry && typeof entry.text === 'string') { + segments.push(entry) + } + } + return segments +} + +module.exports = { flattenSegments } diff --git a/packages/bci-whispercpp/package.json b/packages/bci-whispercpp/package.json index 65dabee614..ba05ac347e 100644 --- a/packages/bci-whispercpp/package.json +++ b/packages/bci-whispercpp/package.json @@ -9,7 +9,7 @@ "scripts": { "lint": "standard \"examples/**/*.js\" \"test/**/*.js\" \"*.js\" \"lib/**/*.js\"", "lint:fix": "standard --fix \"examples/**/*.js\" \"test/**/*.js\" \"*.js\" \"lib/**/*.js\"", - "lint-cpp": "clang-tidy -p build $(find addon -name '*.cpp')", + "lint-cpp": "clang-tidy -p build addon/src/js-interface/JSAdapter.cpp addon/src/js-interface/binding.cpp addon/src/model-interface/bci/BCIConfig.cpp addon/src/model-interface/bci/BCIModel.cpp addon/src/model-interface/bci/NeuralProcessor.cpp", "build": "bare-make generate && bare-make build && bare-make install", "build:pack": "mkdir -p dist && npm pack --pack-destination dist", "test": "npm run test:integration", @@ -55,7 +55,6 @@ "devDependencies": { "@types/node": "^22.15.3", "bare-buffer": "^3.4.2", - "bare-tty": "^5.0.3", "brittle": "^3.17.0", "cmake-bare": "^1.7.5", "cmake-vcpkg": "^1.1.0", @@ -69,8 +68,8 @@ "@qvac/error": "^0.1.0", "@qvac/infer-base": "^0.4.0", "@qvac/logging": "^0.1.0", + "bare-fs": "^4.5.1", "bare-path": "^3.0.0", - "bare-stream": "^2.7.0", "path": "npm:bare-path" }, "exports": { diff --git a/packages/bci-whispercpp/test/integration/addon.test.js b/packages/bci-whispercpp/test/integration/addon.test.js index a25f7d8dd6..cac8d4811b 100644 --- a/packages/bci-whispercpp/test/integration/addon.test.js +++ b/packages/bci-whispercpp/test/integration/addon.test.js @@ -6,6 +6,7 @@ const test = require('brittle') const os = require('bare-os') const BCIWhispercpp = require('../../index') const { getTestPaths, computeWER, detectPlatform } = require('./helpers') +const { flattenSegments } = require('../../lib/util') const platform = detectPlatform() const { manifest, getSamplePath } = getTestPaths() @@ -32,18 +33,6 @@ function bciConfigFor (sample) { return typeof sample?.day_idx === 'number' ? { day_idx: sample.day_idx } : undefined } -function flattenSegments (output) { - const segments = [] - for (const entry of output) { - if (Array.isArray(entry)) { - segments.push(...entry) - } else if (entry && typeof entry.text === 'string') { - segments.push(entry) - } - } - return segments -} - test('[BCI] load and destroy via package interface', { skip: !hasModel, timeout: 120000 }, async (t) => { const bci = new BCIWhispercpp({ files: { model: MODEL_PATH } diff --git a/packages/bci-whispercpp/vcpkg.json b/packages/bci-whispercpp/vcpkg.json index 05d929cba1..d1677848ba 100644 --- a/packages/bci-whispercpp/vcpkg.json +++ b/packages/bci-whispercpp/vcpkg.json @@ -1,6 +1,6 @@ { "name": "bci-whispercpp", - "version-string": "0.1.0", + "version": "0.1.0", "dependencies": [ { "name": "qvac-lib-inference-addon-cpp", From 13bae96d51e7355fe800b538f7858d4c80c2ed78 Mon Sep 17 00:00:00 2001 From: Raju Date: Tue, 21 Apr 2026 16:15:24 +0530 Subject: [PATCH 15/19] fix[api](bci): align with peer addon standards and remove unused code - Add qvac-lint-cpp configure_file block to CMakeLists.txt (copies .clang-format, .clang-tidy, .valgrind.supp from vcpkg into build tree, matching qvac-lib-infer-whispercpp pattern) - Extend lint-cpp script to cover all .hpp header files - Match peer index.d.ts QvacResponse import path (deep import from @qvac/infer-base/src/QvacResponse) - Replace brittle string-matching in _isConfigurationError with structured error detection (TypeError, ERR_ASSERTION code checks) - Remove stale configChecker comments about unimplemented BCI params (smooth_kernel_std, smooth_kernel_size, sample_rate) - Remove unused error codes: FAILED_TO_GET_STATUS, FAILED_TO_RESET, FAILED_TO_PAUSE and their addCodes registrations - Remove unused K_SAMPLES_PER_SECOND constant from BCIModel.cpp - Remove unused include from AddonJs.hpp - Add qvac-lib-inference-addon-cpp to NOTICE C++ dependencies - Add cpp-test-results.xml to .gitignore Made-with: Cursor --- packages/bci-whispercpp/.gitignore | 2 ++ packages/bci-whispercpp/CMakeLists.txt | 8 ++++++++ packages/bci-whispercpp/NOTICE | 6 ++++++ .../bci-whispercpp/addon/src/addon/AddonJs.hpp | 4 +--- .../addon/src/model-interface/bci/BCIModel.cpp | 1 - packages/bci-whispercpp/configChecker.js | 11 ----------- packages/bci-whispercpp/index.d.ts | 2 +- packages/bci-whispercpp/index.js | 15 ++++----------- packages/bci-whispercpp/lib/error.js | 15 --------------- packages/bci-whispercpp/package.json | 2 +- 10 files changed, 23 insertions(+), 43 deletions(-) diff --git a/packages/bci-whispercpp/.gitignore b/packages/bci-whispercpp/.gitignore index 26e50985ea..b79e05d8a5 100644 --- a/packages/bci-whispercpp/.gitignore +++ b/packages/bci-whispercpp/.gitignore @@ -20,6 +20,8 @@ __pycache__/ .pytest_cache/ .vscode +cpp-test-results.xml + .clang-format .clang-tidy .valgrind.supp diff --git a/packages/bci-whispercpp/CMakeLists.txt b/packages/bci-whispercpp/CMakeLists.txt index 773473a14f..6a84dfd2dd 100644 --- a/packages/bci-whispercpp/CMakeLists.txt +++ b/packages/bci-whispercpp/CMakeLists.txt @@ -24,6 +24,14 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Linux") add_link_options(-stdlib=libc++ -static-libstdc++) endif() +find_path(VCPKG_INSTALLED_PATH share/qvac-lint-cpp/.clang-format REQUIRED) +configure_file(${VCPKG_INSTALLED_PATH}/share/qvac-lint-cpp/.clang-format + ${CMAKE_CURRENT_SOURCE_DIR}/.clang-format COPYONLY) +configure_file(${VCPKG_INSTALLED_PATH}/share/qvac-lint-cpp/.clang-tidy + ${CMAKE_CURRENT_SOURCE_DIR}/.clang-tidy COPYONLY) +configure_file(${VCPKG_INSTALLED_PATH}/share/qvac-lint-cpp/.valgrind.supp + ${CMAKE_CURRENT_SOURCE_DIR}/.valgrind.supp COPYONLY) + find_path(QVAC_LIB_INFERENCE_ADDON_CPP_INCLUDE_DIRS "qvac-lib-inference-addon-cpp/ModelInterfaces.hpp") find_package(whisper CONFIG REQUIRED) diff --git a/packages/bci-whispercpp/NOTICE b/packages/bci-whispercpp/NOTICE index 70fbe1dfef..d2d9086b7b 100644 --- a/packages/bci-whispercpp/NOTICE +++ b/packages/bci-whispercpp/NOTICE @@ -66,6 +66,12 @@ PyTorch checkpoints to the GGML + embedder binary format requires: C++ Dependencies ========================================================================= +--- apache-2.0 (Apache License 2.0) --- + + qvac-lib-inference-addon-cpp + https://github.com/tetherto/qvac + Copyright (c) 2024-2026 Tether Data, S.A. de C.V. + --- mit (MIT License) --- whisper-cpp diff --git a/packages/bci-whispercpp/addon/src/addon/AddonJs.hpp b/packages/bci-whispercpp/addon/src/addon/AddonJs.hpp index e010407431..d849efb23e 100644 --- a/packages/bci-whispercpp/addon/src/addon/AddonJs.hpp +++ b/packages/bci-whispercpp/addon/src/addon/AddonJs.hpp @@ -2,12 +2,10 @@ #include #include -#include +#include #include #include -#include - #include #include #include diff --git a/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp b/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp index 37183ef716..40637971c5 100644 --- a/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp +++ b/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp @@ -16,7 +16,6 @@ namespace qvac_lib_inference_addon_bci { namespace { -constexpr double K_SAMPLES_PER_SECOND = 16000.0; constexpr float K_SEGMENT_TIMESTAMP_SCALE = 0.01F; constexpr int K_WARMUP_SAMPLE_COUNT = 8000; constexpr int K_DUMMY_AUDIO_30S = 16000 * 30; diff --git a/packages/bci-whispercpp/configChecker.js b/packages/bci-whispercpp/configChecker.js index 45952b35ce..4c684bd706 100644 --- a/packages/bci-whispercpp/configChecker.js +++ b/packages/bci-whispercpp/configChecker.js @@ -44,17 +44,6 @@ function checkConfig (configObject) { 'caption_enabled' ] - // Only parameters wired through to the C++ addon are accepted. Adding - // smoothing/sample-rate knobs here without consuming them in NeuralProcessor - // would silently drop user intent, so they are kept out until implemented. - // - // `day_idx` is an integer session index. - // - day_idx >= 0 applies the day-specific projection matrices from - // bci-embedder.bin, clamped to the valid range at the C++ layer. - // - day_idx === -1 is a passthrough escape hatch that skips - // preprocessing entirely and treats the input buffer as - // pre-computed 512-bin mel features in frame-major layout. Useful - // for reproducing the Python reference output end-to-end. const validBCIParams = [ 'day_idx' ] diff --git a/packages/bci-whispercpp/index.d.ts b/packages/bci-whispercpp/index.d.ts index e2137d2c3f..9ca0f42542 100644 --- a/packages/bci-whispercpp/index.d.ts +++ b/packages/bci-whispercpp/index.d.ts @@ -1,4 +1,4 @@ -import type { QvacResponse } from '@qvac/infer-base' +import QvacResponse from '@qvac/infer-base/src/QvacResponse' import type { LoggerInterface } from '@qvac/logging' declare interface BCIConfig { diff --git a/packages/bci-whispercpp/index.js b/packages/bci-whispercpp/index.js index fa26bed88a..61fa204c61 100644 --- a/packages/bci-whispercpp/index.js +++ b/packages/bci-whispercpp/index.js @@ -205,17 +205,10 @@ class BCIWhispercpp { } _isConfigurationError (err) { - const message = String(err?.message || '') - return ( - message.includes('object is required') || - message.includes('is not a valid parameter') || - message.includes('bciConfig.day_idx') || - message.includes('Unknown whisperConfig key') || - message.includes('Unknown contextParams key') || - message.includes('Unknown miscConfig key') || - message.includes('error in whisperConfig handler') || - message.includes('error in contextParams handler') - ) + if (err && err.code === 'ERR_ASSERTION') return true + if (err instanceof TypeError) return true + const msg = String(err?.message || '') + return msg.includes('is required') || msg.includes('is not a valid parameter') || msg.includes('must be') } _outputCallback (addon, event, jobId, data, error) { diff --git a/packages/bci-whispercpp/lib/error.js b/packages/bci-whispercpp/lib/error.js index 9c6a2f5791..d3bfacd0f9 100644 --- a/packages/bci-whispercpp/lib/error.js +++ b/packages/bci-whispercpp/lib/error.js @@ -16,11 +16,8 @@ const ERR_CODES = Object.freeze({ FAILED_TO_LOAD_WEIGHTS: 26001, FAILED_TO_CANCEL: 26002, FAILED_TO_APPEND: 26003, - FAILED_TO_GET_STATUS: 26004, FAILED_TO_DESTROY: 26005, FAILED_TO_ACTIVATE: 26006, - FAILED_TO_RESET: 26007, - FAILED_TO_PAUSE: 26008, INVALID_NEURAL_INPUT: 26009, JOB_ALREADY_RUNNING: 26010, MODEL_NOT_LOADED: 26011, @@ -44,10 +41,6 @@ addCodes({ name: 'FAILED_TO_APPEND', message: (message) => `Failed to append data to processing queue, error: ${message}` }, - [ERR_CODES.FAILED_TO_GET_STATUS]: { - name: 'FAILED_TO_GET_STATUS', - message: (message) => `Failed to get addon status, error: ${message}` - }, [ERR_CODES.FAILED_TO_DESTROY]: { name: 'FAILED_TO_DESTROY', message: (message) => `Failed to destroy instance, error: ${message}` @@ -56,14 +49,6 @@ addCodes({ name: 'FAILED_TO_ACTIVATE', message: (message) => `Failed to activate model, error: ${message}` }, - [ERR_CODES.FAILED_TO_RESET]: { - name: 'FAILED_TO_RESET', - message: (message) => `Failed to reset model state, error: ${message}` - }, - [ERR_CODES.FAILED_TO_PAUSE]: { - name: 'FAILED_TO_PAUSE', - message: (message) => `Failed to pause inference, error: ${message}` - }, [ERR_CODES.INVALID_NEURAL_INPUT]: { name: 'INVALID_NEURAL_INPUT', message: (message) => `Invalid neural signal input: ${message}` diff --git a/packages/bci-whispercpp/package.json b/packages/bci-whispercpp/package.json index ba05ac347e..acdc8b134e 100644 --- a/packages/bci-whispercpp/package.json +++ b/packages/bci-whispercpp/package.json @@ -9,7 +9,7 @@ "scripts": { "lint": "standard \"examples/**/*.js\" \"test/**/*.js\" \"*.js\" \"lib/**/*.js\"", "lint:fix": "standard --fix \"examples/**/*.js\" \"test/**/*.js\" \"*.js\" \"lib/**/*.js\"", - "lint-cpp": "clang-tidy -p build addon/src/js-interface/JSAdapter.cpp addon/src/js-interface/binding.cpp addon/src/model-interface/bci/BCIConfig.cpp addon/src/model-interface/bci/BCIModel.cpp addon/src/model-interface/bci/NeuralProcessor.cpp", + "lint-cpp": "clang-tidy -p build addon/src/js-interface/JSAdapter.cpp addon/src/js-interface/binding.cpp addon/src/model-interface/bci/BCIConfig.cpp addon/src/model-interface/bci/BCIModel.cpp addon/src/model-interface/bci/NeuralProcessor.cpp addon/src/addon/AddonJs.hpp addon/src/js-interface/JSAdapter.hpp addon/src/addon/BCIErrors.hpp addon/src/model-interface/BCITypes.hpp addon/src/model-interface/bci/BCIConfig.hpp addon/src/model-interface/bci/BCIModel.hpp addon/src/model-interface/bci/NeuralProcessor.hpp", "build": "bare-make generate && bare-make build && bare-make install", "build:pack": "mkdir -p dist && npm pack --pack-destination dist", "test": "npm run test:integration", From 0720cc0c86d51b58438bb897deaa90ad0e2b754d Mon Sep 17 00:00:00 2001 From: Raju Date: Tue, 21 Apr 2026 17:18:14 +0530 Subject: [PATCH 16/19] chore(bci): remove whisper-cpp overlay, consume v1.8.4.2 from registry The BCI patches (variable conv1 kernel, windowed attention) are now merged into tetherto/qvac-ext-lib-whisper.cpp master and tagged as v1.8.4.2. The local overlay that pinned a specific fork commit is no longer needed. - Delete vcpkg-overlays/whisper-cpp/ (portfile.cmake + vcpkg.json) - Remove VCPKG_OVERLAY_PORTS from CMakeLists.txt - Bump whisper-cpp override from 1.8.4 to 1.8.4.2 - Point vcpkg-configuration.json at personal fork registry (sharmaraju352/qvac-registry-vcpkg) temporarily until tetherto/qvac-registry-vcpkg#125 merges, then swap back - Update README whisper.cpp patches section Verified: clean build from scratch + 18/18 C++ tests + 3/3 integration tests (10/10 asserts, 6.0% avg WER) + batch example all pass. Made-with: Cursor --- packages/bci-whispercpp/CMakeLists.txt | 2 - packages/bci-whispercpp/README.md | 2 +- .../bci-whispercpp/vcpkg-configuration.json | 4 +- .../vcpkg-overlays/whisper-cpp/portfile.cmake | 52 ------------------- .../vcpkg-overlays/whisper-cpp/vcpkg.json | 18 ------- packages/bci-whispercpp/vcpkg.json | 2 +- 6 files changed, 4 insertions(+), 76 deletions(-) delete mode 100644 packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/portfile.cmake delete mode 100644 packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/vcpkg.json diff --git a/packages/bci-whispercpp/CMakeLists.txt b/packages/bci-whispercpp/CMakeLists.txt index 6a84dfd2dd..3b9541eed5 100644 --- a/packages/bci-whispercpp/CMakeLists.txt +++ b/packages/bci-whispercpp/CMakeLists.txt @@ -9,8 +9,6 @@ endif() find_package(cmake-bare REQUIRED PATHS node_modules/cmake-bare) find_package(cmake-vcpkg REQUIRED PATHS node_modules/cmake-vcpkg) -set(VCPKG_OVERLAY_PORTS "${CMAKE_CURRENT_SOURCE_DIR}/vcpkg-overlays;${VCPKG_OVERLAY_PORTS}") - # Prepend the local overlay triplets on every platform and preserve any # externally-set value (matches the other qvac addons). Only the Linux # triplets actually differ from vcpkg's defaults today, but exposing the diff --git a/packages/bci-whispercpp/README.md b/packages/bci-whispercpp/README.md index 4a2bdca8fa..9474d9582d 100644 --- a/packages/bci-whispercpp/README.md +++ b/packages/bci-whispercpp/README.md @@ -177,7 +177,7 @@ VCPKG_ROOT=/path/to/vcpkg npm run test:cpp ## whisper.cpp Patches -The package uses a vcpkg overlay that fetches from the `tetherto/qvac-ext-lib-whisper.cpp` fork (v1.8.4 base) with BCI patches baked in: +The BCI patches live in the `tetherto/qvac-ext-lib-whisper.cpp` fork (v1.8.4.2) and are consumed via the `qvac-registry-vcpkg` port: | Feature | Description | |---------|-------------| diff --git a/packages/bci-whispercpp/vcpkg-configuration.json b/packages/bci-whispercpp/vcpkg-configuration.json index b50ad85757..88498c7094 100644 --- a/packages/bci-whispercpp/vcpkg-configuration.json +++ b/packages/bci-whispercpp/vcpkg-configuration.json @@ -1,8 +1,8 @@ { "default-registry": { "kind": "git", - "baseline": "87ef7179f70122d0cc65a5991b88c20cab59b1e1", - "repository": "https://github.com/tetherto/qvac-registry-vcpkg.git" + "baseline": "3f18e9384f125b43443fd0f59fa2412aee76c631", + "repository": "https://github.com/sharmaraju352/qvac-registry-vcpkg.git" }, "registries": [ { diff --git a/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/portfile.cmake b/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/portfile.cmake deleted file mode 100644 index a3307be8ca..0000000000 --- a/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/portfile.cmake +++ /dev/null @@ -1,52 +0,0 @@ -set(VERSION "2b1e04f20bad9a72321e72df8d6a8c14aae98adc") - -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO tetherto/qvac-ext-lib-whisper.cpp - REF ${VERSION} - SHA512 52c96ab252a4461430740decd4d883a8ef3f9d895d84df348407ec6fddac30116b6056c2e6577866322d2d8aaa729815753bb4c02da755b584ea1205ef2e6259 - HEAD_REF master -) - -set(PLATFORM_OPTIONS) - -if (VCPKG_TARGET_IS_ANDROID) - list(APPEND PLATFORM_OPTIONS -DWHISPER_NO_AVX=ON -DWHISPER_NO_AVX2=ON -DWHISPER_NO_FMA=ON) - list(APPEND PLATFORM_OPTIONS -DGGML_VULKAN=OFF) -endif() - -vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}" - DISABLE_PARALLEL_CONFIGURE - OPTIONS - -DGGML_CCACHE=OFF - -DGGML_OPENMP=OFF - -DGGML_NATIVE=OFF - -DWHISPER_BUILD_TESTS=OFF - -DWHISPER_BUILD_EXAMPLES=OFF - -DWHISPER_BUILD_SERVER=OFF - -DBUILD_SHARED_LIBS=OFF - -DGGML_BUILD_NUMBER=1 - ${PLATFORM_OPTIONS} -) - -vcpkg_cmake_install() - -vcpkg_cmake_config_fixup( - PACKAGE_NAME whisper - CONFIG_PATH share/whisper -) - -vcpkg_fixup_pkgconfig() - -vcpkg_copy_pdbs() - -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") - -if (VCPKG_LIBRARY_LINKAGE MATCHES "static") - file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/bin") - file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/bin") -endif() - -vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE") diff --git a/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/vcpkg.json b/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/vcpkg.json deleted file mode 100644 index 11a4973a9e..0000000000 --- a/packages/bci-whispercpp/vcpkg-overlays/whisper-cpp/vcpkg.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "whisper-cpp", - "version": "1.8.4", - "port-version": 4, - "description": "Port of OpenAI's Whisper model in C/C++ (BCI patched, based on tetherto/qvac-ext-lib-whisper.cpp PR #10)", - "homepage": "https://github.com/tetherto/qvac-ext-lib-whisper.cpp", - "license": "MIT", - "dependencies": [ - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - } - ] -} diff --git a/packages/bci-whispercpp/vcpkg.json b/packages/bci-whispercpp/vcpkg.json index d1677848ba..bb98e06f89 100644 --- a/packages/bci-whispercpp/vcpkg.json +++ b/packages/bci-whispercpp/vcpkg.json @@ -20,7 +20,7 @@ "overrides": [ { "name": "whisper-cpp", - "version": "1.8.4" + "version": "1.8.4.2" } ] } From 7c282b6bd18ce7183ae061f9d800d3d03f4ac662 Mon Sep 17 00:00:00 2001 From: Raju Date: Tue, 21 Apr 2026 17:31:10 +0530 Subject: [PATCH 17/19] chore(bci): point vcpkg registry back to tetherto upstream Registry PR tetherto/qvac-registry-vcpkg#125 has been merged. Swap vcpkg-configuration.json from the personal fork back to the upstream tetherto/qvac-registry-vcpkg and update the baseline to the merge commit. Verified: clean build from scratch + all tests pass on both bci-whispercpp (18/18 C++, 3/3 integration, 6.0% WER) and transcription-whispercpp (106/106 C++, 28/28 unit, 10/10 integration, all extended suites). Made-with: Cursor --- packages/bci-whispercpp/vcpkg-configuration.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/bci-whispercpp/vcpkg-configuration.json b/packages/bci-whispercpp/vcpkg-configuration.json index 88498c7094..91b7470f41 100644 --- a/packages/bci-whispercpp/vcpkg-configuration.json +++ b/packages/bci-whispercpp/vcpkg-configuration.json @@ -1,8 +1,8 @@ { "default-registry": { "kind": "git", - "baseline": "3f18e9384f125b43443fd0f59fa2412aee76c631", - "repository": "https://github.com/sharmaraju352/qvac-registry-vcpkg.git" + "baseline": "acdd94de3e3938d44eea422876adb23c2b33d3a0", + "repository": "https://github.com/tetherto/qvac-registry-vcpkg.git" }, "registries": [ { From 851328740fcb8329669f3cd62cc4b0bec67b6ee0 Mon Sep 17 00:00:00 2001 From: Raju Date: Tue, 21 Apr 2026 18:59:24 +0530 Subject: [PATCH 18/19] =?UTF-8?q?fix(bci):=20address=20Gustavo=20review=20?= =?UTF-8?q?=E2=80=94=20error=20types,=20lifecycle,=20error=20codes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Reset is_warmed_up_ in BCIModel::unload() so re-load triggers warmup - Add FailedToLoadModel and EmbedderWeightsNotFound error codes to BCIErrors.hpp; use them instead of InvalidNeuralSignal for context init failure (BCIModel.cpp:116) and missing embedder (BCIModel.cpp:90) - Wrap addon.activate() in try-catch in index.js _load(), throwing FAILED_TO_ACTIVATE with structured error on failure - Make all JS error codes sequential (26001-26013, no gaps) Made-with: Cursor --- .../addon/src/addon/BCIErrors.hpp | 6 ++++++ .../src/model-interface/bci/BCIModel.cpp | 5 +++-- packages/bci-whispercpp/index.js | 11 +++++++++- packages/bci-whispercpp/lib/error.js | 20 +++++++++---------- 4 files changed, 29 insertions(+), 13 deletions(-) diff --git a/packages/bci-whispercpp/addon/src/addon/BCIErrors.hpp b/packages/bci-whispercpp/addon/src/addon/BCIErrors.hpp index 2c42f1d0ea..15b546b4d6 100644 --- a/packages/bci-whispercpp/addon/src/addon/BCIErrors.hpp +++ b/packages/bci-whispercpp/addon/src/addon/BCIErrors.hpp @@ -13,12 +13,18 @@ namespace qvac_errors { namespace bci_error { enum class Code : std::uint8_t { InvalidNeuralSignal, + FailedToLoadModel, + EmbedderWeightsNotFound, }; inline const char* codeName(Code code) { switch (code) { case Code::InvalidNeuralSignal: return "InvalidNeuralSignal"; + case Code::FailedToLoadModel: + return "FailedToLoadModel"; + case Code::EmbedderWeightsNotFound: + return "EmbedderWeightsNotFound"; } return "BCIError"; } diff --git a/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp b/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp index 40637971c5..9ef986b98c 100644 --- a/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp +++ b/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp @@ -87,7 +87,7 @@ void BCIModel::loadEmbedderIfNeeded() { "Loaded BCI embedder weights from: " + embedderPath); } else { throw qvac_errors::bci_error::makeStatus( - qvac_errors::bci_error::Code::InvalidNeuralSignal, + qvac_errors::bci_error::Code::EmbedderWeightsNotFound, "BCI embedder weights not found at: " + embedderPath + ". This file is required for neural signal preprocessing. " "Generate it with: python3 scripts/convert-model.py --checkpoint "); @@ -113,7 +113,7 @@ void BCIModel::load() { auto* rawCtx = whisper_init_from_file_with_params(modelPath.c_str(), contextParams); if (rawCtx == nullptr) { throw qvac_errors::bci_error::makeStatus( - qvac_errors::bci_error::Code::InvalidNeuralSignal, + qvac_errors::bci_error::Code::FailedToLoadModel, "Failed to initialize Whisper context from: " + modelPath); } @@ -135,6 +135,7 @@ void BCIModel::load() { void BCIModel::unload() { resetContext(); is_loaded_ = false; + is_warmed_up_ = false; } void BCIModel::reload() { diff --git a/packages/bci-whispercpp/index.js b/packages/bci-whispercpp/index.js index 61fa204c61..9cfbcb7c54 100644 --- a/packages/bci-whispercpp/index.js +++ b/packages/bci-whispercpp/index.js @@ -127,7 +127,16 @@ class BCIWhispercpp { }) } - await this.addon.activate() + try { + await this.addon.activate() + } catch (err) { + this.addon = null + throw new QvacErrorAddonBCI({ + code: ERR_CODES.FAILED_TO_ACTIVATE, + adds: err.message, + cause: err + }) + } this.logger.info('BCI addon activated') } diff --git a/packages/bci-whispercpp/lib/error.js b/packages/bci-whispercpp/lib/error.js index d3bfacd0f9..461149eb1f 100644 --- a/packages/bci-whispercpp/lib/error.js +++ b/packages/bci-whispercpp/lib/error.js @@ -16,16 +16,16 @@ const ERR_CODES = Object.freeze({ FAILED_TO_LOAD_WEIGHTS: 26001, FAILED_TO_CANCEL: 26002, FAILED_TO_APPEND: 26003, - FAILED_TO_DESTROY: 26005, - FAILED_TO_ACTIVATE: 26006, - INVALID_NEURAL_INPUT: 26009, - JOB_ALREADY_RUNNING: 26010, - MODEL_NOT_LOADED: 26011, - MODEL_FILE_NOT_FOUND: 26012, - BUFFER_LIMIT_EXCEEDED: 26013, - FAILED_TO_START_JOB: 26014, - INVALID_CONFIG: 26015, - EMBEDDER_WEIGHTS_INVALID: 26016 + FAILED_TO_DESTROY: 26004, + FAILED_TO_ACTIVATE: 26005, + INVALID_NEURAL_INPUT: 26006, + JOB_ALREADY_RUNNING: 26007, + MODEL_NOT_LOADED: 26008, + MODEL_FILE_NOT_FOUND: 26009, + BUFFER_LIMIT_EXCEEDED: 26010, + FAILED_TO_START_JOB: 26011, + INVALID_CONFIG: 26012, + EMBEDDER_WEIGHTS_INVALID: 26013 }) addCodes({ From 26ef7b7c42f105c4661a4c9044a182613267a74e Mon Sep 17 00:00:00 2001 From: Ishan Vohra Date: Wed, 22 Apr 2026 19:14:47 +0530 Subject: [PATCH 19/19] Remove date from changelog --- packages/bci-whispercpp/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/bci-whispercpp/CHANGELOG.md b/packages/bci-whispercpp/CHANGELOG.md index 46d2e6fa9f..d51c53ae3b 100644 --- a/packages/bci-whispercpp/CHANGELOG.md +++ b/packages/bci-whispercpp/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [0.1.0] - 2026-04-15 +## [0.1.0] Initial POC release of `@qvac/bci-whispercpp`, a brain-computer-interface neural signal transcription addon powered by a BCI-patched fork of whisper.cpp.