From 9e098a0229de8b48678f1c95ab89afa35ad14ce7 Mon Sep 17 00:00:00 2001 From: lalalune Date: Thu, 2 Jul 2026 15:11:27 -0700 Subject: [PATCH] fix(omnivoice): reject explicit diarizer gate-order skew --- tools/omnivoice/CMakeLists.txt | 20 +++ .../voice_classifier/voice_diarizer.c | 25 ++- .../voice_classifier/voice_gguf_loader.c | 18 +++ .../voice_classifier/voice_gguf_loader.h | 12 ++ tools/omnivoice/tests/diarizer-metadata.c | 146 ++++++++++++++++++ 5 files changed, 219 insertions(+), 2 deletions(-) create mode 100644 tools/omnivoice/tests/diarizer-metadata.c diff --git a/tools/omnivoice/CMakeLists.txt b/tools/omnivoice/CMakeLists.txt index a5b52ac28..cbe438f05 100644 --- a/tools/omnivoice/CMakeLists.txt +++ b/tools/omnivoice/CMakeLists.txt @@ -396,6 +396,26 @@ target_include_directories(omnivoice-test-abi-c PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) target_link_libraries(omnivoice-test-abi-c PRIVATE omnivoice_lib) +# omnivoice-test-diarizer-metadata : metadata-only GGUF contract test for the +# vendored diarizer reader. It writes tiny GGUF files, so it does not need the +# pyannote model artifact. +add_executable(omnivoice-test-diarizer-metadata tests/diarizer-metadata.c) +set_target_properties(omnivoice-test-diarizer-metadata PROPERTIES + C_STANDARD 99 + C_STANDARD_REQUIRED ON + C_EXTENSIONS ON) +if(MSVC) + target_compile_options(omnivoice-test-diarizer-metadata PRIVATE /W4 /WX) +else() + target_compile_options(omnivoice-test-diarizer-metadata PRIVATE -Wall -Werror -pedantic) +endif() +target_include_directories(omnivoice-test-diarizer-metadata PRIVATE + ${OMNIVOICE_VOICE_CLASSIFIER_INCLUDE_DIRS}) +target_link_libraries(omnivoice-test-diarizer-metadata PRIVATE eliza_voice_classifiers) +if(BUILD_TESTING) + add_test(NAME omnivoice-test-diarizer-metadata COMMAND omnivoice-test-diarizer-metadata) +endif() + # omnivoice-dac-parity : focused CPU numerical parity harness for # dac_conv_t1d's ggml_conv_transpose_1d migration (elizaOS/eliza#7660). # The default run uses deterministic synthetic DAC block shapes; pass diff --git a/tools/omnivoice/src/voice-classifiers/voice_classifier/voice_diarizer.c b/tools/omnivoice/src/voice-classifiers/voice_classifier/voice_diarizer.c index 4f9540c28..a66872fd1 100644 --- a/tools/omnivoice/src/voice-classifiers/voice_classifier/voice_diarizer.c +++ b/tools/omnivoice/src/voice-classifiers/voice_classifier/voice_diarizer.c @@ -70,6 +70,7 @@ #define DIAR_LINEAR0_OUT 128 #define DIAR_LINEAR1_OUT 128 #define DIAR_LEAKY_ALPHA 0.01f +#define DIAR_LSTM_GATE_ORDER "IOFC" /* Cached pointers + buffer struct for one diarizer session. */ struct voice_diarizer_session { @@ -196,8 +197,8 @@ static inline float sigmoidf(float x) { } } -/* One-direction LSTM step. Gates packed in I, F, G, O order - * (matches the converter's reorder). `x_dot_W` is the +/* One-direction LSTM step. Gates packed in I, O, F, C order (matches the + * published diarizer GGUF this fork currently ships). `x_dot_W` is the * pre-computed x @ W_ih^T + b_ih, shape [T, 4H]. */ static void lstm_run_dir(const float *x_dot_W, int T, int H, @@ -393,6 +394,26 @@ int voice_diarizer_open(const char *gguf, voice_diarizer_handle *out) { meta.sample_rate != VOICE_CLASSIFIER_SAMPLE_RATE_HZ) return -EINVAL; if (meta.num_classes != 0 && meta.num_classes != VOICE_DIARIZER_NUM_CLASSES) return -EINVAL; + if (meta.window_samples != 0 && + meta.window_samples != DIAR_WINDOW_SAMPLES) return -EINVAL; + if (meta.frames_per_window != 0 && + meta.frames_per_window != DIAR_FRAMES_PER_WINDOW) return -EINVAL; + if (meta.lstm_layers != 0 && + meta.lstm_layers != DIAR_LSTM_LAYERS) return -EINVAL; + if (meta.lstm_hidden != 0 && + meta.lstm_hidden != DIAR_LSTM_HIDDEN) return -EINVAL; + if (meta.linear0_out != 0 && + meta.linear0_out != DIAR_LINEAR0_OUT) return -EINVAL; + if (meta.linear1_out != 0 && + meta.linear1_out != DIAR_LINEAR1_OUT) return -EINVAL; + if (meta.lstm_gate_order[0] != '\0' && + strcmp(meta.lstm_gate_order, DIAR_LSTM_GATE_ORDER) != 0) { + fprintf(stderr, + "[voice_diarizer] unsupported LSTM gate order '%s'; this fused reader expects %s\n", + meta.lstm_gate_order, + DIAR_LSTM_GATE_ORDER); + return -EINVAL; + } struct voice_diarizer_session *s = (struct voice_diarizer_session *)calloc(1, sizeof(*s)); diff --git a/tools/omnivoice/src/voice-classifiers/voice_classifier/voice_gguf_loader.c b/tools/omnivoice/src/voice-classifiers/voice_classifier/voice_gguf_loader.c index 2272d8b50..7a128f26e 100644 --- a/tools/omnivoice/src/voice-classifiers/voice_classifier/voice_gguf_loader.c +++ b/tools/omnivoice/src/voice-classifiers/voice_classifier/voice_gguf_loader.c @@ -226,6 +226,17 @@ static int vc_gguf_load_state_cb(const char *key, free(str); return 1; } + if (vc_gguf_key_eq(key, s->want_prefix, "lstm_gate_order")) { + if (type != VC_GGUF_TYPE_STRING) return -1; + char *str = NULL; + const int rc = vc_gguf_read_string(f, &str); + if (rc != 0) return -1; + const size_t n = sizeof(s->out->lstm_gate_order) - 1; + strncpy(s->out->lstm_gate_order, str, n); + s->out->lstm_gate_order[n] = '\0'; + free(str); + return 1; + } /* Wav2Small-specific uint32 keys: read into the matching field * and let the per-head opener validate against its expectation. */ struct { @@ -239,6 +250,13 @@ static int vc_gguf_load_state_cb(const char *key, { "ffn_dim", &s->out->ffn_dim }, { "num_layers", &s->out->num_layers }, { "num_heads", &s->out->num_heads }, + { "converter_epoch", &s->out->converter_epoch }, + { "window_samples", &s->out->window_samples }, + { "frames_per_window", &s->out->frames_per_window }, + { "lstm_layers", &s->out->lstm_layers }, + { "lstm_hidden", &s->out->lstm_hidden }, + { "linear0_out", &s->out->linear0_out }, + { "linear1_out", &s->out->linear1_out }, }; for (size_t i = 0; i < sizeof(u32_extras) / sizeof(u32_extras[0]); ++i) { if (vc_gguf_key_eq(key, s->want_prefix, u32_extras[i].suffix)) { diff --git a/tools/omnivoice/src/voice-classifiers/voice_classifier/voice_gguf_loader.h b/tools/omnivoice/src/voice-classifiers/voice_classifier/voice_gguf_loader.h index 30e707702..4a776a412 100644 --- a/tools/omnivoice/src/voice-classifiers/voice_classifier/voice_gguf_loader.h +++ b/tools/omnivoice/src/voice-classifiers/voice_classifier/voice_gguf_loader.h @@ -57,6 +57,18 @@ typedef struct voice_gguf_metadata { int ffn_dim; int num_layers; int num_heads; + /* Diarizer-specific metadata. Older published GGUFs leave these + * zero/empty; the diarizer opener treats missing gate metadata as legacy + * and validates explicit metadata against the reader's compiled contract. + */ + int converter_epoch; + int window_samples; + int frames_per_window; + int lstm_layers; + int lstm_hidden; + int linear0_out; + int linear1_out; + char lstm_gate_order[16]; } voice_gguf_metadata_t; /* GGUF GGML tensor data types — we only support F32 in the per-head diff --git a/tools/omnivoice/tests/diarizer-metadata.c b/tools/omnivoice/tests/diarizer-metadata.c new file mode 100644 index 000000000..983e9159a --- /dev/null +++ b/tools/omnivoice/tests/diarizer-metadata.c @@ -0,0 +1,146 @@ +/* + * Diarizer GGUF metadata contract for the vendored fused reader. + * + * The current pinned fused library reads pyannote LSTM gates in raw ONNX IOFC + * order. Future package-side converters emit explicit epoch/gate metadata, so + * this test pins the fork loader's ability to see that metadata before tensor + * load and reject mismatched artifacts loudly. + */ + +#define _DEFAULT_SOURCE +#define _XOPEN_SOURCE 700 + +#include "voice_classifier/voice_classifier.h" +#include "voice_gguf_loader.h" + +#include +#include +#include +#include +#include + +#define VC_GGUF_MAGIC "GGUF" +#define VC_GGUF_VERSION 3 +#define DIAR_CONVERTER_EPOCH 2 +#define DIAR_WINDOW_SAMPLES 80000 +#define DIAR_FRAMES_PER_WINDOW 293 +#define DIAR_LSTM_LAYERS 4 +#define DIAR_LSTM_HIDDEN 128 +#define DIAR_LINEAR0_OUT 128 +#define DIAR_LINEAR1_OUT 128 + +enum vc_gguf_type { + VC_GGUF_TYPE_UINT32 = 4, + VC_GGUF_TYPE_STRING = 8, +}; + +static void w_u32(FILE * f, uint32_t v) { + fwrite(&v, sizeof(v), 1, f); +} + +static void w_u64(FILE * f, uint64_t v) { + fwrite(&v, sizeof(v), 1, f); +} + +static void w_i64(FILE * f, int64_t v) { + fwrite(&v, sizeof(v), 1, f); +} + +static void w_str(FILE * f, const char * s) { + const uint64_t n = strlen(s); + w_u64(f, n); + fwrite(s, 1, n, f); +} + +static void w_kv_u32(FILE * f, const char * key, uint32_t val) { + w_str(f, key); + w_u32(f, VC_GGUF_TYPE_UINT32); + w_u32(f, val); +} + +static void w_kv_str(FILE * f, const char * key, const char * val) { + w_str(f, key); + w_u32(f, VC_GGUF_TYPE_STRING); + w_str(f, val); +} + +static int write_diarizer_meta(const char * path, uint32_t converter_epoch, const char * gate_order) { + FILE * f = fopen(path, "wb"); + if (!f) { + return -1; + } + + fwrite(VC_GGUF_MAGIC, 1, 4, f); + w_u32(f, VC_GGUF_VERSION); + w_i64(f, 0); + w_i64(f, 11); + + w_kv_u32(f, "voice_diarizer.sample_rate", VOICE_CLASSIFIER_SAMPLE_RATE_HZ); + w_kv_u32(f, "voice_diarizer.num_classes", VOICE_DIARIZER_NUM_CLASSES); + w_kv_u32(f, "voice_diarizer.window_samples", DIAR_WINDOW_SAMPLES); + w_kv_u32(f, "voice_diarizer.frames_per_window", DIAR_FRAMES_PER_WINDOW); + w_kv_u32(f, "voice_diarizer.converter_epoch", converter_epoch); + w_kv_u32(f, "voice_diarizer.lstm_layers", DIAR_LSTM_LAYERS); + w_kv_u32(f, "voice_diarizer.lstm_hidden", DIAR_LSTM_HIDDEN); + w_kv_u32(f, "voice_diarizer.linear0_out", DIAR_LINEAR0_OUT); + w_kv_u32(f, "voice_diarizer.linear1_out", DIAR_LINEAR1_OUT); + w_kv_str(f, "voice_diarizer.variant", "pyannote-segmentation-3.0"); + w_kv_str(f, "voice_diarizer.lstm_gate_order", gate_order); + + fclose(f); + return 0; +} + +int main(void) { + int failures = 0; + char tmpl[] = "/tmp/omnivoice_diarizer_metadata_XXXXXX"; + int fd = mkstemp(tmpl); + if (fd < 0) { + perror("mkstemp"); + return 1; + } + close(fd); + + if (write_diarizer_meta(tmpl, DIAR_CONVERTER_EPOCH, "IOFC") != 0) { + fprintf(stderr, "cannot write IOFC metadata GGUF\n"); + unlink(tmpl); + return 1; + } + + voice_gguf_metadata_t meta; + int rc = voice_gguf_load_metadata(tmpl, "voice_diarizer", &meta); + if (rc != 0) { + fprintf(stderr, "IOFC metadata load returned %d\n", rc); + ++failures; + } + if (meta.converter_epoch != DIAR_CONVERTER_EPOCH || strcmp(meta.lstm_gate_order, "IOFC") != 0) { + fprintf(stderr, "IOFC metadata parsed incorrectly: epoch=%d gate=%s\n", + meta.converter_epoch, meta.lstm_gate_order); + ++failures; + } + if (meta.window_samples != DIAR_WINDOW_SAMPLES || + meta.frames_per_window != DIAR_FRAMES_PER_WINDOW || + meta.lstm_layers != DIAR_LSTM_LAYERS || + meta.lstm_hidden != DIAR_LSTM_HIDDEN || + meta.linear0_out != DIAR_LINEAR0_OUT || + meta.linear1_out != DIAR_LINEAR1_OUT) { + fprintf(stderr, "diarizer shape metadata mismatch\n"); + ++failures; + } + + if (write_diarizer_meta(tmpl, DIAR_CONVERTER_EPOCH, "IFGO") != 0) { + fprintf(stderr, "cannot write IFGO metadata GGUF\n"); + unlink(tmpl); + return 1; + } + memset(&meta, 0, sizeof(meta)); + rc = voice_gguf_load_metadata(tmpl, "voice_diarizer", &meta); + if (rc != 0 || strcmp(meta.lstm_gate_order, "IFGO") != 0) { + fprintf(stderr, "IFGO metadata was not preserved for fail-fast validation\n"); + ++failures; + } + + unlink(tmpl); + printf("omnivoice diarizer metadata failures=%d\n", failures); + return failures == 0 ? 0 : 1; +}