feat: add biometrics UI

Makefile

@@ -394,7 +394,13 @@ protoc:
 .PHONY: protogen-go
 protogen-go: protoc install-go-tools
 	mkdir -p pkg/grpc/proto
-	./protoc --experimental_allow_proto3_optional -Ibackend/ --go_out=pkg/grpc/proto/ --go_opt=paths=source_relative --go-grpc_out=pkg/grpc/proto/ --go-grpc_opt=paths=source_relative \
+	# install-go-tools writes protoc-gen-go and protoc-gen-go-grpc into
+	# $(shell go env GOPATH)/bin, which isn't on every dev's PATH. protoc
+	# resolves its code-gen plugins via PATH, so without this prefix the
+	# generate step fails with "protoc-gen-go: program not found". Prepend
+	# GOPATH/bin so the freshly-installed plugins win without requiring a
+	# shell-profile change.
+	PATH="$$(go env GOPATH)/bin:$$PATH" ./protoc --experimental_allow_proto3_optional -Ibackend/ --go_out=pkg/grpc/proto/ --go_opt=paths=source_relative --go-grpc_out=pkg/grpc/proto/ --go-grpc_opt=paths=source_relative \
     backend/backend.proto
 
 core/config/inference_defaults.json: ## Fetch inference defaults from unsloth (only if missing)

backend/go/local-store/store.go

@@ -4,7 +4,6 @@ package main
 // It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc)
 import (
 	"container/heap"
-	"errors"
 	"fmt"
 	"math"
 	"slices"
@@ -100,9 +99,16 @@ func sortIntoKeySlicese(keys []*pb.StoresKey) [][]float32 {
 }
 
 func (s *Store) Load(opts *pb.ModelOptions) error {
-	if opts.Model != "" {
-		return errors.New("not implemented")
-	}
+	// local-store is an in-memory vector store with no on-disk artefact to
+	// load — opts.Model is just a namespace identifier. The old `!= ""` guard
+	// rejected any non-empty model name with "not implemented", which broke
+	// callers that pass a namespace to isolate embedding spaces (face vs.
+	// voice biometrics both go through local-store but need distinct stores
+	// so ArcFace 512-D and ECAPA-TDNN 192-D don't collide). Namespace
+	// isolation is already handled upstream: ModelLoader spawns a fresh
+	// local-store process per (backend, model) tuple, so each namespace is
+	// its own Store{} instance. Nothing to do here beyond accepting the load.
+	_ = opts
 	return nil
 }
 

backend/python/insightface/engines.py

@@ -173,6 +173,30 @@ def _build_antispoofer(options: dict[str, str], model_dir: str | None) -> Antisp
 
 # ─── InsightFaceEngine ────────────────────────────────────────────────
 
+# Canonical ONNX manifest for each upstream insightface pack (v0.7 release
+# at github.com/deepinsight/insightface/releases). LocalAI's gallery extracts
+# these zips flat into the models directory, so when multiple packs or other
+# backends drop their own ONNX files alongside, the glob-the-directory
+# approach picks up foreign files and insightface's model_zoo.get_model()
+# raises IndexError trying to index `input_shape[2]` on a tensor that isn't
+# shaped like a face model. The manifest lets us pre-filter to only the
+# files that actually belong to the requested pack — deterministic, correct
+# pack choice, no crashes on neighbour ONNX files.
+_KNOWN_PACK_MANIFESTS: dict[str, frozenset[str]] = {
+    "buffalo_l": frozenset({
+        "det_10g.onnx",
+        "w600k_r50.onnx",
+        "genderage.onnx",
+        "2d106det.onnx",
+        "1k3d68.onnx",
+    }),
+    "buffalo_sc": frozenset({
+        "det_500m.onnx",
+        "w600k_mbf.onnx",
+    }),
+}
+
+
 class InsightFaceEngine:
     """Drives insightface's model_zoo directly — no FaceAnalysis wrapper.
 
@@ -222,6 +246,21 @@ def prepare(self, options: dict[str, str]) -> None:
             )
 
         onnx_files = sorted(glob.glob(os.path.join(pack_dir, "*.onnx")))
+        # When the pack extracts flat into a shared models directory it
+        # mixes with ONNX files from other backends (opencv face engine,
+        # MiniFASNet antispoof, WeSpeaker voice embedding, other buffalo
+        # packs installed earlier). Feeding those into model_zoo.get_model()
+        # blows up inside insightface's router — it assumes a 4-D NCHW
+        # input and indexes `input_shape[2]` on tensors that aren't shaped
+        # like a face model, raising IndexError. For the upstream packs we
+        # know the exact ONNX manifest; scoping to it makes the load
+        # deterministic (without it, det_10g.onnx from buffalo_l sorts
+        # before det_500m.onnx from buffalo_sc and silently wins).
+        manifest = _KNOWN_PACK_MANIFESTS.get(self.model_pack)
+        if manifest is not None:
+            scoped = [f for f in onnx_files if os.path.basename(f) in manifest]
+            if scoped:
+                onnx_files = scoped
         if not onnx_files:
             raise ValueError(f"no ONNX files in pack directory: {pack_dir}")
 
@@ -231,14 +270,31 @@ def prepare(self, options: dict[str, str]) -> None:
         self._providers = ["CUDAExecutionProvider", "CPUExecutionProvider"]
 
         self.models = {}
+        skipped: list[tuple[str, str]] = []
         for onnx_file in onnx_files:
-            m = model_zoo.get_model(onnx_file, providers=self._providers)
+            try:
+                m = model_zoo.get_model(onnx_file, providers=self._providers)
+            except Exception as err:
+                # Foreign ONNX (wrong rank/shape, non-insightface model) —
+                # older insightface versions raise IndexError / ValueError
+                # instead of returning None. Keep loading the rest.
+                skipped.append((os.path.basename(onnx_file), str(err)))
+                continue
             if m is None:
+                skipped.append((os.path.basename(onnx_file), "unknown taskname"))
                 continue
             # First occurrence of each taskname wins (matches FaceAnalysis).
             if m.taskname not in self.models:
                 self.models[m.taskname] = m
 
+        if skipped:
+            import sys
+            print(
+                f"[insightface] skipped {len(skipped)} non-pack ONNX file(s) in {pack_dir}: "
+                + ", ".join(f"{n} ({why})" for n, why in skipped),
+                file=sys.stderr,
+            )
+
         if "detection" not in self.models:
             raise ValueError(f"no detector (taskname='detection') found in {pack_dir}")
         self.det_model = self.models["detection"]

backend/python/speaker-recognition/engines.py

@@ -317,8 +317,23 @@ def __init__(self, model_name: str, options: dict[str, str]):
         else:
             provider_list = ["CPUExecutionProvider"]
         self._session = ort.InferenceSession(onnx_path, providers=provider_list)
-        self._input_name = self._session.get_inputs()[0].name
+        input_meta = self._session.get_inputs()[0]
+        self._input_name = input_meta.name
+        # Pre-exported speaker encoders come in two shapes:
+        #   rank-2  [batch, samples]          — some 3D-Speaker exports feed raw waveform.
+        #   rank-3  [batch, frames, n_mels]   — WeSpeaker and most Kaldi-lineage encoders
+        #                                        expect pre-computed Kaldi FBank features.
+        # We detect this at load time and branch in embed(), because feeding raw audio
+        # into a rank-3 graph is exactly what triggered
+        # "Invalid rank for input: feats Got: 2 Expected: 3".
+        self._input_rank = len(input_meta.shape) if input_meta.shape is not None else 2
         self._expected_sr = int(options.get("sample_rate", "16000"))
+        self._fbank_mels = int(options.get("fbank_num_mel_bins", "80"))
+        self._fbank_frame_length_ms = float(options.get("fbank_frame_length_ms", "25"))
+        self._fbank_frame_shift_ms = float(options.get("fbank_frame_shift_ms", "10"))
+        # Per-utterance cepstral mean normalisation — on for WeSpeaker by default,
+        # toggleable for encoders that expect raw FBank.
+        self._fbank_cmn = options.get("fbank_cmn", "true").lower() in ("1", "true", "yes")
         self._analysis = AnalysisHead(options)
 
     def _load_waveform(self, path: str):
@@ -344,11 +359,37 @@ def embed(self, audio_path: str) -> list[float]:
         import numpy as np
 
         audio = self._load_waveform(audio_path)
-        feed = audio.reshape(1, -1)
+        if self._input_rank >= 3:
+            feats = self._extract_fbank(audio)        # [frames, n_mels]
+            feed = feats[np.newaxis, :, :]             # [1, frames, n_mels]
+        else:
+            feed = audio.reshape(1, -1)                # [1, samples]
         out = self._session.run(None, {self._input_name: feed})
         vec = np.asarray(out[0]).reshape(-1)
         return [float(x) for x in vec]
 
+    def _extract_fbank(self, audio):
+        """Compute Kaldi-style 80-dim FBank features for speaker encoders that
+        expect pre-featurised input (WeSpeaker, most 3D-Speaker exports).
+        torchaudio is already a backend dependency for SpeechBrain — no new
+        package required."""
+        import numpy as np
+        import torch  # type: ignore
+        import torchaudio.compliance.kaldi as kaldi  # type: ignore
+
+        tensor = torch.from_numpy(audio).unsqueeze(0)  # [1, samples]
+        feats = kaldi.fbank(
+            tensor,
+            sample_frequency=self._expected_sr,
+            num_mel_bins=self._fbank_mels,
+            frame_length=self._fbank_frame_length_ms,
+            frame_shift=self._fbank_frame_shift_ms,
+            dither=0.0,
+        )  # [frames, n_mels]
+        if self._fbank_cmn:
+            feats = feats - feats.mean(dim=0, keepdim=True)
+        return feats.numpy().astype(np.float32)
+
     def compare(self, audio1: str, audio2: str) -> float:
         return _cosine_distance(self.embed(audio1), self.embed(audio2))
 

core/application/application.go

@@ -81,18 +81,30 @@ func newApplication(appConfig *config.ApplicationConfig) *Application {
 	// The resolver closes over the ModelLoader so the Registry stays
 	// decoupled from loader plumbing; swapping in a postgres-backed
 	// implementation later is a single construction change here.
+	//
+	// `faceStoreName` is the default namespace passed to StoreBackend when
+	// the request doesn't override it. Face and voice MUST use distinct
+	// namespaces — the local-store gRPC surface rejects mixed dimensions
+	// inside one namespace ("Try to add key with length N when existing
+	// length is M"). ArcFace buffalo_l produces 512-dim embeddings while
+	// ECAPA-TDNN produces 192-dim; enrolling one after the other into a
+	// shared namespace is exactly how we hit that error.
+	const (
+		faceStoreName  = "localai-face-biometrics"
+		voiceStoreName = "localai-voice-biometrics"
+	)
 	faceStoreResolver := func(_ context.Context, storeName string) (pkggrpc.Backend, error) {
 		return corebackend.StoreBackend(ml, appConfig, storeName, "")
 	}
-	app.faceRegistry = facerecognition.NewStoreRegistry(faceStoreResolver, "", faceEmbeddingDim)
+	app.faceRegistry = facerecognition.NewStoreRegistry(faceStoreResolver, faceStoreName, faceEmbeddingDim)
 
 	// Voice (speaker) recognition registry — same plumbing, separate
-	// registry so embedding spaces stay isolated (a face vector and a
-	// speaker vector are not comparable).
+	// namespace so embedding spaces stay isolated (a face vector and a
+	// speaker vector are not comparable and differ in dimensionality).
 	voiceStoreResolver := func(_ context.Context, storeName string) (pkggrpc.Backend, error) {
 		return corebackend.StoreBackend(ml, appConfig, storeName, "")
 	}
-	app.voiceRegistry = voicerecognition.NewStoreRegistry(voiceStoreResolver, "", voiceEmbeddingDim)
+	app.voiceRegistry = voicerecognition.NewStoreRegistry(voiceStoreResolver, voiceStoreName, voiceEmbeddingDim)
 
 	return app
 }

core/application/startup.go

@@ -242,6 +242,12 @@ func New(opts ...config.AppOption) (*Application, error) {
 		bmFn := func() galleryop.BackendManager { return application.GalleryService().BackendManager() }
 		uc := NewUpgradeChecker(options, application.ModelLoader(), application.distributedDB(), bmFn)
 		application.upgradeChecker = uc
+		// Refresh the upgrade cache the moment a backend op finishes — otherwise
+		// the UI keeps showing a just-upgraded backend as upgradeable until the
+		// next 6-hour tick. TriggerCheck is non-blocking.
+		if gs := application.GalleryService(); gs != nil {
+			gs.OnBackendOpCompleted = uc.TriggerCheck
+		}
 		go uc.Run(options.Context)
 	}
 

core/backend/stores.go

@@ -11,8 +11,17 @@ func StoreBackend(sl *model.ModelLoader, appConfig *config.ApplicationConfig, st
 	if backend == "" {
 		backend = model.LocalStoreBackend
 	}
+	// ModelLoader caches backend processes by `modelID`, not by the `model`
+	// passed via WithModel. Without a distinct modelID, every StoreBackend
+	// call collapses to the same `modelID=""` cache slot — face (512-D) and
+	// voice (192-D) biometrics would then share the same local-store process
+	// and the second enrollment would fail with
+	//   Try to add key with length N when existing length is M
+	// Use the store namespace as modelID so each namespace gets its own
+	// process instance and its own in-memory Store{}.
 	sc := []model.Option{
 		model.WithBackendString(backend),
+		model.WithModelID(storeName),
 		model.WithModel(storeName),
 	}
 

core/gallery/backends.go

@@ -194,6 +194,20 @@ func InstallBackend(ctx context.Context, systemState *system.SystemState, modelL
 
 	name := config.Name
 	backendPath := filepath.Join(systemState.Backend.BackendsPath, name)
+	// Clean up legacy flat-layout artefacts: earlier dev builds of the
+	// golang backends dropped the compiled binary directly at
+	// `<backendsPath>/<name>` (a plain file) instead of
+	// `<backendsPath>/<name>/<name>` (the nested layout the current code
+	// expects). MkdirAll below returns ENOTDIR when such a stale file
+	// exists, permanently blocking any reinstall or upgrade. Remove the
+	// file first so the install can proceed; the new install will write
+	// the correct nested layout, including metadata.json + run.sh.
+	if fi, statErr := os.Lstat(backendPath); statErr == nil && !fi.IsDir() {
+		xlog.Warn("removing stale non-directory backend artefact to make room for fresh install", "path", backendPath)
+		if rmErr := os.Remove(backendPath); rmErr != nil {
+			return fmt.Errorf("failed to remove stale backend artefact at %s: %w", backendPath, rmErr)
+		}
+	}
 	err = os.MkdirAll(backendPath, 0750)
 	if err != nil {
 		return fmt.Errorf("failed to create base path: %v", err)

core/http/endpoints/localai/audio.go

@@ -14,7 +14,13 @@ import (
 	"github.com/mudler/LocalAI/pkg/utils"
 )
 
-var audioDataURIPattern = regexp.MustCompile(`^data:([^;]+);base64,`)
+// Match `data:<mime>[;param=value...];base64,` — MediaRecorder in the browser
+// produces data URIs like `data:audio/webm;codecs=opus;base64,...`, so the
+// pre-`;base64,` section can contain zero or more parameter segments. The
+// old `([^;]+)` form only matched exactly one segment and left recordings
+// from the React UI's live-capture tab unparsed, which then failed base64
+// decoding on the leading `data:` bytes.
+var audioDataURIPattern = regexp.MustCompile(`^data:[^,]+?;base64,`)
 
 var audioDownloadClient = http.Client{Timeout: 30 * time.Second}