diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py
index aeb9c49960e8..19b5031c7a19 100755
--- a/src/transformers/__init__.py
+++ b/src/transformers/__init__.py
@@ -362,6 +362,7 @@
_import_structure["utils.dummy_pt_objects"] = [name for name in dir(dummy_pt_objects) if not name.startswith("_")]
else:
+ _import_structure["audio_processing_backends"] = ["NumpyAudioBackend", "NumpyBackend", "TorchAudioBackend", "TorchBackend"]
_import_structure["activations"] = []
_import_structure["backbone_utils"] = ["BackboneConfigMixin", "BackboneMixin"]
_import_structure["cache_utils"] = [
@@ -490,6 +491,10 @@
if TYPE_CHECKING:
# All modeling imports
# Models
+ from .audio_processing_backends import NumpyAudioBackend as NumpyAudioBackend
+ from .audio_processing_backends import NumpyBackend as NumpyBackend
+ from .audio_processing_backends import TorchAudioBackend as TorchAudioBackend
+ from .audio_processing_backends import TorchBackend as TorchBackend
from .backbone_utils import BackboneConfigMixin, BackboneMixin
from .cache_utils import Cache as Cache
from .cache_utils import DynamicCache as DynamicCache
diff --git a/src/transformers/audio_processing_backends.py b/src/transformers/audio_processing_backends.py
new file mode 100644
index 000000000000..3bc0b43fdabf
--- /dev/null
+++ b/src/transformers/audio_processing_backends.py
@@ -0,0 +1,573 @@
+# Copyright 2025 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+from typing import Unpack
+
+import numpy as np
+
+from .audio_processing_utils import BaseAudioProcessor
+from .audio_utils import (
+ _create_triangular_filter_bank,
+ hertz_to_mel,
+ mel_to_hertz,
+)
+from .processing_utils import AudioKwargs
+from .utils import is_speech_available, is_torch_available, logging
+
+
+logger = logging.get_logger(__name__)
+
+
+if is_torch_available():
+ import torch
+
+
+class NumpyAudioBackend(BaseAudioProcessor):
+ """NumPy backend for portable CPU-only audio processing."""
+
+ def __init__(self, *args, **kwargs: Unpack[AudioKwargs]):
+ super().__init__(*args, **kwargs)
+ self._set_attributes(**kwargs)
+
+ @property
+ def backend(self) -> str:
+ return "numpy"
+
+ # ── Backend array-API primitives ─────────────────────────────────────
+
+ def _astype(self, x, dtype_name):
+ return x.astype(np.dtype(dtype_name))
+
+ def _amax_over_features(self, x):
+ if x.ndim > 2:
+ return x.max(axis=tuple(range(1, x.ndim)), keepdims=True)
+ return x.max()
+
+ def _zeros_int32(self, shape):
+ return np.zeros(shape, dtype=np.int32)
+
+ def _as_backend_array(self, x):
+ return x if isinstance(x, np.ndarray) else np.asarray(x)
+
+ def _mean_axis0(self, x):
+ return x.mean(axis=0)
+
+ def _squeeze_axis0(self, x):
+ return np.squeeze(x, axis=0)
+
+ def _pad_axis(self, x, left, right, axis, value=0.0):
+ pad_width = [(0, 0)] * x.ndim
+ pad_width[axis] = (left, right)
+ return np.pad(x, pad_width, mode="constant", constant_values=value)
+
+ def _stack(self, seq):
+ return np.stack(seq)
+
+ def _insert_channel_dim(self, batch):
+ return batch[:, np.newaxis, :]
+
+ def _mean_last(self, x):
+ return x.mean(axis=-1, keepdims=True)
+
+ def _concat_last(self, parts):
+ return np.concatenate(parts, axis=-1)
+
+ # ── STFT pipeline ─────────────────────────────────────────────────────
+
+ def _create_stft_window(self, win_length, stft_cfg, audio):
+ if stft_cfg.window_fn == "hann_window_f32":
+ # fixed USM float32 periodic Hann (bit-exact with the legacy Gemma extractors);
+ # ignores `periodic`/`window_dtype`/`wkwargs`
+ arange = np.arange(win_length, dtype=np.float32)
+ return (0.5 * (1 - np.cos(2 * np.pi * arange / win_length))).astype(np.float32)
+ N = win_length + 1 if stft_cfg.periodic else win_length
+ fac = np.linspace(-np.pi, np.pi, N)
+ name = stft_cfg.window_fn
+ if name in ("hann", "hann_window"):
+ w = 0.5 + 0.5 * np.cos(fac)
+ elif name in ("hamming", "hamming_window"):
+ w = 0.54 + 0.46 * np.cos(fac)
+ elif name == "boxcar":
+ w = np.ones(N)
+ elif name == "povey":
+ w = (0.5 + 0.5 * np.cos(fac)) ** 0.85
+ else:
+ raise ValueError(f"Unknown window function '{name}'")
+ return w[:win_length] if stft_cfg.periodic else w
+
+ @staticmethod
+ def _np_frame(x, frame_length, hop_length):
+ """Create overlapping frames using stride tricks (replaces librosa.util.frame)."""
+ n_frames = 1 + (x.shape[-1] - frame_length) // hop_length
+ strides = x.strides[:-1] + (x.strides[-1] * hop_length, x.strides[-1])
+ shape = x.shape[:-1] + (n_frames, frame_length)
+ return np.lib.stride_tricks.as_strided(x, shape=shape, strides=strides)
+
+ def _frame_audio(self, audio, window, frame_length, hop_length, n_fft, stft_cfg):
+ if stft_cfg.center == "left":
+ # semicausal (USM/Gemma): zeros prepended only
+ audio = self._pad_axis(audio, (stft_cfg.win_length or n_fft) // 2, 0, axis=-1)
+ elif stft_cfg.center:
+ pad_width = [(0, 0)] * (audio.ndim - 1) + [(frame_length // 2, frame_length // 2)]
+ audio = np.pad(audio, pad_width, mode=stft_cfg.pad_mode)
+ frames = self._np_frame(np.ascontiguousarray(audio), frame_length, hop_length)
+ compute_dtype = np.result_type(audio.dtype, window.dtype)
+ return frames.astype(compute_dtype, copy=False)
+
+ def _preemphasize_waveform(self, audio, preemphasis, audio_ranges=None):
+ out = audio.copy()
+ out[..., 1:] = audio[..., 1:] - preemphasis * audio[..., :-1] # first sample unchanged
+ if audio_ranges is not None:
+ lengths = np.asarray([end - start for start, end in audio_ranges])
+ mask = np.arange(out.shape[-1])[None, :] < lengths[:, None]
+ out = np.where(mask, out, 0.0).astype(audio.dtype, copy=False)
+ return out
+
+ def _apply_dither(self, audio, audio_ranges=None):
+ return audio + (self.dither * np.random.randn(*audio.shape)).astype(audio.dtype)
+
+ def _window_and_fft(self, frames, window, frame_length, n_fft, stft_cfg, audio_dtype=None):
+ frames = frames * window
+ spec = np.fft.rfft(frames, n=n_fft, axis=-1)
+ if stft_cfg.fft_dtype is None:
+ # librosa contract: FFT output rounded through complex64
+ spec = spec.astype(np.complex64)
+ if stft_cfg.normalized:
+ spec = spec / np.sqrt(np.sum(window**2)).astype(spec.real.dtype)
+ return np.moveaxis(spec, -1, -2)
+
+ def _native_stft(self, audio, window, frame_length, hop_length, n_fft, stft_cfg):
+ # No numpy-native STFT exists; compose the manual framing + FFT leaves. This path
+ # receives the center-padded window and frame_length == n_fft from
+ # `_prepare_window_and_framing`, unlike the manual path (left-aligned window).
+ # `fft_dtype` can't leak in here: `_stft` rejects it on native-STFT configurations.
+ frames = self._frame_audio(audio, window, frame_length, hop_length, n_fft, stft_cfg)
+ return self._window_and_fft(frames, window, frame_length, n_fft, stft_cfg)
+
+ def _compute_magnitudes(self, stft_out, power, spectrogram_config=None):
+ # computation_dtype signals that upstream FE used float64 magnitudes
+ if spectrogram_config and spectrogram_config.computation_dtype:
+ return np.abs(stft_out, dtype=np.float64) ** power
+ return np.abs(stft_out) ** power
+
+ # ── Mel scale & normalization ─────────────────────────────────────────
+ #
+ # The base `_mel_filter_bank` dispatcher (audio_processing_utils) resolves geometry
+ # and dtype; the three leaves below own the numerical construction. Each backend's
+ # leaves deliberately implement their own ecosystem's rounding pattern: these numpy
+ # leaves are bit-exact against librosa and the legacy numpy feature extractors, the
+ # torch leaves against torchaudio / torchaudio.compliance.kaldi. The two backends are
+ # numerically equivalent but NOT bit-identical.
+
+ @staticmethod
+ def _np_triangular_banks(fft_freqs, filter_freqs, computation_dtype):
+ """Triangular bank with the numpy ecosystem's dtype policy.
+
+ With no computation dtype, replicate librosa's per-band float32 rounding:
+ slopes computed in float64 with each band's column cast to float32 on
+ assignment (librosa assigns rows into a float32-initialized array, which
+ rounds differently than casting a float64 matrix at the end). With a dtype,
+ plain full-precision construction cast to that dtype.
+ """
+ if computation_dtype is None:
+ num_frequency_bins = fft_freqs.shape[0]
+ num_mel_filters = filter_freqs.shape[0] - 2
+ filter_diff = np.diff(filter_freqs)
+ ramps = np.subtract.outer(filter_freqs, fft_freqs) # (num_mel_filters+2, num_frequency_bins)
+ mel_filters = np.zeros((num_frequency_bins, num_mel_filters), dtype=np.float32)
+ for i in range(num_mel_filters):
+ lower = -ramps[i] / filter_diff[i]
+ upper = ramps[i + 2] / filter_diff[i + 1]
+ mel_filters[:, i] = np.maximum(0, np.minimum(lower, upper)).astype(np.float32)
+ return mel_filters
+ return _create_triangular_filter_bank(fft_freqs, filter_freqs).astype(np.dtype(computation_dtype), copy=False)
+
+ def _kaldi_exact_mel_banks(
+ self,
+ num_mel_filters,
+ num_frequency_bins,
+ min_frequency,
+ max_frequency,
+ sampling_rate,
+ n_fft,
+ mel_cfg,
+ computation_dtype,
+ ):
+ """Mel-space triangularization, numpy-ecosystem semantics.
+
+ Bit-exact against the legacy numpy feature extractors (e.g. SeamlessM4T):
+ ``hertz_to_mel(mel_scale=...)`` on float64 bin frequencies and filter edges from
+ ``np.linspace`` in mel space. Deliberately NOT torchaudio's hardcoded
+ ``1127 * log`` float32 ``get_mel_banks`` arithmetic — the torch leaf owns that
+ rounding pattern; the two leaves are numerically equivalent, not bit-identical.
+ """
+ mel_min = hertz_to_mel(min_frequency, mel_scale=mel_cfg.mel_scale)
+ mel_max = hertz_to_mel(max_frequency, mel_scale=mel_cfg.mel_scale)
+ filter_freqs = np.linspace(mel_min, mel_max, num_mel_filters + 2)
+ fft_bin_width = sampling_rate / n_fft
+ fft_freqs = hertz_to_mel(fft_bin_width * np.arange(num_frequency_bins), mel_scale=mel_cfg.mel_scale)
+ return self._np_triangular_banks(fft_freqs, filter_freqs, computation_dtype)
+
+ def _kaldi_mel_banks_with_zero_bands(
+ self,
+ num_mel_filters,
+ num_frequency_bins,
+ min_frequency,
+ max_frequency,
+ sampling_rate,
+ n_fft,
+ mel_cfg,
+ computation_dtype,
+ ):
+ """Mel-space triangularization (numpy-ecosystem semantics, see
+ `_kaldi_exact_mel_banks`) with the lowest ``bands_to_zero`` bins zeroed."""
+ mel_min = hertz_to_mel(min_frequency, mel_scale=mel_cfg.mel_scale)
+ mel_max = hertz_to_mel(max_frequency, mel_scale=mel_cfg.mel_scale)
+ filter_freqs = np.linspace(mel_min, mel_max, num_mel_filters + 2)
+ fft_bin_width = sampling_rate / n_fft
+ fft_freqs = hertz_to_mel(
+ fft_bin_width * np.arange(mel_cfg.bands_to_zero, num_frequency_bins), mel_scale=mel_cfg.mel_scale
+ )
+ mel_filters = self._np_triangular_banks(fft_freqs, filter_freqs, computation_dtype)
+ if mel_cfg.bands_to_zero > 0:
+ mel_filters = np.pad(mel_filters, ((mel_cfg.bands_to_zero, 0), (0, 0)))
+ return mel_filters
+
+ def _standard_mel_banks(
+ self,
+ num_mel_filters,
+ num_frequency_bins,
+ min_frequency,
+ max_frequency,
+ sampling_rate,
+ n_fft,
+ mel_cfg,
+ computation_dtype,
+ ):
+ """Standard triangular mel filter bank, numpy-ecosystem semantics.
+
+ Bit-exact against librosa's filters (and the legacy numpy feature extractors
+ built on them): FFT bin frequencies always use the float64
+ ``linspace(0, sr // 2, bins)`` form regardless of ``frequency_bin_mode``, and
+ the slaney area-norm is applied after the dtype policy (i.e. after the per-band
+ float32 cast when no computation dtype is set — librosa's rounding order).
+ The torch leaf implements torchaudio's rounding instead; the two leaves are
+ numerically equivalent, not bit-identical.
+ """
+ mel_min = hertz_to_mel(min_frequency, mel_scale=mel_cfg.mel_scale)
+ mel_max = hertz_to_mel(max_frequency, mel_scale=mel_cfg.mel_scale)
+ mel_freqs = np.linspace(mel_min, mel_max, num_mel_filters + 2)
+ filter_freqs = mel_to_hertz(mel_freqs, mel_scale=mel_cfg.mel_scale)
+ fft_freqs = np.linspace(0, sampling_rate // 2, num_frequency_bins)
+ mel_filters = self._np_triangular_banks(fft_freqs, filter_freqs, computation_dtype)
+ if mel_cfg.norm == "slaney":
+ # Slaney-style mel is scaled to be approx constant energy per channel
+ enorm = 2.0 / (filter_freqs[2 : num_mel_filters + 2] - filter_freqs[:num_mel_filters])
+ mel_filters *= np.expand_dims(enorm, 0)
+ if mel_cfg.bands_to_zero > 0:
+ mel_filters = np.pad(mel_filters, ((mel_cfg.bands_to_zero, 0), (0, 0)))
+ return mel_filters
+
+ def _cast_mel_filters_to_default_float(self, mel_filters):
+ return mel_filters.astype(np.float32, copy=False)
+
+ def _apply_mel_scale(self, features, *, spectrogram_config, **kwargs):
+ mel_filters = self.mel_filters.astype(features.dtype, copy=False)
+ if spectrogram_config.mel_scale_config.matmul_order == "features_first":
+ mel_spec = np.matmul(features.swapaxes(-2, -1), mel_filters)
+ else:
+ mel_spec = np.matmul(mel_filters.T, features)
+ return np.maximum(spectrogram_config.mel_floor, mel_spec)
+
+ # ── Kaldi fbank helper ────────────────────────────────────────────────
+
+ def _kaldi_fbank(self, waveform, num_mel_bins, sample_frequency=None, **kwargs):
+ """Extract kaldi-compatible fbank features using torchaudio (or fallback to base pipeline).
+
+ Returns numpy array of shape (time, num_mel_bins).
+ """
+ if sample_frequency is None:
+ sample_frequency = self.sampling_rate
+
+ if is_speech_available():
+ import torchaudio.compliance.kaldi as ta_kaldi
+
+ waveform_tensor = torch.from_numpy(np.asarray(waveform)).unsqueeze(0)
+ fbank = ta_kaldi.fbank(
+ waveform_tensor, num_mel_bins=num_mel_bins, sample_frequency=sample_frequency, **kwargs
+ )
+ return fbank.numpy()
+
+ waveform = np.squeeze(waveform)
+ features = self.extract_spectrogram([waveform], spectrogram_config=self.spectrogram_config)
+ return features[0].T
+
+
+class TorchAudioBackend(BaseAudioProcessor):
+ """Torch backend for audio processing."""
+
+ def __init__(self, *args, **kwargs: Unpack[AudioKwargs]):
+ super().__init__(*args, **kwargs)
+ self._set_attributes(**kwargs)
+
+ @property
+ def backend(self) -> str:
+ return "torch"
+
+ # ── Backend array-API primitives ─────────────────────────────────────
+
+ def _astype(self, x, dtype_name):
+ return x.to(getattr(torch, dtype_name))
+
+ def _amax_over_features(self, x):
+ return x.amax(dim=(-2, -1), keepdim=True)
+
+ def _zeros_int32(self, shape):
+ return torch.zeros(shape, dtype=torch.int32)
+
+ def _as_backend_array(self, x):
+ return torch.from_numpy(x) if isinstance(x, np.ndarray) else x
+
+ def _mean_axis0(self, x):
+ return x.mean(dim=0)
+
+ def _squeeze_axis0(self, x):
+ return x.squeeze(0)
+
+ def _pad_axis(self, x, left, right, axis, value=0.0):
+ axis = axis % x.ndim
+ pad = [0, 0] * (x.ndim - 1 - axis) + [left, right]
+ return torch.nn.functional.pad(x, pad, "constant", value)
+
+ def _stack(self, seq):
+ return torch.stack(seq)
+
+ def _insert_channel_dim(self, batch):
+ return batch.unsqueeze(1)
+
+ def _mean_last(self, x):
+ return x.mean(dim=-1, keepdim=True)
+
+ def _concat_last(self, parts):
+ return torch.cat(parts, dim=-1)
+
+ # ── STFT pipeline ─────────────────────────────────────────────────────
+
+ def _needs_manual_framing(self, spectrogram_config):
+ return super()._needs_manual_framing(spectrogram_config) or spectrogram_config.stft_config.left_align_fft
+
+ def _create_stft_window(self, win_length, stft_cfg, audio):
+ dtype = getattr(torch, stft_cfg.window_dtype) if stft_cfg.window_dtype else audio.dtype
+ wkwargs = {**(stft_cfg.wkwargs or {}), "dtype": dtype}
+ name = stft_cfg.window_fn
+ if name == "hann_window_f32":
+ # numpy build + convert, so both backends' windows are bit-identical;
+ # ignores `periodic`/`window_dtype`/`wkwargs`
+ arange = np.arange(win_length, dtype=np.float32)
+ window = torch.from_numpy((0.5 * (1 - np.cos(2 * np.pi * arange / win_length))).astype(np.float32))
+ return window.to(device=audio.device)
+ if name in ("hann", "hann_window"):
+ window = torch.hann_window(win_length, periodic=stft_cfg.periodic, **wkwargs)
+ elif name in ("hamming", "hamming_window"):
+ window = torch.hamming_window(win_length, periodic=stft_cfg.periodic, **wkwargs)
+ elif name == "boxcar":
+ window = torch.ones(win_length, dtype=dtype)
+ elif name == "povey":
+ window = torch.hann_window(win_length, periodic=stft_cfg.periodic, **wkwargs).pow(0.85)
+ else:
+ raise ValueError(f"Unknown window function '{name}'")
+ return window.to(device=audio.device)
+
+ def _frame_audio(self, audio, window, frame_length, hop_length, n_fft, stft_cfg):
+ if stft_cfg.center == "left":
+ pad_left = (stft_cfg.win_length or n_fft) // 2
+ audio = torch.nn.functional.pad(audio, (pad_left, 0), mode="constant", value=0.0)
+ elif stft_cfg.center:
+ audio = torch.nn.functional.pad(audio, (frame_length // 2, frame_length // 2), mode=stft_cfg.pad_mode)
+ return audio.unfold(-1, frame_length, hop_length)
+
+ def _preemphasize_waveform(self, audio, preemphasis, audio_ranges=None):
+ audio = torch.cat([audio[..., :1], audio[..., 1:] - preemphasis * audio[..., :-1]], dim=-1)
+ if audio_ranges is not None:
+ lengths = torch.tensor([end - start for start, end in audio_ranges], device=audio.device)
+ mask = torch.arange(audio.shape[-1], device=audio.device).unsqueeze(0) < lengths.unsqueeze(1)
+ audio = audio.masked_fill(~mask, 0.0)
+ return audio
+
+ def _apply_dither(self, audio, audio_ranges=None):
+ noise = torch.randn(audio.shape, dtype=audio.dtype, device=audio.device)
+ return audio + self.dither * noise
+
+ def _window_and_fft(self, frames, window, frame_length, n_fft, stft_cfg, audio_dtype=None):
+ frames = frames * window
+ if stft_cfg.fft_dtype == "float64":
+ frames = frames.to(torch.float64) # mirrors numpy's rfft float64 promotion
+ if frame_length < n_fft:
+ frames = torch.nn.functional.pad(frames, (0, n_fft - frame_length))
+ spec = torch.fft.rfft(frames, n=n_fft)
+ if stft_cfg.normalized:
+ spec = spec / window.pow(2.0).sum().sqrt()
+ return spec.transpose(-2, -1)
+
+ def _native_stft(self, audio, window, frame_length, hop_length, n_fft, stft_cfg):
+ stft_out = torch.stft(
+ audio,
+ n_fft=n_fft,
+ hop_length=hop_length,
+ win_length=frame_length,
+ window=window,
+ center=stft_cfg.center,
+ pad_mode=stft_cfg.pad_mode,
+ normalized=False,
+ return_complex=True,
+ )
+ if stft_cfg.normalized:
+ stft_out = stft_out / window.pow(2.0).sum().sqrt()
+ return stft_out
+
+ def _cast_stft_output(self, magnitudes, spectrogram_config):
+ if spectrogram_config.computation_dtype:
+ return magnitudes
+ return magnitudes.float()
+
+ def _compute_magnitudes(self, stft_out, power, spectrogram_config=None):
+ return stft_out.abs() ** power
+
+ # ── Mel scale & normalization ─────────────────────────────────────────
+ #
+ # The base `_mel_filter_bank` dispatcher (audio_processing_utils) resolves geometry
+ # and dtype; the three leaves below own the numerical construction. Each backend's
+ # leaves deliberately implement their own ecosystem's rounding pattern: these torch
+ # leaves are bit-exact against torchaudio / torchaudio.compliance.kaldi, the numpy
+ # leaves against librosa and the legacy numpy feature extractors. The two backends
+ # are numerically equivalent but NOT bit-identical.
+
+ @staticmethod
+ def _kaldi_exact_mel_banks(
+ num_mel_filters,
+ num_frequency_bins,
+ min_frequency,
+ max_frequency,
+ sampling_rate,
+ n_fft,
+ mel_cfg,
+ computation_dtype,
+ ):
+ """Matches torchaudio.compliance.kaldi.get_mel_banks exactly.
+
+ Hardcoded ``1127 * log`` kaldi mel scale and ``get_mel_banks``'s edge arithmetic,
+ in torch's default float32 when no computation dtype is set. The numpy leaf owns
+ the legacy numpy-FE construction instead (``hertz_to_mel`` + linspace in float64);
+ the two leaves are numerically equivalent, not bit-identical.
+ """
+ dtype = getattr(torch, computation_dtype) if computation_dtype else None
+ num_fft_bins = n_fft // 2
+ fft_bin_width = sampling_rate / n_fft
+ mel_low = 1127.0 * math.log(1.0 + min_frequency / 700.0)
+ mel_high = 1127.0 * math.log(1.0 + max_frequency / 700.0)
+ mel_delta = (mel_high - mel_low) / (num_mel_filters + 1)
+
+ bin_idx = torch.arange(num_mel_filters, dtype=dtype).unsqueeze(1)
+ left_mel = mel_low + bin_idx * mel_delta
+ center_mel = mel_low + (bin_idx + 1.0) * mel_delta
+ right_mel = mel_low + (bin_idx + 2.0) * mel_delta
+
+ mel = 1127.0 * (1.0 + fft_bin_width * torch.arange(num_fft_bins, dtype=dtype) / 700.0).log()
+ mel = mel.unsqueeze(0)
+
+ up_slope = (mel - left_mel) / (center_mel - left_mel)
+ down_slope = (right_mel - mel) / (right_mel - center_mel)
+ banks = torch.max(torch.zeros(1, dtype=dtype), torch.min(up_slope, down_slope))
+ banks = torch.nn.functional.pad(banks, (0, 1), mode="constant", value=0)
+ return banks.T
+
+ @staticmethod
+ def _kaldi_mel_banks_with_zero_bands(
+ num_mel_filters,
+ num_frequency_bins,
+ min_frequency,
+ max_frequency,
+ sampling_rate,
+ n_fft,
+ mel_cfg,
+ computation_dtype,
+ ):
+ """Kaldi-style (triangularize in mel space) with optional zeroed low bands."""
+ dtype = getattr(torch, computation_dtype) if computation_dtype else None
+ mel_min = hertz_to_mel(min_frequency, mel_scale=mel_cfg.mel_scale)
+ mel_max = hertz_to_mel(max_frequency, mel_scale=mel_cfg.mel_scale)
+ filter_freqs = torch.linspace(mel_min, mel_max, num_mel_filters + 2, dtype=dtype)
+
+ fft_bin_width = sampling_rate / n_fft
+ hz_freqs = fft_bin_width * torch.arange(mel_cfg.bands_to_zero, num_frequency_bins, dtype=dtype)
+ fft_freqs = hertz_to_mel(hz_freqs, mel_scale=mel_cfg.mel_scale)
+
+ mel_filters = _create_triangular_filter_bank(fft_freqs, filter_freqs)
+ if mel_cfg.bands_to_zero > 0:
+ mel_filters = torch.nn.functional.pad(mel_filters, (0, 0, mel_cfg.bands_to_zero, 0))
+ return mel_filters
+
+ @staticmethod
+ def _standard_mel_banks(
+ num_mel_filters,
+ num_frequency_bins,
+ min_frequency,
+ max_frequency,
+ sampling_rate,
+ n_fft,
+ mel_cfg,
+ computation_dtype,
+ ):
+ """Standard (non-kaldi) triangular mel filter bank, torchaudio-ecosystem semantics.
+
+ Bit-exact against ``torchaudio.functional.melscale_fbanks`` in the default-dtype
+ case. The numpy leaf owns librosa's rounding pattern instead (float64 linspace
+ bins, per-band float32 casts); the two leaves are numerically equivalent, not
+ bit-identical.
+ """
+ dtype = getattr(torch, computation_dtype) if computation_dtype else None
+ mel_min = hertz_to_mel(min_frequency, mel_scale=mel_cfg.mel_scale)
+ mel_max = hertz_to_mel(max_frequency, mel_scale=mel_cfg.mel_scale)
+ mel_freqs = torch.linspace(mel_min, mel_max, num_mel_filters + 2, dtype=dtype)
+ filter_freqs = mel_to_hertz(mel_freqs, mel_scale=mel_cfg.mel_scale)
+
+ if mel_cfg.frequency_bin_mode == "rfft":
+ fft_freqs = torch.fft.rfftfreq(n=n_fft, d=1.0 / sampling_rate)
+ else:
+ fft_freqs = torch.linspace(0, sampling_rate // 2, num_frequency_bins)
+ if dtype is not None:
+ fft_freqs = fft_freqs.to(dtype)
+
+ mel_filters = _create_triangular_filter_bank(fft_freqs, filter_freqs)
+
+ if mel_cfg.norm == "slaney":
+ enorm = 2.0 / (filter_freqs[2 : num_mel_filters + 2] - filter_freqs[:num_mel_filters])
+ mel_filters = mel_filters * enorm[None, :]
+
+ if mel_cfg.bands_to_zero > 0:
+ mel_filters = torch.nn.functional.pad(mel_filters, (0, 0, mel_cfg.bands_to_zero, 0))
+ return mel_filters
+
+ def _cast_mel_filters_to_default_float(self, mel_filters):
+ return mel_filters.to(torch.get_default_dtype())
+
+ def _apply_mel_scale(self, features, *, spectrogram_config, **kwargs):
+ mel_filters = self.mel_filters.to(device=features.device)
+ if spectrogram_config.mel_scale_config.matmul_order == "features_first":
+ mel_spec = torch.matmul(features.transpose(-2, -1), mel_filters)
+ else:
+ # F.linear matches torchaudio's MelScale implementation exactly
+ mel_spec = torch.nn.functional.linear(features.transpose(-2, -1), mel_filters.T).transpose(-2, -1)
+ return torch.clamp(mel_spec, min=spectrogram_config.mel_floor)
diff --git a/src/transformers/audio_processing_base.py b/src/transformers/audio_processing_base.py
new file mode 100644
index 000000000000..d243d4610d54
--- /dev/null
+++ b/src/transformers/audio_processing_base.py
@@ -0,0 +1,261 @@
+# Copyright 2025 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import warnings
+from typing import Any, ClassVar, TypeVar
+
+from .audio_utils import is_valid_audio, load_audio
+from .preprocessing_base import BatchFeature as BaseBatchFeature
+from .preprocessing_base import PreprocessingMixin
+from .utils import (
+ FEATURE_EXTRACTOR_NAME,
+ copy_func,
+ logging,
+)
+
+
+_LEGACY_KEY_MAP = {
+ "input_features": "audio_features",
+ "input_values": "audio_values",
+ "audio_input_features": "audio_features",
+}
+
+
+AudioProcessorType = TypeVar("AudioProcessorType", bound="AudioProcessingMixin")
+
+
+logger = logging.get_logger(__name__)
+
+
+class BatchFeature(BaseBatchFeature):
+ r"""
+ Holds the output of the audio processor specific `__call__` methods.
+
+ This class is derived from a python dictionary and can be used as a dictionary.
+
+ Args:
+ data (`dict`):
+ Dictionary of lists/arrays/tensors returned by the __call__ method ('input_values', 'input_features', etc.).
+ tensor_type (`Union[None, str, TensorType]`, *optional*):
+ You can give a tensor_type here to convert the lists of integers in PyTorch/Numpy Tensors at
+ initialization.
+ """
+
+ _warned_keys: ClassVar[set] = set()
+
+ def __getitem__(self, item):
+ if isinstance(item, str) and item not in self.data:
+ new_key = self._resolve_legacy_key(item)
+ if new_key is not None and new_key in self.data:
+ if item not in BatchFeature._warned_keys:
+ warnings.warn(
+ f"Accessing '{item}' is deprecated, use '{new_key}' instead.",
+ FutureWarning,
+ stacklevel=2,
+ )
+ BatchFeature._warned_keys.add(item)
+ return self.data[new_key]
+ return super().__getitem__(item)
+
+ def __contains__(self, item):
+ if item in self.data:
+ return True
+ new_key = self._resolve_legacy_key(item)
+ return new_key is not None and new_key in self.data
+
+ def _resolve_legacy_key(self, old_key):
+ if old_key in ("attention_mask", "padding_mask"):
+ if "audio_features_mask" in self.data:
+ return "audio_features_mask"
+ if "audio_values_mask" in self.data:
+ return "audio_values_mask"
+ return None
+ return _LEGACY_KEY_MAP.get(old_key)
+
+
+class AudioProcessingMixin(PreprocessingMixin):
+ """
+ This is an audio processor mixin used to provide saving/loading functionality for audio processors.
+ """
+
+ _config_name = FEATURE_EXTRACTOR_NAME
+ _type_key = "audio_processor_type"
+ _nested_config_keys = ["audio_processor", "feature_extractor"]
+ _auto_class_default = "AutoAudioProcessor"
+ _file_type_label = "audio processor"
+ _excluded_dict_keys = {"mel_filters", "window"}
+ _extra_init_pops = ["feature_extractor_type"]
+ _config_filename_kwarg = "audio_processor_filename"
+ _subfolder_default = ""
+
+ # Legacy hub-config translation. Hub `preprocessor_config.json` files written by the
+ # old `XxxFeatureExtractor` classes use a flat key schema that doesn't match the new
+ # nested `SpectrogramConfig` API. `from_dict` applies `_legacy_field_mapping_base`
+ # first, then any per-model `legacy_field_mapping` last (highest priority). Values:
+ #
+ # - str: dot-path to the nested target (e.g. ``"spectrogram_config.stft_config.hop_length"``)
+ # — `from_dict` walks/creates intermediate dicts and writes the value.
+ # - callable: invoked as ``f(value, config_dict)`` and expected to mutate
+ # ``config_dict`` in place. Used for non-1:1 mappings such as
+ # Whisper's ``chunk_length`` → derived ``max_length = chunk_length * sampling_rate``.
+ # - None: drop the legacy key with no translation.
+ #
+ # The base mapping covers both universal keys (`return_attention_mask`, `feature_extractor_type`,
+ # …) and spectrogram-domain keys (`n_fft`, `hop_length`, …). For non-spectrogram models
+ # the spectrogram keys are simply absent from the hub config — translation is a no-op.
+ # See docs/adr/0002-legacy-field-mapping.md.
+ _legacy_field_mapping_base: dict = {
+ # Universal keys (apply to every audio processor).
+ # NOTE: `sampling_rate` is intentionally not listed — the hub key matches the modern
+ # instance attribute `sampling_rate` verbatim, so it passes through `from_dict` untranslated.
+ "feature_extractor_type": None,
+ "audio_processor_type": None,
+ "processor_class": None,
+ "return_attention_mask": "return_padding_mask",
+ # Spectrogram-domain keys (no-op for non-spectrogram models since hub configs
+ # for raw-audio models don't carry them)
+ "hop_length": "spectrogram_config.stft_config.hop_length",
+ "n_fft": "spectrogram_config.stft_config.n_fft",
+ "win_length": "spectrogram_config.stft_config.win_length",
+ "window_fn": "spectrogram_config.stft_config.window_fn",
+ "power": "spectrogram_config.stft_config.power",
+ "center": "spectrogram_config.stft_config.center",
+ "pad_mode": "spectrogram_config.stft_config.pad_mode",
+ "f_min": "spectrogram_config.mel_scale_config.f_min",
+ "f_max": "spectrogram_config.mel_scale_config.f_max",
+ "preemphasis": "spectrogram_config.preemphasis",
+ "mel_floor": "spectrogram_config.mel_floor",
+ }
+ legacy_field_mapping: dict | None = None
+
+ @classmethod
+ def _apply_legacy_field_mapping(cls, config_dict: dict) -> dict:
+ """Translate legacy hub-config keys to the new nested schema. Mutates and returns ``config_dict``."""
+ merged = {**cls._legacy_field_mapping_base, **(cls.legacy_field_mapping or {})}
+ for legacy_key, target in merged.items():
+ if legacy_key not in config_dict:
+ continue
+ value = config_dict.pop(legacy_key)
+ if target is None:
+ continue
+ if callable(target):
+ target(value, config_dict)
+ continue
+ # Dot-path target: walk/create intermediate dicts. Don't overwrite an
+ # existing modern value if both legacy and modern keys are present.
+ parts = target.split(".")
+ d = config_dict
+ for part in parts[:-1]:
+ next_d = d.get(part)
+ if next_d is None:
+ next_d = {}
+ d[part] = next_d
+ elif not isinstance(next_d, dict):
+ raise TypeError(
+ f"Cannot apply legacy mapping {legacy_key!r}→{target!r}: "
+ f"intermediate key {part!r} is not a dict ({type(next_d).__name__})."
+ )
+ d = next_d
+ if parts[-1] not in d:
+ d[parts[-1]] = value
+ return config_dict
+
+ @classmethod
+ def from_dict(cls, config_dict: dict[str, Any], **kwargs):
+ config_dict = dict(config_dict)
+ cls._apply_legacy_field_mapping(config_dict)
+ return super().from_dict(config_dict, **kwargs)
+
+ @classmethod
+ def get_audio_processor_dict(
+ cls, pretrained_model_name_or_path: str | os.PathLike, **kwargs
+ ) -> tuple[dict[str, Any], dict[str, Any]]:
+ """
+ From a `pretrained_model_name_or_path`, resolve to a dictionary of parameters, to be used for instantiating an
+ audio processor of type [`~audio_processing_base.AudioProcessingMixin`] using `from_dict`.
+
+ Parameters:
+ pretrained_model_name_or_path (`str` or `os.PathLike`):
+ The identifier of the pre-trained checkpoint from which we want the dictionary of parameters.
+ subfolder (`str`, *optional*, defaults to `""`):
+ In case the relevant files are located inside a subfolder of the model repo on huggingface.co, you can
+ specify the folder name here.
+ audio_processor_filename (`str`, *optional*, defaults to `"preprocessor_config.json"`):
+ The name of the file in the model directory to use for the audio processor config.
+
+ Returns:
+ `tuple[Dict, Dict]`: The dictionary(ies) that will be used to instantiate the audio processor object.
+ """
+ return cls._get_config_dict(pretrained_model_name_or_path, **kwargs)
+
+ def fetch_audio(self, audio_url_or_urls: str | list[str] | list[list[str]], sampling_rate: int | None = None):
+ """
+ Convert a single or a list of urls into the corresponding `np.ndarray` objects.
+
+ If a single url is passed, the return value will be a single object. If a list is passed a list of objects is
+ returned.
+ """
+ if sampling_rate is None:
+ sampling_rate = getattr(self, "sampling_rate", 16000)
+ if isinstance(audio_url_or_urls, list):
+ return [self.fetch_audio(x, sampling_rate=sampling_rate) for x in audio_url_or_urls]
+ elif isinstance(audio_url_or_urls, str):
+ return load_audio(audio_url_or_urls, sampling_rate=sampling_rate)
+ elif is_valid_audio(audio_url_or_urls):
+ return audio_url_or_urls
+ else:
+ raise TypeError(f"only a single or a list of entries is supported but got type={type(audio_url_or_urls)}")
+
+
+def make_legacy_audio_processor_alias(new_class: type, legacy_name: str) -> type:
+ """Create a deprecated subclass alias for the legacy ``XxxFeatureExtractor`` name.
+
+ Instantiating the alias emits a ``FutureWarning`` directing users to the new class. The
+ alias overrides ``to_dict`` so that saved configs identify themselves under the new
+ class name (``audio_processor_type: "WhisperAudioProcessor"``, not the legacy name) —
+ a `from_pretrained` followed by `save_pretrained` is enough to migrate a checkpoint.
+
+ Removal target: transformers v5.15. See [ADR 0002](docs/adr/0002-legacy-field-mapping.md).
+ """
+ def __init__(self, *args, **kwargs):
+ warnings.warn(
+ f"`{legacy_name}` is deprecated and will be removed in transformers v5.15. "
+ f"Use `{new_class.__name__}` instead.",
+ FutureWarning,
+ stacklevel=2,
+ )
+ new_class.__init__(self, *args, **kwargs)
+
+ def to_dict(self):
+ output = new_class.to_dict(self)
+ output[self._type_key] = new_class.__name__
+ return output
+
+ return type(
+ legacy_name,
+ (new_class,),
+ {
+ "__init__": __init__,
+ "to_dict": to_dict,
+ "__doc__": f"Deprecated alias for [`{new_class.__name__}`]. Removal: transformers v5.15.",
+ },
+ )
+
+
+AudioProcessingMixin.push_to_hub = copy_func(AudioProcessingMixin.push_to_hub)
+if AudioProcessingMixin.push_to_hub.__doc__ is not None:
+ AudioProcessingMixin.push_to_hub.__doc__ = AudioProcessingMixin.push_to_hub.__doc__.format(
+ object="audio processor", object_class="AutoFeatureExtractor", object_files="audio processor file"
+ )
diff --git a/src/transformers/audio_processing_utils.py b/src/transformers/audio_processing_utils.py
new file mode 100644
index 000000000000..36070072b63f
--- /dev/null
+++ b/src/transformers/audio_processing_utils.py
@@ -0,0 +1,781 @@
+# Copyright 2025 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from dataclasses import fields, replace
+from typing import Any, Unpack
+
+import numpy as np
+
+from .audio_processing_base import AudioProcessingMixin
+from .audio_utils import (
+ AudioInput,
+ SpectrogramConfig,
+ _array_namespace,
+ _clamp_min,
+ amplitude_to_db,
+ make_list_of_audio,
+ power_to_db,
+)
+from .feature_extraction_utils import BatchFeature
+from .processing_utils import AudioKwargs
+from .tokenization_utils_base import TruncationStrategy
+from .utils import PaddingStrategy, TensorType, logging
+
+
+logger = logging.get_logger(__name__)
+
+
+class BaseAudioProcessor(AudioProcessingMixin):
+ valid_kwargs = AudioKwargs
+
+ force_mono: bool = True
+ add_channel_dim: bool = False
+ padding = True
+ padding_side = "right"
+ padding_value = 0.0
+ return_padding_mask = True
+ mask_level = None
+ do_batch_spectrogram = True
+ model_input_names = ["audio"]
+ dither: float = 0.0
+
+ def __init__(
+ self,
+ sampling_rate: int | None = None,
+ **kwargs: Unpack[AudioKwargs],
+ ):
+ if sampling_rate is not None:
+ self.sampling_rate = sampling_rate
+ if self.sampling_rate is None:
+ raise ValueError(
+ f"`sampling_rate` must be set either as a class attribute on {self.__class__.__name__} "
+ "or passed to __init__."
+ )
+
+ super().__init__(**kwargs)
+ # _set_attributes runs in the backend subclasses' __init__, not here, for remote-code BC.
+
+ def _set_attributes(self, **kwargs):
+ """Called from the backend subclasses' ``__init__`` (not the base, for remote-code BC)."""
+ super()._set_attributes(**kwargs)
+ if self.spectrogram_config is not None:
+ if self.spectrogram_config.mel_scale_config is not None and not hasattr(self, "mel_filters"):
+ self.mel_filters = self._mel_filter_bank(self.spectrogram_config)
+ self._cached_stft_window = None
+
+ def _standardize_kwargs(
+ self,
+ **kwargs,
+ ) -> dict:
+ if isinstance(kwargs.get("spectrogram_config"), dict):
+ kwargs["spectrogram_config"] = SpectrogramConfig.from_dict(kwargs["spectrogram_config"])
+ if kwargs.get("spectrogram_config") is not None and kwargs.get("do_extract_spectrogram") is None:
+ kwargs["do_extract_spectrogram"] = True
+ return kwargs
+
+ def _validate_preprocess_kwargs(
+ self,
+ sampling_rate: int | None = None,
+ max_length: int | None = None,
+ truncation: bool | None = None,
+ pad_to_multiple_of: int | None = None,
+ return_tensors: str | TensorType | None = None,
+ **kwargs,
+ ):
+ if truncation and max_length is None:
+ raise ValueError("When setting `truncation=True`, make sure that `max_length` is defined.")
+
+ def _serialize_value(self, key, value):
+ if key == "spectrogram_config" and hasattr(value, "to_dict"):
+ return value.to_dict()
+ return value
+
+ def __call__(self, audio: AudioInput, *args, **kwargs: Unpack[AudioKwargs]) -> BatchFeature:
+ return self.preprocess(audio, *args, **kwargs)
+
+ def preprocess(self, audio: AudioInput, *args, **kwargs: Unpack[AudioKwargs]) -> BatchFeature:
+ return super().preprocess(audio, *args, **kwargs)
+
+ def _preprocess_like_inputs(self, audio: AudioInput, *args, **kwargs) -> BatchFeature:
+ return self._preprocess_audio_like_inputs(audio, *args, **kwargs)
+
+ def _preprocess_audio_like_inputs(
+ self,
+ audio: AudioInput,
+ *args,
+ sampling_rate: int | None = None,
+ **kwargs: Unpack[AudioKwargs],
+ ) -> BatchFeature:
+ audio = self._prepare_audio_like_inputs(audio=audio, sampling_rate=sampling_rate)
+ return self._preprocess(audio, *args, **kwargs)
+
+ def _prepare_audio_like_inputs(self, audio: AudioInput, *args, sampling_rate: int | None = None, **kwargs) -> list:
+ audio = self._prepare_audio_structure(audio, sampling_rate=sampling_rate)
+ audio = [self.process_audio(audio_el) for audio_el in audio]
+ return audio
+
+ def _prepare_audio_structure(self, audio: AudioInput, sampling_rate: int | None = None) -> list:
+ is_url_input = isinstance(audio, str) or (
+ isinstance(audio, (list, tuple)) and all(isinstance(el, str) for el in audio)
+ )
+
+ if is_url_input:
+ audio = self.fetch_audio(audio)
+ else:
+ # `PreprocessingMixin.preprocess` setdefaults `sampling_rate` from `self.sampling_rate`,
+ # so an omitted rate no-ops here; only a genuine caller mismatch raises.
+ if sampling_rate is not None and sampling_rate != self.sampling_rate:
+ raise ValueError(
+ f"The model corresponding to this audio processor: {self.__class__.__name__} was trained using a"
+ f" sampling rate of {self.sampling_rate}. Please make sure that the provided `audio` input"
+ f" was sampled with {self.sampling_rate} and not {sampling_rate}."
+ )
+
+ audio = make_list_of_audio(audio)
+ return audio
+
+ def process_audio(self, *args, **kwargs):
+ return self._process_audio(*args, **kwargs)
+
+ def _preprocess(
+ self,
+ audio: list[np.ndarray] | list["torch.Tensor"],
+ padding: bool | str | PaddingStrategy | None,
+ max_length: int | None,
+ truncation: bool | str | TruncationStrategy | None,
+ pad_to_multiple_of: int | None,
+ return_tensors: str | TensorType | None,
+ spectrogram_config: SpectrogramConfig | None = None,
+ do_extract_spectrogram: bool | None = True,
+ do_batch_spectrogram: bool | None = True,
+ **kwargs: Any,
+ ) -> BatchFeature:
+ # Path 1: per-waveform spectrogram extraction, padded at the feature level.
+ if do_extract_spectrogram and not do_batch_spectrogram:
+ features = self.extract_spectrogram(audio, spectrogram_config=spectrogram_config, **kwargs)
+ feature_lengths = [f.shape[0] for f in features]
+ features = self._postprocess_features(features, feature_lengths)
+ features, feature_ranges = self._pad_features(
+ features,
+ padding,
+ max_length,
+ truncation,
+ pad_to_multiple_of,
+ )
+ output = {"audio_features": self._stack_features(features)}
+ if self.return_padding_mask:
+ output["audio_features_mask"] = self._get_mask(feature_ranges, features[0].shape[0])
+ output = self._postprocess_output(output, feature_ranges=feature_ranges, **kwargs)
+ return BatchFeature(data=output, tensor_type=return_tensors)
+
+ # Path 2: pad audio first, then optionally extract a spectrogram on the padded batch.
+ audio, audio_ranges = self.pad(audio, padding, max_length, truncation, pad_to_multiple_of)
+ padded_length = audio[0].shape[-1]
+ batched = self._to_batch(audio)
+
+ if do_extract_spectrogram:
+ output = {
+ "audio_features": self.extract_spectrogram(
+ batched,
+ spectrogram_config=spectrogram_config,
+ audio_ranges=audio_ranges,
+ **kwargs,
+ )
+ }
+ else:
+ output = {"audio_values": batched}
+
+ if self.return_padding_mask:
+ # Features live on the frame axis: map audio ranges → feature ranges via hop_length,
+ # unless ``mask_level="audio"`` forces an audio-sample-level mask.
+ if do_extract_spectrogram and self.mask_level != "audio":
+ spec_cfg = spectrogram_config or self.spectrogram_config
+ audio_lengths = np.array([end - start for start, end in audio_ranges])
+ feature_lengths = self._get_features_lengths(audio_lengths, spec_cfg)
+ mask_ranges = [(0, int(length)) for length in feature_lengths]
+ mask_length = int(self._get_features_lengths(padded_length, spec_cfg, include_center_frame=True))
+ else:
+ mask_ranges = audio_ranges
+ mask_length = padded_length
+ mask_key = "audio_features_mask" if do_extract_spectrogram else "audio_values_mask"
+ output[mask_key] = self._get_mask(mask_ranges, mask_length)
+
+ output = self._postprocess_output(output, audio_ranges=audio_ranges, **kwargs)
+ return BatchFeature(data=output, tensor_type=return_tensors)
+
+ def _postprocess_features(self, features, feature_lengths):
+ """Hook: per-utterance feature processing after extraction, before feature-level padding.
+ Override for normalization that must happen on unpadded features
+ """
+ return features
+
+ def _postprocess_output(self, output, audio_ranges=None, feature_ranges=None, **kwargs):
+ """Hook: augment or modify the output dict after main processing.
+ Override to add custom fields (e.g., audio_embed_sizes) or post-hoc normalization on the stacked/batched output.
+ """
+ return output
+
+ def _get_padding_strategies(self, padding=False, max_length=None):
+ if padding is not False:
+ if padding is True:
+ padding_strategy = PaddingStrategy.LONGEST
+ elif not isinstance(padding, PaddingStrategy):
+ padding_strategy = PaddingStrategy(padding)
+ elif isinstance(padding, PaddingStrategy):
+ padding_strategy = padding
+ else:
+ padding_strategy = PaddingStrategy.DO_NOT_PAD
+
+ if max_length is None:
+ if padding_strategy == PaddingStrategy.MAX_LENGTH:
+ raise ValueError(
+ f"When setting ``padding={PaddingStrategy.MAX_LENGTH}``, make sure that max_length is defined"
+ )
+
+ if padding_strategy != PaddingStrategy.DO_NOT_PAD and (self.padding_value is None):
+ raise ValueError(
+ "Asking to pad but the feature_extractor does not have a padding value. Please select a value to use"
+ " as `padding_value`. For example: `feature_extractor.padding_value = 0.0`."
+ )
+
+ return padding_strategy
+
+ def pad(
+ self,
+ audio: list[np.ndarray] | list["torch.Tensor"],
+ padding: bool | str | PaddingStrategy = True,
+ max_length: int | None = None,
+ truncation: bool = False,
+ pad_to_multiple_of: int | None = None,
+ ) -> tuple[list, list[tuple[int, int]]]:
+ padding_strategy = self._get_padding_strategies(padding=padding, max_length=max_length)
+
+ if truncation:
+ trunc_length = max_length
+ if pad_to_multiple_of is not None and (trunc_length % pad_to_multiple_of != 0):
+ trunc_length = ((trunc_length // pad_to_multiple_of) + 1) * pad_to_multiple_of
+ audio = [self._truncate_single(audio_el, max_length=trunc_length) for audio_el in audio]
+
+ if padding_strategy == PaddingStrategy.LONGEST:
+ max_length = max(audio_el.shape[-1] for audio_el in audio)
+ padding_strategy = PaddingStrategy.MAX_LENGTH
+
+ if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
+ max_length = ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of
+
+ actual_lengths = [audio_el.shape[-1] for audio_el in audio]
+
+ if padding_strategy != PaddingStrategy.DO_NOT_PAD:
+ audio = [self._pad_single(audio_el, max_length=max_length) for audio_el in audio]
+
+ audio_ranges = []
+ for i, length in enumerate(actual_lengths):
+ padded_length = audio[i].shape[-1]
+ if self.padding_side == "left":
+ audio_ranges.append((padded_length - length, padded_length))
+ else:
+ audio_ranges.append((0, length))
+
+ return audio, audio_ranges
+
+ def _truncate_single(self, audio_el, max_length: int):
+ return audio_el[..., :max_length] if audio_el.shape[-1] > max_length else audio_el
+
+ def _pad_features(self, features, padding, max_length, truncation, pad_to_multiple_of):
+ padding_strategy = self._get_padding_strategies(padding=padding, max_length=max_length)
+ if truncation and max_length is not None:
+ features = [f[:max_length] for f in features]
+ actual_lengths = [f.shape[0] for f in features]
+ if padding_strategy == PaddingStrategy.LONGEST:
+ max_length = max(actual_lengths)
+ padding_strategy = PaddingStrategy.MAX_LENGTH
+ if max_length is not None and pad_to_multiple_of is not None and max_length % pad_to_multiple_of != 0:
+ max_length = ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of
+ if padding_strategy == PaddingStrategy.MAX_LENGTH and max_length is not None:
+ features = [f if f.shape[0] >= max_length else self._pad_feature_single(f, max_length) for f in features]
+ return features, [(0, length) for length in actual_lengths]
+
+ def _pad_feature_single(self, feature, max_length):
+ """Right-pad one feature array/tensor along its first (time) axis with `padding_value`."""
+ return self._pad_axis(feature, 0, max_length - feature.shape[0], axis=0, value=self.padding_value)
+
+ def _process_audio(self, audio_el):
+ audio_el = self._as_backend_array(audio_el)
+ if audio_el.ndim > 1:
+ if self.force_mono and audio_el.shape[0] > 1:
+ audio_el = self._mean_axis0(audio_el)
+ elif audio_el.shape[0] == 1:
+ audio_el = self._squeeze_axis0(audio_el)
+ else:
+ raise ValueError("Audio has more than one channel but force_mono is False")
+ return audio_el
+
+ def _to_batch(self, audio):
+ batch = self._stack(audio)
+ if self.add_channel_dim:
+ batch = self._insert_channel_dim(batch)
+ return batch
+
+ def _pad_single(self, audio, max_length: int) -> AudioInput:
+ current_length = audio.shape[-1]
+ if current_length >= max_length:
+ return audio
+ pad = max_length - current_length
+ if self.padding_side == "right":
+ left, right = 0, pad
+ elif self.padding_side == "left":
+ left, right = pad, 0
+ else:
+ raise ValueError(f"Invalid padding side: {self.padding_side}")
+ return self._pad_axis(audio, left, right, axis=-1, value=self.padding_value)
+
+ def _stack_features(self, features):
+ return self._stack(features)
+
+ def _get_mask(self, ranges, padded_length):
+ mask = self._zeros_int32((len(ranges), padded_length))
+ for i, (start, end) in enumerate(ranges):
+ mask[i, start:end] = 1
+ return mask
+
+ def _masked_mean_var_normalize(self, features, feature_lengths, epsilon=1e-5):
+ """NeMo/Cohere-style per-utterance mean/variance normalization over the first
+ `feature_lengths` frames of `features` (batch, frames, feature_dim), zeroing padded
+ frames. `feature_lengths` is a CPU sequence/array of float32-representable counts."""
+ xp = _array_namespace(features)
+ lengths = self._astype(self._as_backend_array(np.asarray(feature_lengths)), "float32")
+ mask = (xp.arange(features.shape[1])[None, :] < lengths[:, None])[..., None]
+ masked = features * mask
+ mean = (masked.sum(axis=1) / lengths[:, None])[:, None, :]
+ variance = (((masked - mean) ** 2) * mask).sum(axis=1) / (lengths - 1)[:, None]
+ std = xp.sqrt(variance)[:, None, :]
+ return (features - mean) / (std + epsilon) * mask
+
+ # ── Spectrogram core ─────────────────────────────────────────────────
+
+ def extract_spectrogram(self, audio, *, spectrogram_config: SpectrogramConfig | None = None, **kwargs):
+ if spectrogram_config is None:
+ spectrogram_config = self.spectrogram_config
+
+ config_field_names = {f.name for f in fields(SpectrogramConfig)}
+ overrides = {k: kwargs.pop(k) for k in list(kwargs) if k in config_field_names}
+ if overrides:
+ spectrogram_config = replace(spectrogram_config, **overrides)
+
+ norm_kwargs = {k: v for k, v in kwargs.items() if k not in ("audio_ranges", "feature_ranges")}
+
+ if isinstance(audio, list):
+ features = [self._extract_spectrogram(a, spectrogram_config=spectrogram_config, **kwargs) for a in audio]
+ if spectrogram_config.mel_scale_config is not None:
+ features = [
+ self._apply_mel_scale(f, spectrogram_config=spectrogram_config, **kwargs) for f in features
+ ]
+ features = [
+ self._normalize_magnitude(f, spectrogram_config=spectrogram_config, **norm_kwargs) for f in features
+ ]
+ else:
+ features = self._extract_spectrogram(audio, spectrogram_config=spectrogram_config, **kwargs)
+ if spectrogram_config.mel_scale_config is not None:
+ features = self._apply_mel_scale(features, spectrogram_config=spectrogram_config, **kwargs)
+ features = self._normalize_magnitude(features, spectrogram_config=spectrogram_config, **norm_kwargs)
+
+ return features
+
+ def _extract_spectrogram(self, audio, *, spectrogram_config, **kwargs):
+ return self._stft(audio, spectrogram_config=spectrogram_config, **kwargs)
+
+ def _stft(self, audio, *, spectrogram_config, **kwargs):
+ stft_cfg = spectrogram_config.stft_config
+ needs_manual_framing = self._needs_manual_framing(spectrogram_config)
+ if stft_cfg.frame_extension:
+ if stft_cfg.frame_extension != 1:
+ raise ValueError(f"Only frame_extension=1 is supported, got {stft_cfg.frame_extension}.")
+ if stft_cfg.center is True:
+ raise ValueError("frame_extension requires center=False or center='left', not symmetric centering.")
+ if spectrogram_config.remove_dc_offset:
+ raise ValueError("remove_dc_offset is not supported with frame_extension.")
+ elif spectrogram_config.preemphasis_mode == "htk_per_frame":
+ raise ValueError("preemphasis_mode='htk_per_frame' requires frame_extension=1.")
+ if stft_cfg.fft_dtype is not None:
+ if stft_cfg.fft_dtype not in ("float64", "native"):
+ raise ValueError(f"fft_dtype must be None, 'float64' or 'native', got {stft_cfg.fft_dtype!r}.")
+ if not needs_manual_framing:
+ raise ValueError(
+ "fft_dtype applies to the manual-framing path only; this configuration uses the native STFT."
+ )
+ n_fft = stft_cfg.n_fft
+ win_length = stft_cfg.win_length or n_fft
+ hop_length = stft_cfg.hop_length or win_length // 2
+
+ if spectrogram_config.computation_dtype and stft_cfg.fft_dtype is None:
+ # with `fft_dtype`, the cast happens at the FFT boundary instead
+ dtype_str = spectrogram_config.computation_dtype
+ if isinstance(audio, np.ndarray):
+ audio = audio.astype(dtype_str)
+ else:
+ import torch
+
+ audio = audio.to(getattr(torch, dtype_str))
+ if self.dither > 0:
+ audio = self._apply_dither(audio, kwargs.get("audio_ranges"))
+ if spectrogram_config.waveform_scale is not None:
+ audio = audio * spectrogram_config.waveform_scale
+ if spectrogram_config.preemphasis is not None and spectrogram_config.preemphasis_mode == "waveform":
+ audio = self._preemphasize_waveform(audio, spectrogram_config.preemphasis, kwargs.get("audio_ranges"))
+
+ # Cache window on first call; reuse on subsequent calls with same config
+ if self._cached_stft_window is not None and spectrogram_config is self.spectrogram_config:
+ window, frame_length = self._cached_stft_window
+ else:
+ window = self._create_stft_window(win_length, stft_cfg, audio)
+ window, frame_length = self._prepare_window_and_framing(window, win_length, n_fft, needs_manual_framing)
+ if spectrogram_config is self.spectrogram_config:
+ self._cached_stft_window = (window, frame_length)
+
+ if needs_manual_framing:
+ audio_dtype = audio.dtype
+ frames = self._frame_audio(
+ audio, window, frame_length + stft_cfg.frame_extension, hop_length, n_fft, stft_cfg
+ )
+ frames = self._apply_frame_processing(frames, spectrogram_config=spectrogram_config, **kwargs)
+ stft_out = self._window_and_fft(frames, window, frame_length, n_fft, stft_cfg, audio_dtype=audio_dtype)
+ else:
+ stft_out = self._native_stft(audio, window, frame_length, hop_length, n_fft, stft_cfg)
+
+ magnitudes = self._compute_magnitudes(stft_out, stft_cfg.power, spectrogram_config=spectrogram_config)
+ return self._cast_stft_output(magnitudes, spectrogram_config)
+
+ # ── Spectrogram hooks ────────────────────────────────────────────────
+
+ def _needs_manual_framing(self, spectrogram_config):
+ """Whether the STFT requires manual framing (unfold-based) instead of a native STFT."""
+ return (
+ (
+ spectrogram_config.preemphasis is not None
+ and spectrogram_config.preemphasis_mode in ("per_frame", "htk_per_frame")
+ )
+ or spectrogram_config.remove_dc_offset
+ or bool(spectrogram_config.stft_config.frame_extension)
+ or spectrogram_config.stft_config.center == "left" # truthy string would center-pad natively
+ )
+
+ def _cast_stft_output(self, magnitudes, spectrogram_config):
+ """Cast STFT output to the desired output dtype. Default: no-op."""
+ return magnitudes
+
+ def _get_features_lengths(self, audio_lengths, spectrogram_config, include_center_frame=False):
+ """
+ Convert raw audio sample lengths to the number of feature frames after spectrogram extraction.
+
+ For centered STFT returns `audio_lengths // hop_length` (plus 1 when
+ `include_center_frame=True`); for non-centered STFT returns the exact frame count
+ `(audio_lengths - win_length) // hop_length + 1`.
+
+ Override this method in subclasses that use non-standard STFT framing (e.g.,
+ unfold-based with extra samples, or model-specific frame counting).
+ """
+ stft_cfg = spectrogram_config.stft_config
+ win_length = stft_cfg.win_length or stft_cfg.n_fft
+ hop_length = stft_cfg.hop_length or win_length // 2
+ if stft_cfg.center == "left":
+ lengths = (audio_lengths + win_length // 2 - (win_length + stft_cfg.frame_extension)) // hop_length + 1
+ return max(0, lengths) if isinstance(lengths, int) else lengths.clip(min=0)
+ if not stft_cfg.center:
+ return (audio_lengths - win_length) // hop_length + 1
+ lengths = audio_lengths // hop_length
+ if include_center_frame:
+ lengths = lengths + 1
+ return lengths
+
+ # ── Spectrogram backend ──────────────────────────────────────────────
+
+ def _create_stft_window(self, win_length, stft_cfg, audio):
+ raise NotImplementedError
+
+ def _prepare_window_and_framing(self, window, win_length, n_fft, needs_manual_framing):
+ if needs_manual_framing and win_length < n_fft:
+ return window, win_length
+ if win_length < n_fft:
+ left_pad = (n_fft - win_length) // 2
+ right_pad = n_fft - win_length - left_pad
+ window = self._pad_axis(window, left_pad, right_pad, axis=-1, value=0.0)
+ return window, n_fft
+
+ def _frame_audio(self, audio, window, frame_length, hop_length, n_fft, stft_cfg):
+ """Extract overlapping frames from the audio signal.
+
+ Handles center padding and dtype promotion. Returns frames of shape
+ (..., num_frames, frame_length). Implemented by backend subclasses.
+ """
+ raise NotImplementedError
+
+ def _apply_frame_processing(self, frames, *, spectrogram_config, **kwargs):
+ """Hook: per-frame signal conditioning after frame extraction.
+
+ Called after framing, before windowing and FFT. Applies DC-offset removal, per-frame
+ (kaldi-style) preemphasis, and USM/HTK-style extended-frame preemphasis when
+ ``stft_config.frame_extension`` is set. Override for non-standard frame processing
+ that doesn't fit these knobs, e.g. boundary-frame masking (Phi4-multimodal).
+ """
+ if spectrogram_config.stft_config.frame_extension:
+ # USM-style extended frames: preemphasis consumes the extra trailing sample,
+ # reducing the frame back to `win_length`.
+ preemphasis = spectrogram_config.preemphasis
+ if preemphasis is None or preemphasis <= 0.0:
+ return frames[..., :-1]
+ if spectrogram_config.preemphasis_mode == "htk_per_frame":
+ # HTK flavor: first sample scaled by (1 - p) instead of replicate-padded
+ first = frames[..., :1] * (1.0 - preemphasis)
+ rest = frames[..., 1:-1] - preemphasis * frames[..., :-2]
+ return self._concat_last([first, rest])
+ if spectrogram_config.preemphasis_mode == "per_frame":
+ return frames[..., 1:] - preemphasis * frames[..., :-1]
+ return frames[..., :-1] # "waveform" mode: already applied upstream
+ if spectrogram_config.remove_dc_offset:
+ frames = frames - self._mean_last(frames)
+ preemphasis = spectrogram_config.preemphasis
+ if preemphasis is not None and spectrogram_config.preemphasis_mode == "per_frame":
+ # Replicate-pad first sample (x0 - p*x0, not x0*(1-p)): bit-exact with kaldi.
+ first = frames[..., :1] - preemphasis * frames[..., :1]
+ rest = frames[..., 1:] - preemphasis * frames[..., :-1]
+ frames = self._concat_last([first, rest])
+ return frames
+
+ def _preemphasize_waveform(self, audio, preemphasis, audio_ranges=None):
+ """Waveform-level preemphasis (first sample unchanged), zeroing padded samples via
+ ``audio_ranges``. Used when ``spectrogram_config.preemphasis_mode == "waveform"``
+ (ASR models: Parakeet/Cohere/Nemotron). Implemented by backend subclasses."""
+ raise NotImplementedError
+
+ def _window_and_fft(self, frames, window, frame_length, n_fft, stft_cfg, audio_dtype=None):
+ """Apply window, zero-pad, FFT, and normalize. Returns complex STFT of shape (..., freq, time).
+ Implemented by backend subclasses."""
+ raise NotImplementedError
+
+ def _apply_dither(self, audio, audio_ranges=None):
+ """Additive dither, applied when ``self.dither`` is nonzero. Backend defaults add
+ unseeded Gaussian noise; deterministic implementations (Cohere-ASR) override.
+ ``audio_ranges`` is ``None`` on unbatched calls. Implemented by backend subclasses."""
+ raise NotImplementedError
+
+ def _native_stft(self, audio, window, frame_length, hop_length, n_fft, stft_cfg):
+ """Native STFT (e.g. torch.stft). Returns complex output. Implemented by backend subclasses."""
+ raise NotImplementedError
+
+ def _compute_magnitudes(self, stft_out, power, spectrogram_config=None):
+ """Convert complex STFT output to a real-valued magnitude spectrogram.
+ Implemented by backend subclasses. Overridable for custom magnitude computation (e.g. Parakeet)."""
+ raise NotImplementedError
+
+ def _apply_mel_scale(self, *args, **kwargs):
+ """Apply mel filterbank to spectrogram features."""
+ raise NotImplementedError
+
+ def _normalize_magnitude(
+ self, features, *, spectrogram_config, reference=1.0, min_value=1e-10, db_range=None, **kwargs
+ ):
+ log_mel = spectrogram_config.log_mode
+ if log_mel is None:
+ return self._astype(features, "float32")
+
+ if spectrogram_config.pre_log_offset is not None:
+ result = features + spectrogram_config.pre_log_offset
+ else:
+ result = _clamp_min(features, spectrogram_config.mel_floor)
+
+ if log_mel == "log":
+ result = self._astype(_array_namespace(result).log(result), "float32")
+ elif log_mel == "log10":
+ result = self._astype(_array_namespace(result).log10(result), "float32")
+ elif log_mel == "dB":
+ power = spectrogram_config.stft_config.power
+ if power == 2.0:
+ result = power_to_db(result, reference, min_value, db_range)
+ elif power == 1.0:
+ result = amplitude_to_db(result, reference, min_value, db_range)
+ else:
+ raise ValueError(f"Cannot use log_mel option 'dB' with power {power}")
+ result = self._astype(result, "float32")
+ else:
+ raise ValueError(f"Unknown log_mel option: {log_mel}")
+
+ if spectrogram_config.skip_last_frame:
+ result = result[..., :-1]
+ return self._apply_post_log_normalization(result, spectrogram_config)
+
+ def _apply_post_log_normalization(self, result, spectrogram_config):
+ if spectrogram_config.clip_max_offset is not None:
+ max_vals = self._amax_over_features(result)
+ result = _array_namespace(result).maximum(result, max_vals - spectrogram_config.clip_max_offset)
+ if spectrogram_config.post_log_shift is not None:
+ result = result + spectrogram_config.post_log_shift
+ if spectrogram_config.post_log_scale is not None:
+ result = result * spectrogram_config.post_log_scale
+ return result
+
+ # ── Backend array-API primitives ─────────────────────────────────────
+
+ def _astype(self, x, dtype_name):
+ raise NotImplementedError
+
+ def _amax_over_features(self, x):
+ raise NotImplementedError
+
+ def _zeros_int32(self, shape):
+ raise NotImplementedError
+
+ def _as_backend_array(self, x):
+ raise NotImplementedError
+
+ def _mean_axis0(self, x):
+ raise NotImplementedError
+
+ def _squeeze_axis0(self, x):
+ raise NotImplementedError
+
+ def _pad_axis(self, x, left, right, axis, value=0.0):
+ raise NotImplementedError
+
+ def _stack(self, seq):
+ raise NotImplementedError
+
+ def _insert_channel_dim(self, batch):
+ raise NotImplementedError
+
+ def _mean_last(self, x):
+ raise NotImplementedError
+
+ def _concat_last(self, parts):
+ raise NotImplementedError
+
+ def _mel_filter_bank(self, spectrogram_config: SpectrogramConfig):
+ """Build the mel filter bank described by ``spectrogram_config.mel_scale_config``.
+
+ Backend-agnostic dispatcher: derives the geometry (number of frequency bins, FFT
+ size, frequency range) and the computation dtype, then delegates the numerical
+ construction to one of three backend leaves:
+
+ - ``_kaldi_exact_mel_banks``: ``triangularize_in_mel_space`` with no zeroed bands
+ - ``_kaldi_mel_banks_with_zero_bands``: ``triangularize_in_mel_space`` with ``bands_to_zero``
+ - ``_standard_mel_banks``: standard triangular (librosa/torchaudio-style) filters
+
+ Dtype policy: the dtype name is resolved as ``mel_scale_config.computation_dtype``
+ falling back to the top-level ``spectrogram_config.computation_dtype`` (pipelines
+ that run in float64 for legacy-FE parity need float64 filters as well), and passed
+ to the leaves as a string (or None); each backend resolves it natively. When only
+ the mel-level dtype is set, the filters are built in that dtype and cast back to
+ the backend's default float afterwards, since the rest of the pipeline runs in
+ default precision.
+ """
+ stft_cfg = spectrogram_config.stft_config
+ mel_cfg = spectrogram_config.mel_scale_config
+ n_fft = stft_cfg.n_fft
+ num_frequency_bins = 1 + n_fft // 2
+ min_frequency = mel_cfg.f_min
+ max_frequency = mel_cfg.f_max if mel_cfg.f_max is not None else self.sampling_rate / 2
+ computation_dtype = mel_cfg.computation_dtype or spectrogram_config.computation_dtype
+
+ if mel_cfg.triangularize_in_mel_space and mel_cfg.bands_to_zero == 0:
+ mel_filters = self._kaldi_exact_mel_banks(
+ mel_cfg.n_mels,
+ num_frequency_bins,
+ min_frequency,
+ max_frequency,
+ self.sampling_rate,
+ n_fft,
+ mel_cfg,
+ computation_dtype,
+ )
+ elif mel_cfg.triangularize_in_mel_space:
+ mel_filters = self._kaldi_mel_banks_with_zero_bands(
+ mel_cfg.n_mels,
+ num_frequency_bins,
+ min_frequency,
+ max_frequency,
+ self.sampling_rate,
+ n_fft,
+ mel_cfg,
+ computation_dtype,
+ )
+ else:
+ mel_filters = self._standard_mel_banks(
+ mel_cfg.n_mels,
+ num_frequency_bins,
+ min_frequency,
+ max_frequency,
+ self.sampling_rate,
+ n_fft,
+ mel_cfg,
+ computation_dtype,
+ )
+
+ # Cast back when only the mel-level dtype requested higher precision.
+ if mel_cfg.computation_dtype is not None and not spectrogram_config.computation_dtype:
+ mel_filters = self._cast_mel_filters_to_default_float(mel_filters)
+ return mel_filters
+
+ def _kaldi_exact_mel_banks(
+ self,
+ num_mel_filters,
+ num_frequency_bins,
+ min_frequency,
+ max_frequency,
+ sampling_rate,
+ n_fft,
+ mel_cfg,
+ computation_dtype,
+ ):
+ """Mel filter bank triangularized in mel space, without zeroed bands.
+
+ Each backend leaf owns its ecosystem's rounding pattern: the torch leaf matches
+ ``torchaudio.compliance.kaldi.get_mel_banks`` arithmetic, the numpy leaf matches
+ the legacy numpy feature extractors. Numerically equivalent, not bit-identical.
+ Implemented by backend subclasses."""
+ raise NotImplementedError
+
+ def _kaldi_mel_banks_with_zero_bands(
+ self,
+ num_mel_filters,
+ num_frequency_bins,
+ min_frequency,
+ max_frequency,
+ sampling_rate,
+ n_fft,
+ mel_cfg,
+ computation_dtype,
+ ):
+ """Mel filter bank triangularized in mel space with the lowest ``bands_to_zero``
+ frequency bins zeroed out. Implemented by backend subclasses."""
+ raise NotImplementedError
+
+ def _standard_mel_banks(
+ self,
+ num_mel_filters,
+ num_frequency_bins,
+ min_frequency,
+ max_frequency,
+ sampling_rate,
+ n_fft,
+ mel_cfg,
+ computation_dtype,
+ ):
+ """Standard (non-kaldi) triangular mel filter bank. Each backend leaf owns its
+ ecosystem's rounding pattern (numpy: librosa/legacy numpy FEs; torch: torchaudio).
+ Implemented by backend subclasses."""
+ raise NotImplementedError
+
+ def _cast_mel_filters_to_default_float(self, mel_filters):
+ """Cast a built filter bank to the backend's default float dtype (torch:
+ ``torch.get_default_dtype()``, numpy: float32). Implemented by backend subclasses."""
+ raise NotImplementedError
diff --git a/src/transformers/audio_utils.py b/src/transformers/audio_utils.py
index ece33f081799..5a01801227cd 100644
--- a/src/transformers/audio_utils.py
+++ b/src/transformers/audio_utils.py
@@ -19,9 +19,11 @@
import base64
import importlib
import io
+import math
import os
import warnings
from collections.abc import Sequence
+from dataclasses import dataclass, field, fields
from io import BytesIO
from typing import TYPE_CHECKING, Any, Union
from urllib.parse import urlparse
@@ -34,6 +36,7 @@
is_librosa_available,
is_numpy_array,
is_soundfile_available,
+ is_torch_available,
is_torch_tensor,
is_torchaudio_available,
is_torchcodec_available,
@@ -63,6 +66,119 @@
AudioInput = Union[np.ndarray, "torch.Tensor", Sequence[np.ndarray], Sequence["torch.Tensor"]]
+@dataclass(frozen=True)
+class StftConfig:
+ n_fft: int = 400
+ win_length: int | None = None
+ hop_length: int | None = None
+ window_fn: str = "hann_window"
+ wkwargs: dict | None = None
+ power: float = 2.0
+ # True: symmetric center padding; False: none; "left": semicausal (USM/Gemma),
+ # `win_length // 2` zeros prepended — forces manual framing.
+ center: bool | str = True
+ pad_mode: str = "reflect"
+ normalized: bool = False
+ onesided: bool | None = None
+ periodic: bool = True
+ left_align_fft: bool = False
+ window_dtype: str | None = None
+ # USM-style extended framing: frame at `win_length + 1`, reduced back to `win_length`
+ # by the per-frame preemphasis. Only 1 is supported.
+ frame_extension: int = 0
+ # Manual-framing FFT dtype. None: legacy rounding (numpy complex64, torch float32);
+ # "float64": both backends; "native": each backend's own (numpy float64, torch float32).
+ # Complements `computation_dtype` (which still controls magnitude dtype).
+ fft_dtype: str | None = None
+
+ def to_dict(self) -> dict:
+ return {f.name: getattr(self, f.name) for f in fields(self) if getattr(self, f.name) is not None}
+
+ @classmethod
+ def from_dict(cls, d: dict) -> "StftConfig":
+ valid_keys = {f.name for f in fields(cls)}
+ return cls(**{k: v for k, v in d.items() if k in valid_keys})
+
+
+@dataclass(frozen=True)
+class MelScaleConfig:
+ n_mels: int = 128
+ f_min: float = 0.0
+ f_max: float | None = None
+ mel_scale: str = "htk"
+ norm: str | None = None
+ triangularize_in_mel_space: bool = False
+ frequency_bin_mode: str = "rfft"
+ computation_dtype: str | None = None
+ bands_to_zero: int = 0
+ # Precision knob only; `_apply_mel_scale` input is always `(..., freq, time)`.
+ matmul_order: str = "filters_first"
+
+ def to_dict(self) -> dict:
+ return {f.name: getattr(self, f.name) for f in fields(self) if getattr(self, f.name) is not None}
+
+ @classmethod
+ def from_dict(cls, d: dict) -> "MelScaleConfig":
+ valid_keys = {f.name for f in fields(cls)}
+ return cls(**{k: v for k, v in d.items() if k in valid_keys})
+
+
+@dataclass(frozen=True)
+class SpectrogramConfig:
+ stft_config: StftConfig = field(default_factory=StftConfig)
+ mel_scale_config: MelScaleConfig | None = None
+ log_mode: str = "log10"
+ chunk_length: int | None = None
+ preemphasis: float | None = None
+ preemphasis_mode: str = "per_frame"
+ remove_dc_offset: bool = False
+ mel_floor: float = 1e-10
+ # When set, the log stage computes log(x + pre_log_offset) instead of
+ # log(clamp(x, mel_floor)) — the guard form used by NeMo-style extractors (ADR 0004).
+ pre_log_offset: float | None = None
+ waveform_scale: float | None = None
+ computation_dtype: str | None = None
+ skip_last_frame: bool = False
+ clip_max_offset: float | None = None
+ post_log_shift: float | None = None
+ post_log_scale: float | None = None
+
+ def __getitem__(self, key):
+ if hasattr(self, key):
+ return getattr(self, key)
+ raise KeyError(f"Key {key} not found in SpectrogramConfig.")
+
+ def __iter__(self):
+ for f in fields(self):
+ val = getattr(self, f.name)
+ if val is not None:
+ if hasattr(val, "to_dict"):
+ yield f.name, val.to_dict()
+ else:
+ yield f.name, val
+
+ def __eq__(self, other):
+ if isinstance(other, dict):
+ return dict(self) == other
+ if isinstance(other, SpectrogramConfig):
+ return tuple(getattr(self, f.name) for f in fields(self)) == tuple(
+ getattr(other, f.name) for f in fields(self)
+ )
+ return NotImplemented
+
+ def to_dict(self) -> dict:
+ return dict(self)
+
+ @classmethod
+ def from_dict(cls, d: dict) -> "SpectrogramConfig":
+ kwargs = {k: v for k, v in d.items() if k in {f.name for f in fields(cls)}}
+ if "stft_config" in kwargs and isinstance(kwargs["stft_config"], dict):
+ kwargs["stft_config"] = StftConfig.from_dict(kwargs["stft_config"])
+ if "mel_scale_config" in kwargs and isinstance(kwargs["mel_scale_config"], dict):
+ kwargs["mel_scale_config"] = MelScaleConfig.from_dict(kwargs["mel_scale_config"])
+ return cls(**kwargs)
+
+
@retry(exceptions=(httpx.HTTPError,))
def _fetch_audio_bytes(url: str, timeout: float | None = 10.0) -> bytes:
"""Fetch audio bytes from a URL with automatic retry and exponential backoff."""
@@ -445,78 +561,6 @@ def make_list_of_audio_chat_template(
return make_list_of_audio(audio)
-def hertz_to_mel(freq: float | np.ndarray, mel_scale: str = "htk") -> float | np.ndarray:
- """
- Convert frequency from hertz to mels.
-
- Args:
- freq (`float` or `np.ndarray`):
- The frequency, or multiple frequencies, in hertz (Hz).
- mel_scale (`str`, *optional*, defaults to `"htk"`):
- The mel frequency scale to use, `"htk"`, `"kaldi"` or `"slaney"`.
-
- Returns:
- `float` or `np.ndarray`: The frequencies on the mel scale.
- """
-
- if mel_scale not in ["slaney", "htk", "kaldi"]:
- raise ValueError('mel_scale should be one of "htk", "slaney" or "kaldi".')
-
- if mel_scale == "htk":
- return 2595.0 * np.log10(1.0 + (freq / 700.0))
- elif mel_scale == "kaldi":
- return 1127.0 * np.log(1.0 + (freq / 700.0))
-
- min_log_hertz = 1000.0
- min_log_mel = 15.0
- logstep = 27.0 / np.log(6.4)
- mels = 3.0 * freq / 200.0
-
- if isinstance(freq, np.ndarray):
- log_region = freq >= min_log_hertz
- mels[log_region] = min_log_mel + np.log(freq[log_region] / min_log_hertz) * logstep
- elif freq >= min_log_hertz:
- mels = min_log_mel + np.log(freq / min_log_hertz) * logstep
-
- return mels
-
-
-def mel_to_hertz(mels: float | np.ndarray, mel_scale: str = "htk") -> float | np.ndarray:
- """
- Convert frequency from mels to hertz.
-
- Args:
- mels (`float` or `np.ndarray`):
- The frequency, or multiple frequencies, in mels.
- mel_scale (`str`, *optional*, `"htk"`):
- The mel frequency scale to use, `"htk"`, `"kaldi"` or `"slaney"`.
-
- Returns:
- `float` or `np.ndarray`: The frequencies in hertz.
- """
-
- if mel_scale not in ["slaney", "htk", "kaldi"]:
- raise ValueError('mel_scale should be one of "htk", "slaney" or "kaldi".')
-
- if mel_scale == "htk":
- return 700.0 * (np.power(10, mels / 2595.0) - 1.0)
- elif mel_scale == "kaldi":
- return 700.0 * (np.exp(mels / 1127.0) - 1.0)
-
- min_log_hertz = 1000.0
- min_log_mel = 15.0
- logstep = np.log(6.4) / 27.0
- freq = 200.0 * mels / 3.0
-
- if isinstance(mels, np.ndarray):
- log_region = mels >= min_log_mel
- freq[log_region] = min_log_hertz * np.exp(logstep * (mels[log_region] - min_log_mel))
- elif mels >= min_log_mel:
- freq = min_log_hertz * np.exp(logstep * (mels - min_log_mel))
-
- return freq
-
-
def hertz_to_octave(freq: float | np.ndarray, tuning: float = 0.0, bins_per_octave: int = 12):
"""
Convert frequency from hertz to fractional octave numbers.
@@ -538,28 +582,6 @@ def hertz_to_octave(freq: float | np.ndarray, tuning: float = 0.0, bins_per_octa
return octave
-def _create_triangular_filter_bank(fft_freqs: np.ndarray, filter_freqs: np.ndarray) -> np.ndarray:
- """
- Creates a triangular filter bank.
-
- Adapted from *torchaudio* and *librosa*.
-
- Args:
- fft_freqs (`np.ndarray` of shape `(num_frequency_bins,)`):
- Discrete frequencies of the FFT bins in Hz.
- filter_freqs (`np.ndarray` of shape `(num_mel_filters,)`):
- Center frequencies of the triangular filters to create, in Hz.
-
- Returns:
- `np.ndarray` of shape `(num_frequency_bins, num_mel_filters)`
- """
- filter_diff = np.diff(filter_freqs)
- slopes = np.expand_dims(filter_freqs, 0) - np.expand_dims(fft_freqs, 1)
- down_slopes = -slopes[:, :-2] / filter_diff[:-1]
- up_slopes = slopes[:, 2:] / filter_diff[1:]
- return np.maximum(np.zeros(1), np.minimum(down_slopes, up_slopes))
-
-
def chroma_filter_bank(
num_frequency_bins: int,
num_chroma: int,
@@ -635,100 +657,6 @@ def chroma_filter_bank(
return np.ascontiguousarray(chroma_filters[:, : int(1 + num_frequency_bins / 2)])
-def mel_filter_bank(
- num_frequency_bins: int,
- num_mel_filters: int,
- min_frequency: float,
- max_frequency: float,
- sampling_rate: int,
- norm: str | None = None,
- mel_scale: str = "htk",
- triangularize_in_mel_space: bool = False,
-) -> np.ndarray:
- """
- Creates a frequency bin conversion matrix used to obtain a mel spectrogram. This is called a *mel filter bank*, and
- various implementation exist, which differ in the number of filters, the shape of the filters, the way the filters
- are spaced, the bandwidth of the filters, and the manner in which the spectrum is warped. The goal of these
- features is to approximate the non-linear human perception of the variation in pitch with respect to the frequency.
-
- Different banks of mel filters were introduced in the literature. The following variations are supported:
-
- - MFCC FB-20: introduced in 1980 by Davis and Mermelstein, it assumes a sampling frequency of 10 kHz and a speech
- bandwidth of `[0, 4600]` Hz.
- - MFCC FB-24 HTK: from the Cambridge HMM Toolkit (HTK) (1995) uses a filter bank of 24 filters for a speech
- bandwidth of `[0, 8000]` Hz. This assumes sampling rate ≥ 16 kHz.
- - MFCC FB-40: from the Auditory Toolbox for MATLAB written by Slaney in 1998, assumes a sampling rate of 16 kHz and
- speech bandwidth of `[133, 6854]` Hz. This version also includes area normalization.
- - HFCC-E FB-29 (Human Factor Cepstral Coefficients) of Skowronski and Harris (2004), assumes a sampling rate of
- 12.5 kHz and speech bandwidth of `[0, 6250]` Hz.
-
- This code is adapted from *torchaudio* and *librosa*. Note that the default parameters of torchaudio's
- `melscale_fbanks` implement the `"htk"` filters while librosa uses the `"slaney"` implementation.
-
- Args:
- num_frequency_bins (`int`):
- Number of frequency bins (should be the same as `n_fft // 2 + 1` where `n_fft` is the size of the Fourier Transform used to compute the spectrogram).
- num_mel_filters (`int`):
- Number of mel filters to generate.
- min_frequency (`float`):
- Lowest frequency of interest in Hz.
- max_frequency (`float`):
- Highest frequency of interest in Hz. This should not exceed `sampling_rate / 2`.
- sampling_rate (`int`):
- Sample rate of the audio waveform.
- norm (`str`, *optional*):
- If `"slaney"`, divide the triangular mel weights by the width of the mel band (area normalization).
- mel_scale (`str`, *optional*, defaults to `"htk"`):
- The mel frequency scale to use, `"htk"`, `"kaldi"` or `"slaney"`.
- triangularize_in_mel_space (`bool`, *optional*, defaults to `False`):
- If this option is enabled, the triangular filter is applied in mel space rather than frequency space. This
- should be set to `true` in order to get the same results as `torchaudio` when computing mel filters.
-
- Returns:
- `np.ndarray` of shape (`num_frequency_bins`, `num_mel_filters`): Triangular filter bank matrix. This is a
- projection matrix to go from a spectrogram to a mel spectrogram.
- """
- if norm is not None and norm != "slaney":
- raise ValueError('norm must be one of None or "slaney"')
-
- if num_frequency_bins < 2:
- raise ValueError(f"Require num_frequency_bins: {num_frequency_bins} >= 2")
-
- if min_frequency > max_frequency:
- raise ValueError(f"Require min_frequency: {min_frequency} <= max_frequency: {max_frequency}")
-
- # center points of the triangular mel filters
- mel_min = hertz_to_mel(min_frequency, mel_scale=mel_scale)
- mel_max = hertz_to_mel(max_frequency, mel_scale=mel_scale)
- mel_freqs = np.linspace(mel_min, mel_max, num_mel_filters + 2)
- filter_freqs = mel_to_hertz(mel_freqs, mel_scale=mel_scale)
-
- if triangularize_in_mel_space:
- # frequencies of FFT bins in Hz, but filters triangularized in mel space
- fft_bin_width = sampling_rate / ((num_frequency_bins - 1) * 2)
- fft_freqs = hertz_to_mel(fft_bin_width * np.arange(num_frequency_bins), mel_scale=mel_scale)
- filter_freqs = mel_freqs
- else:
- # frequencies of FFT bins in Hz
- fft_freqs = np.linspace(0, sampling_rate // 2, num_frequency_bins)
-
- mel_filters = _create_triangular_filter_bank(fft_freqs, filter_freqs)
-
- if norm is not None and norm == "slaney":
- # Slaney-style mel is scaled to be approx constant energy per channel
- enorm = 2.0 / (filter_freqs[2 : num_mel_filters + 2] - filter_freqs[:num_mel_filters])
- mel_filters *= np.expand_dims(enorm, 0)
-
- if (mel_filters.max(axis=0) == 0.0).any():
- warnings.warn(
- "At least one mel filter has all zero values. "
- f"The value for `num_mel_filters` ({num_mel_filters}) may be set too high. "
- f"Or, the value for `num_frequency_bins` ({num_frequency_bins}) may be set too low."
- )
-
- return mel_filters
-
-
def optimal_fft_length(window_length: int) -> int:
"""
Finds the best FFT input size for a given `window_length`. This function takes a given window length and, if not
@@ -805,435 +733,175 @@ def window_function(
return padded_window
-# Note: This method processes a single waveform. For batch processing, use spectrogram_batch().
-def spectrogram(
- waveform: np.ndarray,
- window: np.ndarray,
- frame_length: int,
- hop_length: int,
- fft_length: int | None = None,
- power: float | None = 1.0,
- center: bool = True,
- pad_mode: str = "reflect",
- onesided: bool = True,
- dither: float = 0.0,
- preemphasis: float | None = None,
- mel_filters: np.ndarray | None = None,
- mel_floor: float = 1e-10,
- log_mel: str | None = None,
- reference: float = 1.0,
- min_value: float = 1e-10,
- db_range: float | None = None,
- remove_dc_offset: bool = False,
- dtype: np.dtype = np.float32,
-) -> np.ndarray:
- """
- Calculates a spectrogram over one waveform using the Short-Time Fourier Transform.
-
- This function can create the following kinds of spectrograms:
+# ═══════════════════════════════════════════════════════════════════════════════
+# Audio math helpers (numpy/torch agnostic)
+# ═══════════════════════════════════════════════════════════════════════════════
- - amplitude spectrogram (`power = 1.0`)
- - power spectrogram (`power = 2.0`)
- - complex-valued spectrogram (`power = None`)
- - log spectrogram (use `log_mel` argument)
- - mel spectrogram (provide `mel_filters`)
- - log-mel spectrogram (provide `mel_filters` and `log_mel`)
- How this works:
+def _array_namespace(x):
+ """Return the array module (``numpy`` or ``torch``) matching ``x``.
- 1. The input waveform is split into frames of size `frame_length` that are partially overlapping by `frame_length
- - hop_length` samples.
- 2. Each frame is multiplied by the window and placed into a buffer of size `fft_length`.
- 3. The DFT is taken of each windowed frame.
- 4. The results are stacked into a spectrogram.
-
- We make a distinction between the following "blocks" of sample data, each of which may have a different lengths:
-
- - The analysis frame. This is the size of the time slices that the input waveform is split into.
- - The window. Each analysis frame is multiplied by the window to avoid spectral leakage.
- - The FFT input buffer. The length of this determines how many frequency bins are in the spectrogram.
-
- In this implementation, the window is assumed to be zero-padded to have the same size as the analysis frame. A
- padded window can be obtained from `window_function()`. The FFT input buffer may be larger than the analysis frame,
- typically the next power of two.
-
- Note: This function is not optimized for speed yet. It should be mostly compatible with `librosa.stft` and
- `torchaudio.functional.transforms.Spectrogram`, although it is more flexible due to the different ways spectrograms
- can be constructed.
-
- Args:
- waveform (`np.ndarray` of shape `(length,)`):
- The input waveform. This must be a single real-valued, mono waveform.
- window (`np.ndarray` of shape `(frame_length,)`):
- The windowing function to apply, including zero-padding if necessary. The actual window length may be
- shorter than `frame_length`, but we're assuming the array has already been zero-padded.
- frame_length (`int`):
- The length of the analysis frames in samples. With librosa this is always equal to `fft_length` but we also
- allow smaller sizes.
- hop_length (`int`):
- The stride between successive analysis frames in samples.
- fft_length (`int`, *optional*):
- The size of the FFT buffer in samples. This determines how many frequency bins the spectrogram will have.
- For optimal speed, this should be a power of two. If `None`, uses `frame_length`.
- power (`float`, *optional*, defaults to 1.0):
- If 1.0, returns the amplitude spectrogram. If 2.0, returns the power spectrogram. If `None`, returns
- complex numbers.
- center (`bool`, *optional*, defaults to `True`):
- Whether to pad the waveform so that frame `t` is centered around time `t * hop_length`. If `False`, frame
- `t` will start at time `t * hop_length`.
- pad_mode (`str`, *optional*, defaults to `"reflect"`):
- Padding mode used when `center` is `True`. Possible values are: `"constant"` (pad with zeros), `"edge"`
- (pad with edge values), `"reflect"` (pads with mirrored values).
- onesided (`bool`, *optional*, defaults to `True`):
- If True, only computes the positive frequencies and returns a spectrogram containing `fft_length // 2 + 1`
- frequency bins. If False, also computes the negative frequencies and returns `fft_length` frequency bins.
- dither (`float`, *optional*, defaults to 0.0):
- Adds dithering. In other words, adds a small Gaussian noise to each frame.
- E.g. use 4.0 to add dithering with a normal distribution centered
- around 0.0 with standard deviation 4.0, 0.0 means no dithering.
- Dithering has similar effect as `mel_floor`. It reduces the high log_mel_fbank
- values for signals with hard-zero sections, when VAD cutoff is present in the signal.
- preemphasis (`float`, *optional*)
- Coefficient for a low-pass filter that applies pre-emphasis before the DFT.
- mel_filters (`np.ndarray` of shape `(num_freq_bins, num_mel_filters)`, *optional*):
- The mel filter bank. If supplied, applies a this filter bank to create a mel spectrogram.
- mel_floor (`float`, *optional*, defaults to 1e-10):
- Minimum value of mel frequency banks.
- log_mel (`str`, *optional*):
- How to convert the spectrogram to log scale. Possible options are: `None` (don't convert), `"log"` (take
- the natural logarithm) `"log10"` (take the base-10 logarithm), `"dB"` (convert to decibels). Can only be
- used when `power` is not `None`.
- reference (`float`, *optional*, defaults to 1.0):
- Sets the input spectrogram value that corresponds to 0 dB. For example, use `np.max(spectrogram)` to set
- the loudest part to 0 dB. Must be greater than zero.
- min_value (`float`, *optional*, defaults to `1e-10`):
- The spectrogram will be clipped to this minimum value before conversion to decibels, to avoid taking
- `log(0)`. For a power spectrogram, the default of `1e-10` corresponds to a minimum of -100 dB. For an
- amplitude spectrogram, the value `1e-5` corresponds to -100 dB. Must be greater than zero.
- db_range (`float`, *optional*):
- Sets the maximum dynamic range in decibels. For example, if `db_range = 80`, the difference between the
- peak value and the smallest value will never be more than 80 dB. Must be greater than zero.
- remove_dc_offset (`bool`, *optional*):
- Subtract mean from waveform on each frame, applied before pre-emphasis. This should be set to `true` in
- order to get the same results as `torchaudio.compliance.kaldi.fbank` when computing mel filters.
- dtype (`np.dtype`, *optional*, defaults to `np.float32`):
- Data type of the spectrogram tensor. If `power` is None, this argument is ignored and the dtype will be
- `np.complex64`.
-
- Returns:
- `nd.array` containing a spectrogram of shape `(num_frequency_bins, length)` for a regular spectrogram or shape
- `(num_mel_filters, length)` for a mel spectrogram.
+ Raises ``TypeError`` for unknown types. Use :func:`_xp_or_math` instead when
+ Python scalars are also valid input.
"""
- window_length = len(window)
-
- if fft_length is None:
- fft_length = frame_length
-
- if frame_length > fft_length:
- raise ValueError(f"frame_length ({frame_length}) may not be larger than fft_length ({fft_length})")
+ if isinstance(x, np.ndarray):
+ return np
+ if is_torch_available():
+ import torch
- if window_length != frame_length:
- raise ValueError(f"Length of the window ({window_length}) must equal frame_length ({frame_length})")
+ if isinstance(x, torch.Tensor):
+ return torch
+ raise TypeError(f"Unsupported array type: {type(x)}")
- if hop_length <= 0:
- raise ValueError("hop_length must be greater than zero")
- if waveform.ndim != 1:
- raise ValueError(f"Input waveform must have only one dimension, shape is {waveform.shape}")
+def _xp_or_math(x):
+ """Like :func:`_array_namespace` but returns ``math`` for Python scalars.
- if np.iscomplexobj(waveform):
- raise ValueError("Complex-valued input waveforms are not currently supported")
-
- if power is None and mel_filters is not None:
- raise ValueError(
- "You have provided `mel_filters` but `power` is `None`. Mel spectrogram computation is not yet supported for complex-valued spectrogram."
- "Specify `power` to fix this issue."
- )
+ Lets scalar-or-array math be written once: ``math.log10`` has the right
+ signature for Python floats; numpy and torch use the same names on arrays.
+ """
+ if isinstance(x, (int, float)):
+ return math
+ return _array_namespace(x)
- # center pad the waveform
- if center:
- padding = [(int(frame_length // 2), int(frame_length // 2))]
- waveform = np.pad(waveform, padding, mode=pad_mode)
- # promote to float64, since np.fft uses float64 internally
- waveform = waveform.astype(np.float64)
- window = window.astype(np.float64)
+def _clamp_min(x, min_value):
+ """Element-wise ``max(x, min_value)`` for numpy arrays or torch tensors.
- # split waveform into frames of frame_length size
- num_frames = int(1 + np.floor((waveform.size - frame_length) / hop_length))
+ Needed because ``np.maximum(arr, scalar)`` accepts a Python scalar but
+ ``torch.maximum(tensor, scalar)`` does not — and ``torch.clamp(x, min=)``
+ has a different kwarg name than ``np.clip(x, a_min=)``.
+ """
+ if isinstance(x, np.ndarray):
+ return np.maximum(x, min_value)
+ return x.clamp(min=min_value)
- num_frequency_bins = (fft_length // 2) + 1 if onesided else fft_length
- spectrogram = np.empty((num_frames, num_frequency_bins), dtype=np.complex64)
- # rfft is faster than fft
- fft_func = np.fft.rfft if onesided else np.fft.fft
- buffer = np.zeros(fft_length)
+def hertz_to_mel(freq: float | np.ndarray, mel_scale: str = "htk"):
+ """
+ Convert frequency from hertz to mels.
- timestep = 0
- for frame_idx in range(num_frames):
- buffer[:frame_length] = waveform[timestep : timestep + frame_length]
+ Args:
+ freq (`float`, `np.ndarray`, or `torch.Tensor`):
+ The frequency, or multiple frequencies, in hertz (Hz).
+ mel_scale (`str`, *optional*, defaults to `"htk"`):
+ The mel frequency scale to use, `"htk"`, `"kaldi"` or `"slaney"`.
- if dither != 0.0:
- buffer[:frame_length] += dither * np.random.randn(frame_length)
+ Returns:
+ The frequencies on the mel scale, in the same form as the input.
+ """
+ if mel_scale not in ("htk", "kaldi", "slaney"):
+ raise ValueError('mel_scale should be one of "htk", "slaney" or "kaldi".')
- if remove_dc_offset:
- buffer[:frame_length] = buffer[:frame_length] - buffer[:frame_length].mean()
+ xp = _xp_or_math(freq)
- if preemphasis is not None:
- buffer[1:frame_length] -= preemphasis * buffer[: frame_length - 1]
- buffer[0] *= 1 - preemphasis
+ if mel_scale == "htk":
+ return 2595.0 * xp.log10(1.0 + freq / 700.0)
+ if mel_scale == "kaldi":
+ return 1127.0 * xp.log(1.0 + freq / 700.0)
- buffer[:frame_length] *= window
+ # slaney: linear below 1000 Hz, logarithmic above. The constants are written
+ # differently per backend to preserve bit-exact parity with librosa (numpy) and
+ # torchaudio (torch) — they use different float32 rounding paths.
+ min_log_hertz = 1000.0
+ min_log_mel = 15.0
- spectrogram[frame_idx] = fft_func(buffer)
- timestep += hop_length
+ if xp is math:
+ if freq >= min_log_hertz:
+ return min_log_mel + math.log(freq / min_log_hertz) * 27.0 / math.log(6.4)
+ return 3.0 * freq / 200.0
- # note: ** is much faster than np.power
- if power is not None:
- spectrogram = np.abs(spectrogram, dtype=np.float64) ** power
-
- spectrogram = spectrogram.T
-
- if mel_filters is not None:
- spectrogram = np.maximum(mel_floor, np.dot(mel_filters.T, spectrogram))
-
- if power is not None and log_mel is not None:
- if log_mel == "log":
- spectrogram = np.log(spectrogram)
- elif log_mel == "log10":
- spectrogram = np.log10(spectrogram)
- elif log_mel == "dB":
- if power == 1.0:
- spectrogram = amplitude_to_db(spectrogram, reference, min_value, db_range)
- elif power == 2.0:
- spectrogram = power_to_db(spectrogram, reference, min_value, db_range)
- else:
- raise ValueError(f"Cannot use log_mel option '{log_mel}' with power {power}")
- else:
- raise ValueError(f"Unknown log_mel option: {log_mel}")
+ if xp is np:
+ linear = 3.0 * freq / 200.0
+ logstep = 27.0 / np.log(6.4)
+ else: # torch — float32-tensor logstep matches torchaudio
+ import torch
- spectrogram = np.asarray(spectrogram, dtype)
+ linear = freq / (200.0 / 3.0)
+ logstep = 27.0 / torch.log(torch.tensor(6.4))
- return spectrogram
+ # Guard log against discarded-branch values; xp.where evaluates both branches.
+ safe = _clamp_min(freq, min_log_hertz)
+ log_branch = min_log_mel + xp.log(safe / min_log_hertz) * logstep
+ return xp.where(freq >= min_log_hertz, log_branch, linear)
-def spectrogram_batch(
- waveform_list: list[np.ndarray],
- window: np.ndarray,
- frame_length: int,
- hop_length: int,
- fft_length: int | None = None,
- power: float | None = 1.0,
- center: bool = True,
- pad_mode: str = "reflect",
- onesided: bool = True,
- dither: float = 0.0,
- preemphasis: float | None = None,
- mel_filters: np.ndarray | None = None,
- mel_floor: float = 1e-10,
- log_mel: str | None = None,
- reference: float = 1.0,
- min_value: float = 1e-10,
- db_range: float | None = None,
- remove_dc_offset: bool = False,
- dtype: np.dtype = np.float32,
-) -> list[np.ndarray]:
+def mel_to_hertz(mels: float | np.ndarray, mel_scale: str = "htk"):
"""
- Calculates spectrograms for a list of waveforms using the Short-Time Fourier Transform, optimized for batch processing.
- This function extends the capabilities of the `spectrogram` function to handle multiple waveforms efficiently by leveraging broadcasting.
-
- It supports generating various types of spectrograms:
-
- - amplitude spectrogram (`power = 1.0`)
- - power spectrogram (`power = 2.0`)
- - complex-valued spectrogram (`power = None`)
- - log spectrogram (use `log_mel` argument)
- - mel spectrogram (provide `mel_filters`)
- - log-mel spectrogram (provide `mel_filters` and `log_mel`)
-
- How this works:
-
- 1. The input waveform is split into frames of size `frame_length` that are partially overlapping by `frame_length
- - hop_length` samples.
- 2. Each frame is multiplied by the window and placed into a buffer of size `fft_length`.
- 3. The DFT is taken of each windowed frame.
- 4. The results are stacked into a spectrogram.
-
- We make a distinction between the following "blocks" of sample data, each of which may have a different lengths:
-
- - The analysis frame. This is the size of the time slices that the input waveform is split into.
- - The window. Each analysis frame is multiplied by the window to avoid spectral leakage.
- - The FFT input buffer. The length of this determines how many frequency bins are in the spectrogram.
-
- In this implementation, the window is assumed to be zero-padded to have the same size as the analysis frame. A
- padded window can be obtained from `window_function()`. The FFT input buffer may be larger than the analysis frame,
- typically the next power of two.
-
- Note: This function is designed for efficient batch processing of multiple waveforms but retains compatibility with individual waveform processing methods like `librosa.stft`.
+ Convert frequency from mels to hertz.
Args:
- waveform_list (`list[np.ndarray]` with arrays of shape `(length,)`):
- The list of input waveforms, each a single-channel (mono) signal.
- window (`np.ndarray` of shape `(frame_length,)`):
- The windowing function to apply, including zero-padding if necessary.
- frame_length (`int`):
- The length of each frame for analysis.
- hop_length (`int`):
- The step size between successive frames.
- fft_length (`int`, *optional*):
- The size of the FFT buffer, defining frequency bin resolution.
- power (`float`, *optional*, defaults to 1.0):
- Determines the type of spectrogram: 1.0 for amplitude, 2.0 for power, None for complex.
- center (`bool`, *optional*, defaults to `True`):
- Whether to center-pad the waveform frames.
- pad_mode (`str`, *optional*, defaults to `"reflect"`):
- The padding strategy when `center` is `True`.
- onesided (`bool`, *optional*, defaults to `True`):
- If True, returns a one-sided spectrogram for real input signals.
- dither (`float`, *optional*, defaults to 0.0):
- Adds dithering. In other words, adds a small Gaussian noise to each frame.
- E.g. use 4.0 to add dithering with a normal distribution centered
- around 0.0 with standard deviation 4.0, 0.0 means no dithering.
- preemphasis (`float`, *optional*):
- Applies a pre-emphasis filter to each frame.
- mel_filters (`np.ndarray`, *optional*):
- Mel filter bank for converting to mel spectrogram.
- mel_floor (`float`, *optional*, defaults to 1e-10):
- Floor value for mel spectrogram to avoid log(0).
- log_mel (`str`, *optional*):
- Specifies log scaling strategy; options are None, "log", "log10", "dB".
- reference (`float`, *optional*, defaults to 1.0):
- Reference value for dB conversion in log_mel.
- min_value (`float`, *optional*, defaults to 1e-10):
- Minimum floor value for log scale conversions.
- db_range (`float`, *optional*):
- Dynamic range for dB scale spectrograms.
- remove_dc_offset (`bool`, *optional*):
- Whether to remove the DC offset from each frame.
- dtype (`np.dtype`, *optional*, defaults to `np.float32`):
- Data type of the output spectrogram.
+ mels (`float`, `np.ndarray`, or `torch.Tensor`):
+ The frequency, or multiple frequencies, in mels.
+ mel_scale (`str`, *optional*, defaults to `"htk"`):
+ The mel frequency scale to use, `"htk"`, `"kaldi"` or `"slaney"`.
Returns:
- list[`np.ndarray`]: A list of spectrogram arrays, one for each input waveform.
+ The frequencies in hertz, in the same form as the input.
"""
- window_length = len(window)
-
- if fft_length is None:
- fft_length = frame_length
-
- if frame_length > fft_length:
- raise ValueError(f"frame_length ({frame_length}) may not be larger than fft_length ({fft_length})")
-
- if window_length != frame_length:
- raise ValueError(f"Length of the window ({window_length}) must equal frame_length ({frame_length})")
-
- if hop_length <= 0:
- raise ValueError("hop_length must be greater than zero")
-
- # Check the dimensions of the waveform , and if waveform is complex
- for waveform in waveform_list:
- if waveform.ndim != 1:
- raise ValueError(f"Input waveform must have only one dimension, shape is {waveform.shape}")
- if np.iscomplexobj(waveform):
- raise ValueError("Complex-valued input waveforms are not currently supported")
- # Center pad the waveform
- if center:
- padding = [(int(frame_length // 2), int(frame_length // 2))]
- waveform_list = [
- np.pad(
- waveform,
- padding,
- mode=pad_mode,
- )
- for waveform in waveform_list
- ]
- original_waveform_lengths = [
- len(waveform) for waveform in waveform_list
- ] # these lengths will be used to remove padding later
-
- # Batch pad the waveform
- max_length = max(original_waveform_lengths)
- padded_waveform_batch = np.array(
- [
- np.pad(waveform, (0, max_length - len(waveform)), mode="constant", constant_values=0)
- for waveform in waveform_list
- ],
- dtype=dtype,
- )
-
- # Promote to float64, since np.fft uses float64 internally
- padded_waveform_batch = padded_waveform_batch.astype(np.float64)
- window = window.astype(np.float64)
-
- # Split waveform into frames of frame_length size
- num_frames = int(1 + np.floor((padded_waveform_batch.shape[1] - frame_length) / hop_length))
- # these lengths will be used to remove padding later
- true_num_frames = [int(1 + np.floor((length - frame_length) / hop_length)) for length in original_waveform_lengths]
- num_batches = padded_waveform_batch.shape[0]
-
- num_frequency_bins = (fft_length // 2) + 1 if onesided else fft_length
- spectrogram = np.empty((num_batches, num_frames, num_frequency_bins), dtype=np.complex64)
+ if mel_scale not in ("htk", "kaldi", "slaney"):
+ raise ValueError('mel_scale should be one of "htk", "slaney" or "kaldi".')
- # rfft is faster than fft
- fft_func = np.fft.rfft if onesided else np.fft.fft
- buffer = np.zeros((num_batches, fft_length))
+ xp = _xp_or_math(mels)
- for frame_idx in range(num_frames):
- timestep = frame_idx * hop_length
- buffer[:, :frame_length] = padded_waveform_batch[:, timestep : timestep + frame_length]
+ if mel_scale == "htk":
+ return 700.0 * (10.0 ** (mels / 2595.0) - 1.0)
+ if mel_scale == "kaldi":
+ return 700.0 * (xp.exp(mels / 1127.0) - 1.0)
- if dither != 0.0:
- buffer[:, :frame_length] += dither * np.random.randn(*buffer[:, :frame_length].shape)
+ # slaney — see note in hertz_to_mel; constants are written per-backend for
+ # bit-exact parity with librosa (numpy) and torchaudio (torch).
+ min_log_hertz = 1000.0
+ min_log_mel = 15.0
- if remove_dc_offset:
- buffer[:, :frame_length] -= buffer[:, :frame_length].mean(axis=1, keepdims=True)
+ if xp is math:
+ if mels >= min_log_mel:
+ return min_log_hertz * math.exp(math.log(6.4) / 27.0 * (mels - min_log_mel))
+ return 200.0 * mels / 3.0
- if preemphasis is not None:
- buffer[:, 1:frame_length] -= preemphasis * buffer[:, : frame_length - 1]
- buffer[:, 0] *= 1 - preemphasis
+ if xp is np:
+ linear = 200.0 * mels / 3.0
+ logstep = np.log(6.4) / 27.0
+ else: # torch — match old per-backend precision (Python-float logstep here,
+ # though the reciprocal in hertz_to_mel uses a float32 tensor — old code
+ # was inconsistent and we preserve that for bit-exact parity).
+ linear = (200.0 / 3.0) * mels
+ logstep = math.log(6.4) / 27.0
- buffer[:, :frame_length] *= window
+ log_branch = min_log_hertz * xp.exp(logstep * (mels - min_log_mel))
+ return xp.where(mels >= min_log_mel, log_branch, linear)
- spectrogram[:, frame_idx] = fft_func(buffer)
- # Note: ** is much faster than np.power
- if power is not None:
- spectrogram = np.abs(spectrogram, dtype=np.float64) ** power
-
- # Apply mel filters if provided
- if mel_filters is not None:
- result = np.tensordot(spectrogram, mel_filters.T, axes=([2], [1]))
- spectrogram = np.maximum(mel_floor, result)
-
- # Convert to log scale if specified
- if power is not None and log_mel is not None:
- if log_mel == "log":
- spectrogram = np.log(spectrogram)
- elif log_mel == "log10":
- spectrogram = np.log10(spectrogram)
- elif log_mel == "dB":
- if power == 1.0:
- spectrogram = amplitude_to_db_batch(spectrogram, reference, min_value, db_range)
- elif power == 2.0:
- spectrogram = power_to_db_batch(spectrogram, reference, min_value, db_range)
- else:
- raise ValueError(f"Cannot use log_mel option '{log_mel}' with power {power}")
- else:
- raise ValueError(f"Unknown log_mel option: {log_mel}")
+def _create_triangular_filter_bank(fft_freqs, filter_freqs):
+ """
+ Triangular filter bank from FFT bin frequencies and filter center frequencies.
- spectrogram = np.asarray(spectrogram, dtype)
+ Adapted from *torchaudio* and *librosa*. Works on numpy or torch inputs.
- spectrogram_list = [spectrogram[i, : true_num_frames[i], :].T for i in range(len(true_num_frames))]
+ Args:
+ fft_freqs (array of shape `(num_frequency_bins,)`):
+ Discrete frequencies of the FFT bins (in Hz, or in mel space when
+ ``triangularize_in_mel_space=True``).
+ filter_freqs (array of shape `(num_mel_filters + 2,)`):
+ Edges and center frequencies of the triangular filters.
- return spectrogram_list
+ Returns:
+ Filter bank of shape `(num_frequency_bins, num_mel_filters)`.
+ """
+ xp = _array_namespace(fft_freqs)
+ filter_diff = filter_freqs[1:] - filter_freqs[:-1]
+ slopes = filter_freqs[None, :] - fft_freqs[:, None]
+ down_slopes = -slopes[:, :-2] / filter_diff[:-1]
+ up_slopes = slopes[:, 2:] / filter_diff[1:]
+ return _clamp_min(xp.minimum(down_slopes, up_slopes), 0)
def power_to_db(
- spectrogram: np.ndarray,
+ spectrogram,
reference: float = 1.0,
min_value: float = 1e-10,
db_range: float | None = None,
-) -> np.ndarray:
+):
"""
Converts a power spectrogram to the decibel scale. This computes `10 * log10(spectrogram / reference)`, using basic
logarithm properties for numerical stability.
@@ -1243,10 +911,10 @@ def power_to_db(
This means that large variations in energy may not sound all that different if the sound is loud to begin with.
This compression operation makes the (mel) spectrogram features match more closely what humans actually hear.
- Based on the implementation of `librosa.power_to_db`.
+ Based on the implementation of `librosa.power_to_db`. Works on numpy or torch inputs.
Args:
- spectrogram (`np.ndarray`):
+ spectrogram (`np.ndarray` or `torch.Tensor`):
The input power (mel) spectrogram. Note that a power spectrogram has the amplitudes squared!
reference (`float`, *optional*, defaults to 1.0):
Sets the input spectrogram value that corresponds to 0 dB. For example, use `np.max(spectrogram)` to set
@@ -1259,54 +927,7 @@ def power_to_db(
peak value and the smallest value will never be more than 80 dB. Must be greater than zero.
Returns:
- `np.ndarray`: the spectrogram in decibels
- """
- if reference <= 0.0:
- raise ValueError("reference must be greater than zero")
- if min_value <= 0.0:
- raise ValueError("min_value must be greater than zero")
-
- reference = max(min_value, reference)
-
- spectrogram = np.clip(spectrogram, a_min=min_value, a_max=None)
- spectrogram = 10.0 * (np.log10(spectrogram) - np.log10(reference))
-
- if db_range is not None:
- if db_range <= 0.0:
- raise ValueError("db_range must be greater than zero")
- spectrogram = np.clip(spectrogram, a_min=spectrogram.max() - db_range, a_max=None)
-
- return spectrogram
-
-
-def power_to_db_batch(
- spectrogram: np.ndarray,
- reference: float = 1.0,
- min_value: float = 1e-10,
- db_range: float | None = None,
-) -> np.ndarray:
- """
- Converts a batch of power spectrograms to the decibel scale. This computes `10 * log10(spectrogram / reference)`,
- using basic logarithm properties for numerical stability.
-
- This function supports batch processing, where each item in the batch is an individual power (mel) spectrogram.
-
- Args:
- spectrogram (`np.ndarray`):
- The input batch of power (mel) spectrograms. Expected shape is (batch_size, *spectrogram_shape).
- Note that a power spectrogram has the amplitudes squared!
- reference (`float`, *optional*, defaults to 1.0):
- Sets the input spectrogram value that corresponds to 0 dB. For example, use `np.max(spectrogram)` to set
- the loudest part to 0 dB. Must be greater than zero.
- min_value (`float`, *optional*, defaults to `1e-10`):
- The spectrogram will be clipped to this minimum value before conversion to decibels, to avoid taking
- `log(0)`. The default of `1e-10` corresponds to a minimum of -100 dB. Must be greater than zero.
- db_range (`float`, *optional*):
- Sets the maximum dynamic range in decibels. For example, if `db_range = 80`, the difference between the
- peak value and the smallest value will never be more than 80 dB. Must be greater than zero.
-
- Returns:
- `np.ndarray`: the batch of spectrograms in decibels
+ The spectrogram in decibels, same array type as the input.
"""
if reference <= 0.0:
raise ValueError("reference must be greater than zero")
@@ -1315,25 +936,23 @@ def power_to_db_batch(
reference = max(min_value, reference)
- spectrogram = np.clip(spectrogram, a_min=min_value, a_max=None)
- spectrogram = 10.0 * (np.log10(spectrogram) - np.log10(reference))
+ spectrogram = _clamp_min(spectrogram, min_value)
+ spectrogram = 10.0 * (_array_namespace(spectrogram).log10(spectrogram) - math.log10(reference))
if db_range is not None:
if db_range <= 0.0:
raise ValueError("db_range must be greater than zero")
- # Apply db_range clipping per batch item
- max_values = spectrogram.max(axis=(1, 2), keepdims=True)
- spectrogram = np.clip(spectrogram, a_min=max_values - db_range, a_max=None)
+ spectrogram = _clamp_min(spectrogram, spectrogram.max() - db_range)
return spectrogram
def amplitude_to_db(
- spectrogram: np.ndarray,
+ spectrogram,
reference: float = 1.0,
min_value: float = 1e-5,
db_range: float | None = None,
-) -> np.ndarray:
+):
"""
Converts an amplitude spectrogram to the decibel scale. This computes `20 * log10(spectrogram / reference)`, using
basic logarithm properties for numerical stability.
@@ -1343,52 +962,11 @@ def amplitude_to_db(
This means that large variations in energy may not sound all that different if the sound is loud to begin with.
This compression operation makes the (mel) spectrogram features match more closely what humans actually hear.
- Args:
- spectrogram (`np.ndarray`):
- The input amplitude (mel) spectrogram.
- reference (`float`, *optional*, defaults to 1.0):
- Sets the input spectrogram value that corresponds to 0 dB. For example, use `np.max(spectrogram)` to set
- the loudest part to 0 dB. Must be greater than zero.
- min_value (`float`, *optional*, defaults to `1e-5`):
- The spectrogram will be clipped to this minimum value before conversion to decibels, to avoid taking
- `log(0)`. The default of `1e-5` corresponds to a minimum of -100 dB. Must be greater than zero.
- db_range (`float`, *optional*):
- Sets the maximum dynamic range in decibels. For example, if `db_range = 80`, the difference between the
- peak value and the smallest value will never be more than 80 dB. Must be greater than zero.
-
- Returns:
- `np.ndarray`: the spectrogram in decibels
- """
- if reference <= 0.0:
- raise ValueError("reference must be greater than zero")
- if min_value <= 0.0:
- raise ValueError("min_value must be greater than zero")
-
- reference = max(min_value, reference)
-
- spectrogram = np.clip(spectrogram, a_min=min_value, a_max=None)
- spectrogram = 20.0 * (np.log10(spectrogram) - np.log10(reference))
-
- if db_range is not None:
- if db_range <= 0.0:
- raise ValueError("db_range must be greater than zero")
- spectrogram = np.clip(spectrogram, a_min=spectrogram.max() - db_range, a_max=None)
-
- return spectrogram
-
-
-def amplitude_to_db_batch(
- spectrogram: np.ndarray, reference: float = 1.0, min_value: float = 1e-5, db_range: float | None = None
-) -> np.ndarray:
- """
- Converts a batch of amplitude spectrograms to the decibel scale. This computes `20 * log10(spectrogram / reference)`,
- using basic logarithm properties for numerical stability.
-
- The function supports batch processing, where each item in the batch is an individual amplitude (mel) spectrogram.
+ Works on numpy or torch inputs.
Args:
- spectrogram (`np.ndarray`):
- The input batch of amplitude (mel) spectrograms. Expected shape is (batch_size, *spectrogram_shape).
+ spectrogram (`np.ndarray` or `torch.Tensor`):
+ The input amplitude (mel) spectrogram.
reference (`float`, *optional*, defaults to 1.0):
Sets the input spectrogram value that corresponds to 0 dB. For example, use `np.max(spectrogram)` to set
the loudest part to 0 dB. Must be greater than zero.
@@ -1400,7 +978,7 @@ def amplitude_to_db_batch(
peak value and the smallest value will never be more than 80 dB. Must be greater than zero.
Returns:
- `np.ndarray`: the batch of spectrograms in decibels
+ The spectrogram in decibels, same array type as the input.
"""
if reference <= 0.0:
raise ValueError("reference must be greater than zero")
@@ -1409,14 +987,12 @@ def amplitude_to_db_batch(
reference = max(min_value, reference)
- spectrogram = np.clip(spectrogram, a_min=min_value, a_max=None)
- spectrogram = 20.0 * (np.log10(spectrogram) - np.log10(reference))
+ spectrogram = _clamp_min(spectrogram, min_value)
+ spectrogram = 20.0 * (_array_namespace(spectrogram).log10(spectrogram) - math.log10(reference))
if db_range is not None:
if db_range <= 0.0:
raise ValueError("db_range must be greater than zero")
- # Apply db_range clipping per batch item
- max_values = spectrogram.max(axis=(1, 2), keepdims=True)
- spectrogram = np.clip(spectrogram, a_min=max_values - db_range, a_max=None)
+ spectrogram = _clamp_min(spectrogram, spectrogram.max() - db_range)
return spectrogram
diff --git a/src/transformers/feature_extraction_sequence_utils.py b/src/transformers/feature_extraction_sequence_utils.py
index 7613fe8e2d36..b5b30401f94a 100644
--- a/src/transformers/feature_extraction_sequence_utils.py
+++ b/src/transformers/feature_extraction_sequence_utils.py
@@ -12,376 +12,67 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""
-Sequence feature extraction class for common feature extractors to preprocess sequences.
+Deprecated sequence feature extraction class, kept as a thin shim over [`BaseAudioProcessor`].
"""
-import numpy as np
+import warnings
+from dataclasses import replace
-from .audio_utils import is_valid_audio, load_audio
-from .feature_extraction_utils import BatchFeature, FeatureExtractionMixin
-from .utils import PaddingStrategy, TensorType, is_torch_tensor, logging, to_numpy
+from .audio_processing_utils import BaseAudioProcessor
+from .utils import logging
logger = logging.get_logger(__name__)
-class SequenceFeatureExtractor(FeatureExtractionMixin):
+class SequenceFeatureExtractor(BaseAudioProcessor):
"""
- This is a general feature extraction class for speech recognition.
+ Deprecated base class for speech feature extractors. Subclass [`BaseAudioProcessor`] (through
+ `NumpyAudioBackend` or `TorchAudioBackend`) instead; every in-library `XxxFeatureExtractor` is
+ now a deprecated alias of the corresponding `XxxAudioProcessor`.
+
+ Padding, truncation, masking and audio fetching all come from [`BaseAudioProcessor`]. Note that
+ the inherited `pad` operates on raw audio (as used by the audio-processor pipeline) rather than
+ on already-extracted feature dicts like the legacy `SequenceFeatureExtractor.pad` did.
Args:
- feature_size (`int`):
- The feature dimension of the extracted features.
- sampling_rate (`int`):
- The sampling rate at which the audio files should be digitalized expressed in hertz (Hz).
- padding_value (`float`):
+ feature_size (`int`, *optional*):
+ The feature dimension of the extracted features. Translated to
+ `spectrogram_config.mel_scale_config.n_mels` when the subclass defines a mel configuration.
+ sampling_rate (`int`, *optional*):
+ The sampling rate at which the audio files should be digitalized, expressed in hertz (Hz).
+ padding_value (`float`, *optional*, defaults to 0.0):
The value that is used to fill the padding values / vectors.
"""
- def __init__(self, feature_size: int, sampling_rate: int, padding_value: float, **kwargs):
- self.feature_size = feature_size
- self.sampling_rate = sampling_rate
- self.padding_value = padding_value
-
- self.padding_side = kwargs.pop("padding_side", "right")
- self.return_attention_mask = kwargs.pop("return_attention_mask", True)
-
- super().__init__(**kwargs)
-
- def pad(
+ def __init__(
self,
- processed_features: BatchFeature
- | list[BatchFeature]
- | dict[str, BatchFeature]
- | dict[str, list[BatchFeature]]
- | list[dict[str, BatchFeature]],
- padding: bool | str | PaddingStrategy = True,
- max_length: int | None = None,
- truncation: bool = False,
- pad_to_multiple_of: int | None = None,
- return_attention_mask: bool | None = None,
- return_tensors: str | TensorType | None = None,
- ) -> BatchFeature:
- """
- Pad input values / input vectors or a batch of input values / input vectors up to predefined length or to the
- max sequence length in the batch.
-
- Padding side (left/right) padding values are defined at the feature extractor level (with `self.padding_side`,
- `self.padding_value`)
-
-
-
- If the `processed_features` passed are dictionary of numpy arrays or PyTorch tensors the
- result will use the same type unless you provide a different tensor type with `return_tensors`. In the case of
- PyTorch tensors, you will lose the specific device of your tensors however.
-
-
-
- Args:
- processed_features ([`BatchFeature`], list of [`BatchFeature`], `dict[str, list[float]]`, `dict[str, list[list[float]]` or `list[dict[str, list[float]]]`):
- Processed inputs. Can represent one input ([`BatchFeature`] or `dict[str, list[float]]`) or a batch of
- input values / vectors (list of [`BatchFeature`], *dict[str, list[list[float]]]* or *list[dict[str,
- list[float]]]*) so you can use this method during preprocessing as well as in a PyTorch Dataloader
- collate function.
-
- Instead of `list[float]` you can have tensors (numpy arrays or PyTorch tensors),
- see the note above for the return type.
- padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `True`):
- Select a strategy to pad the returned sequences (according to the model's padding side and padding
- index) among:
-
- - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
- sequence if provided).
- - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
- acceptable input length for the model if that argument is not provided.
- - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
- lengths).
- max_length (`int`, *optional*):
- Maximum length of the returned list and optionally padding length (see above).
- truncation (`bool`):
- Activates truncation to cut input sequences longer than `max_length` to `max_length`.
- pad_to_multiple_of (`int`, *optional*):
- If set will pad the sequence to a multiple of the provided value.
-
- This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability
- `>= 7.5` (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128.
- return_attention_mask (`bool`, *optional*):
- Whether to return the attention mask. If left to the default, will return the attention mask according
- to the specific feature_extractor's default.
-
- [What are attention masks?](../glossary#attention-mask)
- return_tensors (`str` or [`~utils.TensorType`], *optional*):
- If set, will return tensors instead of list of python integers. Acceptable values are:
-
- - `'pt'`: Return PyTorch `torch.Tensor` objects.
- - `'np'`: Return Numpy `np.ndarray` objects.
- """
- # If we have a list of dicts, let's convert it in a dict of lists
- # We do this to allow using this method as a collate_fn function in PyTorch Dataloader
- if isinstance(processed_features, (list, tuple)) and isinstance(processed_features[0], (dict, BatchFeature)):
- # Call .keys() explicitly for compatibility with TensorDict and other Mapping subclasses
- processed_features = {
- key: [example[key] for example in processed_features] for key in processed_features[0].keys()
- }
-
- # The model's main input name, usually `input_values`, has be passed for padding
- if self.model_input_names[0] not in processed_features:
- raise ValueError(
- "You should supply an instance of `transformers.BatchFeature` or list of `transformers.BatchFeature`"
- f" to this method that includes {self.model_input_names[0]}, but you provided"
- f" {list(processed_features.keys())}"
- )
-
- required_input = processed_features[self.model_input_names[0]]
- return_attention_mask = (
- return_attention_mask if return_attention_mask is not None else self.return_attention_mask
- )
-
- if len(required_input) == 0:
- if return_attention_mask:
- processed_features["attention_mask"] = []
- return processed_features
-
- # If we have PyTorch tensors or lists as inputs, we cast them as Numpy arrays
- # and rebuild them afterwards if no return_tensors is specified
- # Note that we lose the specific device the tensor may be on for PyTorch
-
- first_element = required_input[0]
- if isinstance(first_element, (list, tuple)):
- # first_element might be an empty list/tuple in some edge cases so we grab the first non empty element.
- index = 0
- while len(required_input[index]) == 0:
- index += 1
- if index < len(required_input):
- first_element = required_input[index][0]
-
- if return_tensors is None:
- if is_torch_tensor(first_element):
- return_tensors = "pt"
- elif isinstance(first_element, (int, float, list, tuple, np.ndarray)):
- return_tensors = "np"
- else:
- raise ValueError(
- f"type of {first_element} unknown: {type(first_element)}. "
- "Should be one of a python, numpy, or pytorch object."
- )
-
- for key, value in processed_features.items():
- if isinstance(value[0], (int, float)):
- processed_features[key] = to_numpy(value)
- elif not isinstance(value, np.ndarray):
- # An already-batched numpy array can be used as-is; splitting it
- # into a list of per-example arrays is pure overhead and is very
- # slow for large inputs (e.g. long audio).
- processed_features[key] = [to_numpy(v) for v in value]
-
- # Convert padding_strategy in PaddingStrategy
- padding_strategy = self._get_padding_strategies(padding=padding, max_length=max_length)
-
- required_input = processed_features[self.model_input_names[0]]
-
- batch_size = len(required_input)
- if not all(len(v) == batch_size for v in processed_features.values()):
- raise ValueError("Some items in the output dictionary have a different batch size than others.")
-
- truncated_inputs = []
- for i in range(batch_size):
- inputs = {k: v[i] for k, v in processed_features.items()}
- # truncation
- inputs_slice = self._truncate(
- inputs,
- max_length=max_length,
- pad_to_multiple_of=pad_to_multiple_of,
- truncation=truncation,
- )
- truncated_inputs.append(inputs_slice)
-
- if padding_strategy == PaddingStrategy.LONGEST:
- # make sure that `max_length` cannot be longer than the longest truncated length
- max_length = max(len(input_slice[self.model_input_names[0]]) for input_slice in truncated_inputs)
- padding_strategy = PaddingStrategy.MAX_LENGTH
-
- batch_outputs = {}
- for i in range(batch_size):
- # padding
- outputs = self._pad(
- truncated_inputs[i],
- max_length=max_length,
- padding_strategy=padding_strategy,
- pad_to_multiple_of=pad_to_multiple_of,
- return_attention_mask=return_attention_mask,
- )
-
- for key, value in outputs.items():
- if key not in batch_outputs:
- batch_outputs[key] = []
- if value.dtype is np.dtype(np.float64):
- value = value.astype(np.float32)
- batch_outputs[key].append(value)
-
- return BatchFeature(batch_outputs, tensor_type=return_tensors)
-
- def _pad(
- self,
- processed_features: dict[str, np.ndarray] | BatchFeature,
- max_length: int | None = None,
- padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
- pad_to_multiple_of: int | None = None,
- return_attention_mask: bool | None = None,
- ) -> dict:
- """
- Pad inputs (on left/right and up to predefined length or max length in the batch)
-
- Args:
- processed_features (`Union[dict[str, np.ndarray], BatchFeature]`):
- Dictionary of input values (`np.ndarray[float]`) / input vectors (`list[np.ndarray[float]]`) or batch
- of inputs values (`list[np.ndarray[int]]`) / input vectors (`list[np.ndarray[int]]`)
- max_length (`int`, *optional*):
- Maximum length of the returned list and optionally padding length (see below)
- padding_strategy (`PaddingStrategy`, *optional*, default to `PaddingStrategy.DO_NOT_PAD`):
- PaddingStrategy to use for padding.
-
- - PaddingStrategy.LONGEST Pad to the longest sequence in the batch
- - PaddingStrategy.MAX_LENGTH: Pad to the max length (default)
- - PaddingStrategy.DO_NOT_PAD: Do not pad
- The feature_extractor padding sides are defined in self.padding_side:
-
- - 'left': pads on the left of the sequences
- - 'right': pads on the right of the sequences
- pad_to_multiple_of (`int`, *optional*):
- Integer if set will pad the sequence to a multiple of the provided value. This is especially useful to
- enable the use of Tensor Core on NVIDIA hardware with compute capability `>= 7.5` (Volta), or on TPUs
- which benefit from having sequence lengths be a multiple of 128.
- return_attention_mask (`bool`, *optional*):
- Set to False to avoid returning attention mask (default: set to model specifics)
- """
- required_input = processed_features[self.model_input_names[0]]
-
- if padding_strategy == PaddingStrategy.LONGEST:
- max_length = len(required_input)
-
- if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
- max_length = ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of
-
- needs_to_be_padded = padding_strategy != PaddingStrategy.DO_NOT_PAD and len(required_input) < max_length
-
- if return_attention_mask and "attention_mask" not in processed_features:
- processed_features["attention_mask"] = np.ones(len(required_input), dtype=np.int32)
-
- if needs_to_be_padded:
- difference = max_length - len(required_input)
- if self.padding_side == "right":
- if return_attention_mask:
- processed_features["attention_mask"] = np.pad(
- processed_features["attention_mask"], (0, difference)
- )
- padding_shape = ((0, difference), (0, 0)) if self.feature_size > 1 else (0, difference)
- processed_features[self.model_input_names[0]] = np.pad(
- required_input, padding_shape, "constant", constant_values=self.padding_value
- )
- elif self.padding_side == "left":
- if return_attention_mask:
- processed_features["attention_mask"] = np.pad(
- processed_features["attention_mask"], (difference, 0)
- )
- padding_shape = ((difference, 0), (0, 0)) if self.feature_size > 1 else (difference, 0)
- processed_features[self.model_input_names[0]] = np.pad(
- required_input, padding_shape, "constant", constant_values=self.padding_value
- )
- else:
- raise ValueError("Invalid padding strategy:" + str(self.padding_side))
-
- return processed_features
-
- def _truncate(
- self,
- processed_features: dict[str, np.ndarray] | BatchFeature,
- max_length: int | None = None,
- pad_to_multiple_of: int | None = None,
- truncation: bool | None = None,
+ feature_size: int | None = None,
+ sampling_rate: int | None = None,
+ padding_value: float = 0.0,
+ **kwargs,
):
- """
- Truncate inputs to predefined length or max length in the batch
-
- Args:
- processed_features(`Union[dict[str, np.ndarray], BatchFeature]`):
- Dictionary of input values (`np.ndarray[float]`) / input vectors (`list[np.ndarray[float]]`) or batch
- of inputs values (`list[np.ndarray[int]]`) / input vectors (`list[np.ndarray[int]]`)
- max_length (`int`, *optional*):
- maximum length of the returned list and optionally padding length (see below)
- pad_to_multiple_of (`int`, *optional*) :
- Integer if set will pad the sequence to a multiple of the provided value. This is especially useful to
- enable the use of Tensor Core on NVIDIA hardware with compute capability `>= 7.5` (Volta), or on TPUs
- which benefit from having sequence lengths be a multiple of 128.
- truncation (`bool`, *optional*):
- Activates truncation to cut input sequences longer than `max_length` to `max_length`.
- """
- if not truncation:
- return processed_features
- elif truncation and max_length is None:
- raise ValueError("When setting ``truncation=True``, make sure that ``max_length`` is defined.")
-
- required_input = processed_features[self.model_input_names[0]]
-
- # find `max_length` that fits `pad_to_multiple_of`
- if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
- max_length = ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of
-
- needs_to_be_truncated = len(required_input) > max_length
-
- if needs_to_be_truncated:
- processed_features[self.model_input_names[0]] = processed_features[self.model_input_names[0]][:max_length]
- if "attention_mask" in processed_features:
- processed_features["attention_mask"] = processed_features["attention_mask"][:max_length]
-
- return processed_features
-
- def _get_padding_strategies(self, padding=False, max_length=None):
- """
- Find the correct padding strategy
- """
+ warnings.warn(
+ "`SequenceFeatureExtractor` is deprecated and will be removed in transformers v5.15. Use the "
+ "model's `XxxAudioProcessor` (a `BaseAudioProcessor` subclass) instead.",
+ FutureWarning,
+ )
- # Get padding strategy
- if padding is not False:
- if padding is True:
- padding_strategy = PaddingStrategy.LONGEST # Default to pad to the longest sequence in the batch
- elif not isinstance(padding, PaddingStrategy):
- padding_strategy = PaddingStrategy(padding)
- elif isinstance(padding, PaddingStrategy):
- padding_strategy = padding
- else:
- padding_strategy = PaddingStrategy.DO_NOT_PAD
+ # Legacy `return_attention_mask` drives the modern `return_padding_mask` (same meaning, same
+ # default), mirroring the `_legacy_field_mapping_base` translation used for hub configs.
+ if "return_attention_mask" in kwargs:
+ kwargs.setdefault("return_padding_mask", kwargs["return_attention_mask"])
- # Set max length if needed
- if max_length is None:
- if padding_strategy == PaddingStrategy.MAX_LENGTH:
- raise ValueError(
- f"When setting ``padding={PaddingStrategy.MAX_LENGTH}``, make sure that max_length is defined"
- )
+ super().__init__(sampling_rate=sampling_rate, padding_value=padding_value, **kwargs)
- # Test if we have a padding value
- if padding_strategy != PaddingStrategy.DO_NOT_PAD and (self.padding_value is None):
- raise ValueError(
- "Asking to pad but the feature_extractor does not have a padding value. Please select a value to use"
- " as `padding_value`. For example: `feature_extractor.padding_value = 0.0`."
+ # Legacy `feature_size` is the number of mel bins; it only has a target when the subclass
+ # provides a mel configuration (raw-audio processors have no `n_mels` to set).
+ mel_scale_config = getattr(self.spectrogram_config, "mel_scale_config", None)
+ if feature_size is not None and mel_scale_config is not None:
+ self.spectrogram_config = replace(
+ self.spectrogram_config,
+ mel_scale_config=replace(mel_scale_config, n_mels=feature_size),
)
- return padding_strategy
-
- def fetch_audio(self, audio_url_or_urls: str | list[str] | list[list[str]], sampling_rate: int | None = None):
- """
- Convert a single or a list of urls into the corresponding `np.ndarray` objects.
- If a single url is passed, the return value will be a single object. If a list is passed a list of objects is
- returned.
- """
- # Accepted input types for `raw_audio`: "np.ndarray | list[float] | list[np.ndarray] | list[list[float]]"
- sampling_rate = sampling_rate if sampling_rate else self.sampling_rate
- if isinstance(audio_url_or_urls, list) and not isinstance(audio_url_or_urls[0], float):
- return [self.fetch_audio(x, sampling_rate=sampling_rate) for x in audio_url_or_urls]
- elif isinstance(audio_url_or_urls, str):
- return load_audio(audio_url_or_urls, sampling_rate=sampling_rate)
- elif is_valid_audio(audio_url_or_urls):
- return audio_url_or_urls
- else:
- raise TypeError(f"only a single or a list of entries is supported but got type={type(audio_url_or_urls)}")
+__all__ = ["SequenceFeatureExtractor"]
diff --git a/src/transformers/feature_extraction_utils.py b/src/transformers/feature_extraction_utils.py
index 0b03555be9fb..c69cb557d783 100644
--- a/src/transformers/feature_extraction_utils.py
+++ b/src/transformers/feature_extraction_utils.py
@@ -15,32 +15,17 @@
Feature extraction saving/loading class for common feature extractors.
"""
-import copy
-import json
import os
-from collections import UserDict
+import warnings
from typing import TYPE_CHECKING, Any, TypeVar, Union
-import numpy as np
-from huggingface_hub import is_offline_mode
-
-from .dynamic_module_utils import custom_object_save
+from .preprocessing_base import BatchFeature as BatchFeature
+from .preprocessing_base import PreprocessingMixin
from .utils import (
FEATURE_EXTRACTOR_NAME,
- PROCESSOR_NAME,
- PushToHubMixin,
- TensorType,
- _is_tensor_or_array_like,
copy_func,
- is_numpy_array,
- is_torch_available,
- is_torch_device,
- is_torch_dtype,
logging,
- requires_backends,
- safe_load_json_file,
)
-from .utils.hub import cached_file, hf_api
if TYPE_CHECKING:
@@ -55,378 +40,32 @@
SpecificFeatureExtractorType = TypeVar("SpecificFeatureExtractorType", bound="FeatureExtractionMixin")
-class BatchFeature(UserDict):
- r"""
- Holds the output of the [`~SequenceFeatureExtractor.pad`] and feature extractor specific `__call__` methods.
-
- This class is derived from a python dictionary and can be used as a dictionary.
-
- Args:
- data (`dict`, *optional*):
- Dictionary of lists/arrays/tensors returned by the __call__/pad methods ('input_values', 'attention_mask',
- etc.).
- tensor_type (`Union[None, str, TensorType]`, *optional*):
- You can give a tensor_type here to convert the lists of integers in PyTorch/Numpy Tensors at
- initialization.
- skip_tensor_conversion (`list[str]` or `set[str]`, *optional*):
- List or set of keys that should NOT be converted to tensors, even when `tensor_type` is specified.
- """
-
- def __init__(
- self,
- data: dict[str, Any] | None = None,
- tensor_type: None | str | TensorType = None,
- skip_tensor_conversion: list[str] | set[str] | None = None,
- ):
- super().__init__(data)
- self.skip_tensor_conversion = skip_tensor_conversion
- self.convert_to_tensors(tensor_type=tensor_type)
-
- def __getitem__(self, item: str) -> Any:
- """
- If the key is a string, returns the value of the dict associated to `key` ('input_values', 'attention_mask',
- etc.).
- """
- if isinstance(item, str):
- return self.data[item]
- else:
- raise KeyError("Indexing with integers is not available when using Python based feature extractors")
-
- def __getattr__(self, item: str):
- try:
- return self.data[item]
- except KeyError:
- raise AttributeError
-
- def __getstate__(self):
- return {"data": self.data}
-
- def __setstate__(self, state):
- if "data" in state:
- self.data = state["data"]
-
- def _get_is_as_tensor_fns(self, tensor_type: str | TensorType | None = None):
- if tensor_type is None:
- return None, None
-
- # Convert to TensorType
- if not isinstance(tensor_type, TensorType):
- tensor_type = TensorType(tensor_type)
-
- if tensor_type == TensorType.PYTORCH:
- if not is_torch_available():
- raise ImportError("Unable to convert output to PyTorch tensors format, PyTorch is not installed.")
- import torch
-
- def as_tensor(value):
- if torch.is_tensor(value):
- return value
-
- # stack list of tensors if tensor_type is PyTorch (# torch.tensor() does not support list of tensors)
- if isinstance(value, (list, tuple)) and len(value) > 0 and torch.is_tensor(value[0]):
- return torch.stack(value)
-
- # convert list of numpy arrays to numpy array (stack) if tensor_type is Numpy
- if isinstance(value, (list, tuple)) and len(value) > 0:
- if isinstance(value[0], np.ndarray):
- value = np.array(value)
- elif (
- isinstance(value[0], (list, tuple))
- and len(value[0]) > 0
- and isinstance(value[0][0], np.ndarray)
- ):
- value = np.array(value)
- if isinstance(value, np.ndarray):
- return torch.from_numpy(value)
- else:
- return torch.tensor(value)
-
- is_tensor = torch.is_tensor
- else:
-
- def as_tensor(value, dtype=None):
- if isinstance(value, (list, tuple)) and isinstance(value[0], (list, tuple, np.ndarray)):
- value_lens = [len(val) for val in value]
- if len(set(value_lens)) > 1 and dtype is None:
- # we have a ragged list so handle explicitly
- value = as_tensor([np.asarray(val) for val in value], dtype=object)
- return np.asarray(value, dtype=dtype)
-
- is_tensor = is_numpy_array
- return is_tensor, as_tensor
-
- def convert_to_tensors(
- self,
- tensor_type: str | TensorType | None = None,
- skip_tensor_conversion: list[str] | set[str] | None = None,
- ):
- """
- Convert the inner content to tensors.
-
- Args:
- tensor_type (`str` or [`~utils.TensorType`], *optional*):
- The type of tensors to use. If `str`, should be one of the values of the enum [`~utils.TensorType`]. If
- `None`, no modification is done.
- skip_tensor_conversion (`list[str]` or `set[str]`, *optional*):
- List or set of keys that should NOT be converted to tensors, even when `tensor_type` is specified.
-
- Note:
- Values that don't have an array-like structure (e.g., strings, dicts, lists of strings) are
- automatically skipped and won't be converted to tensors. Ragged arrays (lists of arrays with
- different lengths) are still attempted, though they may raise errors during conversion.
- """
- if tensor_type is None:
- return self
-
- is_tensor, as_tensor = self._get_is_as_tensor_fns(tensor_type)
- skip_tensor_conversion = (
- skip_tensor_conversion if skip_tensor_conversion is not None else self.skip_tensor_conversion
- )
-
- # Do the tensor conversion in batch
- for key, value in self.items():
- # Skip keys explicitly marked for no conversion
- if skip_tensor_conversion and key in skip_tensor_conversion:
- continue
-
- # Skip values that are not array-like
- if not _is_tensor_or_array_like(value):
- continue
-
- try:
- if not is_tensor(value):
- tensor = as_tensor(value)
- self[key] = tensor
- except Exception as e:
- if key == "overflowing_values":
- raise ValueError(
- f"Unable to create tensor for '{key}' with overflowing values of different lengths. "
- f"Original error: {str(e)}"
- ) from e
- raise ValueError(
- f"Unable to convert output '{key}' (type: {type(value).__name__}) to tensor: {str(e)}\n"
- f"You can try:\n"
- f" 1. Use padding=True to ensure all outputs have the same shape\n"
- f" 2. Set return_tensors=None to return Python objects instead of tensors"
- ) from e
-
- return self
-
- def to(self, *args, **kwargs) -> "BatchFeature":
- """
- Send all values to device by calling `v.to(*args, **kwargs)` (PyTorch only). This should support casting in
- different `dtypes` and sending the `BatchFeature` to a different `device`.
-
- Args:
- args (`Tuple`):
- Will be passed to the `to(...)` function of the tensors.
- kwargs (`Dict`, *optional*):
- Will be passed to the `to(...)` function of the tensors.
- To enable asynchronous data transfer, set the `non_blocking` flag in `kwargs` (defaults to `False`).
-
- Returns:
- [`BatchFeature`]: The same instance after modification.
- """
- requires_backends(self, ["torch"])
- import torch
-
- device = kwargs.get("device")
- non_blocking = kwargs.get("non_blocking", False)
- # Check if the args are a device or a dtype
- if device is None and len(args) > 0:
- # device should be always the first argument
- arg = args[0]
- if is_torch_dtype(arg):
- # The first argument is a dtype
- pass
- elif isinstance(arg, str) or is_torch_device(arg) or isinstance(arg, int):
- device = arg
- else:
- # it's something else
- raise ValueError(f"Attempting to cast a BatchFeature to type {str(arg)}. This is not supported.")
-
- # We cast only floating point tensors to avoid issues with tokenizers casting `LongTensor` to `FloatTensor`
- def maybe_to(v):
- # check if v is a floating point tensor
- if isinstance(v, torch.Tensor) and torch.is_floating_point(v):
- # cast and send to device
- return v.to(*args, **kwargs)
- elif isinstance(v, torch.Tensor) and device is not None:
- return v.to(device=device, non_blocking=non_blocking)
- # recursively handle lists and tuples
- elif isinstance(v, (list, tuple)):
- return type(v)(maybe_to(item) for item in v)
- else:
- return v
-
- self.data = {k: maybe_to(v) for k, v in self.items()}
- return self
-
-
-class FeatureExtractionMixin(PushToHubMixin):
+class FeatureExtractionMixin(PreprocessingMixin):
"""
- This is a feature extraction mixin used to provide saving/loading functionality for sequential and audio feature
- extractors.
+ Deprecated saving/loading mixin for feature extractors. Subclass [`PreprocessingMixin`] directly
+ (or, for audio, [`BaseAudioProcessor`]) and set the identity attributes below on your own class.
"""
- _auto_class = None
+ _config_name = FEATURE_EXTRACTOR_NAME
+ _type_key = "feature_extractor_type"
+ _nested_config_keys = ["feature_extractor", "audio_processor"]
+ _auto_class_default = "AutoFeatureExtractor"
+ _file_type_label = "feature extractor"
+ _excluded_dict_keys = {"mel_filters", "window"}
+ # Legacy feature extractors serialize every field (including None-valued ones), unlike the
+ # modern processor classes which slim their config — keep that contract.
+ _filter_none_class_defaults = False
+ _extra_init_pops = []
+ _config_filename_kwarg = None
+ _subfolder_default = None
def __init__(self, **kwargs):
- """Set elements of `kwargs` as attributes."""
- # Pop "processor_class", it should not be saved in feature extractor config
- kwargs.pop("processor_class", None)
- # Additional attributes without default values
- for key, value in kwargs.items():
- try:
- setattr(self, key, value)
- except AttributeError as err:
- logger.error(f"Can't set {key} with value {value} for {self}")
- raise err
-
- @classmethod
- def from_pretrained(
- cls: type[SpecificFeatureExtractorType],
- pretrained_model_name_or_path: str | os.PathLike,
- cache_dir: str | os.PathLike | None = None,
- force_download: bool = False,
- local_files_only: bool = False,
- token: str | bool | None = None,
- revision: str = "main",
- **kwargs,
- ) -> SpecificFeatureExtractorType:
- r"""
- Instantiate a type of [`~feature_extraction_utils.FeatureExtractionMixin`] from a feature extractor, *e.g.* a
- derived class of [`SequenceFeatureExtractor`].
-
- Args:
- pretrained_model_name_or_path (`str` or `os.PathLike`):
- This can be either:
-
- - a string, the *model id* of a pretrained feature_extractor hosted inside a model repo on
- huggingface.co.
- - a path to a *directory* containing a feature extractor file saved using the
- [`~feature_extraction_utils.FeatureExtractionMixin.save_pretrained`] method, e.g.,
- `./my_model_directory/`.
- - a path to a saved feature extractor JSON *file*, e.g.,
- `./my_model_directory/preprocessor_config.json`.
- cache_dir (`str` or `os.PathLike`, *optional*):
- Path to a directory in which a downloaded pretrained model feature extractor should be cached if the
- standard cache should not be used.
- force_download (`bool`, *optional*, defaults to `False`):
- Whether or not to force to (re-)download the feature extractor files and override the cached versions
- if they exist.
- proxies (`dict[str, str]`, *optional*):
- A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
- 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request.
- token (`str` or `bool`, *optional*):
- The token to use as HTTP bearer authorization for remote files. If `True`, or not specified, will use
- the token generated when running `hf auth login` (stored in `~/.huggingface`).
- revision (`str`, *optional*, defaults to `"main"`):
- The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
- git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
- identifier allowed by git.
-
-
-
-
- To test a pull request you made on the Hub, you can pass `revision="refs/pr/"`.
-
-
-
- return_unused_kwargs (`bool`, *optional*, defaults to `False`):
- If `False`, then this function returns just the final feature extractor object. If `True`, then this
- functions returns a `Tuple(feature_extractor, unused_kwargs)` where *unused_kwargs* is a dictionary
- consisting of the key/value pairs whose keys are not feature extractor attributes: i.e., the part of
- `kwargs` which has not been used to update `feature_extractor` and is otherwise ignored.
- kwargs (`dict[str, Any]`, *optional*):
- The values in kwargs of any keys which are feature extractor attributes will be used to override the
- loaded values. Behavior concerning key/value pairs whose keys are *not* feature extractor attributes is
- controlled by the `return_unused_kwargs` keyword parameter.
-
- Returns:
- A feature extractor of type [`~feature_extraction_utils.FeatureExtractionMixin`].
-
- Examples:
-
- ```python
- # We can't instantiate directly the base class *FeatureExtractionMixin* nor *SequenceFeatureExtractor* so let's show the examples on a
- # derived class: *Wav2Vec2FeatureExtractor*
- feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(
- "facebook/wav2vec2-base-960h"
- ) # Download feature_extraction_config from huggingface.co and cache.
- feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(
- "./test/saved_model/"
- ) # E.g. feature_extractor (or model) was saved using *save_pretrained('./test/saved_model/')*
- feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained("./test/saved_model/preprocessor_config.json")
- feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(
- "facebook/wav2vec2-base-960h", return_attention_mask=False, foo=False
+ warnings.warn(
+ "`FeatureExtractionMixin` is deprecated and will be removed in transformers v5.15. Subclass "
+ "`PreprocessingMixin` instead (or `BaseAudioProcessor` for audio models).",
+ FutureWarning,
)
- assert feature_extractor.return_attention_mask is False
- feature_extractor, unused_kwargs = Wav2Vec2FeatureExtractor.from_pretrained(
- "facebook/wav2vec2-base-960h", return_attention_mask=False, foo=False, return_unused_kwargs=True
- )
- assert feature_extractor.return_attention_mask is False
- assert unused_kwargs == {"foo": False}
- ```"""
- kwargs["cache_dir"] = cache_dir
- kwargs["force_download"] = force_download
- kwargs["local_files_only"] = local_files_only
- kwargs["revision"] = revision
-
- if token is not None:
- kwargs["token"] = token
-
- feature_extractor_dict, kwargs = cls.get_feature_extractor_dict(pretrained_model_name_or_path, **kwargs)
-
- return cls.from_dict(feature_extractor_dict, **kwargs)
-
- def save_pretrained(self, save_directory: str | os.PathLike, push_to_hub: bool = False, **kwargs):
- """
- Save a feature_extractor object to the directory `save_directory`, so that it can be re-loaded using the
- [`~feature_extraction_utils.FeatureExtractionMixin.from_pretrained`] class method.
-
- Args:
- save_directory (`str` or `os.PathLike`):
- Directory where the feature extractor JSON file will be saved (will be created if it does not exist).
- push_to_hub (`bool`, *optional*, defaults to `False`):
- Whether or not to push your model to the Hugging Face model hub after saving it. You can specify the
- repository you want to push to with `repo_id` (will default to the name of `save_directory` in your
- namespace).
- kwargs (`dict[str, Any]`, *optional*):
- Additional key word arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method.
- """
- if os.path.isfile(save_directory):
- raise AssertionError(f"Provided path ({save_directory}) should be a directory, not a file")
-
- os.makedirs(save_directory, exist_ok=True)
-
- if push_to_hub:
- commit_message = kwargs.pop("commit_message", None)
- repo_id = kwargs.pop("repo_id", save_directory.split(os.path.sep)[-1])
- repo_id = hf_api().create_repo(repo_id, exist_ok=True, **kwargs).repo_id
- files_timestamps = self._get_files_timestamps(save_directory)
-
- # If we have a custom config, we copy the file defining it in the folder and set the attributes so it can be
- # loaded from the Hub.
- if self._auto_class is not None:
- custom_object_save(self, save_directory, config=self)
-
- # If we save using the predefined names, we can load using `from_pretrained`
- output_feature_extractor_file = os.path.join(save_directory, FEATURE_EXTRACTOR_NAME)
-
- self.to_json_file(output_feature_extractor_file)
- logger.info(f"Feature extractor saved in {output_feature_extractor_file}")
-
- if push_to_hub:
- self._upload_modified_files(
- save_directory,
- repo_id,
- files_timestamps,
- commit_message=commit_message,
- token=kwargs.get("token"),
- )
-
- return [output_feature_extractor_file]
+ super().__init__(**kwargs)
@classmethod
def get_feature_extractor_dict(
@@ -443,104 +82,7 @@ def get_feature_extractor_dict(
Returns:
`tuple[Dict, Dict]`: The dictionary(ies) that will be used to instantiate the feature extractor object.
"""
- cache_dir = kwargs.pop("cache_dir", None)
- force_download = kwargs.pop("force_download", False)
- proxies = kwargs.pop("proxies", None)
- subfolder = kwargs.pop("subfolder", None)
- token = kwargs.pop("token", None)
- local_files_only = kwargs.pop("local_files_only", False)
- revision = kwargs.pop("revision", None)
-
- from_pipeline = kwargs.pop("_from_pipeline", None)
- from_auto_class = kwargs.pop("_from_auto", False)
-
- user_agent = {"file_type": "feature extractor", "from_auto_class": from_auto_class}
- if from_pipeline is not None:
- user_agent["using_pipeline"] = from_pipeline
-
- if is_offline_mode() and not local_files_only:
- logger.info("Offline mode: forcing local_files_only=True")
- local_files_only = True
-
- pretrained_model_name_or_path = str(pretrained_model_name_or_path)
- is_local = os.path.isdir(pretrained_model_name_or_path)
- if os.path.isdir(pretrained_model_name_or_path):
- feature_extractor_file = os.path.join(pretrained_model_name_or_path, FEATURE_EXTRACTOR_NAME)
- if os.path.isfile(pretrained_model_name_or_path):
- resolved_feature_extractor_file = pretrained_model_name_or_path
- resolved_processor_file = None
- is_local = True
- else:
- feature_extractor_file = FEATURE_EXTRACTOR_NAME
- try:
- # Load from local folder or from cache or download from model Hub and cache
- resolved_processor_file = cached_file(
- pretrained_model_name_or_path,
- filename=PROCESSOR_NAME,
- cache_dir=cache_dir,
- force_download=force_download,
- proxies=proxies,
- local_files_only=local_files_only,
- token=token,
- user_agent=user_agent,
- revision=revision,
- subfolder=subfolder,
- _raise_exceptions_for_missing_entries=False,
- )
- resolved_feature_extractor_file = cached_file(
- pretrained_model_name_or_path,
- filename=feature_extractor_file,
- cache_dir=cache_dir,
- force_download=force_download,
- proxies=proxies,
- local_files_only=local_files_only,
- token=token,
- user_agent=user_agent,
- revision=revision,
- subfolder=subfolder,
- _raise_exceptions_for_missing_entries=False,
- )
- except OSError:
- # Raise any environment error raise by `cached_file`. It will have a helpful error message adapted to
- # the original exception.
- raise
- except Exception:
- # For any other exception, we throw a generic error.
- raise OSError(
- f"Can't load feature extractor for '{pretrained_model_name_or_path}'. If you were trying to load"
- " it from 'https://huggingface.co/models', make sure you don't have a local directory with the"
- f" same name. Otherwise, make sure '{pretrained_model_name_or_path}' is the correct path to a"
- f" directory containing a {FEATURE_EXTRACTOR_NAME} file"
- )
-
- # Load feature_extractor dict. Priority goes as (nested config if found -> image processor config)
- # We are downloading both configs because almost all models have a `processor_config.json` but
- # not all of these are nested. We need to check if it was saved recebtly as nested or if it is legacy style
- feature_extractor_dict = None
- if resolved_processor_file is not None:
- processor_dict = safe_load_json_file(resolved_processor_file)
- if "feature_extractor" in processor_dict or "audio_processor" in processor_dict:
- feature_extractor_dict = processor_dict.get("feature_extractor", processor_dict.get("audio_processor"))
-
- if resolved_feature_extractor_file is not None and feature_extractor_dict is None:
- feature_extractor_dict = safe_load_json_file(resolved_feature_extractor_file)
-
- if feature_extractor_dict is None:
- raise OSError(
- f"Can't load feature extractor for '{pretrained_model_name_or_path}'. If you were trying to load"
- " it from 'https://huggingface.co/models', make sure you don't have a local directory with the"
- f" same name. Otherwise, make sure '{pretrained_model_name_or_path}' is the correct path to a"
- f" directory containing a {feature_extractor_file} file"
- )
-
- if is_local:
- logger.info(f"loading configuration file {resolved_feature_extractor_file}")
- else:
- logger.info(
- f"loading configuration file {feature_extractor_file} from cache at {resolved_feature_extractor_file}"
- )
-
- return feature_extractor_dict, kwargs
+ return cls._get_config_dict(pretrained_model_name_or_path, **kwargs)
@classmethod
def from_dict(
@@ -581,89 +123,6 @@ def from_dict(
else:
return feature_extractor
- def to_dict(self) -> dict[str, Any]:
- """
- Serializes this instance to a Python dictionary. Returns:
- `dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance.
- """
- output = copy.deepcopy(self.__dict__)
- output["feature_extractor_type"] = self.__class__.__name__
- if "mel_filters" in output:
- del output["mel_filters"]
- if "window" in output:
- del output["window"]
- return output
-
- @classmethod
- def from_json_file(cls, json_file: str | os.PathLike) -> "FeatureExtractionMixin":
- """
- Instantiates a feature extractor of type [`~feature_extraction_utils.FeatureExtractionMixin`] from the path to
- a JSON file of parameters.
-
- Args:
- json_file (`str` or `os.PathLike`):
- Path to the JSON file containing the parameters.
-
- Returns:
- A feature extractor of type [`~feature_extraction_utils.FeatureExtractionMixin`]: The feature_extractor
- object instantiated from that JSON file.
- """
- with open(json_file, encoding="utf-8") as reader:
- text = reader.read()
- feature_extractor_dict = json.loads(text)
- return cls(**feature_extractor_dict)
-
- def to_json_string(self) -> str:
- """
- Serializes this instance to a JSON string.
-
- Returns:
- `str`: String containing all the attributes that make up this feature_extractor instance in JSON format.
- """
- dictionary = self.to_dict()
-
- for key, value in dictionary.items():
- if isinstance(value, np.ndarray):
- dictionary[key] = value.tolist()
-
- return json.dumps(dictionary, indent=2, sort_keys=True) + "\n"
-
- def to_json_file(self, json_file_path: str | os.PathLike):
- """
- Save this instance to a JSON file.
-
- Args:
- json_file_path (`str` or `os.PathLike`):
- Path to the JSON file in which this feature_extractor instance's parameters will be saved.
- """
- with open(json_file_path, "w", encoding="utf-8") as writer:
- writer.write(self.to_json_string())
-
- def __repr__(self):
- return f"{self.__class__.__name__} {self.to_json_string()}"
-
- @classmethod
- def register_for_auto_class(cls, auto_class="AutoFeatureExtractor"):
- """
- Register this class with a given auto class. This should only be used for custom feature extractors as the ones
- in the library are already mapped with `AutoFeatureExtractor`.
-
-
-
- Args:
- auto_class (`str` or `type`, *optional*, defaults to `"AutoFeatureExtractor"`):
- The auto class to register this new feature extractor with.
- """
- if not isinstance(auto_class, str):
- auto_class = auto_class.__name__
-
- import transformers.models.auto as auto_module
-
- if not hasattr(auto_module, auto_class):
- raise ValueError(f"{auto_class} is not a valid auto class.")
-
- cls._auto_class = auto_class
-
FeatureExtractionMixin.push_to_hub = copy_func(FeatureExtractionMixin.push_to_hub)
if FeatureExtractionMixin.push_to_hub.__doc__ is not None:
diff --git a/src/transformers/image_processing_base.py b/src/transformers/image_processing_base.py
index f5d362944ced..7c541f76d130 100644
--- a/src/transformers/image_processing_base.py
+++ b/src/transformers/image_processing_base.py
@@ -12,26 +12,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import copy
-import json
import os
from typing import Any, TypeVar
-import numpy as np
-from huggingface_hub import is_offline_mode
-
-from .dynamic_module_utils import custom_object_save
-from .feature_extraction_utils import BatchFeature as BaseBatchFeature
from .image_utils import is_valid_image, load_image
+from .preprocessing_base import BatchFeature as BaseBatchFeature
+from .preprocessing_base import PreprocessingMixin
from .utils import (
IMAGE_PROCESSOR_NAME,
- PROCESSOR_NAME,
- PushToHubMixin,
copy_func,
logging,
- safe_load_json_file,
)
-from .utils.hub import cached_file, hf_api
ImageProcessorType = TypeVar("ImageProcessorType", bound="ImageProcessingMixin")
@@ -58,175 +49,21 @@ class BatchFeature(BaseBatchFeature):
# TODO: (Amy) - factor out the common parts of this and the feature extractor
-class ImageProcessingMixin(PushToHubMixin):
+class ImageProcessingMixin(PreprocessingMixin):
"""
This is an image processor mixin used to provide saving/loading functionality for sequential and image feature
extractors.
"""
- _auto_class = None
-
- def __init__(self, **kwargs):
- """Set elements of `kwargs` as attributes."""
- # This key was saved while we still used `XXXFeatureExtractor` for image processing. Now we use
- # `XXXImageProcessor`, this attribute and its value are misleading.
- kwargs.pop("feature_extractor_type", None)
- # Pop "processor_class", should not be saved with image processing config anymore
- kwargs.pop("processor_class", None)
- # Additional attributes without default values
- for key, value in kwargs.items():
- try:
- setattr(self, key, value)
- except AttributeError as err:
- logger.error(f"Can't set {key} with value {value} for {self}")
- raise err
-
- @classmethod
- def from_pretrained(
- cls: type[ImageProcessorType],
- pretrained_model_name_or_path: str | os.PathLike,
- cache_dir: str | os.PathLike | None = None,
- force_download: bool = False,
- local_files_only: bool = False,
- token: str | bool | None = None,
- revision: str = "main",
- **kwargs,
- ) -> ImageProcessorType:
- r"""
- Instantiate a type of [`~image_processing_utils.ImageProcessingMixin`] from an image processor.
-
- Args:
- pretrained_model_name_or_path (`str` or `os.PathLike`):
- This can be either:
-
- - a string, the *model id* of a pretrained image_processor hosted inside a model repo on
- huggingface.co.
- - a path to a *directory* containing a image processor file saved using the
- [`~image_processing_utils.ImageProcessingMixin.save_pretrained`] method, e.g.,
- `./my_model_directory/`.
- - a path to a saved image processor JSON *file*, e.g.,
- `./my_model_directory/preprocessor_config.json`.
- cache_dir (`str` or `os.PathLike`, *optional*):
- Path to a directory in which a downloaded pretrained model image processor should be cached if the
- standard cache should not be used.
- force_download (`bool`, *optional*, defaults to `False`):
- Whether or not to force to (re-)download the image processor files and override the cached versions if
- they exist.
- proxies (`dict[str, str]`, *optional*):
- A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
- 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request.
- token (`str` or `bool`, *optional*):
- The token to use as HTTP bearer authorization for remote files. If `True`, or not specified, will use
- the token generated when running `hf auth login` (stored in `~/.huggingface`).
- revision (`str`, *optional*, defaults to `"main"`):
- The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
- git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
- identifier allowed by git.
-
-
-
-
- To test a pull request you made on the Hub, you can pass `revision="refs/pr/"`.
-
-
-
- return_unused_kwargs (`bool`, *optional*, defaults to `False`):
- If `False`, then this function returns just the final image processor object. If `True`, then this
- functions returns a `Tuple(image_processor, unused_kwargs)` where *unused_kwargs* is a dictionary
- consisting of the key/value pairs whose keys are not image processor attributes: i.e., the part of
- `kwargs` which has not been used to update `image_processor` and is otherwise ignored.
- subfolder (`str`, *optional*, defaults to `""`):
- In case the relevant files are located inside a subfolder of the model repo on huggingface.co, you can
- specify the folder name here.
- kwargs (`dict[str, Any]`, *optional*):
- The values in kwargs of any keys which are image processor attributes will be used to override the
- loaded values. Behavior concerning key/value pairs whose keys are *not* image processor attributes is
- controlled by the `return_unused_kwargs` keyword parameter.
-
- Returns:
- A image processor of type [`~image_processing_utils.ImageProcessingMixin`].
-
- Examples:
-
- ```python
- # We can't instantiate directly the base class *ImageProcessingMixin* so let's show the examples on a
- # derived class: *CLIPImageProcessor*
- image_processor = CLIPImageProcessor.from_pretrained(
- "openai/clip-vit-base-patch32"
- ) # Download image_processing_config from huggingface.co and cache.
- image_processor = CLIPImageProcessor.from_pretrained(
- "./test/saved_model/"
- ) # E.g. image processor (or model) was saved using *save_pretrained('./test/saved_model/')*
- image_processor = CLIPImageProcessor.from_pretrained("./test/saved_model/preprocessor_config.json")
- image_processor = CLIPImageProcessor.from_pretrained(
- "openai/clip-vit-base-patch32", do_normalize=False, foo=False
- )
- assert image_processor.do_normalize is False
- image_processor, unused_kwargs = CLIPImageProcessor.from_pretrained(
- "openai/clip-vit-base-patch32", do_normalize=False, foo=False, return_unused_kwargs=True
- )
- assert image_processor.do_normalize is False
- assert unused_kwargs == {"foo": False}
- ```"""
- kwargs["cache_dir"] = cache_dir
- kwargs["force_download"] = force_download
- kwargs["local_files_only"] = local_files_only
- kwargs["revision"] = revision
-
- if token is not None:
- kwargs["token"] = token
-
- image_processor_dict, kwargs = cls.get_image_processor_dict(pretrained_model_name_or_path, **kwargs)
-
- return cls.from_dict(image_processor_dict, **kwargs)
-
- def save_pretrained(self, save_directory: str | os.PathLike, push_to_hub: bool = False, **kwargs):
- """
- Save an image processor object to the directory `save_directory`, so that it can be re-loaded using the
- [`~image_processing_utils.ImageProcessingMixin.from_pretrained`] class method.
-
- Args:
- save_directory (`str` or `os.PathLike`):
- Directory where the image processor JSON file will be saved (will be created if it does not exist).
- push_to_hub (`bool`, *optional*, defaults to `False`):
- Whether or not to push your model to the Hugging Face model hub after saving it. You can specify the
- repository you want to push to with `repo_id` (will default to the name of `save_directory` in your
- namespace).
- kwargs (`dict[str, Any]`, *optional*):
- Additional key word arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method.
- """
- if os.path.isfile(save_directory):
- raise AssertionError(f"Provided path ({save_directory}) should be a directory, not a file")
-
- os.makedirs(save_directory, exist_ok=True)
-
- if push_to_hub:
- commit_message = kwargs.pop("commit_message", None)
- repo_id = kwargs.pop("repo_id", save_directory.split(os.path.sep)[-1])
- repo_id = hf_api().create_repo(repo_id, exist_ok=True, **kwargs).repo_id
- files_timestamps = self._get_files_timestamps(save_directory)
-
- # If we have a custom config, we copy the file defining it in the folder and set the attributes so it can be
- # loaded from the Hub.
- if self._auto_class is not None:
- custom_object_save(self, save_directory, config=self)
-
- # If we save using the predefined names, we can load using `from_pretrained`
- output_image_processor_file = os.path.join(save_directory, IMAGE_PROCESSOR_NAME)
-
- self.to_json_file(output_image_processor_file)
- logger.info(f"Image processor saved in {output_image_processor_file}")
-
- if push_to_hub:
- self._upload_modified_files(
- save_directory,
- repo_id,
- files_timestamps,
- commit_message=commit_message,
- token=kwargs.get("token"),
- )
-
- return [output_image_processor_file]
+ _config_name = IMAGE_PROCESSOR_NAME
+ _type_key = "image_processor_type"
+ _nested_config_keys = ["image_processor"]
+ _auto_class_default = "AutoImageProcessor"
+ _file_type_label = "image processor"
+ _excluded_dict_keys = set()
+ _extra_init_pops = ["feature_extractor_type"]
+ _config_filename_kwarg = "image_processor_filename"
+ _subfolder_default = ""
@classmethod
def get_image_processor_dict(
@@ -248,227 +85,7 @@ def get_image_processor_dict(
Returns:
`tuple[Dict, Dict]`: The dictionary(ies) that will be used to instantiate the image processor object.
"""
- cache_dir = kwargs.pop("cache_dir", None)
- force_download = kwargs.pop("force_download", False)
- proxies = kwargs.pop("proxies", None)
- token = kwargs.pop("token", None)
- local_files_only = kwargs.pop("local_files_only", False)
- revision = kwargs.pop("revision", None)
- subfolder = kwargs.pop("subfolder", "")
- image_processor_filename = kwargs.pop("image_processor_filename", IMAGE_PROCESSOR_NAME)
-
- from_pipeline = kwargs.pop("_from_pipeline", None)
- from_auto_class = kwargs.pop("_from_auto", False)
-
- user_agent = {"file_type": "image processor", "from_auto_class": from_auto_class}
- if from_pipeline is not None:
- user_agent["using_pipeline"] = from_pipeline
-
- if is_offline_mode() and not local_files_only:
- logger.info("Offline mode: forcing local_files_only=True")
- local_files_only = True
-
- pretrained_model_name_or_path = str(pretrained_model_name_or_path)
- is_local = os.path.isdir(pretrained_model_name_or_path)
- if os.path.isdir(pretrained_model_name_or_path):
- image_processor_file = os.path.join(pretrained_model_name_or_path, image_processor_filename)
- if os.path.isfile(pretrained_model_name_or_path):
- resolved_image_processor_file = pretrained_model_name_or_path
- resolved_processor_file = None
- is_local = True
- else:
- image_processor_file = image_processor_filename
- try:
- resolved_processor_file = cached_file(
- pretrained_model_name_or_path,
- filename=PROCESSOR_NAME,
- cache_dir=cache_dir,
- force_download=force_download,
- proxies=proxies,
- local_files_only=local_files_only,
- token=token,
- user_agent=user_agent,
- revision=revision,
- subfolder=subfolder,
- _raise_exceptions_for_missing_entries=False,
- )
- resolved_image_processor_file = cached_file(
- pretrained_model_name_or_path,
- filename=image_processor_file,
- cache_dir=cache_dir,
- force_download=force_download,
- proxies=proxies,
- local_files_only=local_files_only,
- token=token,
- user_agent=user_agent,
- revision=revision,
- subfolder=subfolder,
- _raise_exceptions_for_missing_entries=False,
- )
- except OSError:
- # Raise any environment error raise by `cached_file`. It will have a helpful error message adapted to
- # the original exception.
- raise
- except Exception:
- # For any other exception, we throw a generic error.
- raise OSError(
- f"Can't load image processor for '{pretrained_model_name_or_path}'. If you were trying to load"
- " it from 'https://huggingface.co/models', make sure you don't have a local directory with the"
- f" same name. Otherwise, make sure '{pretrained_model_name_or_path}' is the correct path to a"
- f" directory containing a {image_processor_filename} file"
- )
-
- # Load image_processor dict. Priority goes as (nested config if found -> image processor config)
- # We are downloading both configs because almost all models have a `processor_config.json` but
- # not all of these are nested. We need to check if it was saved recebtly as nested or if it is legacy style
- image_processor_dict = None
- if resolved_processor_file is not None:
- processor_dict = safe_load_json_file(resolved_processor_file)
- if "image_processor" in processor_dict:
- image_processor_dict = processor_dict["image_processor"]
-
- if resolved_image_processor_file is not None and image_processor_dict is None:
- image_processor_dict = safe_load_json_file(resolved_image_processor_file)
-
- if image_processor_dict is None:
- raise OSError(
- f"Can't load image processor for '{pretrained_model_name_or_path}'. If you were trying to load"
- " it from 'https://huggingface.co/models', make sure you don't have a local directory with the"
- f" same name. Otherwise, make sure '{pretrained_model_name_or_path}' is the correct path to a"
- f" directory containing a {image_processor_filename} file"
- )
-
- if is_local:
- logger.info(f"loading configuration file {resolved_image_processor_file}")
- else:
- logger.info(
- f"loading configuration file {image_processor_file} from cache at {resolved_image_processor_file}"
- )
-
- return image_processor_dict, kwargs
-
- @classmethod
- def from_dict(cls, image_processor_dict: dict[str, Any], **kwargs):
- """
- Instantiates a type of [`~image_processing_utils.ImageProcessingMixin`] from a Python dictionary of parameters.
-
- Args:
- image_processor_dict (`dict[str, Any]`):
- Dictionary that will be used to instantiate the image processor object. Such a dictionary can be
- retrieved from a pretrained checkpoint by leveraging the
- [`~image_processing_utils.ImageProcessingMixin.to_dict`] method.
- kwargs (`dict[str, Any]`):
- Additional parameters from which to initialize the image processor object.
-
- Returns:
- [`~image_processing_utils.ImageProcessingMixin`]: The image processor object instantiated from those
- parameters.
- """
- image_processor_dict = image_processor_dict.copy()
- return_unused_kwargs = kwargs.pop("return_unused_kwargs", False)
- image_processor_dict.update({k: v for k, v in kwargs.items() if k in cls.valid_kwargs.__annotations__})
- image_processor = cls(**image_processor_dict)
-
- # Apply extra kwargs to instance (BC for remote code, e.g. phi4_multimodal)
- extra_keys = []
- for key in reversed(list(kwargs.keys())):
- if hasattr(image_processor, key) and key not in cls.valid_kwargs.__annotations__:
- setattr(image_processor, key, kwargs.pop(key, None))
- extra_keys.append(key)
- if extra_keys:
- logger.warning_once(
- f"Image processor {cls.__name__}: kwargs {extra_keys} were applied for backward compatibility. "
- f"To avoid this warning, add them to valid_kwargs: create a custom TypedDict extending "
- f"ImagesKwargs with these keys and set it as the `valid_kwargs` class attribute."
- )
-
- logger.info(f"Image processor {image_processor}")
- if return_unused_kwargs:
- return image_processor, kwargs
- else:
- return image_processor
-
- def to_dict(self) -> dict[str, Any]:
- """
- Serializes this instance to a Python dictionary.
-
- Returns:
- `dict[str, Any]`: Dictionary of all the attributes that make up this image processor instance.
- """
- output = copy.deepcopy(self.__dict__)
- output["image_processor_type"] = self.__class__.__name__
-
- return output
-
- @classmethod
- def from_json_file(cls, json_file: str | os.PathLike):
- """
- Instantiates a image processor of type [`~image_processing_utils.ImageProcessingMixin`] from the path to a JSON
- file of parameters.
-
- Args:
- json_file (`str` or `os.PathLike`):
- Path to the JSON file containing the parameters.
-
- Returns:
- A image processor of type [`~image_processing_utils.ImageProcessingMixin`]: The image_processor object
- instantiated from that JSON file.
- """
- with open(json_file, encoding="utf-8") as reader:
- text = reader.read()
- image_processor_dict = json.loads(text)
- return cls(**image_processor_dict)
-
- def to_json_string(self) -> str:
- """
- Serializes this instance to a JSON string.
-
- Returns:
- `str`: String containing all the attributes that make up this feature_extractor instance in JSON format.
- """
- dictionary = self.to_dict()
-
- for key, value in dictionary.items():
- if isinstance(value, np.ndarray):
- dictionary[key] = value.tolist()
-
- return json.dumps(dictionary, indent=2, sort_keys=True) + "\n"
-
- def to_json_file(self, json_file_path: str | os.PathLike):
- """
- Save this instance to a JSON file.
-
- Args:
- json_file_path (`str` or `os.PathLike`):
- Path to the JSON file in which this image_processor instance's parameters will be saved.
- """
- with open(json_file_path, "w", encoding="utf-8") as writer:
- writer.write(self.to_json_string())
-
- def __repr__(self):
- return f"{self.__class__.__name__} {self.to_json_string()}"
-
- @classmethod
- def register_for_auto_class(cls, auto_class="AutoImageProcessor"):
- """
- Register this class with a given auto class. This should only be used for custom image processors as the ones
- in the library are already mapped with `AutoImageProcessor `.
-
-
-
- Args:
- auto_class (`str` or `type`, *optional*, defaults to `"AutoImageProcessor "`):
- The auto class to register this new image processor with.
- """
- if not isinstance(auto_class, str):
- auto_class = auto_class.__name__
-
- import transformers.models.auto as auto_module
-
- if not hasattr(auto_module, auto_class):
- raise ValueError(f"{auto_class} is not a valid auto class.")
-
- cls._auto_class = auto_class
+ return cls._get_config_dict(pretrained_model_name_or_path, **kwargs)
def fetch_images(self, image_url_or_urls: str | list[str] | list[list[str]]):
"""
diff --git a/src/transformers/image_processing_utils.py b/src/transformers/image_processing_utils.py
index 0e7948ffd08a..7faccf9a8466 100644
--- a/src/transformers/image_processing_utils.py
+++ b/src/transformers/image_processing_utils.py
@@ -14,12 +14,10 @@
import math
from collections.abc import Iterable
-from copy import deepcopy
from functools import partial
from typing import Any
import numpy as np
-from huggingface_hub.dataclasses import validate_typed_dict
from .image_processing_base import BatchFeature, ImageProcessingMixin
from .image_transforms import center_crop, normalize, rescale
@@ -191,27 +189,17 @@ class MyImageProcessor(TorchvisionBackend):
default_to_square = True
rescale_factor = 1 / 255
model_input_names = ["pixel_values"]
+ _excluded_dict_keys = {"_valid_processor_keys"}
+
+ def _serialize_value(self, key, value):
+ # Coerce SizeDict attributes to plain dicts for JSON persistence.
+ return dict(value) if isinstance(value, SizeDict) else value
def __init__(self, **kwargs: Unpack[ImagesKwargs]):
super().__init__(**kwargs)
# We don't call self._set_attributes in BaseImageProcessor for backward compatibility with remote code
# We call it instead in the backend subclasses' __init__ methods.
- def _set_attributes(self, **kwargs):
- """Resolve and set instance attributes from kwargs and class-level defaults for all valid kwargs."""
- attributes = {}
- for key in self.valid_kwargs.__annotations__:
- kwarg = kwargs.pop(key, None)
- if kwarg is not None:
- attributes[key] = kwarg
- else:
- attributes[key] = deepcopy(getattr(self, key, None))
- attributes = self._standardize_kwargs(**attributes)
- for key, value in attributes.items():
- setattr(self, key, value)
-
- self._valid_kwargs_names = list(self.valid_kwargs.__annotations__.keys())
-
def __call__(self, images: ImageInput, *args, **kwargs: Unpack[ImagesKwargs]) -> BatchFeature:
"""Preprocess an image or a batch of images."""
return self.preprocess(images, *args, **kwargs)
@@ -384,41 +372,13 @@ def preprocess(self, images: ImageInput, *args, **kwargs: Unpack[ImagesKwargs])
"""
Preprocess an image or a batch of images.
"""
- # Perform type validation on received kwargs
- validate_typed_dict(self.valid_kwargs, kwargs)
-
- # Set default kwargs from self
- for kwarg_name in self._valid_kwargs_names:
- kwargs.setdefault(kwarg_name, getattr(self, kwarg_name, None))
-
- # Update kwargs that need further processing before being validated
- kwargs = self._standardize_kwargs(**kwargs)
-
- # Validate kwargs
- self._validate_preprocess_kwargs(**kwargs)
+ # Common validate/setdefault/standardize/dispatch logic lives in `PreprocessingMixin.preprocess`.
+ return super().preprocess(images, *args, **kwargs)
+ def _preprocess_like_inputs(self, images: ImageInput, *args, **kwargs) -> BatchFeature:
+ """Dispatch hook called by `PreprocessingMixin.preprocess` with validated kwargs."""
return self._preprocess_image_like_inputs(images, *args, **kwargs)
- def to_dict(self) -> dict[str, Any]:
- processor_dict = super().to_dict()
-
- # Filter out None values that are class defaults
- filtered_dict = {}
- for key, value in processor_dict.items():
- if isinstance(value, SizeDict):
- value = dict(value)
- if value is None:
- class_default = getattr(type(self), key, "NOT_FOUND")
- # Keep None if user explicitly set it (class default is non-None)
- if class_default != "NOT_FOUND" and class_default is not None:
- filtered_dict[key] = value
- else:
- filtered_dict[key] = value
-
- filtered_dict.pop("_valid_processor_keys", None)
- filtered_dict.pop("_valid_kwargs_names", None)
- return filtered_dict
-
def rescale(
self,
image: np.ndarray,
diff --git a/src/transformers/models/audio_spectrogram_transformer/__init__.py b/src/transformers/models/audio_spectrogram_transformer/__init__.py
index 618fceef70d3..ab5d63fb6a73 100644
--- a/src/transformers/models/audio_spectrogram_transformer/__init__.py
+++ b/src/transformers/models/audio_spectrogram_transformer/__init__.py
@@ -18,6 +18,8 @@
if TYPE_CHECKING:
+ from .audio_processing_audio_spectrogram_transformer import *
+ from .audio_processing_numpy_audio_spectrogram_transformer import *
from .configuration_audio_spectrogram_transformer import *
from .feature_extraction_audio_spectrogram_transformer import *
from .modeling_audio_spectrogram_transformer import *
diff --git a/src/transformers/models/audio_spectrogram_transformer/audio_processing_audio_spectrogram_transformer.py b/src/transformers/models/audio_spectrogram_transformer/audio_processing_audio_spectrogram_transformer.py
new file mode 100644
index 000000000000..c5825dddc55b
--- /dev/null
+++ b/src/transformers/models/audio_spectrogram_transformer/audio_processing_audio_spectrogram_transformer.py
@@ -0,0 +1,54 @@
+# Copyright 2025 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ...audio_processing_backends import TorchAudioBackend
+from .audio_processing_numpy_audio_spectrogram_transformer import AudioSpectrogramTransformerAudioProcessorNumpy
+
+
+class AudioSpectrogramTransformerAudioProcessor(TorchAudioBackend):
+ sampling_rate = 16000
+ force_mono = True
+ return_padding_mask = False
+ do_batch_spectrogram = False
+
+ max_length_frames = 1024
+ do_normalize = True
+
+ # AudioSet normalization constants
+ ast_mean = -4.2677393
+ ast_std = 4.5689974
+
+
+ spectrogram_config = AudioSpectrogramTransformerAudioProcessorNumpy.spectrogram_config
+
+ def extract_spectrogram(self, audio, **kwargs):
+ # Native kaldi-exact pipeline (bit-equal to `torchaudio.compliance.kaldi.fbank`),
+ # transposed to kaldi's (time, num_mel_bins) orientation expected downstream.
+ features = super().extract_spectrogram(audio, **kwargs)
+ return [f.transpose(-2, -1) for f in features]
+
+ def _pad_features(self, features, padding, max_length, truncation, pad_to_multiple_of):
+ # Always pad/truncate to max_length_frames regardless of caller's padding args
+ return super()._pad_features(features, "max_length", self.max_length_frames, True, pad_to_multiple_of)
+
+ def _postprocess_output(self, output, **kwargs):
+ # Rename to audio_values (AST convention) and apply AudioSet normalization
+ features = output.pop("audio_features")
+ if self.do_normalize:
+ features = (features - self.ast_mean) / (self.ast_std * 2)
+ output["audio_values"] = features
+ return output
+
+
+__all__ = ["AudioSpectrogramTransformerAudioProcessor"]
diff --git a/src/transformers/models/audio_spectrogram_transformer/audio_processing_numpy_audio_spectrogram_transformer.py b/src/transformers/models/audio_spectrogram_transformer/audio_processing_numpy_audio_spectrogram_transformer.py
new file mode 100644
index 000000000000..92f1581dda65
--- /dev/null
+++ b/src/transformers/models/audio_spectrogram_transformer/audio_processing_numpy_audio_spectrogram_transformer.py
@@ -0,0 +1,75 @@
+# Copyright 2026 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ...audio_processing_backends import NumpyAudioBackend
+from ...audio_utils import MelScaleConfig, SpectrogramConfig, StftConfig
+
+
+class AudioSpectrogramTransformerAudioProcessorNumpy(NumpyAudioBackend):
+ """NumPy sibling of [`AudioSpectrogramTransformerAudioProcessor`]. Uses kaldi-compatible
+ fbank features via `_kaldi_fbank` (which delegates to torchaudio under the hood)."""
+
+ sampling_rate = 16000
+ force_mono = True
+ return_padding_mask = False
+ do_batch_spectrogram = False
+
+ max_length_frames = 1024
+ do_normalize = True
+
+ # AudioSet normalization constants
+ ast_mean = -4.2677393
+ ast_std = 4.5689974
+
+ spectrogram_config = SpectrogramConfig(
+ stft_config=StftConfig(
+ n_fft=512,
+ win_length=400,
+ hop_length=160,
+ window_fn="hann_window",
+ power=2.0,
+ center=False,
+ periodic=False,
+ left_align_fft=True,
+ ),
+ mel_scale_config=MelScaleConfig(
+ n_mels=128,
+ f_min=20.0,
+ f_max=8000.0,
+ mel_scale="kaldi",
+ triangularize_in_mel_space=True,
+ ),
+ log_mode="log",
+ preemphasis=0.97,
+ remove_dc_offset=True,
+ mel_floor=1.192092955078125e-07,
+ )
+
+ def extract_spectrogram(self, audio, **kwargs):
+ return [self._kaldi_fbank(waveform, num_mel_bins=128, window_type="hanning") for waveform in audio]
+
+ def _pad_features(self, features, padding, max_length, truncation, pad_to_multiple_of):
+ # Always pad/truncate to max_length_frames regardless of caller's padding args
+ return super()._pad_features(features, "max_length", self.max_length_frames, True, pad_to_multiple_of)
+
+ def _postprocess_output(self, output, **kwargs):
+ # Rename to audio_values (AST convention) and apply AudioSet normalization
+ features = output.pop("audio_features")
+ if self.do_normalize:
+ features = (features - self.ast_mean) / (self.ast_std * 2)
+ output["audio_values"] = features
+ return output
+
+
+__all__ = ["AudioSpectrogramTransformerAudioProcessorNumpy"]
diff --git a/src/transformers/models/audio_spectrogram_transformer/feature_extraction_audio_spectrogram_transformer.py b/src/transformers/models/audio_spectrogram_transformer/feature_extraction_audio_spectrogram_transformer.py
index ee69d1d0b991..aa52242bd864 100644
--- a/src/transformers/models/audio_spectrogram_transformer/feature_extraction_audio_spectrogram_transformer.py
+++ b/src/transformers/models/audio_spectrogram_transformer/feature_extraction_audio_spectrogram_transformer.py
@@ -1,235 +1,20 @@
-# Copyright 2022 The HuggingFace Inc. team.
+# Copyright 2026 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""
-Feature extractor class for Audio Spectrogram Transformer.
+"""Backwards-compatibility shim: re-exports the legacy ``ASTFeatureExtractor`` name as a
+deprecated alias of [`AudioSpectrogramTransformerAudioProcessor`]. Importing or instantiating the alias emits a
+``FutureWarning``; the alias is removed in transformers v5.15 (see ADR 0002).
"""
-import numpy as np
-
-from ...audio_utils import mel_filter_bank, spectrogram, window_function
-from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
-from ...feature_extraction_utils import BatchFeature
-from ...utils import TensorType, is_speech_available, is_torch_available, logging
-
-
-if is_speech_available():
- import torchaudio.compliance.kaldi as ta_kaldi
-
-if is_torch_available():
- import torch
-
-
-logger = logging.get_logger(__name__)
-
-
-class ASTFeatureExtractor(SequenceFeatureExtractor):
- r"""
- Constructs a Audio Spectrogram Transformer (AST) feature extractor.
-
- This feature extractor inherits from [`~feature_extraction_sequence_utils.SequenceFeatureExtractor`] which contains
- most of the main methods. Users should refer to this superclass for more information regarding those methods.
-
- This class extracts mel-filter bank features from raw speech using TorchAudio if installed or using numpy
- otherwise, pads/truncates them to a fixed length and normalizes them using a mean and standard deviation.
-
- Args:
- feature_size (`int`, *optional*, defaults to 1):
- The feature dimension of the extracted features.
- sampling_rate (`int`, *optional*, defaults to 16000):
- The sampling rate at which the audio files should be digitalized expressed in hertz (Hz).
- num_mel_bins (`int`, *optional*, defaults to 128):
- Number of Mel-frequency bins.
- max_length (`int`, *optional*, defaults to 1024):
- Maximum length to which to pad/truncate the extracted features.
- do_normalize (`bool`, *optional*, defaults to `True`):
- Whether or not to normalize the log-Mel features using `mean` and `std`.
- mean (`float`, *optional*, defaults to -4.2677393):
- The mean value used to normalize the log-Mel features. Uses the AudioSet mean by default.
- std (`float`, *optional*, defaults to 4.5689974):
- The standard deviation value used to normalize the log-Mel features. Uses the AudioSet standard deviation
- by default.
- return_attention_mask (`bool`, *optional*, defaults to `False`):
- Whether or not [`~ASTFeatureExtractor.__call__`] should return `attention_mask`.
- """
-
- model_input_names = ["input_values", "attention_mask"]
-
- def __init__(
- self,
- feature_size=1,
- sampling_rate=16000,
- num_mel_bins=128,
- max_length=1024,
- padding_value=0.0,
- do_normalize=True,
- mean=-4.2677393,
- std=4.5689974,
- return_attention_mask=False,
- **kwargs,
- ):
- super().__init__(feature_size=feature_size, sampling_rate=sampling_rate, padding_value=padding_value, **kwargs)
- self.num_mel_bins = num_mel_bins
- self.max_length = max_length
- self.do_normalize = do_normalize
- self.mean = mean
- self.std = std
- self.return_attention_mask = return_attention_mask
-
- if not is_speech_available():
- mel_filters = mel_filter_bank(
- num_frequency_bins=257,
- num_mel_filters=self.num_mel_bins,
- min_frequency=20,
- max_frequency=sampling_rate // 2,
- sampling_rate=sampling_rate,
- norm=None,
- mel_scale="kaldi",
- triangularize_in_mel_space=True,
- )
-
- self.mel_filters = mel_filters
- self.window = window_function(400, "hann", periodic=False)
-
- def _extract_fbank_features(
- self,
- waveform: np.ndarray,
- max_length: int,
- ) -> np.ndarray:
- """
- Get mel-filter bank features using TorchAudio. Note that TorchAudio requires 16-bit signed integers as inputs
- and hence the waveform should not be normalized before feature extraction.
- """
- # waveform = waveform * (2**15) # Kaldi compliance: 16-bit signed integers
- if is_speech_available():
- waveform = torch.from_numpy(waveform).unsqueeze(0)
- fbank = ta_kaldi.fbank(
- waveform,
- sample_frequency=self.sampling_rate,
- window_type="hanning",
- num_mel_bins=self.num_mel_bins,
- )
- else:
- waveform = np.squeeze(waveform)
- fbank = spectrogram(
- waveform,
- self.window,
- frame_length=400,
- hop_length=160,
- fft_length=512,
- power=2.0,
- center=False,
- preemphasis=0.97,
- mel_filters=self.mel_filters,
- log_mel="log",
- mel_floor=1.192092955078125e-07,
- remove_dc_offset=True,
- ).T
-
- fbank = torch.from_numpy(fbank)
-
- n_frames = fbank.shape[0]
- difference = max_length - n_frames
-
- # pad or truncate, depending on difference
- if difference > 0:
- pad_module = torch.nn.ZeroPad2d((0, 0, 0, difference))
- fbank = pad_module(fbank)
- elif difference < 0:
- fbank = fbank[0:max_length, :]
-
- fbank = fbank.numpy()
-
- return fbank
-
- def normalize(self, input_values: np.ndarray) -> np.ndarray:
- return (input_values - (self.mean)) / (self.std * 2)
-
- def __call__(
- self,
- raw_speech: np.ndarray | list[float] | list[np.ndarray] | list[list[float]],
- sampling_rate: int | None = None,
- return_tensors: str | TensorType | None = None,
- **kwargs,
- ) -> BatchFeature:
- """
- Main method to featurize and prepare for the model one or several sequence(s).
-
- Args:
- raw_speech (`np.ndarray`, `list[float]`, `list[np.ndarray]`, `list[list[float]]`):
- The sequence or batch of sequences to be padded. Each sequence can be a numpy array, a list of float
- values, a list of numpy arrays or a list of list of float values. Must be mono channel audio, not
- stereo, i.e. single float per timestep.
- sampling_rate (`int`, *optional*):
- The sampling rate at which the `raw_speech` input was sampled. It is strongly recommended to pass
- `sampling_rate` at the forward call to prevent silent errors.
- return_tensors (`str` or [`~utils.TensorType`], *optional*):
- If set, will return tensors instead of list of python integers. Acceptable values are:
-
- - `'pt'`: Return PyTorch `torch.Tensor` objects.
- - `'np'`: Return Numpy `np.ndarray` objects.
- """
-
- if sampling_rate is not None:
- if sampling_rate != self.sampling_rate:
- raise ValueError(
- f"The model corresponding to this feature extractor: {self} was trained using a sampling rate of"
- f" {self.sampling_rate}. Please make sure that the provided `raw_speech` input was sampled with"
- f" {self.sampling_rate} and not {sampling_rate}."
- )
- else:
- logger.warning(
- f"It is strongly recommended to pass the `sampling_rate` argument to `{self.__class__.__name__}()`. "
- "Failing to do so can result in silent errors that might be hard to debug."
- )
-
- is_batched_numpy = isinstance(raw_speech, np.ndarray) and len(raw_speech.shape) > 1
- if is_batched_numpy and len(raw_speech.shape) > 2:
- raise ValueError(f"Only mono-channel audio is supported for input to {self}")
- is_batched = is_batched_numpy or (
- isinstance(raw_speech, (list, tuple)) and (isinstance(raw_speech[0], (np.ndarray, tuple, list)))
- )
-
- if is_batched:
- raw_speech = [np.asarray(speech, dtype=np.float32) for speech in raw_speech]
- elif not is_batched and not isinstance(raw_speech, np.ndarray):
- raw_speech = np.asarray(raw_speech, dtype=np.float32)
- elif isinstance(raw_speech, np.ndarray) and raw_speech.dtype is np.dtype(np.float64):
- raw_speech = raw_speech.astype(np.float32)
-
- # always return batch
- if not is_batched:
- raw_speech = [raw_speech]
-
- # extract fbank features and pad/truncate to max_length
- features = [self._extract_fbank_features(waveform, max_length=self.max_length) for waveform in raw_speech]
-
- # convert into BatchFeature
- padded_inputs = BatchFeature({"input_values": features})
-
- # make sure list is in array format
- input_values = padded_inputs.get("input_values")
- if isinstance(input_values[0], list):
- padded_inputs["input_values"] = [np.asarray(feature, dtype=np.float32) for feature in input_values]
-
- # normalization
- if self.do_normalize:
- padded_inputs["input_values"] = [self.normalize(feature) for feature in input_values]
+from ...audio_processing_base import make_legacy_audio_processor_alias
+from .audio_processing_audio_spectrogram_transformer import AudioSpectrogramTransformerAudioProcessor
- if return_tensors is not None:
- padded_inputs = padded_inputs.convert_to_tensors(return_tensors)
- return padded_inputs
+ASTFeatureExtractor = make_legacy_audio_processor_alias(AudioSpectrogramTransformerAudioProcessor, "ASTFeatureExtractor")
__all__ = ["ASTFeatureExtractor"]
diff --git a/src/transformers/models/auto/feature_extraction_auto.py b/src/transformers/models/auto/feature_extraction_auto.py
index eba04a7c799d..09de641b53fd 100644
--- a/src/transformers/models/auto/feature_extraction_auto.py
+++ b/src/transformers/models/auto/feature_extraction_auto.py
@@ -11,19 +11,18 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-"""AutoFeatureExtractor class."""
+"""AutoAudioProcessor and (deprecated) AutoFeatureExtractor classes."""
import importlib
import os
+import warnings
from collections import OrderedDict
-# Build the list of all feature extractors
+from ...audio_processing_base import AudioProcessingMixin
from ...configuration_utils import PreTrainedConfig
from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code
-from ...feature_extraction_utils import FeatureExtractionMixin
from ...utils import CONFIG_NAME, FEATURE_EXTRACTOR_NAME, PROCESSOR_NAME, cached_file, logging, safe_load_json_file
from .auto_factory import _LazyAutoMapping
-from .auto_mappings import FEATURE_EXTRACTOR_MAPPING_NAMES
from .configuration_auto import (
CONFIG_MAPPING_NAMES,
AutoConfig,
@@ -34,64 +33,154 @@
logger = logging.get_logger(__name__)
-MISSING_FEATURE_EXTRACTOR_MAPPING_NAMES = OrderedDict(
+# Each entry maps a `model_type` to a dict of backend → audio-processor class name. The torch
+# entry is the default returned by `AutoAudioProcessor.from_pretrained`; the numpy entry, when
+# present, is the bit-exact CPU-only sibling (see docs/adr/0001-bit-exact-backend-parity.md).
+# Non-audio feature extractors (e.g. MarkupLM) keep a single-key dict for back-compat.
+FEATURE_EXTRACTOR_MAPPING_NAMES = OrderedDict(
[
- ("audioflamingo3", "WhisperFeatureExtractor"),
- ("csm", "EncodecFeatureExtractor"),
- ("data2vec-audio", "Wav2Vec2FeatureExtractor"),
- ("glmasr", "WhisperFeatureExtractor"),
- ("granite_speech_plus", "GraniteSpeechFeatureExtractor"),
- ("higgs_audio_v2_tokenizer", "DacFeatureExtractor"),
- ("hubert", "Wav2Vec2FeatureExtractor"),
- ("inkling_mm_model", "InklingFeatureExtractor"),
- ("lasr_ctc", "LasrFeatureExtractor"),
- ("lasr_encoder", "LasrFeatureExtractor"),
- ("mimi", "EncodecFeatureExtractor"),
- ("moonshine", "Wav2Vec2FeatureExtractor"),
- ("moshi", "EncodecFeatureExtractor"),
- ("musicgen", "EncodecFeatureExtractor"),
- ("nemotron3_5_asr", "NemotronAsrStreamingFeatureExtractor"),
- ("nemotron_asr_streaming_encoder", "NemotronAsrStreamingFeatureExtractor"),
- ("parakeet_ctc", "ParakeetFeatureExtractor"),
- ("parakeet_encoder", "ParakeetFeatureExtractor"),
- ("parakeet_rnnt", "ParakeetFeatureExtractor"),
- ("parakeet_tdt", "ParakeetFeatureExtractor"),
- ("pe_audio_video", "PeAudioFeatureExtractor"),
- ("qwen2_5_omni", "WhisperFeatureExtractor"),
- ("qwen2_audio", "WhisperFeatureExtractor"),
- ("qwen3_omni_moe", "WhisperFeatureExtractor"),
- ("seamless_m4t_v2", "SeamlessM4TFeatureExtractor"),
- ("sew", "Wav2Vec2FeatureExtractor"),
- ("sew-d", "Wav2Vec2FeatureExtractor"),
- ("unispeech", "Wav2Vec2FeatureExtractor"),
- ("unispeech-sat", "Wav2Vec2FeatureExtractor"),
- ("vibevoice_asr", "VibeVoiceAcousticTokenizerFeatureExtractor"),
- ("voxtral", "WhisperFeatureExtractor"),
- ("wav2vec2-bert", "Wav2Vec2FeatureExtractor"),
- ("wav2vec2-conformer", "Wav2Vec2FeatureExtractor"),
- ("wavlm", "Wav2Vec2FeatureExtractor"),
- ("xcodec", "DacFeatureExtractor"),
+ # The backend label reflects the actual base class of the registered processor. When a
+ # model has only one sibling today the other backend's lookup falls back via
+ # `_load_class_with_fallback` with a warning. Whisper is the first model with both.
+ (
+ "audio-spectrogram-transformer",
+ {
+ "torch": "AudioSpectrogramTransformerAudioProcessor",
+ "numpy": "AudioSpectrogramTransformerAudioProcessorNumpy",
+ },
+ ),
+ ("audioflamingo3", {"torch": "WhisperAudioProcessor", "numpy": "WhisperAudioProcessorNumpy"}),
+ ("clap", {"torch": "ClapAudioProcessor", "numpy": "ClapAudioProcessorNumpy"}),
+ ("clvp", {"torch": "ClvpAudioProcessor", "numpy": "ClvpAudioProcessorNumpy"}),
+ ("cohere_asr", {"torch": "CohereAsrAudioProcessor", "numpy": "CohereAsrAudioProcessorNumpy"}),
+ ("csm", {"torch": "EncodecAudioProcessor", "numpy": "EncodecAudioProcessorNumpy"}),
+ ("dac", {"torch": "DacAudioProcessor", "numpy": "DacAudioProcessorNumpy"}),
+ ("data2vec-audio", {"torch": "Wav2Vec2AudioProcessor", "numpy": "Wav2Vec2AudioProcessorNumpy"}),
+ ("dia", {"torch": "DiaAudioProcessor", "numpy": "DiaAudioProcessorNumpy"}),
+ ("encodec", {"torch": "EncodecAudioProcessor", "numpy": "EncodecAudioProcessorNumpy"}),
+ ("gemma3n", {"torch": "Gemma3nAudioProcessor", "numpy": "Gemma3nAudioProcessorNumpy"}),
+ ("gemma4", {"torch": "Gemma4AudioProcessor", "numpy": "Gemma4AudioProcessorNumpy"}),
+ (
+ "gemma4_unified",
+ {"torch": "Gemma4UnifiedAudioProcessor", "numpy": "Gemma4UnifiedAudioProcessorNumpy"},
+ ),
+ ("glmasr", {"torch": "WhisperAudioProcessor", "numpy": "WhisperAudioProcessorNumpy"}),
+ ("granite_speech", {"torch": "GraniteSpeechAudioProcessor"}),
+ ("granite_speech_plus", {"torch": "GraniteSpeechAudioProcessor"}),
+ ("higgs_audio_v2_tokenizer", {"torch": "DacAudioProcessor", "numpy": "DacAudioProcessorNumpy"}),
+ ("hubert", {"torch": "Wav2Vec2AudioProcessor", "numpy": "Wav2Vec2AudioProcessorNumpy"}),
+ ("inkling_mm_model", {"torch": "InklingAudioProcessor"}),
+ (
+ "kyutai_speech_to_text",
+ {"torch": "KyutaiSpeechToTextAudioProcessor", "numpy": "KyutaiSpeechToTextAudioProcessorNumpy"},
+ ),
+ ("lasr_ctc", {"torch": "LasrAudioProcessor"}),
+ ("lasr_encoder", {"torch": "LasrAudioProcessor"}),
+ ("markuplm", {"torch": "MarkupLMFeatureExtractor"}),
+ ("mimi", {"torch": "EncodecAudioProcessor", "numpy": "EncodecAudioProcessorNumpy"}),
+ ("moonshine", {"torch": "Wav2Vec2AudioProcessor", "numpy": "Wav2Vec2AudioProcessorNumpy"}),
+ ("moshi", {"torch": "EncodecAudioProcessor", "numpy": "EncodecAudioProcessorNumpy"}),
+ ("musicgen", {"torch": "EncodecAudioProcessor", "numpy": "EncodecAudioProcessorNumpy"}),
+ ("musicgen_melody", {"torch": "MusicgenMelodyAudioProcessor"}),
+ ("nemotron3_5_asr", {"torch": "NemotronAsrStreamingAudioProcessor"}),
+ ("nemotron_asr_streaming_encoder", {"torch": "NemotronAsrStreamingAudioProcessor"}),
+ ("parakeet_ctc", {"torch": "ParakeetAudioProcessor", "numpy": "ParakeetAudioProcessorNumpy"}),
+ ("parakeet_encoder", {"torch": "ParakeetAudioProcessor", "numpy": "ParakeetAudioProcessorNumpy"}),
+ ("parakeet_rnnt", {"torch": "ParakeetAudioProcessor", "numpy": "ParakeetAudioProcessorNumpy"}),
+ ("parakeet_tdt", {"torch": "ParakeetAudioProcessor", "numpy": "ParakeetAudioProcessorNumpy"}),
+ ("pe_audio", {"torch": "PeAudioAudioProcessor", "numpy": "PeAudioAudioProcessorNumpy"}),
+ ("pe_audio_video", {"torch": "PeAudioAudioProcessor", "numpy": "PeAudioAudioProcessorNumpy"}),
+ ("phi4_multimodal", {"torch": "Phi4MultimodalAudioProcessor"}),
+ ("pop2piano", {"torch": "Pop2PianoAudioProcessor", "numpy": "Pop2PianoAudioProcessorNumpy"}),
+ ("qwen2_5_omni", {"torch": "WhisperAudioProcessor", "numpy": "WhisperAudioProcessorNumpy"}),
+ ("qwen2_audio", {"torch": "WhisperAudioProcessor", "numpy": "WhisperAudioProcessorNumpy"}),
+ ("qwen3_asr", {"torch": "Qwen3ASRAudioProcessor", "numpy": "Qwen3ASRAudioProcessorNumpy"}),
+ ("qwen3_omni_moe", {"torch": "WhisperAudioProcessor", "numpy": "WhisperAudioProcessorNumpy"}),
+ ("seamless_m4t", {"torch": "SeamlessM4tAudioProcessor", "numpy": "SeamlessM4tAudioProcessorNumpy"}),
+ ("seamless_m4t_v2", {"torch": "SeamlessM4tAudioProcessor", "numpy": "SeamlessM4tAudioProcessorNumpy"}),
+ ("sew", {"torch": "Wav2Vec2AudioProcessor", "numpy": "Wav2Vec2AudioProcessorNumpy"}),
+ ("sew-d", {"torch": "Wav2Vec2AudioProcessor", "numpy": "Wav2Vec2AudioProcessorNumpy"}),
+ ("speech_to_text", {"torch": "SpeechToTextAudioProcessor", "numpy": "SpeechToTextAudioProcessorNumpy"}),
+ ("speecht5", {"torch": "SpeechT5AudioProcessor", "numpy": "SpeechT5AudioProcessorNumpy"}),
+ ("unispeech", {"torch": "Wav2Vec2AudioProcessor", "numpy": "Wav2Vec2AudioProcessorNumpy"}),
+ ("unispeech-sat", {"torch": "Wav2Vec2AudioProcessor", "numpy": "Wav2Vec2AudioProcessorNumpy"}),
+ ("univnet", {"torch": "UnivNetAudioProcessor", "numpy": "UnivNetAudioProcessorNumpy"}),
+ ("vibevoice_acoustic_tokenizer", {"torch": "VibevoiceAcousticTokenizerAudioProcessor"}),
+ ("vibevoice_asr", {"torch": "VibevoiceAcousticTokenizerAudioProcessor"}),
+ ("voxtral", {"torch": "WhisperAudioProcessor", "numpy": "WhisperAudioProcessorNumpy"}),
+ ("voxtral_realtime", {"torch": "VoxtralRealtimeAudioProcessor"}),
+ ("wav2vec2", {"torch": "Wav2Vec2AudioProcessor", "numpy": "Wav2Vec2AudioProcessorNumpy"}),
+ ("wav2vec2-bert", {"torch": "Wav2Vec2AudioProcessor", "numpy": "Wav2Vec2AudioProcessorNumpy"}),
+ ("wav2vec2-conformer", {"torch": "Wav2Vec2AudioProcessor", "numpy": "Wav2Vec2AudioProcessorNumpy"}),
+ ("wavlm", {"torch": "Wav2Vec2AudioProcessor", "numpy": "Wav2Vec2AudioProcessorNumpy"}),
+ ("whisper", {"torch": "WhisperAudioProcessor", "numpy": "WhisperAudioProcessorNumpy"}),
+ ("xcodec", {"torch": "DacAudioProcessor", "numpy": "DacAudioProcessorNumpy"}),
+ ("xcodec2", {"torch": "Xcodec2AudioProcessor", "numpy": "Xcodec2AudioProcessorNumpy"}),
]
)
-FEATURE_EXTRACTOR_MAPPING_NAMES.update(MISSING_FEATURE_EXTRACTOR_MAPPING_NAMES)
+# Irregular legacy `XxxFeatureExtractor` names that do not derive from the new
+# `XxxAudioProcessor` by simple suffix substitution. Used by
+# `feature_extractor_class_from_name` to resolve hub JSON `feature_extractor_type` strings.
+LEGACY_FEATURE_EXTRACTOR_NAME_MAP = {
+ "ASTFeatureExtractor": "AudioSpectrogramTransformerAudioProcessor",
+ "Gemma3nAudioFeatureExtractor": "Gemma3nAudioProcessor",
+ "Gemma4AudioFeatureExtractor": "Gemma4AudioProcessor",
+ "Gemma4UnifiedAudioFeatureExtractor": "Gemma4UnifiedAudioProcessor",
+ "Speech2TextFeatureExtractor": "SpeechToTextAudioProcessor",
+ "SeamlessM4TFeatureExtractor": "SeamlessM4tAudioProcessor",
+ "VibeVoiceAcousticTokenizerFeatureExtractor": "VibevoiceAcousticTokenizerAudioProcessor",
+ "PeAudioFeatureExtractor": "PeAudioAudioProcessor",
+}
+
+
FEATURE_EXTRACTOR_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, FEATURE_EXTRACTOR_MAPPING_NAMES)
-def feature_extractor_class_from_name(class_name: str):
- for module_name, extractors in FEATURE_EXTRACTOR_MAPPING_NAMES.items():
- if class_name in extractors:
- module_name = model_type_to_module_name(module_name)
+def _legacy_name_candidates(class_name: str) -> list[str]:
+ """Translate a legacy `XxxFeatureExtractor` name into modern candidates.
+
+ Hub `preprocessor_config.json` files have `feature_extractor_type: "WhisperFeatureExtractor"`
+ (legacy) or `audio_processor_type: "WhisperAudioProcessor"` (new). When loading legacy
+ configs we need to find the new class by name. Tries, in order:
+ 1. The name as-is (modern names already match)
+ 2. An explicit override in `LEGACY_FEATURE_EXTRACTOR_NAME_MAP`
+ 3. The simple `FeatureExtractor → AudioProcessor` suffix substitution
+ """
+ candidates = [class_name]
+ if class_name in LEGACY_FEATURE_EXTRACTOR_NAME_MAP:
+ # Explicit override wins over the generic suffix substitution
+ candidates.append(LEGACY_FEATURE_EXTRACTOR_NAME_MAP[class_name])
+ elif class_name.endswith("FeatureExtractor"):
+ candidates.append(class_name.replace("FeatureExtractor", "AudioProcessor"))
+ # De-duplicate while preserving order
+ seen = set()
+ return [c for c in candidates if not (c in seen or seen.add(c))]
+
- module = importlib.import_module(f".{module_name}", "transformers.models")
- try:
- return getattr(module, class_name)
- except AttributeError:
- continue
+def feature_extractor_class_from_name(class_name: str):
+ """Resolve an audio-processor or legacy feature-extractor name to its class object.
- for extractor in FEATURE_EXTRACTOR_MAPPING._extra_content.values():
- if getattr(extractor, "__name__", None) == class_name:
- return extractor
+ Handles both modern names (`WhisperAudioProcessor`, `WhisperAudioProcessorNumpy`) and the
+ legacy `XxxFeatureExtractor` names still found in hub `preprocessor_config.json` files.
+ """
+ for candidate in _legacy_name_candidates(class_name):
+ for model_type, extractors_dict in FEATURE_EXTRACTOR_MAPPING_NAMES.items():
+ if candidate in extractors_dict.values():
+ module_name = model_type_to_module_name(model_type)
+ module = importlib.import_module(f".{module_name}", "transformers.models")
+ try:
+ return getattr(module, candidate)
+ except AttributeError:
+ continue
+
+ for mapping in FEATURE_EXTRACTOR_MAPPING._extra_content.values():
+ if isinstance(mapping, dict):
+ for cls in mapping.values():
+ if getattr(cls, "__name__", None) == class_name:
+ return cls
+ elif getattr(mapping, "__name__", None) == class_name:
+ return mapping
# We did not find the class, but maybe it's because a dep is missing. In that case, the class will be in the main
# init and we return the proper dummy to get an appropriate error message.
@@ -102,6 +191,92 @@ def feature_extractor_class_from_name(class_name: str):
return None
+def _resolve_audio_backend(backend: str | None) -> str:
+ """Resolve raw backend input to a concrete backend name (`'torch'` or `'numpy'`).
+
+ Default is `'torch'`; `'numpy'` is the bit-exact CPU-only sibling. If a model has no
+ sibling for the requested backend, `_load_class_with_fallback` warns and falls back to
+ the available one.
+ """
+ if backend is None:
+ return "torch"
+ if backend not in {"torch", "numpy"}:
+ raise ValueError(f"Unknown audio-processor backend: {backend!r}. Expected 'torch' or 'numpy'.")
+ return backend
+
+
+def _load_class_with_fallback(mapping, backend):
+ """Load an audio-processor class from a backend→class mapping, with fallback.
+
+ Tries the requested backend first; if the model has no sibling for it, falls back to any
+ other available backend and warns. Returns `None` if the mapping is empty.
+ """
+ backends_to_try = [backend] + [b for b in mapping if b != backend]
+
+ for b in backends_to_try:
+ value = mapping.get(b)
+ if value is None:
+ continue
+
+ if isinstance(value, type):
+ processor_class = value
+ else:
+ processor_class = feature_extractor_class_from_name(value)
+
+ if processor_class is None or getattr(processor_class, "is_dummy", False):
+ continue
+
+ if b != backend:
+ logger.warning_once(
+ f"Requested audio-processor backend {backend!r} is not available for this model. "
+ f"Falling back to {b!r} backend."
+ )
+ return processor_class
+
+ return None
+
+
+def _find_mapping_for_audio_processor(base_class_name: str) -> dict | None:
+ """Find the backend→class mapping that contains `base_class_name` in its values."""
+
+ def _value_matches(val, name: str) -> bool:
+ if val is None:
+ return False
+ if isinstance(val, str):
+ return val == name
+ if isinstance(val, type):
+ return getattr(val, "__name__", None) == name
+ return False
+
+ for mapping_dict in FEATURE_EXTRACTOR_MAPPING_NAMES.values():
+ if any(_value_matches(v, base_class_name) for v in mapping_dict.values()):
+ return mapping_dict
+
+ for content in FEATURE_EXTRACTOR_MAPPING._extra_content.values():
+ if isinstance(content, dict) and any(_value_matches(v, base_class_name) for v in content.values()):
+ return content
+
+ return None
+
+
+def _load_backend_class(base_class_name: str, backend: str):
+ """Load an audio-processor class for the requested backend, with fallback."""
+ mapping = _find_mapping_for_audio_processor(base_class_name)
+ if mapping is None:
+ # Unknown class name (e.g. remote code, custom registration): default to the literal name
+ mapping = {"torch": base_class_name}
+ return _load_class_with_fallback(mapping, backend)
+
+
+def _resolve_auto_map_class_ref(auto_map, backend: str) -> str:
+ """Extract the class reference string from an `auto_map` entry based on backend preference."""
+ if isinstance(auto_map, dict):
+ return auto_map.get(backend) or next(iter(auto_map.values()))
+ if isinstance(auto_map, (list, tuple)):
+ return auto_map[0]
+ return auto_map
+
+
def get_feature_extractor_config(
pretrained_model_name_or_path: str | os.PathLike,
cache_dir: str | os.PathLike | None = None,
@@ -113,7 +288,7 @@ def get_feature_extractor_config(
**kwargs,
):
"""
- Loads the feature extractor configuration from a pretrained model feature extractor configuration.
+ Loads the audio-processor / feature-extractor configuration from a pretrained model.
Args:
pretrained_model_name_or_path (`str` or `os.PathLike`):
@@ -122,7 +297,7 @@ def get_feature_extractor_config(
- a string, the *model id* of a pretrained model configuration hosted inside a model repo on
huggingface.co.
- a path to a *directory* containing a configuration file saved using the
- [`~FeatureExtractionMixin.save_pretrained`] method, e.g., `./my_model_directory/`.
+ [`~AudioProcessingMixin.save_pretrained`] method, e.g., `./my_model_directory/`.
cache_dir (`str` or `os.PathLike`, *optional*):
Path to a directory in which a downloaded pretrained model configuration should be cached if the standard
@@ -141,32 +316,11 @@ def get_feature_extractor_config(
git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
identifier allowed by git.
local_files_only (`bool`, *optional*, defaults to `False`):
- If `True`, will only try to load the feature extractor configuration from local files.
-
-
-
- Passing `token=True` is required when you want to use a private model.
-
-
+ If `True`, will only try to load the audio-processor configuration from local files.
Returns:
- `Dict`: The configuration of the feature extractor.
-
- Examples:
-
- ```python
- # Download configuration from huggingface.co and cache.
- feature_extractor_config = get_feature_extractor_config("facebook/wav2vec2-base-960h")
- # This model does not have a feature extractor config so the result will be an empty dict.
- feature_extractor_config = get_feature_extractor_config("FacebookAI/xlm-roberta-base")
-
- # Save a pretrained feature extractor locally and you can reload its config
- from transformers import AutoFeatureExtractor
-
- feature_extractor = AutoFeatureExtractor.from_pretrained("facebook/wav2vec2-base-960h")
- feature_extractor.save_pretrained("feature-extractor-test")
- feature_extractor_config = get_feature_extractor_config("feature-extractor-test")
- ```"""
+ `Dict`: The configuration of the audio processor.
+ """
# Load with a priority given to the nested processor config, if available in repo
resolved_processor_file = cached_file(
pretrained_model_name_or_path,
@@ -193,178 +347,186 @@ def get_feature_extractor_config(
_raise_exceptions_for_missing_entries=False,
)
- # An empty list if none of the possible files is found in the repo
if not resolved_feature_extractor_file and not resolved_processor_file:
- logger.info("Could not locate the feature extractor configuration file.")
+ logger.info("Could not locate the audio-processor configuration file.")
return {}
- # Load feature_extractor dict. Priority goes as (nested config if found -> feature extractor config)
- # We are downloading both configs because almost all models have a `processor_config.json` but
- # not all of these are nested. We need to check if it was saved recently as nested or if it is legacy style
feature_extractor_dict = {}
if resolved_processor_file is not None:
processor_dict = safe_load_json_file(resolved_processor_file)
- if "feature_extractor" in processor_dict:
+ # New nested key takes priority; legacy `feature_extractor` key is the fallback.
+ if "audio_processor" in processor_dict:
+ feature_extractor_dict = processor_dict["audio_processor"]
+ elif "feature_extractor" in processor_dict:
feature_extractor_dict = processor_dict["feature_extractor"]
- if resolved_feature_extractor_file is not None and feature_extractor_dict is None:
+ if resolved_feature_extractor_file is not None and not feature_extractor_dict:
feature_extractor_dict = safe_load_json_file(resolved_feature_extractor_file)
return feature_extractor_dict
-class AutoFeatureExtractor:
+def _resolve_audio_processor_from_pretrained(pretrained_model_name_or_path, *, backend: str, **kwargs):
+ """Shared resolution logic used by `AutoAudioProcessor` and the deprecated `AutoFeatureExtractor`.
+
+ Reads the hub config, identifies the class via the new `audio_processor_type` key, falling
+ back to the legacy `feature_extractor_type` key, then picks the right backend sibling.
+ """
+ config = kwargs.pop("config", None)
+ trust_remote_code = kwargs.pop("trust_remote_code", None)
+ kwargs["_from_auto"] = True
+
+ config_dict, _ = AudioProcessingMixin.get_audio_processor_dict(pretrained_model_name_or_path, **kwargs)
+
+ class_name_in_config = config_dict.get("audio_processor_type") or config_dict.get("feature_extractor_type")
+ auto_map = config_dict.get("auto_map") or {}
+ audio_processor_auto_map = auto_map.get("AutoAudioProcessor") or auto_map.get("AutoFeatureExtractor")
+
+ if class_name_in_config is None and audio_processor_auto_map is None:
+ if not isinstance(config, PreTrainedConfig):
+ config = AutoConfig.from_pretrained(
+ pretrained_model_name_or_path, trust_remote_code=trust_remote_code, **kwargs
+ )
+ class_name_in_config = getattr(config, "audio_processor_type", None) or getattr(
+ config, "feature_extractor_type", None
+ )
+ if hasattr(config, "auto_map"):
+ audio_processor_auto_map = config.auto_map.get("AutoAudioProcessor") or config.auto_map.get(
+ "AutoFeatureExtractor"
+ )
+
+ audio_processor_class = None
+ if class_name_in_config is not None:
+ # Translate legacy → modern, then dispatch to the requested backend
+ modern_name = _legacy_name_candidates(class_name_in_config)[-1]
+ audio_processor_class = _load_backend_class(modern_name, backend)
+
+ has_remote_code = audio_processor_auto_map is not None
+ has_local_code = audio_processor_class is not None or type(config) in FEATURE_EXTRACTOR_MAPPING
+ if has_local_code and audio_processor_class is None:
+ audio_processor_class = _load_class_with_fallback(FEATURE_EXTRACTOR_MAPPING[type(config)], backend)
+ explicit_local_code = (
+ has_local_code
+ and audio_processor_class is not None
+ and not audio_processor_class.__module__.startswith("transformers.")
+ )
+
+ if has_remote_code:
+ class_ref = _resolve_auto_map_class_ref(audio_processor_auto_map, backend)
+ upstream_repo = class_ref.split("--")[0] if "--" in class_ref else None
+ trust_remote_code = resolve_trust_remote_code(
+ trust_remote_code, pretrained_model_name_or_path, has_local_code, has_remote_code, upstream_repo
+ )
+
+ if has_remote_code and trust_remote_code and not explicit_local_code:
+ audio_processor_class = get_class_from_dynamic_module(class_ref, pretrained_model_name_or_path, **kwargs)
+ _ = kwargs.pop("code_revision", None)
+ audio_processor_class.register_for_auto_class()
+ return audio_processor_class.from_pretrained(pretrained_model_name_or_path, **kwargs)
+
+ if audio_processor_class is not None:
+ return audio_processor_class.from_pretrained(pretrained_model_name_or_path, **kwargs)
+
+ raise ValueError(
+ f"Unrecognized audio processor in {pretrained_model_name_or_path}. Should have an "
+ f"`audio_processor_type` or `feature_extractor_type` key in its {FEATURE_EXTRACTOR_NAME} or {CONFIG_NAME}, "
+ f"or one of the following `model_type` keys in its {CONFIG_NAME}: "
+ f"{', '.join(c for c in FEATURE_EXTRACTOR_MAPPING_NAMES)}"
+ )
+
+
+class AutoAudioProcessor:
r"""
- This is a generic feature extractor class that will be instantiated as one of the feature extractor classes of the
- library when created with the [`AutoFeatureExtractor.from_pretrained`] class method.
+ This is a generic audio processor class that will be instantiated as one of the
+ backend-specific [`~audio_processing_base.AudioProcessingMixin`] subclasses when created with the
+ [`AutoAudioProcessor.from_pretrained`] class method.
This class cannot be instantiated directly using `__init__()` (throws an error).
"""
def __init__(self):
raise OSError(
- "AutoFeatureExtractor is designed to be instantiated "
- "using the `AutoFeatureExtractor.from_pretrained(pretrained_model_name_or_path)` method."
+ "AutoAudioProcessor is designed to be instantiated "
+ "using the `AutoAudioProcessor.from_pretrained(pretrained_model_name_or_path)` method."
)
@classmethod
@replace_list_option_in_docstrings(FEATURE_EXTRACTOR_MAPPING_NAMES)
- def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
+ def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs):
r"""
- Instantiate one of the feature extractor classes of the library from a pretrained model vocabulary.
+ Instantiate one of the audio processor classes of the library from a pretrained model.
- The feature extractor class to instantiate is selected based on the `model_type` property of the config object
- (either passed as an argument or loaded from `pretrained_model_name_or_path` if possible), or when it's
- missing, by falling back to using pattern matching on `pretrained_model_name_or_path`:
-
- List options
+ The class to instantiate is selected by reading the `audio_processor_type`
+ (or legacy `feature_extractor_type`) entry in the hub `preprocessor_config.json`, then
+ picking the right backend sibling (`backend="torch"` by default; `"numpy"` for the
+ bit-exact CPU-only variant — see [`~docs/adr/0001-bit-exact-backend-parity`]).
Params:
pretrained_model_name_or_path (`str` or `os.PathLike`):
- This can be either:
-
- - a string, the *model id* of a pretrained feature_extractor hosted inside a model repo on
- huggingface.co.
- - a path to a *directory* containing a feature extractor file saved using the
- [`~feature_extraction_utils.FeatureExtractionMixin.save_pretrained`] method, e.g.,
- `./my_model_directory/`.
- - a path to a saved feature extractor JSON *file*, e.g.,
- `./my_model_directory/preprocessor_config.json`.
- cache_dir (`str` or `os.PathLike`, *optional*):
- Path to a directory in which a downloaded pretrained model feature extractor should be cached if the
- standard cache should not be used.
- force_download (`bool`, *optional*, defaults to `False`):
- Whether or not to force to (re-)download the feature extractor files and override the cached versions
- if they exist.
- proxies (`dict[str, str]`, *optional*):
- A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
- 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request.
- token (`str` or *bool*, *optional*):
- The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated
- when running `hf auth login` (stored in `~/.huggingface`).
- revision (`str`, *optional*, defaults to `"main"`):
- The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
- git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
- identifier allowed by git.
- return_unused_kwargs (`bool`, *optional*, defaults to `False`):
- If `False`, then this function returns just the final feature extractor object. If `True`, then this
- functions returns a `Tuple(feature_extractor, unused_kwargs)` where *unused_kwargs* is a dictionary
- consisting of the key/value pairs whose keys are not feature extractor attributes: i.e., the part of
- `kwargs` which has not been used to update `feature_extractor` and is otherwise ignored.
- trust_remote_code (`bool`, *optional*, defaults to `False`):
- Whether or not to allow for custom models defined on the Hub in their own modeling files. This option
- should only be set to `True` for repositories you trust and in which you have read the code, as it will
- execute code present on the Hub on your local machine.
- kwargs (`dict[str, Any]`, *optional*):
- The values in kwargs of any keys which are feature extractor attributes will be used to override the
- loaded values. Behavior concerning key/value pairs whose keys are *not* feature extractor attributes is
- controlled by the `return_unused_kwargs` keyword parameter.
-
-
-
- Passing `token=True` is required when you want to use a private model.
-
-
-
- Examples:
-
- ```python
- >>> from transformers import AutoFeatureExtractor
-
- >>> # Download feature extractor from huggingface.co and cache.
- >>> feature_extractor = AutoFeatureExtractor.from_pretrained("facebook/wav2vec2-base-960h")
-
- >>> # If feature extractor files are in a directory (e.g. feature extractor was saved using *save_pretrained('./test/saved_model/')*)
- >>> # feature_extractor = AutoFeatureExtractor.from_pretrained("./test/saved_model/")
- ```"""
- config = kwargs.pop("config", None)
- trust_remote_code = kwargs.pop("trust_remote_code", None)
- kwargs["_from_auto"] = True
-
- config_dict, _ = FeatureExtractionMixin.get_feature_extractor_dict(pretrained_model_name_or_path, **kwargs)
- feature_extractor_class = config_dict.get("feature_extractor_type", None)
- feature_extractor_auto_map = None
- if "AutoFeatureExtractor" in config_dict.get("auto_map", {}):
- feature_extractor_auto_map = config_dict["auto_map"]["AutoFeatureExtractor"]
-
- # If we don't find the feature extractor class in the feature extractor config, let's try the model config.
- if feature_extractor_class is None and feature_extractor_auto_map is None:
- if not isinstance(config, PreTrainedConfig):
- config = AutoConfig.from_pretrained(
- pretrained_model_name_or_path, trust_remote_code=trust_remote_code, **kwargs
- )
- # It could be in `config.feature_extractor_type``
- feature_extractor_class = getattr(config, "feature_extractor_type", None)
- if hasattr(config, "auto_map") and "AutoFeatureExtractor" in config.auto_map:
- feature_extractor_auto_map = config.auto_map["AutoFeatureExtractor"]
-
- if feature_extractor_class is not None:
- feature_extractor_class = feature_extractor_class_from_name(feature_extractor_class)
-
- has_remote_code = feature_extractor_auto_map is not None
- has_local_code = feature_extractor_class is not None or type(config) in FEATURE_EXTRACTOR_MAPPING
- explicit_local_code = has_local_code and not (
- feature_extractor_class or FEATURE_EXTRACTOR_MAPPING[type(config)]
- ).__module__.startswith("transformers.")
- if has_remote_code:
- if "--" in feature_extractor_auto_map:
- upstream_repo = feature_extractor_auto_map.split("--")[0]
- else:
- upstream_repo = None
- trust_remote_code = resolve_trust_remote_code(
- trust_remote_code, pretrained_model_name_or_path, has_local_code, has_remote_code, upstream_repo
- )
+ A model identifier on huggingface.co, a path to a saved model directory, or a path to
+ a saved audio-processor JSON file.
+ backend (`str`, *optional*, defaults to `"torch"`):
+ Which backend sibling to load. `"torch"` returns the `XxxAudioProcessor` class;
+ `"numpy"` returns `XxxAudioProcessorNumpy` when it exists. If the requested
+ backend has no sibling for the resolved model, falls back to the available one
+ with a warning.
- if has_remote_code and trust_remote_code and not explicit_local_code:
- feature_extractor_class = get_class_from_dynamic_module(
- feature_extractor_auto_map, pretrained_model_name_or_path, **kwargs
- )
- _ = kwargs.pop("code_revision", None)
- feature_extractor_class.register_for_auto_class()
- return feature_extractor_class.from_pretrained(pretrained_model_name_or_path, **kwargs)
- elif feature_extractor_class is not None:
- return feature_extractor_class.from_pretrained(pretrained_model_name_or_path, **kwargs)
- # Last try: we use the FEATURE_EXTRACTOR_MAPPING.
- elif type(config) in FEATURE_EXTRACTOR_MAPPING:
- feature_extractor_class = FEATURE_EXTRACTOR_MAPPING[type(config)]
- return feature_extractor_class.from_pretrained(pretrained_model_name_or_path, **kwargs)
-
- raise ValueError(
- f"Unrecognized feature extractor in {pretrained_model_name_or_path}. Should have a "
- f"`feature_extractor_type` key in its {FEATURE_EXTRACTOR_NAME} of {CONFIG_NAME}, or one of the following "
- f"`model_type` keys in its {CONFIG_NAME}: {', '.join(c for c in FEATURE_EXTRACTOR_MAPPING_NAMES)}"
- )
+ List options
+ """
+ backend = _resolve_audio_backend(kwargs.pop("backend", None))
+ return _resolve_audio_processor_from_pretrained(pretrained_model_name_or_path, backend=backend, **kwargs)
@staticmethod
- def register(config_class, feature_extractor_class, exist_ok=False):
+ def register(config_class, audio_processor_class, exist_ok=False):
"""
- Register a new feature extractor for this class.
+ Register a new audio processor for this class.
Args:
config_class ([`PreTrainedConfig`]):
The configuration corresponding to the model to register.
- feature_extractor_class ([`FeatureExtractorMixin`]): The feature extractor to register.
+ audio_processor_class ([`AudioProcessingMixin`] or `dict[str, type]`):
+ Either a single class (treated as the torch backend) or a backend→class dict.
"""
- FEATURE_EXTRACTOR_MAPPING.register(config_class, feature_extractor_class, exist_ok=exist_ok)
+ if not isinstance(audio_processor_class, dict):
+ audio_processor_class = {"torch": audio_processor_class}
+ FEATURE_EXTRACTOR_MAPPING.register(config_class, audio_processor_class, exist_ok=exist_ok)
+
+
+class AutoFeatureExtractor:
+ r"""
+ Deprecated alias for [`AutoAudioProcessor`].
+
+ Returns the same class as `AutoAudioProcessor.from_pretrained` (torch backend by default).
+ Removal target: transformers v5.15. See [ADR 0002](docs/adr/0002-legacy-field-mapping.md).
+ """
+
+ def __init__(self):
+ raise OSError(
+ "AutoFeatureExtractor is designed to be instantiated "
+ "using the `AutoFeatureExtractor.from_pretrained(pretrained_model_name_or_path)` method."
+ )
+
+ @classmethod
+ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
+ r"""Deprecated alias for [`AutoAudioProcessor.from_pretrained`]."""
+ warnings.warn(
+ "`AutoFeatureExtractor` is deprecated and will be removed in transformers v5.15. "
+ "Use `AutoAudioProcessor` instead.",
+ FutureWarning,
+ stacklevel=2,
+ )
+ backend = _resolve_audio_backend(kwargs.pop("backend", None))
+ return _resolve_audio_processor_from_pretrained(pretrained_model_name_or_path, backend=backend, **kwargs)
+
+ @staticmethod
+ def register(config_class, feature_extractor_class, exist_ok=False):
+ """Deprecated alias for [`AutoAudioProcessor.register`]."""
+ warnings.warn(
+ "`AutoFeatureExtractor.register` is deprecated and will be removed in transformers v5.15. "
+ "Use `AutoAudioProcessor.register` instead.",
+ FutureWarning,
+ stacklevel=2,
+ )
+ AutoAudioProcessor.register(config_class, feature_extractor_class, exist_ok=exist_ok)
-__all__ = ["FEATURE_EXTRACTOR_MAPPING", "AutoFeatureExtractor"]
+__all__ = ["FEATURE_EXTRACTOR_MAPPING", "AutoAudioProcessor", "AutoFeatureExtractor"]
diff --git a/src/transformers/models/clap/__init__.py b/src/transformers/models/clap/__init__.py
index 6d54ee86aece..62219ce52dcf 100644
--- a/src/transformers/models/clap/__init__.py
+++ b/src/transformers/models/clap/__init__.py
@@ -18,6 +18,8 @@
if TYPE_CHECKING:
+ from .audio_processing_clap import *
+ from .audio_processing_numpy_clap import *
from .configuration_clap import *
from .feature_extraction_clap import *
from .modeling_clap import *
diff --git a/src/transformers/models/clap/audio_processing_clap.py b/src/transformers/models/clap/audio_processing_clap.py
new file mode 100644
index 000000000000..312af4c7a9c6
--- /dev/null
+++ b/src/transformers/models/clap/audio_processing_clap.py
@@ -0,0 +1,44 @@
+# Copyright 2025 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+
+from ...audio_processing_backends import TorchAudioBackend
+from .audio_processing_numpy_clap import ClapAudioProcessorMixin
+
+
+class ClapAudioProcessor(ClapAudioProcessorMixin, TorchAudioBackend):
+ """Torch sibling of [`ClapAudioProcessorNumpy`]. See the mixin for the pipeline."""
+
+ def _native_stft(self, audio, window, frame_length, hop_length, n_fft, stft_cfg):
+ stft_out = super()._native_stft(audio, window, frame_length, hop_length, n_fft, stft_cfg)
+ # round-trip through complex64 like the legacy FE, so float64 magnitudes match bit-exactly
+ return stft_out.to(torch.complex64).to(torch.complex128)
+
+ def _apply_mel_scale(self, features, *, spectrogram_config, **kwargs):
+ # cast mel_filters to the features' dtype, matching the numpy sibling's float64 path
+ mel_filters = self.mel_filters.to(device=features.device, dtype=features.dtype)
+ mel_spec = torch.nn.functional.linear(features.transpose(-2, -1), mel_filters.T).transpose(-2, -1)
+ return torch.clamp(mel_spec, min=spectrogram_config.mel_floor)
+
+ def _bilinear_shrink(self, mel, chunk_frames):
+ # legacy torch dtype path: round-trip through float32 (numpy sibling stays float64)
+ mel_tensor = mel.unsqueeze(0).unsqueeze(0).to(torch.float32)
+ mel_shrink = torch.nn.functional.interpolate(
+ mel_tensor, size=[chunk_frames, 64], mode="bilinear", align_corners=False
+ )
+ return mel_shrink[0][0].to(mel.dtype)
+
+
+__all__ = ["ClapAudioProcessor"]
diff --git a/src/transformers/models/clap/audio_processing_numpy_clap.py b/src/transformers/models/clap/audio_processing_numpy_clap.py
new file mode 100644
index 000000000000..d9468fab9f8f
--- /dev/null
+++ b/src/transformers/models/clap/audio_processing_numpy_clap.py
@@ -0,0 +1,156 @@
+# Copyright 2026 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from ...audio_processing_backends import NumpyAudioBackend
+from ...audio_utils import MelScaleConfig, SpectrogramConfig, StftConfig
+from ...utils import PaddingStrategy
+
+
+class ClapAudioProcessorMixin:
+ """Backend-agnostic CLAP logic shared by the numpy and torch siblings: `rand_trunc`
+ (single view) and `fusion` (4-view chunking with a bilinear-downsampled global view)
+ truncation modes. Random offsets come from `np.random` on both backends."""
+
+ sampling_rate = 48000
+ force_mono = True
+ max_length = 480000
+ truncation_mode = "rand_trunc" # "fusion" or "rand_trunc"
+ return_padding_mask = False # CLAP returns is_longer instead of a padding mask
+
+ # computation_dtype="float64": the legacy FE builds its filter banks in float64
+ _mel_configs = {
+ "rand_trunc": MelScaleConfig(
+ n_mels=64,
+ f_min=50,
+ f_max=14000,
+ mel_scale="slaney",
+ norm="slaney",
+ frequency_bin_mode="linspace",
+ computation_dtype="float64",
+ ),
+ "fusion": MelScaleConfig(
+ n_mels=64,
+ f_min=50,
+ f_max=14000,
+ mel_scale="htk",
+ frequency_bin_mode="linspace",
+ computation_dtype="float64",
+ ),
+ }
+
+ def _set_attributes(self, **kwargs):
+ # an explicitly passed spectrogram_config wins over the per-mode default
+ if kwargs.get("spectrogram_config") is None:
+ self.spectrogram_config = SpectrogramConfig(
+ stft_config=StftConfig(n_fft=1024, hop_length=480, power=2.0),
+ mel_scale_config=self._mel_configs[self.truncation_mode],
+ log_mode="dB",
+ computation_dtype="float64",
+ )
+ super()._set_attributes(**kwargs)
+ # fusion extracts the full mel then chunks, so no pre-truncation
+ self.truncation = self.truncation_mode == "rand_trunc"
+
+ def _get_padding_strategies(self, padding=False, max_length=None):
+ # CLAP always pads to max_length, not to the longest in the batch
+ if padding is True and max_length is not None:
+ return PaddingStrategy.MAX_LENGTH
+ return super()._get_padding_strategies(padding=padding, max_length=max_length)
+
+ def pad(self, audio, *args, **kwargs):
+ self._is_longer_flags = []
+ return super().pad(audio, *args, **kwargs)
+
+ def _truncate_single(self, audio_el, max_length):
+ """Random-offset truncation for rand_trunc mode, also tracks which samples were longer."""
+ self._is_longer_flags.append(audio_el.shape[-1] > max_length)
+ if audio_el.shape[-1] > max_length:
+ idx = np.random.randint(0, audio_el.shape[-1] - max_length + 1)
+ return audio_el[..., idx : idx + max_length]
+ return audio_el
+
+ def extract_spectrogram(self, audio, *, spectrogram_config=None, audio_ranges=None, **kwargs):
+ """Extract mel spectrogram and shape output (1 view for rand_trunc, 4 for fusion)."""
+ is_fusion = self.truncation_mode == "fusion"
+ chunk_frames = self.max_length // self.spectrogram_config.stft_config.hop_length + 1
+
+ if not isinstance(audio, list):
+ audio = list(audio) if audio.ndim == 2 else [audio]
+ waveforms = [self._as_backend_array(w) for w in audio]
+
+ mels = []
+ is_longer = []
+ for waveform in waveforms:
+ mel = super().extract_spectrogram(waveform, spectrogram_config=self.spectrogram_config).swapaxes(-2, -1)
+ total_frames = mel.shape[0]
+
+ if is_fusion and total_frames > chunk_frames:
+ mels.append(self._random_mel_fusion(mel, total_frames, chunk_frames))
+ is_longer.append(True)
+ elif is_fusion:
+ mels.append(self._stack([mel, mel, mel, mel]))
+ is_longer.append(False)
+ else:
+ mels.append(mel[None])
+ is_longer.append(False)
+
+ if is_fusion:
+ self._is_longer_flags = is_longer
+ return mels
+
+ def _random_mel_fusion(self, mel, total_frames, chunk_frames):
+ ranges = np.array_split(list(range(0, total_frames - chunk_frames + 1)), 3)
+ if len(ranges[1]) == 0:
+ ranges[1] = [0]
+ if len(ranges[2]) == 0:
+ ranges[2] = [0]
+ idx_front = np.random.choice(ranges[0])
+ idx_middle = np.random.choice(ranges[1])
+ idx_back = np.random.choice(ranges[2])
+
+ mel_chunk_front = mel[idx_front : idx_front + chunk_frames, :]
+ mel_chunk_middle = mel[idx_middle : idx_middle + chunk_frames, :]
+ mel_chunk_back = mel[idx_back : idx_back + chunk_frames, :]
+ mel_shrink = self._bilinear_shrink(mel, chunk_frames) # downsampled "global" view
+ return self._stack([mel_shrink, mel_chunk_front, mel_chunk_middle, mel_chunk_back])
+
+ def _postprocess_output(self, output, audio_ranges=None, feature_ranges=None, **kwargs):
+ """Add CLAP's is_longer flag to the output (returned instead of a standard attention mask)."""
+ ranges = audio_ranges if audio_ranges is not None else feature_ranges
+ is_longer = getattr(self, "_is_longer_flags", None) or [False] * len(ranges)
+ if self.truncation_mode == "fusion" and sum(is_longer) == 0:
+ rand_idx = np.random.randint(0, len(is_longer))
+ is_longer[rand_idx] = True
+ output["is_longer"] = [[longer] for longer in is_longer]
+ return output
+
+
+class ClapAudioProcessorNumpy(ClapAudioProcessorMixin, NumpyAudioBackend):
+ """NumPy sibling of [`ClapAudioProcessor`]."""
+
+ def _bilinear_shrink(self, mel, chunk_frames):
+ # legacy numpy dtype path: float64 straight through interpolate (torch sibling
+ # round-trips through float32 instead)
+ import torch
+
+ mel_tensor = torch.tensor(mel[None, None, :])
+ mel_shrink = torch.nn.functional.interpolate(
+ mel_tensor, size=[chunk_frames, 64], mode="bilinear", align_corners=False
+ )
+ return mel_shrink[0][0].numpy()
+
+
+__all__ = ["ClapAudioProcessorNumpy"]
diff --git a/src/transformers/models/clap/feature_extraction_clap.py b/src/transformers/models/clap/feature_extraction_clap.py
index 8f0a34d2cf4e..2d07fcadf4ed 100644
--- a/src/transformers/models/clap/feature_extraction_clap.py
+++ b/src/transformers/models/clap/feature_extraction_clap.py
@@ -1,364 +1,20 @@
-# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
+# Copyright 2026 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Feature extractor class for CLAP."""
-
-import copy
-from typing import Any
-
-import numpy as np
-import torch
-
-from ...audio_utils import mel_filter_bank, spectrogram, window_function
-from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
-from ...feature_extraction_utils import BatchFeature
-from ...utils import TensorType, logging
-from ...utils.import_utils import requires
-
-
-logger = logging.get_logger(__name__)
-
-
-@requires(backends=("torch",))
-class ClapFeatureExtractor(SequenceFeatureExtractor):
- r"""
- Constructs a CLAP feature extractor.
-
- This feature extractor inherits from [`~feature_extraction_sequence_utils.SequenceFeatureExtractor`] which contains
- most of the main methods. Users should refer to this superclass for more information regarding those methods.
-
- This class extracts mel-filter bank features from raw speech using a custom numpy implementation of the *Short Time
- Fourier Transform* (STFT) which should match pytorch's `torch.stft` equivalent.
-
- Args:
- feature_size (`int`, *optional*, defaults to 64):
- The feature dimension of the extracted Mel spectrograms. This corresponds to the number of mel filters
- (`n_mels`).
- sampling_rate (`int`, *optional*, defaults to 48000):
- The sampling rate at which the audio files should be digitalized expressed in hertz (Hz). This only serves
- to warn users if the audio fed to the feature extractor does not have the same sampling rate.
- hop_length (`int`,*optional*, defaults to 480):
- Length of the overlapping windows for the STFT used to obtain the Mel Spectrogram. The audio will be split
- in smaller `frames` with a step of `hop_length` between each frame.
- max_length_s (`int`, *optional*, defaults to 10):
- The maximum input length of the model in seconds. This is used to pad the audio.
- fft_window_size (`int`, *optional*, defaults to 1024):
- Size of the window (in samples) on which the Fourier transform is applied. This controls the frequency
- resolution of the spectrogram. 400 means that the fourier transform is computed on windows of 400 samples.
- padding_value (`float`, *optional*, defaults to 0.0):
- Padding value used to pad the audio. Should correspond to silences.
- return_attention_mask (`bool`, *optional*, defaults to `False`):
- Whether or not the model should return the attention masks corresponding to the input.
- frequency_min (`float`, *optional*, defaults to 0):
- The lowest frequency of interest. The STFT will not be computed for values below this.
- frequency_max (`float`, *optional*, defaults to 14000):
- The highest frequency of interest. The STFT will not be computed for values above this.
- top_db (`float`, *optional*):
- The highest decibel value used to convert the mel spectrogram to the log scale. For more details see the
- `audio_utils.power_to_db` function
- truncation (`str`, *optional*, defaults to `"fusion"`):
- Truncation pattern for long audio inputs. Two patterns are available:
- - `fusion` will use `_random_mel_fusion`, which stacks 3 random crops from the mel spectrogram and a
- downsampled version of the entire mel spectrogram.
- If `config.fusion` is set to True, shorter audios also need to return 4 mels, which will just be a copy
- of the original mel obtained from the padded audio.
- - `rand_trunc` will select a random crop of the mel spectrogram.
- padding (`str`, *optional*, defaults to `"repeatpad"`):
- Padding pattern for shorter audio inputs. Three patterns were originally implemented:
- - `repeatpad`: the audio is repeated, and then padded to fit the `max_length`.
- - `repeat`: the audio is repeated and then cut to fit the `max_length`
- - `pad`: the audio is padded.
- """
-
- model_input_names = ["input_features", "is_longer"]
-
- def __init__(
- self,
- feature_size=64,
- sampling_rate=48_000,
- hop_length=480,
- max_length_s=10,
- fft_window_size=1024,
- padding_value=0.0,
- return_attention_mask=False, # pad inputs to max length with silence token (zero) and no attention mask
- frequency_min: float = 0,
- frequency_max: float = 14_000,
- top_db: int | None = None,
- truncation: str = "fusion",
- padding: str = "repeatpad",
- **kwargs,
- ):
- super().__init__(
- feature_size=feature_size,
- sampling_rate=sampling_rate,
- padding_value=padding_value,
- return_attention_mask=return_attention_mask,
- **kwargs,
- )
- self.top_db = top_db
- self.truncation = truncation
- self.padding = padding
- self.fft_window_size = fft_window_size
- self.nb_frequency_bins = (fft_window_size >> 1) + 1
- self.hop_length = hop_length
- self.max_length_s = max_length_s
- self.nb_max_samples = max_length_s * sampling_rate
- self.sampling_rate = sampling_rate
- self.frequency_min = frequency_min
- self.frequency_max = frequency_max
- self.mel_filters = mel_filter_bank(
- num_frequency_bins=self.nb_frequency_bins,
- num_mel_filters=feature_size,
- min_frequency=frequency_min,
- max_frequency=frequency_max,
- sampling_rate=sampling_rate,
- norm=None,
- mel_scale="htk",
- )
- self.mel_filters_slaney = mel_filter_bank(
- num_frequency_bins=self.nb_frequency_bins,
- num_mel_filters=feature_size,
- min_frequency=frequency_min,
- max_frequency=frequency_max,
- sampling_rate=sampling_rate,
- norm="slaney",
- mel_scale="slaney",
- )
-
- def to_dict(self) -> dict[str, Any]:
- """
- Serializes this instance to a Python dictionary.
-
- Returns:
- `dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance, except for the
- mel filter banks, which do not need to be saved or printed as they are too long.
- """
- output = copy.deepcopy(self.__dict__)
- output["feature_extractor_type"] = self.__class__.__name__
- if "mel_filters" in output:
- del output["mel_filters"]
- if "mel_filters_slaney" in output:
- del output["mel_filters_slaney"]
- return output
-
- def _np_extract_fbank_features(self, waveform: np.ndarray, mel_filters: np.ndarray | None = None) -> np.ndarray:
- """
- Compute the log-mel spectrogram of the provided `waveform` using the Hann window. In CLAP, two different filter
- banks are used depending on the truncation pattern:
- - `self.mel_filters`: they correspond to the default parameters of `torchaudio` which can be obtained from
- calling `torchaudio.transforms.MelSpectrogram().mel_scale.fb`. These filters are used when `truncation`
- is set to `"fusion"`.
- - `self.mel_filteres_slaney` : they correspond to the default parameters of `librosa` which used
- `librosa.filters.mel` when computing the mel spectrogram. These filters were only used in the original
- implementation when the truncation mode is not `"fusion"`.
- """
- log_mel_spectrogram = spectrogram(
- waveform,
- window_function(self.fft_window_size, "hann"),
- frame_length=self.fft_window_size,
- hop_length=self.hop_length,
- power=2.0,
- mel_filters=mel_filters,
- log_mel="dB",
- )
- return log_mel_spectrogram.T
-
- def _random_mel_fusion(self, mel, total_frames, chunk_frames):
- ranges = np.array_split(list(range(0, total_frames - chunk_frames + 1)), 3)
- if len(ranges[1]) == 0:
- # if the audio is too short, we just use the first chunk
- ranges[1] = [0]
- if len(ranges[2]) == 0:
- # if the audio is too short, we just use the first chunk
- ranges[2] = [0]
- # randomly choose index for each part
- idx_front = np.random.choice(ranges[0])
- idx_middle = np.random.choice(ranges[1])
- idx_back = np.random.choice(ranges[2])
-
- mel_chunk_front = mel[idx_front : idx_front + chunk_frames, :]
- mel_chunk_middle = mel[idx_middle : idx_middle + chunk_frames, :]
- mel_chunk_back = mel[idx_back : idx_back + chunk_frames, :]
-
- mel = torch.tensor(mel[None, None, :])
- mel_shrink = torch.nn.functional.interpolate(
- mel, size=[chunk_frames, 64], mode="bilinear", align_corners=False
- )
- mel_shrink = mel_shrink[0][0].numpy()
- mel_fusion = np.stack([mel_shrink, mel_chunk_front, mel_chunk_middle, mel_chunk_back], axis=0)
- return mel_fusion
-
- def _get_input_mel(self, waveform: np.ndarray, max_length, truncation, padding) -> np.ndarray:
- """
- Extracts the mel spectrogram and prepares it for the mode based on the `truncation` and `padding` arguments.
- Four different path are possible:
- - `truncation="fusion"` and the length of the waveform is greater than the max length: the mel spectrogram
- will be computed on the entire audio. 3 random crops and a dowsampled version of the full mel spectrogram
- are then stacked together. They will later be used for `feature_fusion`.
- - `truncation="rand_trunc"` and the length of the waveform is smaller than the max length: the audio is
- padded based on `padding`.
- - `truncation="fusion"` and the length of the waveform is smaller than the max length: the audio is padded
- based on `padding`, and is repeated `4` times.
- - `truncation="rand_trunc"` and the length of the waveform is greater than the max length: the mel
- spectrogram will be computed on a random crop of the waveform.
-
- """
- if waveform.shape[0] > max_length:
- if truncation == "rand_trunc":
- longer = True
- # random crop to max_length (for compatibility) -> this should be handled by self.pad
- overflow = len(waveform) - max_length
- idx = np.random.randint(0, overflow + 1)
- waveform = waveform[idx : idx + max_length]
- input_mel = self._np_extract_fbank_features(waveform, self.mel_filters_slaney)[None, :]
- elif truncation == "fusion":
- mel = self._np_extract_fbank_features(waveform, self.mel_filters)
- chunk_frames = max_length // self.hop_length + 1 # the +1 related to how the spectrogram is computed
- total_frames = mel.shape[0]
- if chunk_frames == total_frames:
- # there is a corner case where the audio length is larger than max_length but smaller than max_length+hop_length.
- # In this case, we just use the whole audio.
- input_mel = np.stack([mel, mel, mel, mel], axis=0)
- longer = False
- else:
- input_mel = self._random_mel_fusion(mel, total_frames, chunk_frames)
- longer = True
- else:
- raise NotImplementedError(f"data_truncating {truncation} not implemented")
-
- else:
- longer = False
- # only use repeat as a new possible value for padding. you repeat the audio before applying the usual max_length padding
- if waveform.shape[0] < max_length:
- if padding == "repeat":
- n_repeat = int(max_length / len(waveform))
- waveform = np.tile(waveform, n_repeat + 1)[:max_length]
- if padding == "repeatpad":
- n_repeat = int(max_length / len(waveform))
- waveform = np.tile(waveform, n_repeat)
- waveform = np.pad(waveform, (0, max_length - waveform.shape[0]), mode="constant", constant_values=0)
-
- if truncation == "fusion":
- input_mel = self._np_extract_fbank_features(waveform, self.mel_filters)
- input_mel = np.stack([input_mel, input_mel, input_mel, input_mel], axis=0)
- else:
- input_mel = self._np_extract_fbank_features(waveform, self.mel_filters_slaney)[None, :]
-
- return input_mel, longer
-
- def __call__(
- self,
- raw_speech: np.ndarray | list[float] | list[np.ndarray] | list[list[float]],
- truncation: str | None = None,
- padding: str | None = None,
- max_length: int | None = None,
- sampling_rate: int | None = None,
- return_tensors: str | TensorType | None = None,
- **kwargs,
- ) -> BatchFeature:
- """
- Main method to featurize and prepare for the model one or several sequence(s).
-
- Args:
- raw_speech (`np.ndarray`, `list[float]`, `list[np.ndarray]`, `list[list[float]]`):
- The sequence or batch of sequences to be padded. Each sequence can be a numpy array, a list of float
- values, a list of numpy arrays or a list of list of float values. Must be mono channel audio, not
- stereo, i.e. single float per timestep.
- truncation (`str`, *optional*):
- Truncation pattern for long audio inputs. Two patterns are available:
- - `fusion` will use `_random_mel_fusion`, which stacks 3 random crops from the mel spectrogram and
- a downsampled version of the entire mel spectrogram.
- If `config.fusion` is set to True, shorter audios also need to return 4 mels, which will just be a
- copy of the original mel obtained from the padded audio.
- - `rand_trunc` will select a random crop of the mel spectrogram.
- padding (`str`, *optional*):
- Padding pattern for shorter audio inputs. Three patterns were originally implemented:
- - `repeatpad`: the audio is repeated, and then padded to fit the `max_length`.
- - `repeat`: the audio is repeated and then cut to fit the `max_length`
- - `pad`: the audio is padded.
- return_tensors (`str` or [`~utils.TensorType`], *optional*):
- If set, will return tensors instead of list of python integers. Acceptable values are:
- - `'pt'`: Return PyTorch `torch.np.array` objects.
- - `'np'`: Return Numpy `np.ndarray` objects.
- sampling_rate (`int`, *optional*):
- The sampling rate at which the `raw_speech` input was sampled. It is strongly recommended to pass
- `sampling_rate` at the forward call to prevent silent errors and allow automatic speech recognition
- pipeline.
- """
- truncation = truncation if truncation is not None else self.truncation
- padding = padding if padding else self.padding
-
- if sampling_rate is not None:
- if sampling_rate != self.sampling_rate:
- raise ValueError(
- f"The model corresponding to this feature extractor: {self.__class__.__name__} was trained using a"
- f" sampling rate of {self.sampling_rate}. Please make sure that the provided `raw_speech` input"
- f" was sampled with {self.sampling_rate} and not {sampling_rate}."
- )
- else:
- logger.warning(
- f"It is strongly recommended to pass the `sampling_rate` argument to `{self.__class__.__name__}()`. "
- "Failing to do so can result in silent errors that might be hard to debug."
- )
-
- is_batched_numpy = isinstance(raw_speech, np.ndarray) and len(raw_speech.shape) > 1
- if is_batched_numpy and len(raw_speech.shape) > 2:
- raise ValueError(f"Only mono-channel audio is supported for input to {self}")
- is_batched = is_batched_numpy or (
- isinstance(raw_speech, (list, tuple)) and (isinstance(raw_speech[0], (np.ndarray, tuple, list)))
- )
-
- if is_batched:
- raw_speech = [np.asarray(speech, dtype=np.float64) for speech in raw_speech]
- elif not is_batched and not isinstance(raw_speech, np.ndarray):
- raw_speech = np.asarray(raw_speech, dtype=np.float64)
- elif isinstance(raw_speech, np.ndarray) and raw_speech.dtype is np.dtype(np.float64):
- raw_speech = raw_speech.astype(np.float64)
-
- # always return batch
- if not is_batched:
- raw_speech = [np.asarray(raw_speech)]
-
- # convert to mel spectrogram, truncate and pad if needed.
- padded_inputs = [
- self._get_input_mel(waveform, max_length if max_length else self.nb_max_samples, truncation, padding)
- for waveform in raw_speech
- ]
-
- input_mel = []
- is_longer = []
- for mel, longer in padded_inputs:
- input_mel.append(mel)
- is_longer.append(longer)
-
- if truncation == "fusion" and sum(is_longer) == 0:
- # if no audio is longer than 10s, then randomly select one audio to be longer
- rand_idx = np.random.randint(0, len(input_mel))
- is_longer[rand_idx] = True
-
- if isinstance(input_mel[0], list):
- input_mel = [np.asarray(feature, dtype=np.float64) for feature in input_mel]
-
- # is_longer is a list of bool
- is_longer = [[longer] for longer in is_longer]
+"""Backwards-compatibility shim: re-exports the legacy ``ClapFeatureExtractor`` name as a
+deprecated alias of [`ClapAudioProcessor`]. Importing or instantiating the alias emits a
+``FutureWarning``; the alias is removed in transformers v5.15 (see ADR 0002).
+"""
- input_features = {"input_features": input_mel, "is_longer": is_longer}
- input_features = BatchFeature(input_features)
+from ...audio_processing_base import make_legacy_audio_processor_alias
+from .audio_processing_clap import ClapAudioProcessor
- if return_tensors is not None:
- input_features = input_features.convert_to_tensors(return_tensors)
- return input_features
+ClapFeatureExtractor = make_legacy_audio_processor_alias(ClapAudioProcessor, "ClapFeatureExtractor")
__all__ = ["ClapFeatureExtractor"]
diff --git a/src/transformers/models/clvp/__init__.py b/src/transformers/models/clvp/__init__.py
index 986e185ff777..53af7d8dea88 100644
--- a/src/transformers/models/clvp/__init__.py
+++ b/src/transformers/models/clvp/__init__.py
@@ -18,6 +18,8 @@
if TYPE_CHECKING:
+ from .audio_processing_clvp import *
+ from .audio_processing_numpy_clvp import *
from .configuration_clvp import *
from .feature_extraction_clvp import *
from .modeling_clvp import *
diff --git a/src/transformers/models/clvp/audio_processing_clvp.py b/src/transformers/models/clvp/audio_processing_clvp.py
new file mode 100644
index 000000000000..95f41b773938
--- /dev/null
+++ b/src/transformers/models/clvp/audio_processing_clvp.py
@@ -0,0 +1,66 @@
+# Copyright 2025 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+
+from ...audio_processing_backends import TorchAudioBackend
+from .audio_processing_numpy_clvp import ClvpAudioProcessorNumpy
+
+
+class ClvpAudioProcessor(TorchAudioBackend):
+ """Torch sibling of [`ClvpAudioProcessorNumpy`]. Applies log compression and an optional
+ per-mel-bin normalization (``mel_norms`` ~ per-bin stddev with implicit zero mean)."""
+
+ sampling_rate = 22050
+ force_mono = True
+ max_length = 132300 # 6 seconds at 22050 Hz
+ truncation = True
+ mask_level = "audio"
+
+
+ spectrogram_config = ClvpAudioProcessorNumpy.spectrogram_config
+
+ def __init__(self, mel_norms=None, **kwargs):
+ super().__init__(**kwargs)
+ self.mel_norms = mel_norms
+
+ # Mel filters: the base dispatcher resolves the top-level `computation_dtype="float64"`
+ # into float64 torch-native filters (matching the legacy FE's float64 numpy build
+ # within ~1e-16), kept float64 for the mel matmul below.
+
+ def _compute_magnitudes(self, stft_out, power, spectrogram_config=None):
+ # The legacy FE stores the STFT in a complex64 buffer before taking float64 magnitudes
+ # (`np.abs(spectrogram, dtype=np.float64) ** power`). Replicate that rounding step so the
+ # float64 power spectrum is bit-identical (mirrors the numpy sibling's complex64 cast).
+ return stft_out.to(torch.complex64).to(torch.complex128).abs() ** power
+
+ def _apply_mel_scale(self, features, *, spectrogram_config, **kwargs):
+ # Cast mel_filters to the features' dtype so the float64 spectrogram path matches the
+ # numpy sibling, which casts via `mel_filters.astype(features.dtype, copy=False)`.
+ mel_filters = self.mel_filters.to(device=features.device, dtype=features.dtype)
+ mel_spec = torch.nn.functional.linear(features.transpose(-2, -1), mel_filters.T).transpose(-2, -1)
+ return torch.clamp(mel_spec, min=spectrogram_config.mel_floor)
+
+ def _normalize_magnitude(self, features, *, spectrogram_config, **kwargs):
+ # Compute log and mel_norms division in float64 before casting to float32
+ # to match the legacy feature extractor's precision (same recipe as the numpy sibling).
+ mel_floor = spectrogram_config.mel_floor
+ features = torch.log(torch.maximum(torch.tensor(mel_floor, dtype=features.dtype, device=features.device), features))
+ if self.mel_norms is not None:
+ mel_norms = torch.as_tensor(self.mel_norms, dtype=features.dtype, device=features.device)[:, None]
+ features = features / mel_norms
+ return features.to(torch.float32)
+
+
+__all__ = ["ClvpAudioProcessor"]
diff --git a/src/transformers/models/clvp/audio_processing_numpy_clvp.py b/src/transformers/models/clvp/audio_processing_numpy_clvp.py
new file mode 100644
index 000000000000..19fec37ac3b1
--- /dev/null
+++ b/src/transformers/models/clvp/audio_processing_numpy_clvp.py
@@ -0,0 +1,65 @@
+# Copyright 2026 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from ...audio_processing_backends import NumpyAudioBackend
+from ...audio_utils import MelScaleConfig, SpectrogramConfig, StftConfig
+
+
+class ClvpAudioProcessorNumpy(NumpyAudioBackend):
+ """NumPy sibling of [`ClvpAudioProcessor`]. Bit-exact to the legacy `ClvpFeatureExtractor`
+ via float64 log + per-mel-norm division before float32 cast (ADR 0001)."""
+
+ sampling_rate = 22050
+ force_mono = True
+ max_length = 132300 # 6 seconds at 22050 Hz
+ truncation = True
+ mask_level = "audio"
+
+ spectrogram_config = SpectrogramConfig(
+ stft_config=StftConfig(
+ n_fft=1024,
+ hop_length=256,
+ window_fn="hann_window",
+ power=2.0,
+ ),
+ mel_scale_config=MelScaleConfig(
+ n_mels=80,
+ f_min=0.0,
+ f_max=8000.0,
+ norm="slaney",
+ mel_scale="htk",
+ frequency_bin_mode="linspace",
+ ),
+ log_mode="log",
+ mel_floor=1e-5,
+ computation_dtype="float64",
+ )
+
+ def __init__(self, mel_norms=None, **kwargs):
+ super().__init__(**kwargs)
+ self.mel_norms = mel_norms
+
+ def _normalize_magnitude(self, features, *, spectrogram_config, **kwargs):
+ # Compute log and mel_norms division in float64 before casting to float32
+ # to match the legacy feature extractor's precision
+ mel_floor = spectrogram_config.mel_floor
+ features = np.log(np.maximum(mel_floor, features))
+ if self.mel_norms is not None:
+ features = features / np.array(self.mel_norms)[:, None]
+ return features.astype(np.float32)
+
+
+__all__ = ["ClvpAudioProcessorNumpy"]
diff --git a/src/transformers/models/clvp/feature_extraction_clvp.py b/src/transformers/models/clvp/feature_extraction_clvp.py
index cc39e6aca677..301918552095 100644
--- a/src/transformers/models/clvp/feature_extraction_clvp.py
+++ b/src/transformers/models/clvp/feature_extraction_clvp.py
@@ -1,237 +1,20 @@
-# Copyright 2023 The HuggingFace Inc. team.
+# Copyright 2026 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-Feature extractor class for CLVP
+"""Backwards-compatibility shim: re-exports the legacy ``ClvpFeatureExtractor`` name as a
+deprecated alias of [`ClvpAudioProcessor`]. Importing or instantiating the alias emits a
+``FutureWarning``; the alias is removed in transformers v5.15 (see ADR 0002).
"""
-import numpy as np
-
-from ...audio_utils import mel_filter_bank, spectrogram, window_function
-from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
-from ...feature_extraction_utils import BatchFeature
-from ...utils import TensorType, logging
-
-
-logger = logging.get_logger(__name__)
-
-
-class ClvpFeatureExtractor(SequenceFeatureExtractor):
- r"""
- Constructs a CLVP feature extractor.
-
- This feature extractor inherits from [`~feature_extraction_sequence_utils.SequenceFeatureExtractor`] which contains
- most of the main methods. Users should refer to this superclass for more information regarding those methods.
-
- This class extracts log-mel-spectrogram features from raw speech using a custom numpy implementation of the `Short
- Time Fourier Transform` which should match pytorch's `torch.stft` equivalent.
-
- Args:
- feature_size (`int`, *optional*, defaults to 80):
- The feature dimension of the extracted features.
- sampling_rate (`int`, *optional*, defaults to 22050):
- The sampling rate at which the audio files should be digitalized expressed in hertz (Hz).
- default_audio_length (`int`, *optional*, defaults to 6):
- The default length of raw audio in seconds. If `max_length` is not set during `__call__` then it will
- automatically be set to default_audio_length * `self.sampling_rate`.
- hop_length (`int`, *optional*, defaults to 256):
- Length of the overlapping windows for the STFT used to obtain the Mel Frequency coefficients.
- chunk_length (`int`, *optional*, defaults to 30):
- The maximum number of chunks of `sampling_rate` samples used to trim and pad longer or shorter audio
- sequences.
- n_fft (`int`, *optional*, defaults to 1024):
- Size of the Fourier transform.
- padding_value (`float`, *optional*, defaults to 0.0):
- Padding value used to pad the audio. Should correspond to silences.
- mel_norms (`list` of length `feature_size`, *optional*):
- If `mel_norms` is provided then it will be used to normalize the log-mel spectrograms along each
- mel-filter.
- return_attention_mask (`bool`, *optional*, defaults to `False`):
- Whether to return the attention mask. If left to the default, it will return the attention mask.
-
- [What are attention masks?](../glossary#attention-mask)
- """
-
- model_input_names = ["input_features", "attention_mask"]
-
- def __init__(
- self,
- feature_size=80,
- sampling_rate=22050,
- default_audio_length=6,
- hop_length=256,
- chunk_length=30,
- n_fft=1024,
- padding_value=0.0,
- mel_norms=None,
- return_attention_mask=False, # pad inputs to max length with silence token (zero) and no attention mask
- **kwargs,
- ):
- super().__init__(
- feature_size=feature_size,
- sampling_rate=sampling_rate,
- padding_value=padding_value,
- return_attention_mask=return_attention_mask,
- **kwargs,
- )
- self.n_fft = n_fft
- self.hop_length = hop_length
- self.chunk_length = chunk_length
- self.n_samples = chunk_length * sampling_rate
- self.nb_max_frames = self.n_samples // hop_length
- self.sampling_rate = sampling_rate
- self.default_audio_length = default_audio_length
- self.mel_norms = mel_norms
- self.mel_filters = mel_filter_bank(
- num_frequency_bins=1 + (n_fft // 2),
- num_mel_filters=feature_size,
- min_frequency=0.0,
- max_frequency=8000.0,
- sampling_rate=sampling_rate,
- norm="slaney",
- mel_scale="htk",
- )
-
- def _np_extract_fbank_features(self, waveform: np.ndarray) -> np.ndarray:
- """
- This method first computes the log-mel spectrogram of the provided audio then applies normalization along the
- each mel-filterbank, if `mel_norms` is provided.
- """
- log_spec = spectrogram(
- waveform,
- window_function(self.n_fft, "hann"),
- frame_length=self.n_fft,
- hop_length=self.hop_length,
- power=2.0,
- mel_filters=self.mel_filters,
- log_mel=None,
- )
-
- log_spec = np.log(np.clip(log_spec, a_min=1e-5, a_max=None))
-
- if self.mel_norms is not None:
- log_spec = log_spec / np.array(self.mel_norms)[:, None]
-
- return log_spec
-
- def __call__(
- self,
- raw_speech: np.ndarray | list[float] | list[np.ndarray] | list[list[float]],
- sampling_rate: int | None = None,
- truncation: bool = True,
- pad_to_multiple_of: int | None = None,
- return_tensors: str | TensorType | None = None,
- return_attention_mask: bool | None = True,
- padding: str | None = "max_length",
- max_length: int | None = None,
- **kwargs,
- ) -> BatchFeature:
- """
- `ClvpFeatureExtractor` is used to extract various voice specific properties such as the pitch and tone of the
- voice, speaking speed, and even speaking defects like a lisp or stuttering from a sample voice or `raw_speech`.
-
- First the voice is padded or truncated in a way such that it becomes a waveform of `self.default_audio_length`
- seconds long and then the log-mel spectrogram is extracted from it.
-
- Args:
- raw_speech (`np.ndarray`, `list[float]`, `list[np.ndarray]`, `list[list[float]]`):
- The sequence or batch of sequences to be padded. Each sequence can be a numpy array, a list of float
- values, a list of numpy arrays or a list of list of float values. Must be mono channel audio, not
- stereo, i.e. single float per timestep.
- sampling_rate (`int`, *optional*):
- The sampling rate at which the `raw_speech` input was sampled. It is strongly recommended to pass
- `sampling_rate` at the forward call to prevent silent errors and allow automatic speech recognition
- pipeline.
- truncation (`bool`, *optional*, default to `True`):
- Activates truncation to cut input sequences longer than *max_length* to *max_length*.
- pad_to_multiple_of (`int`, *optional*):
- If set will pad the sequence to a multiple of the provided value.
-
- This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability
- `>= 7.5` (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128.
- return_attention_mask (`bool`, *optional*, defaults to `True`):
- Whether to return the attention mask. If left to the default, it will return the attention mask.
-
- [What are attention masks?](../glossary#attention-mask)
- return_tensors (`str` or [`~utils.TensorType`], *optional*):
- If set, will return tensors instead of list of python integers. Acceptable values are:
-
- - `'pt'`: Return PyTorch `torch.Tensor` objects.
- - `'np'`: Return Numpy `np.ndarray` objects.
- padding_value (`float`, *optional*, defaults to 0.0):
- The value that is used to fill the padding values / vectors.
- max_length (`int`, *optional*):
- The maximum input length of the inputs.
- """
-
- if sampling_rate is not None:
- if sampling_rate != self.sampling_rate:
- raise ValueError(
- f"The model corresponding to this feature extractor: {self.__class__.__name__} was trained using a"
- f" sampling rate of {self.sampling_rate}. Please make sure that the provided `raw_speech` input"
- f" was sampled with {self.sampling_rate} and not {sampling_rate}."
- )
- else:
- logger.warning(
- f"It is strongly recommended to pass the `sampling_rate` argument to `{self.__class__.__name__}()`. "
- "Failing to do so can result in silent errors that might be hard to debug."
- )
-
- is_batched_numpy = isinstance(raw_speech, np.ndarray) and len(raw_speech.shape) > 1
- if is_batched_numpy and len(raw_speech.shape) > 2:
- raise ValueError(f"Only mono-channel audio is supported for input to {self}")
- is_batched = is_batched_numpy or (
- isinstance(raw_speech, (list, tuple)) and (isinstance(raw_speech[0], (np.ndarray, tuple, list)))
- )
-
- if is_batched:
- raw_speech = [np.asarray([speech], dtype=np.float32).T for speech in raw_speech]
- elif not is_batched and not isinstance(raw_speech, np.ndarray):
- raw_speech = np.asarray(raw_speech, dtype=np.float32)
- elif isinstance(raw_speech, np.ndarray) and raw_speech.dtype is np.dtype(np.float64):
- raw_speech = raw_speech.astype(np.float32)
-
- # always return batch
- if not is_batched:
- raw_speech = [np.asarray([raw_speech]).T]
-
- batched_speech = BatchFeature({"input_features": raw_speech})
-
- max_length = self.default_audio_length * self.sampling_rate if max_length is None else max_length
-
- padded_inputs = self.pad(
- batched_speech,
- padding=padding,
- max_length=max_length,
- truncation=truncation,
- pad_to_multiple_of=pad_to_multiple_of,
- return_attention_mask=return_attention_mask,
- )
-
- # make sure list is in array format
- input_features = padded_inputs.get("input_features").transpose(2, 0, 1)
-
- input_features = [
- self._np_extract_fbank_features(waveform).astype(np.float32) for waveform in input_features[0]
- ]
+from ...audio_processing_base import make_legacy_audio_processor_alias
+from .audio_processing_clvp import ClvpAudioProcessor
- if isinstance(input_features[0], list):
- padded_inputs["input_features"] = [np.asarray(feature) for feature in input_features]
- else:
- padded_inputs["input_features"] = input_features
- return padded_inputs.convert_to_tensors(return_tensors)
+ClvpFeatureExtractor = make_legacy_audio_processor_alias(ClvpAudioProcessor, "ClvpFeatureExtractor")
__all__ = ["ClvpFeatureExtractor"]
diff --git a/src/transformers/models/cohere_asr/__init__.py b/src/transformers/models/cohere_asr/__init__.py
index 64734198de9e..e5e507ba27e6 100644
--- a/src/transformers/models/cohere_asr/__init__.py
+++ b/src/transformers/models/cohere_asr/__init__.py
@@ -19,6 +19,8 @@
if TYPE_CHECKING:
+ from .audio_processing_cohere_asr import *
+ from .audio_processing_numpy_cohere_asr import *
from .configuration_cohere_asr import *
from .feature_extraction_cohere_asr import *
from .modeling_cohere_asr import *
diff --git a/src/transformers/models/cohere_asr/audio_processing_cohere_asr.py b/src/transformers/models/cohere_asr/audio_processing_cohere_asr.py
new file mode 100644
index 000000000000..fccffa643786
--- /dev/null
+++ b/src/transformers/models/cohere_asr/audio_processing_cohere_asr.py
@@ -0,0 +1,80 @@
+# Copyright 2026 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+
+from ...audio_processing_backends import TorchAudioBackend
+from ...audio_utils import _create_triangular_filter_bank, hertz_to_mel, mel_to_hertz
+from .audio_processing_numpy_cohere_asr import CohereAsrAudioProcessorMixin
+
+
+class CohereAsrAudioProcessor(CohereAsrAudioProcessorMixin, TorchAudioBackend):
+ """Torch sibling of [`CohereAsrAudioProcessorNumpy`]: energy-based long-audio chunking,
+ deterministic dither, waveform preemphasis, ``log(mel @ |X|^2 + 2^-24)`` features with
+ per-utterance mean/variance normalization."""
+
+ def _standard_mel_banks(
+ self,
+ num_mel_filters,
+ num_frequency_bins,
+ min_frequency,
+ max_frequency,
+ sampling_rate,
+ n_fft,
+ mel_cfg,
+ computation_dtype,
+ ):
+ """Torch-native build of librosa's per-band float32 rounding: float64 weights cast
+ to float32, slaney norm applied *after* that cast with a second float32 rounding —
+ the only order that reproduces the legacy filters bit-exactly."""
+ mel_min = hertz_to_mel(min_frequency, mel_scale=mel_cfg.mel_scale)
+ mel_max = hertz_to_mel(max_frequency, mel_scale=mel_cfg.mel_scale)
+ mel_freqs = torch.linspace(mel_min, mel_max, num_mel_filters + 2, dtype=torch.float64)
+ filter_freqs = mel_to_hertz(mel_freqs, mel_scale=mel_cfg.mel_scale)
+ fft_freqs = torch.linspace(0, sampling_rate // 2, num_frequency_bins, dtype=torch.float64)
+ mel_filters = _create_triangular_filter_bank(fft_freqs, filter_freqs).to(torch.float32)
+ if mel_cfg.norm == "slaney":
+ enorm = 2.0 / (filter_freqs[2 : num_mel_filters + 2] - filter_freqs[:num_mel_filters])
+ mel_filters = (mel_filters * enorm[None, :]).to(torch.float32)
+ return mel_filters
+
+ def _apply_dither(self, audio, audio_ranges=None):
+ """Deterministic per-utterance dither: each row is seeded by its valid sample count,
+ so dither is invariant to batch composition (matches the legacy FE)."""
+ if self.dither <= 0 or audio_ranges is None:
+ return audio
+ audio = audio.clone()
+ generator = torch.Generator(device=audio.device)
+ for i, (start, end) in enumerate(audio_ranges):
+ valid_samples = min(end - start, audio.shape[1])
+ if valid_samples <= 0:
+ continue
+ generator.manual_seed(valid_samples)
+ noise = torch.randn(valid_samples, dtype=audio.dtype, device=audio.device, generator=generator)
+ audio[i, :valid_samples] = audio[i, :valid_samples] + self.dither * noise
+ return audio
+
+ def _compute_magnitudes(self, stft_out, power, spectrogram_config=None):
+ # legacy view_as_real + sqrt(real² + imag²) ** power pattern
+ magnitudes = torch.view_as_real(stft_out)
+ magnitudes = torch.sqrt(magnitudes.pow(2).sum(-1))
+ if power != 1.0:
+ magnitudes = magnitudes.pow(power)
+ return magnitudes
+
+ def _apply_mel_scale(self, features, *, spectrogram_config, **kwargs):
+ return torch.matmul(self.mel_filters.T, features)
+
+
+__all__ = ["CohereAsrAudioProcessor"]
diff --git a/src/transformers/models/cohere_asr/audio_processing_numpy_cohere_asr.py b/src/transformers/models/cohere_asr/audio_processing_numpy_cohere_asr.py
new file mode 100644
index 000000000000..07f0b71b8d7d
--- /dev/null
+++ b/src/transformers/models/cohere_asr/audio_processing_numpy_cohere_asr.py
@@ -0,0 +1,179 @@
+# Copyright 2026 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from ...audio_processing_backends import NumpyAudioBackend
+from ...audio_utils import MelScaleConfig, SpectrogramConfig, StftConfig, _array_namespace
+
+
+EPSILON = 1e-5
+
+
+class CohereAsrAudioProcessorMixin:
+ """Backend-agnostic Cohere-ASR logic shared by the numpy and torch siblings; only the
+ RNG-dependent dither and the torch mel/magnitude leaves live in the sibling classes."""
+
+ sampling_rate = 16000
+ force_mono = True
+ padding = "longest"
+
+ dither: float = 1e-5
+ max_audio_clip_s: float = 35.0
+ overlap_chunk_second: float = 5.0
+ min_energy_window_samples: int = 1600
+
+ legacy_field_mapping = {
+ "feature_size": "spectrogram_config.mel_scale_config.n_mels",
+ }
+
+ spectrogram_config = SpectrogramConfig(
+ stft_config=StftConfig(
+ n_fft=512,
+ hop_length=160,
+ win_length=400,
+ window_fn="hann_window",
+ power=2.0,
+ pad_mode="constant",
+ periodic=False,
+ ),
+ mel_scale_config=MelScaleConfig(
+ n_mels=128,
+ f_min=0.0,
+ norm="slaney",
+ mel_scale="slaney",
+ ),
+ preemphasis=0.97,
+ preemphasis_mode="waveform",
+ log_mode="log",
+ mel_floor=0.0, # no clamp; the log guard is pre_log_offset
+ pre_log_offset=2**-24,
+ )
+
+ def _normalize_magnitude(self, features, *, spectrogram_config, **kwargs):
+ # transpose to (batch, frames, mels)
+ features = super()._normalize_magnitude(features, spectrogram_config=spectrogram_config, **kwargs)
+ return features.swapaxes(-2, -1)
+
+ def _postprocess_output(self, output, audio_ranges=None, **kwargs):
+ if audio_ranges is None or "audio_features" not in output:
+ return output
+ stft_cfg = self.spectrogram_config.stft_config
+ feature_lengths = [
+ (end - start + stft_cfg.n_fft // 2 * 2 - stft_cfg.n_fft) // stft_cfg.hop_length
+ for start, end in audio_ranges
+ ]
+ output["audio_features"] = self._masked_mean_var_normalize(
+ output["audio_features"], feature_lengths, epsilon=EPSILON
+ )
+ return output
+
+ def _preprocess_audio_like_inputs(self, audio, *args, sampling_rate=None, **kwargs):
+ # long-audio chunking (1 audio → N chunks) happens before padding/extraction
+ prepared = self._prepare_audio_like_inputs(audio=audio, sampling_rate=sampling_rate)
+ chunked, audio_chunk_index = self._split_audio_chunks(prepared)
+ result = self._preprocess(chunked, *args, **kwargs)
+ result["audio_chunk_index"] = self._encode_chunk_index(audio_chunk_index, kwargs.get("return_tensors"))
+ return result
+
+ def _encode_chunk_index(self, audio_chunk_index, return_tensors):
+ # integer-encode so it survives `convert_to_tensors`; no-chunking marker None -> -1
+ encoded = [[s, -1 if c is None else c] for s, c in audio_chunk_index]
+ if return_tensors == "pt":
+ import torch
+
+ return torch.tensor(encoded, dtype=torch.long)
+ return np.asarray(encoded, dtype=np.int64)
+
+ def _split_audio_chunks(self, prepared_audio):
+ """Split audio longer than ``max_audio_clip_s - overlap_chunk_second`` at the
+ quietest window. Returns (chunks, [(sample_idx, chunk_idx or None)])."""
+ fast_path_threshold_s = max(0.0, self.max_audio_clip_s - self.overlap_chunk_second)
+ chunked: list = []
+ audio_chunk_index: list[tuple[int, int | None]] = []
+ for sample_idx, waveform in enumerate(prepared_audio):
+ duration_s = waveform.shape[0] / self.sampling_rate
+ if duration_s <= fast_path_threshold_s:
+ chunked.append(waveform)
+ audio_chunk_index.append((sample_idx, None))
+ else:
+ for chunk_idx, chunk in enumerate(self._split_single_audio(waveform)):
+ chunked.append(chunk)
+ audio_chunk_index.append((sample_idx, chunk_idx))
+ return chunked, audio_chunk_index
+
+ def _split_single_audio(self, waveform):
+ chunk_size = max(1, int(round(self.max_audio_clip_s * self.sampling_rate)))
+ boundary_context_size = max(1, int(round(self.overlap_chunk_second * self.sampling_rate)))
+ total_samples = waveform.shape[0]
+ if total_samples <= chunk_size:
+ return [waveform]
+
+ chunks_meta: list[tuple[int, int]] = []
+ idx = 0
+ while idx < total_samples:
+ if idx + chunk_size >= total_samples:
+ chunks_meta.append((idx, total_samples))
+ break
+ search_start = max(idx, idx + chunk_size - boundary_context_size)
+ search_end = min(idx + chunk_size, total_samples)
+ split_point = self._find_split_point_energy(waveform, search_start, search_end)
+ split_point = max(idx + 1, min(split_point, total_samples))
+ chunks_meta.append((idx, split_point))
+ idx = split_point
+
+ return [waveform[start:end] for start, end in chunks_meta if end > start]
+
+ def _find_split_point_energy(self, waveform, start_idx: int, end_idx: int) -> int:
+ segment = waveform[start_idx:end_idx]
+ if segment.shape[0] <= self.min_energy_window_samples:
+ return (start_idx + end_idx) // 2
+
+ xp = _array_namespace(segment)
+ min_energy = float("inf")
+ quietest_idx = start_idx
+ upper = segment.shape[0] - self.min_energy_window_samples
+ for i in range(0, upper, self.min_energy_window_samples):
+ window = segment[i : i + self.min_energy_window_samples]
+ energy = float(xp.sqrt(xp.mean(window * window)))
+ if energy < min_energy:
+ min_energy = energy
+ quietest_idx = start_idx + i
+ return quietest_idx
+
+
+class CohereAsrAudioProcessorNumpy(CohereAsrAudioProcessorMixin, NumpyAudioBackend):
+ """NumPy sibling of [`CohereAsrAudioProcessor`]. Bit-exact to the torch sibling within
+ the float32 noise floor when ``dither=0`` — the deterministic torch-RNG dither cannot
+ be reproduced bit-exactly with numpy's RNG, so the parity test disables it. See
+ [`CohereAsrAudioProcessor`] for the full pipeline description."""
+
+ def _apply_dither(self, audio, audio_ranges=None):
+ """Deterministic per-utterance dither, seeded by valid sample count. Numpy and torch
+ RNGs differ, so the parity fixture sets ``dither=0``. Runs before `waveform_scale`
+ and waveform preemphasis in the base `_stft` (ordering is load-bearing)."""
+ if self.dither <= 0 or audio_ranges is None:
+ return audio
+ audio = audio.copy()
+ for i, (start, end) in enumerate(audio_ranges):
+ valid_samples = min(end - start, audio.shape[1])
+ if valid_samples <= 0:
+ continue
+ rng = np.random.RandomState(valid_samples)
+ noise = rng.standard_normal(valid_samples).astype(audio.dtype)
+ audio[i, :valid_samples] = audio[i, :valid_samples] + self.dither * noise
+ return audio
+
+
+__all__ = ["CohereAsrAudioProcessorNumpy"]
diff --git a/src/transformers/models/cohere_asr/feature_extraction_cohere_asr.py b/src/transformers/models/cohere_asr/feature_extraction_cohere_asr.py
index 1192be10606d..32b32db45382 100644
--- a/src/transformers/models/cohere_asr/feature_extraction_cohere_asr.py
+++ b/src/transformers/models/cohere_asr/feature_extraction_cohere_asr.py
@@ -1,374 +1,20 @@
-# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
+# Copyright 2026 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import numpy as np
-import torch
-
-from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
-from ...feature_extraction_utils import BatchFeature
-from ...utils import TensorType, is_librosa_available, logging
-from ...utils.import_utils import requires
-
-
-if is_librosa_available():
- import librosa
-
-
-EPSILON = 1e-5
-LOG_ZERO_GUARD_VALUE = 2**-24
-
-
-logger = logging.get_logger(__name__)
-
-
-@requires(backends=("torch", "librosa"))
-class CohereAsrFeatureExtractor(SequenceFeatureExtractor):
- r"""
- Constructs a CohereAsr feature extractor.
-
- This feature extractor inherits from [`~feature_extraction_sequence_utils.SequenceFeatureExtractor`] which contains
- most of the main methods. Users should refer to this superclass for more information regarding those methods.
-
- This class extracts mel-filter bank features from raw speech using a custom numpy implementation of the `Short Time
- Fourier Transform` which should match pytorch's `torch.stft` equivalent.
-
- Args:
- feature_size (`int`, *optional*, defaults to 128):
- The feature dimension of the extracted features.
- sampling_rate (`int`, *optional*, defaults to 16000):
- The sampling rate at which the audio files should be digitalized expressed in hertz (Hz).
- hop_length (`int`, *optional*, defaults to 160):
- Length of the overlapping windows for the STFT used to obtain the Mel Frequency coefficients.
- n_fft (`int`, *optional*, defaults to 512):
- Size of the Fourier transform.
- win_length (`int`, *optional*, defaults to 400):
- The window length for the STFT computation.
- preemphasis (`float`, *optional*, defaults to 0.97):
- A preemphasis filter coefficient. 0.0 means no preemphasis filter.
- padding_value (`float`, *optional*, defaults to 0.0):
- Padding value used to pad the audio. Should correspond to silences.
- dither (`float`, *optional*, defaults to 1e-05):
- Amount of deterministic dither noise to add before feature extraction. Each sample is seeded by its
- valid waveform length so that dither is batch-composition invariant. Set to 0.0 to disable.
- max_audio_clip_s (`float`, *optional*, defaults to 35.0):
- Maximum duration in seconds for a single audio chunk. Audio longer than
- `max_audio_clip_s - overlap_chunk_second` is split at energy-based boundaries.
- overlap_chunk_second (`float`, *optional*, defaults to 5.0):
- Size in seconds of the boundary search window used when splitting long audio. This is not actual
- overlap between chunks — it defines how far back from the chunk boundary to search for a quiet
- split point.
- min_energy_window_samples (`int`, *optional*, defaults to 1600):
- Size in samples of the sliding window used to find the quietest point when splitting audio chunks.
- """
-
- model_input_names = ["input_features", "attention_mask"]
-
- def __init__(
- self,
- feature_size=128,
- sampling_rate=16000,
- hop_length=160,
- n_fft=512,
- win_length=400,
- preemphasis=0.97,
- padding_value=0.0,
- dither=1e-5,
- max_audio_clip_s=35.0,
- overlap_chunk_second=5.0,
- min_energy_window_samples=1600,
- **kwargs,
- ):
- super().__init__(feature_size=feature_size, sampling_rate=sampling_rate, padding_value=padding_value, **kwargs)
-
- self.hop_length = hop_length
- self.n_fft = n_fft
- self.win_length = win_length
- self.preemphasis = preemphasis
- self.dither = dither
- self.max_audio_clip_s = max_audio_clip_s
- self.overlap_chunk_second = overlap_chunk_second
- self.min_energy_window_samples = min_energy_window_samples
-
- # TODO: @eustlb, for now we use librosa to compute the mel filters
- # indeed mel_filter_bank uses np.float64 (while librosa uses np.float32), giving numerical differences
- mel_filters = librosa.filters.mel(
- sr=sampling_rate, n_fft=n_fft, n_mels=feature_size, fmin=0.0, fmax=sampling_rate / 2, norm="slaney"
- )
- self.mel_filters = torch.from_numpy(mel_filters).to(torch.float32)
-
- def _find_split_point_energy(self, waveform: torch.Tensor, start_idx: int, end_idx: int) -> int:
- segment = waveform[start_idx:end_idx]
- if segment.shape[0] <= self.min_energy_window_samples:
- return (start_idx + end_idx) // 2
-
- min_energy = float("inf")
- quietest_idx = start_idx
- upper = segment.shape[0] - self.min_energy_window_samples
- for i in range(0, upper, self.min_energy_window_samples):
- window = segment[i : i + self.min_energy_window_samples]
- energy = torch.sqrt(torch.mean(window * window)).item()
- if energy < min_energy:
- min_energy = energy
- quietest_idx = start_idx + i
- return quietest_idx
-
- def _split_audio_chunks_energy(self, waveform: torch.Tensor) -> list[torch.Tensor]:
- chunk_size = max(1, int(round(self.max_audio_clip_s * self.sampling_rate)))
- boundary_context_size = max(1, int(round(self.overlap_chunk_second * self.sampling_rate)))
- total_samples = waveform.shape[0]
-
- if total_samples <= chunk_size:
- return [waveform]
-
- chunks_meta: list[tuple[int, int]] = []
- idx = 0
- while idx < total_samples:
- if idx + chunk_size >= total_samples:
- chunks_meta.append((idx, total_samples))
- break
-
- search_start = max(idx, idx + chunk_size - boundary_context_size)
- search_end = min(idx + chunk_size, total_samples)
- if search_end <= search_start:
- split_point = idx + chunk_size
- else:
- split_point = self._find_split_point_energy(waveform, search_start, search_end)
-
- split_point = max(idx + 1, min(split_point, total_samples))
- chunks_meta.append((idx, split_point))
- idx = split_point
-
- return [waveform[start:end] for start, end in chunks_meta if end > start]
-
- def _apply_dither(self, waveform: torch.Tensor, audio_lengths: torch.Tensor) -> torch.Tensor:
- if self.dither <= 0:
- return waveform
- generator = torch.Generator(device=waveform.device)
- for i in range(waveform.shape[0]):
- valid_samples = min(int(audio_lengths[i].item()), waveform.shape[1])
- if valid_samples <= 0:
- continue
- generator.manual_seed(valid_samples)
- noise = torch.randn(valid_samples, dtype=waveform.dtype, device=waveform.device, generator=generator)
- waveform[i, :valid_samples] += self.dither * noise
- return waveform
-
- def _torch_extract_fbank_features(self, waveform, device="cpu"):
- # spectrogram
- window = torch.hann_window(self.win_length, periodic=False, device=device)
- stft = torch.stft(
- waveform,
- self.n_fft,
- hop_length=self.hop_length,
- win_length=self.win_length,
- window=window,
- return_complex=True,
- pad_mode="constant",
- )
- # Let's match original implementation
- magnitudes = torch.view_as_real(stft)
- magnitudes = torch.sqrt(magnitudes.pow(2).sum(-1))
- magnitudes = magnitudes.pow(2)
-
- # log mel spectrogram
- mel_filters = self.mel_filters.to(device)
- mel_spec = mel_filters @ magnitudes
- mel_spec = torch.log(mel_spec + LOG_ZERO_GUARD_VALUE)
-
- # (batch_size, num_mel_filters, num_frames) -> (batch_size, num_frames, num_mel_filters)
- mel_spec = mel_spec.permute(0, 2, 1)
-
- return mel_spec
-
- def __call__(
- self,
- raw_speech: np.ndarray | list[float] | list[np.ndarray] | list[list[float]],
- truncation: bool = False,
- pad_to_multiple_of: int | None = None,
- return_tensors: str | TensorType | None = None,
- return_attention_mask: bool | None = None,
- padding: str | None = "longest",
- max_length: int | None = None,
- sampling_rate: int | None = None,
- do_normalize: bool | None = None,
- device: str | None = "cpu",
- return_token_timestamps: bool | None = None,
- **kwargs,
- ) -> BatchFeature:
- """
- Main method to featurize and prepare for the model one or several sequence(s). Implementation uses PyTorch for
- the STFT computation if available, otherwise a slower NumPy based one.
-
- Args:
- raw_speech (`np.ndarray`, `list[float]`, `list[np.ndarray]`, `list[list[float]]`):
- The sequence or batch of sequences to be padded. Each sequence can be a numpy array, a list of float
- values, a list of numpy arrays or a list of list of float values. Must be mono channel audio, not
- stereo, i.e. single float per timestep.
- truncation (`bool`, *optional*, default to `True`):
- Activates truncation to cut input sequences longer than *max_length* to *max_length*.
- pad_to_multiple_of (`int`, *optional*, defaults to None):
- If set will pad the sequence to a multiple of the provided value.
-
- This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability
- `>= 7.5` (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128.
- return_attention_mask (`bool`, *optional*):
- Whether to return the attention mask. If left to the default, will return the attention mask according
- to the specific feature_extractor's default.
-
- [What are attention masks?](../glossary#attention-mask)
-
-
-
- For CohereAsr models, `attention_mask` should always be passed for batched inference, to avoid subtle
- bugs.
-
-
-
- return_tensors (`str` or [`~utils.TensorType`], *optional*):
- If set, will return tensors instead of list of python integers. Acceptable values are:
-
- - `'tf'`: Return TensorFlow `tf.constant` objects.
- - `'pt'`: Return PyTorch `torch.Tensor` objects.
- - `'np'`: Return Numpy `np.ndarray` objects.
- sampling_rate (`int`, *optional*):
- The sampling rate at which the `raw_speech` input was sampled. It is strongly recommended to pass
- `sampling_rate` at the forward call to prevent silent errors and allow automatic speech recognition
- pipeline.
- padding_value (`float`, *optional*, defaults to 0.0):
- The value that is used to fill the padding values / vectors.
- do_normalize (`bool`, *optional*, defaults to `False`):
- Whether or not to zero-mean unit-variance normalize the input. Normalizing can help to significantly
- improve the performance of the model.
- device (`str`, *optional*, defaults to `'cpu'`):
- Specifies the device for computation of the log-mel spectrogram of audio signals in the
- `_torch_extract_fbank_features` method. (e.g., "cpu", "cuda")
- return_token_timestamps (`bool`, *optional*, defaults to `None`):
- Deprecated. Use `return_attention_mask` instead from which the number of frames can be inferred.
-
- Whether or not to return the number of frames of the input raw_speech.
- These num_frames can be used by the model to compute word level timestamps.
- """
- if sampling_rate is not None:
- if sampling_rate != self.sampling_rate:
- raise ValueError(
- f"The model corresponding to this feature extractor: {self.__class__.__name__} was trained using a"
- f" sampling rate of {self.sampling_rate}. Please make sure that the provided `raw_speech` input"
- f" was sampled with {self.sampling_rate} and not {sampling_rate}."
- )
- else:
- logger.warning(
- f"It is strongly recommended to pass the `sampling_rate` argument to `{self.__class__.__name__}()`. "
- "Failing to do so can result in silent errors that might be hard to debug."
- )
-
- # Convert to torch tensor
- if isinstance(raw_speech, np.ndarray):
- raw_speech = torch.tensor(raw_speech)
- elif isinstance(raw_speech, (list, tuple)) and isinstance(raw_speech[0], np.ndarray):
- raw_speech = [torch.tensor(speech) for speech in raw_speech]
-
- is_batched_torch = isinstance(raw_speech, torch.Tensor) and len(raw_speech.shape) > 1
- if is_batched_torch and len(raw_speech.shape) > 2:
- logger.warning(
- f"Only mono-channel audio is supported for input to {self.__class__.__name__}. "
- "We will take the mean of the channels to convert to mono."
- )
- raw_speech = raw_speech.mean(-1)
-
- is_batched_sequence = isinstance(raw_speech, (list, tuple))
- if is_batched_sequence:
- for speech in raw_speech:
- if len(speech.shape) > 1:
- logger.warning(
- f"Only mono-channel audio is supported for input to {self.__class__.__name__}. "
- "We will take the mean of the channels to convert to mono."
- )
- speech = speech.mean(-1)
-
- if is_batched_torch or is_batched_sequence:
- raw_speech = [speech.to(torch.float32) for speech in raw_speech]
- else:
- raw_speech = [raw_speech.to(torch.float32)]
-
- # Chunk long audio at energy-based boundaries
- fast_path_threshold_s = max(0.0, self.max_audio_clip_s - self.overlap_chunk_second)
- audio_chunk_index: list[tuple[int, int | None]] = []
- chunked_speech: list[torch.Tensor] = []
- for sample_idx, speech in enumerate(raw_speech):
- duration_s = speech.shape[0] / self.sampling_rate
- if duration_s <= fast_path_threshold_s:
- chunked_speech.append(speech)
- audio_chunk_index.append((sample_idx, None))
- else:
- chunks = self._split_audio_chunks_energy(speech)
- for chunk_idx, chunk in enumerate(chunks):
- chunked_speech.append(chunk)
- audio_chunk_index.append((sample_idx, chunk_idx))
-
- raw_speech = [speech[:, None] for speech in chunked_speech]
-
- audio_lengths = [len(speech) for speech in raw_speech]
- batched_speech = BatchFeature({"input_features": raw_speech, "audio_lengths": audio_lengths})
-
- padded_inputs = self.pad(
- batched_speech,
- padding=padding,
- max_length=max_length,
- truncation=truncation,
- pad_to_multiple_of=pad_to_multiple_of,
- return_tensors="pt",
- )
- input_features = padded_inputs.input_features.squeeze(-1)
-
- # dithering
- input_features = self._apply_dither(input_features, padded_inputs.audio_lengths)
-
- # preemphasis
- if self.preemphasis is not None:
- timemask = torch.arange(input_features.shape[1], device=input_features.device).unsqueeze(
- 0
- ) < padded_inputs.audio_lengths.unsqueeze(1)
- input_features = torch.cat(
- [input_features[:, :1], input_features[:, 1:] - self.preemphasis * input_features[:, :-1]], dim=1
- )
- input_features = input_features.masked_fill(~timemask, 0.0)
+"""Backwards-compatibility shim: re-exports the legacy ``CohereAsrFeatureExtractor`` name as a
+deprecated alias of [`CohereAsrAudioProcessor`]. Importing or instantiating the alias emits a
+``FutureWarning``; the alias is removed in transformers v5.15 (see ADR 0002).
+"""
- input_features = self._torch_extract_fbank_features(input_features, device)
- features_lengths = torch.floor_divide(
- padded_inputs.audio_lengths + self.n_fft // 2 * 2 - self.n_fft, self.hop_length
- )
- attention_mask = torch.arange(input_features.shape[1], device=device)[None, :] < features_lengths[:, None]
+from ...audio_processing_base import make_legacy_audio_processor_alias
+from .audio_processing_cohere_asr import CohereAsrAudioProcessor
- # normalize mel features, ignoring padding
- mask = attention_mask.unsqueeze(-1)
- input_features_masked = input_features * mask
- mean = input_features_masked.sum(dim=1) / features_lengths.unsqueeze(-1)
- mean = mean.unsqueeze(1)
- variance = ((input_features_masked - mean) ** 2 * mask).sum(dim=1) / (features_lengths - 1).unsqueeze(-1)
- std = torch.sqrt(variance).unsqueeze(1)
- input_features = (input_features - mean) / (std + EPSILON)
- input_features *= mask
- result = BatchFeature(
- data={
- "input_features": input_features,
- "attention_mask": attention_mask,
- },
- tensor_type=return_tensors,
- )
- result["audio_chunk_index"] = audio_chunk_index
- return result
+CohereAsrFeatureExtractor = make_legacy_audio_processor_alias(CohereAsrAudioProcessor, "CohereAsrFeatureExtractor")
__all__ = ["CohereAsrFeatureExtractor"]
diff --git a/src/transformers/models/dac/__init__.py b/src/transformers/models/dac/__init__.py
index 40f84ccb59be..8265d11f07b2 100644
--- a/src/transformers/models/dac/__init__.py
+++ b/src/transformers/models/dac/__init__.py
@@ -18,6 +18,8 @@
if TYPE_CHECKING:
+ from .audio_processing_dac import *
+ from .audio_processing_numpy_dac import *
from .configuration_dac import *
from .feature_extraction_dac import *
from .modeling_dac import *
diff --git a/src/transformers/models/dac/audio_processing_dac.py b/src/transformers/models/dac/audio_processing_dac.py
new file mode 100644
index 000000000000..bff4c90959bb
--- /dev/null
+++ b/src/transformers/models/dac/audio_processing_dac.py
@@ -0,0 +1,24 @@
+# Copyright 2025 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ...audio_processing_backends import NumpyAudioBackend
+
+
+class DacAudioProcessor(NumpyAudioBackend):
+ sampling_rate = 16000
+ force_mono = True
+ add_channel_dim = True
+
+
+__all__ = ["DacAudioProcessor"]
diff --git a/src/transformers/models/dac/audio_processing_numpy_dac.py b/src/transformers/models/dac/audio_processing_numpy_dac.py
new file mode 100644
index 000000000000..6d7e480dbf6b
--- /dev/null
+++ b/src/transformers/models/dac/audio_processing_numpy_dac.py
@@ -0,0 +1,27 @@
+# Copyright 2026 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ...audio_processing_backends import NumpyAudioBackend
+
+
+class DacAudioProcessorNumpy(NumpyAudioBackend):
+ """NumPy sibling of [`DacAudioProcessor`]. Pure-config: no spectrogram extraction, just
+ raw-audio passthrough with mono coercion and a leading channel axis (ADR 0001)."""
+
+ sampling_rate = 16000
+ force_mono = True
+ add_channel_dim = True
+
+
+__all__ = ["DacAudioProcessorNumpy"]
diff --git a/src/transformers/models/dac/feature_extraction_dac.py b/src/transformers/models/dac/feature_extraction_dac.py
index 7f910f57f09f..49658683c1e3 100644
--- a/src/transformers/models/dac/feature_extraction_dac.py
+++ b/src/transformers/models/dac/feature_extraction_dac.py
@@ -1,170 +1,20 @@
-# Copyright 2024 Descript and The HuggingFace Inc. team. All rights reserved.
+# Copyright 2026 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Feature extractor class for DAC"""
-
-import numpy as np
-
-from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
-from ...feature_extraction_utils import BatchFeature
-from ...utils import PaddingStrategy, TensorType, logging
-
-
-logger = logging.get_logger(__name__)
-
-
-class DacFeatureExtractor(SequenceFeatureExtractor):
- r"""
- Constructs an Dac feature extractor.
-
- This feature extractor inherits from [`~feature_extraction_sequence_utils.SequenceFeatureExtractor`] which contains
- most of the main methods. Users should refer to this superclass for more information regarding those methods.
-
- Args:
- feature_size (`int`, *optional*, defaults to 1):
- The feature dimension of the extracted features. Use 1 for mono, 2 for stereo.
- sampling_rate (`int`, *optional*, defaults to 16000):
- The sampling rate at which the audio waveform should be digitalized, expressed in hertz (Hz).
- padding_value (`float`, *optional*, defaults to 0.0):
- The value that is used for padding.
- hop_length (`int`, *optional*, defaults to 512):
- Overlap length between successive windows.
- """
-
- model_input_names = ["input_values", "n_quantizers"]
-
- def __init__(
- self,
- feature_size: int = 1,
- sampling_rate: int = 16000,
- padding_value: float = 0.0,
- hop_length: int = 512,
- **kwargs,
- ):
- super().__init__(feature_size=feature_size, sampling_rate=sampling_rate, padding_value=padding_value, **kwargs)
- self.hop_length = hop_length
-
- def __call__(
- self,
- raw_audio: np.ndarray | list[float] | list[np.ndarray] | list[list[float]],
- padding: bool | str | PaddingStrategy | None = None,
- truncation: bool | None = False,
- max_length: int | None = None,
- return_tensors: str | TensorType | None = None,
- sampling_rate: int | None = None,
- ) -> BatchFeature:
- """
- Main method to featurize and prepare for the model one or several sequence(s).
-
- Args:
- raw_audio (`np.ndarray`, `list[float]`, `list[np.ndarray]`, `list[list[float]]`):
- The sequence or batch of sequences to be processed. Each sequence can be a numpy array, a list of float
- values, a list of numpy arrays or a list of list of float values. The numpy array must be of shape
- `(num_samples,)` for mono audio (`feature_size = 1`), or `(2, num_samples)` for stereo audio
- (`feature_size = 2`).
- padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `True`):
- Select a strategy to pad the returned sequences (according to the model's padding side and padding
- index) among:
-
- - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
- sequence if provided).
- - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
- acceptable input length for the model if that argument is not provided.
- - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
- lengths).
- truncation (`bool`, *optional*, defaults to `False`):
- Activates truncation to cut input sequences longer than `max_length` to `max_length`.
- max_length (`int`, *optional*):
- Maximum length of the returned list and optionally padding length (see above).
- return_tensors (`str` or [`~utils.TensorType`], *optional*, default to 'pt'):
- If set, will return tensors instead of list of python integers. Acceptable values are:
-
- - `'pt'`: Return PyTorch `torch.Tensor` objects.
- - `'np'`: Return Numpy `np.ndarray` objects.
- sampling_rate (`int`, *optional*):
- The sampling rate at which the `audio` input was sampled. It is strongly recommended to pass
- `sampling_rate` at the forward call to prevent silent errors.
- """
- if sampling_rate is not None:
- if sampling_rate != self.sampling_rate:
- raise ValueError(
- f"The model corresponding to this feature extractor: {self} was trained using a sampling rate of"
- f" {self.sampling_rate}. Please make sure that the provided audio input was sampled with"
- f" {self.sampling_rate} and not {sampling_rate}."
- )
- else:
- logger.warning(
- f"It is strongly recommended to pass the `sampling_rate` argument to `{self.__class__.__name__}()`. "
- "Failing to do so can result in silent errors that might be hard to debug."
- )
-
- if padding and truncation:
- raise ValueError("Both padding and truncation were set. Make sure you only set one.")
- elif padding is None:
- # by default let's pad the inputs
- padding = True
-
- is_batched = bool(
- isinstance(raw_audio, (list, tuple)) and (isinstance(raw_audio[0], (np.ndarray, tuple, list)))
- )
-
- if is_batched:
- raw_audio = [np.asarray(audio, dtype=np.float32).T for audio in raw_audio]
- elif not is_batched and not isinstance(raw_audio, np.ndarray):
- raw_audio = np.asarray(raw_audio, dtype=np.float32)
- elif isinstance(raw_audio, np.ndarray) and raw_audio.dtype is np.dtype(np.float64):
- raw_audio = raw_audio.astype(np.float32)
-
- # always return batch
- if not is_batched:
- raw_audio = [np.asarray(raw_audio).T]
-
- # verify inputs are valid
- for idx, example in enumerate(raw_audio):
- if example.ndim > 2:
- raise ValueError(f"Expected input shape (channels, length) but got shape {example.shape}")
- if self.feature_size == 1 and example.ndim != 1:
- raise ValueError(f"Expected mono audio but example has {example.shape[-1]} channels")
- if self.feature_size == 2:
- raise ValueError("Stereo audio isn't supported for now")
-
- input_values = BatchFeature({"input_values": raw_audio})
-
- # normal padding on batch
- padded_inputs = self.pad(
- input_values,
- max_length=max_length,
- truncation=truncation,
- padding=padding,
- return_attention_mask=padding,
- pad_to_multiple_of=self.hop_length,
- )
- if padding:
- padded_inputs["padding_mask"] = padded_inputs.pop("attention_mask")
- if padding:
- padded_inputs.input_values = padded_inputs.input_values[:, np.newaxis, :]
+"""Backwards-compatibility shim: re-exports the legacy ``DacFeatureExtractor`` name as a
+deprecated alias of [`DacAudioProcessor`]. Importing or instantiating the alias emits a
+``FutureWarning``; the alias is removed in transformers v5.15 (see ADR 0002).
+"""
- input_values = []
- for example in padded_inputs.pop("input_values"):
- if self.feature_size == 1:
- example = example[..., None]
- input_values.append(example.T)
+from ...audio_processing_base import make_legacy_audio_processor_alias
+from .audio_processing_dac import DacAudioProcessor
- padded_inputs["input_values"] = input_values
- if return_tensors is not None:
- padded_inputs = padded_inputs.convert_to_tensors(return_tensors)
- return padded_inputs
+DacFeatureExtractor = make_legacy_audio_processor_alias(DacAudioProcessor, "DacFeatureExtractor")
__all__ = ["DacFeatureExtractor"]
diff --git a/src/transformers/models/dia/__init__.py b/src/transformers/models/dia/__init__.py
index d738fbc08788..d603ab729c30 100644
--- a/src/transformers/models/dia/__init__.py
+++ b/src/transformers/models/dia/__init__.py
@@ -18,6 +18,8 @@
if TYPE_CHECKING:
+ from .audio_processing_dia import *
+ from .audio_processing_numpy_dia import *
from .configuration_dia import *
from .feature_extraction_dia import *
from .generation_dia import *
diff --git a/src/transformers/models/dia/audio_processing_dia.py b/src/transformers/models/dia/audio_processing_dia.py
new file mode 100644
index 000000000000..aabfa0075716
--- /dev/null
+++ b/src/transformers/models/dia/audio_processing_dia.py
@@ -0,0 +1,25 @@
+# Copyright 2025 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ...audio_processing_backends import TorchAudioBackend
+
+
+class DiaAudioProcessor(TorchAudioBackend):
+ sampling_rate = 44100
+ force_mono = True
+ add_channel_dim = True
+ pad_to_multiple_of = 512
+
+
+__all__ = ["DiaAudioProcessor"]
diff --git a/src/transformers/models/dia/audio_processing_numpy_dia.py b/src/transformers/models/dia/audio_processing_numpy_dia.py
new file mode 100644
index 000000000000..259aa336470b
--- /dev/null
+++ b/src/transformers/models/dia/audio_processing_numpy_dia.py
@@ -0,0 +1,27 @@
+# Copyright 2026 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ...audio_processing_backends import NumpyAudioBackend
+
+
+class DiaAudioProcessorNumpy(NumpyAudioBackend):
+ """NumPy sibling of [`DiaAudioProcessor`]. Bit-exact to the torch sibling (ADR 0001)."""
+
+ sampling_rate = 44100
+ force_mono = True
+ add_channel_dim = True
+ pad_to_multiple_of = 512
+
+
+__all__ = ["DiaAudioProcessorNumpy"]
diff --git a/src/transformers/models/dia/feature_extraction_dia.py b/src/transformers/models/dia/feature_extraction_dia.py
index eda1ead6e014..f883b1c27d86 100644
--- a/src/transformers/models/dia/feature_extraction_dia.py
+++ b/src/transformers/models/dia/feature_extraction_dia.py
@@ -1,179 +1,20 @@
-# Copyright 2025 The Nari Labs and HuggingFace Inc. team. All rights reserved.
+# Copyright 2026 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Feature extractor class for Dia"""
-
-import numpy as np
-
-from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
-from ...feature_extraction_utils import BatchFeature
-from ...utils import PaddingStrategy, TensorType, logging
-
-
-logger = logging.get_logger(__name__)
-
-
-class DiaFeatureExtractor(SequenceFeatureExtractor):
- r"""
- Constructs an Dia feature extractor.
-
- This feature extractor inherits from [`~feature_extraction_sequence_utils.SequenceFeatureExtractor`] which contains
- most of the main methods. Users should refer to this superclass for more information regarding those methods.
-
- Args:
- feature_size (`int`, *optional*, defaults to 1):
- The feature dimension of the extracted features. Use 1 for mono, 2 for stereo.
- sampling_rate (`int`, *optional*, defaults to 16000):
- The sampling rate at which the audio waveform should be digitalized, expressed in hertz (Hz).
- padding_value (`float`, *optional*, defaults to 0.0):
- The value that is used for padding.
- hop_length (`int`, *optional*, defaults to 512):
- Overlap length between successive windows.
- """
-
- model_input_names = ["input_values", "n_quantizers"]
-
- def __init__(
- self,
- feature_size: int = 1,
- sampling_rate: int = 16000,
- padding_value: float = 0.0,
- hop_length: int = 512,
- **kwargs,
- ):
- super().__init__(feature_size=feature_size, sampling_rate=sampling_rate, padding_value=padding_value, **kwargs)
- self.hop_length = hop_length
-
- def __call__(
- self,
- raw_audio: np.ndarray | list[float] | list[np.ndarray] | list[list[float]],
- padding: bool | str | PaddingStrategy | None = None,
- truncation: bool | None = False,
- max_length: int | None = None,
- return_tensors: str | TensorType | None = None,
- sampling_rate: int | None = None,
- ) -> BatchFeature:
- """
- Main method to featurize and prepare for the model one or several sequence(s).
-
- Args:
- raw_audio (`np.ndarray`, `list[float]`, `list[np.ndarray]`, `list[list[float]]`):
- The sequence or batch of sequences to be processed. Each sequence can be a numpy array, a list of float
- values, a list of numpy arrays or a list of list of float values. The numpy array must be of shape
- `(num_samples,)` for mono audio (`feature_size = 1`), or `(2, num_samples)` for stereo audio
- (`feature_size = 2`).
- padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `True`):
- Select a strategy to pad the returned sequences (according to the model's padding side and padding
- index) among:
-
- - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
- sequence if provided).
- - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
- acceptable input length for the model if that argument is not provided.
- - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
- lengths).
- truncation (`bool`, *optional*, defaults to `False`):
- Activates truncation to cut input sequences longer than `max_length` to `max_length`.
- max_length (`int`, *optional*):
- Maximum length of the returned list and optionally padding length (see above).
- return_tensors (`str` or [`~utils.TensorType`], *optional*, default to 'pt'):
- If set, will return tensors instead of list of python integers. Acceptable values are:
-
- - `'pt'`: Return PyTorch `torch.Tensor` objects.
- - `'np'`: Return Numpy `np.ndarray` objects.
- sampling_rate (`int`, *optional*):
- The sampling rate at which the `audio` input was sampled. It is strongly recommended to pass
- `sampling_rate` at the forward call to prevent silent errors.
- """
- if sampling_rate is not None:
- if sampling_rate != self.sampling_rate:
- raise ValueError(
- f"The model corresponding to this feature extractor: {self} was trained using a sampling rate of"
- f" {self.sampling_rate}. Please make sure that the provided audio input was sampled with"
- f" {self.sampling_rate} and not {sampling_rate}."
- )
- else:
- logger.warning(
- f"It is strongly recommended to pass the `sampling_rate` argument to `{self.__class__.__name__}()`. "
- "Failing to do so can result in silent errors that might be hard to debug."
- )
-
- if padding and truncation:
- raise ValueError("Both padding and truncation were set. Make sure you only set one.")
- elif padding is None:
- # by default let's pad the inputs
- padding = True
-
- is_batched = bool(
- isinstance(raw_audio, (list, tuple)) and (isinstance(raw_audio[0], (np.ndarray, tuple, list)))
- )
-
- if is_batched:
- raw_audio = [np.asarray(audio, dtype=np.float32).T for audio in raw_audio]
- elif not is_batched and not isinstance(raw_audio, np.ndarray):
- raw_audio = np.asarray(raw_audio, dtype=np.float32)
- elif isinstance(raw_audio, np.ndarray) and raw_audio.dtype is np.dtype(np.float64):
- raw_audio = raw_audio.astype(np.float32)
-
- # always return batch
- if not is_batched:
- raw_audio = [np.asarray(raw_audio).T]
-
- # convert stereo to mono if necessary, unique to Dia
- for idx, example in enumerate(raw_audio):
- if self.feature_size == 2 and example.ndim == 2:
- raw_audio[idx] = np.mean(example, -1)
-
- # verify inputs are valid
- for idx, example in enumerate(raw_audio):
- if example.ndim > 2:
- raise ValueError(f"Expected input shape (channels, length) but got shape {example.shape}")
- if self.feature_size == 1 and example.ndim != 1:
- raise ValueError(f"Expected mono audio but example has {example.shape[-1]} channels")
- if self.feature_size == 2 and example.ndim != 1: # note the conversion before
- raise ValueError(f"Expected stereo audio but example has {example.shape[-1]} channels")
-
- input_values = BatchFeature({"input_values": raw_audio})
-
- # temporarily treat it as if we were mono as we also convert stereo to mono
- original_feature_size = self.feature_size
- self.feature_size = 1
-
- # normal padding on batch
- padded_inputs = self.pad(
- input_values,
- max_length=max_length,
- truncation=truncation,
- padding=padding,
- return_attention_mask=True,
- pad_to_multiple_of=self.hop_length,
- )
- padded_inputs["padding_mask"] = padded_inputs.pop("attention_mask")
-
- input_values = []
- for example in padded_inputs.pop("input_values"):
- if self.feature_size == 1:
- example = example[..., None]
- input_values.append(example.T)
+"""Backwards-compatibility shim: re-exports the legacy ``DiaFeatureExtractor`` name as a
+deprecated alias of [`DiaAudioProcessor`]. Importing or instantiating the alias emits a
+``FutureWarning``; the alias is removed in transformers v5.15 (see ADR 0002).
+"""
- padded_inputs["input_values"] = input_values
- if return_tensors is not None:
- padded_inputs = padded_inputs.convert_to_tensors(return_tensors)
+from ...audio_processing_base import make_legacy_audio_processor_alias
+from .audio_processing_dia import DiaAudioProcessor
- # rewrite back to original feature size
- self.feature_size = original_feature_size
- return padded_inputs
+DiaFeatureExtractor = make_legacy_audio_processor_alias(DiaAudioProcessor, "DiaFeatureExtractor")
__all__ = ["DiaFeatureExtractor"]
diff --git a/src/transformers/models/encodec/__init__.py b/src/transformers/models/encodec/__init__.py
index 3adeea056604..9a6800819e92 100644
--- a/src/transformers/models/encodec/__init__.py
+++ b/src/transformers/models/encodec/__init__.py
@@ -18,6 +18,8 @@
if TYPE_CHECKING:
+ from .audio_processing_encodec import *
+ from .audio_processing_numpy_encodec import *
from .configuration_encodec import *
from .feature_extraction_encodec import *
from .modeling_encodec import *
diff --git a/src/transformers/models/encodec/audio_processing_encodec.py b/src/transformers/models/encodec/audio_processing_encodec.py
new file mode 100644
index 000000000000..746f7d6b8d89
--- /dev/null
+++ b/src/transformers/models/encodec/audio_processing_encodec.py
@@ -0,0 +1,24 @@
+# Copyright 2025 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ...audio_processing_backends import TorchAudioBackend
+
+
+class EncodecAudioProcessor(TorchAudioBackend):
+ sampling_rate = 24000
+ force_mono = True
+ add_channel_dim = True
+
+
+__all__ = ["EncodecAudioProcessor"]
diff --git a/src/transformers/models/encodec/audio_processing_numpy_encodec.py b/src/transformers/models/encodec/audio_processing_numpy_encodec.py
new file mode 100644
index 000000000000..2b46f7fb1eb8
--- /dev/null
+++ b/src/transformers/models/encodec/audio_processing_numpy_encodec.py
@@ -0,0 +1,26 @@
+# Copyright 2026 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ...audio_processing_backends import NumpyAudioBackend
+
+
+class EncodecAudioProcessorNumpy(NumpyAudioBackend):
+ """NumPy sibling of [`EncodecAudioProcessor`]. Bit-exact to the torch sibling (ADR 0001)."""
+
+ sampling_rate = 24000
+ force_mono = True
+ add_channel_dim = True
+
+
+__all__ = ["EncodecAudioProcessorNumpy"]
diff --git a/src/transformers/models/encodec/feature_extraction_encodec.py b/src/transformers/models/encodec/feature_extraction_encodec.py
index 383936000243..36d7347d6526 100644
--- a/src/transformers/models/encodec/feature_extraction_encodec.py
+++ b/src/transformers/models/encodec/feature_extraction_encodec.py
@@ -1,205 +1,20 @@
-# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
+# Copyright 2026 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Feature extractor class for EnCodec."""
-
-import numpy as np
-
-from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
-from ...feature_extraction_utils import BatchFeature
-from ...utils import PaddingStrategy, TensorType, logging
-
-
-logger = logging.get_logger(__name__)
-
-
-class EncodecFeatureExtractor(SequenceFeatureExtractor):
- r"""
- Constructs an EnCodec feature extractor.
-
- This feature extractor inherits from [`~feature_extraction_sequence_utils.SequenceFeatureExtractor`] which contains
- most of the main methods. Users should refer to this superclass for more information regarding those methods.
-
- Instantiating a feature extractor with the defaults will yield a similar configuration to that of the
- [facebook/encodec_24khz](https://huggingface.co/facebook/encodec_24khz) architecture.
-
- Args:
- feature_size (`int`, *optional*, defaults to 1):
- The feature dimension of the extracted features. Use 1 for mono, 2 for stereo.
- sampling_rate (`int`, *optional*, defaults to 24000):
- The sampling rate at which the audio waveform should be digitalized expressed in hertz (Hz).
- padding_value (`float`, *optional*, defaults to 0.0):
- The value that is used to fill the padding values.
- chunk_length_s (`float`, *optional*):
- If defined the audio is pre-processed into chunks of lengths `chunk_length_s` and then encoded.
- overlap (`float`, *optional*):
- Defines the overlap between each chunk. It is used to compute the `chunk_stride` using the following
- formulae : `int((1.0 - self.overlap) * self.chunk_length)`.
- """
-
- model_input_names = ["input_values", "padding_mask"]
-
- def __init__(
- self,
- feature_size: int = 1,
- sampling_rate: int = 24000,
- padding_value: float = 0.0,
- chunk_length_s: float | None = None,
- overlap: float | None = None,
- **kwargs,
- ):
- super().__init__(feature_size=feature_size, sampling_rate=sampling_rate, padding_value=padding_value, **kwargs)
- self.chunk_length_s = chunk_length_s
- self.overlap = overlap
-
- # This is a property because you might want to change the chunk_length_s on the fly
- @property
- def chunk_length(self) -> int | None:
- if self.chunk_length_s is None:
- return None
- else:
- return int(self.chunk_length_s * self.sampling_rate)
-
- # This is a property because you might want to change the chunk_length_s on the fly
- @property
- def chunk_stride(self) -> int | None:
- if self.chunk_length_s is None or self.overlap is None:
- return None
- else:
- return max(1, int((1.0 - self.overlap) * self.chunk_length))
-
- def __call__(
- self,
- raw_audio: np.ndarray | list[float] | list[np.ndarray] | list[list[float]],
- padding: bool | str | PaddingStrategy | None = None,
- truncation: bool | None = False,
- max_length: int | None = None,
- return_tensors: str | TensorType | None = None,
- sampling_rate: int | None = None,
- ) -> BatchFeature:
- """
- Main method to featurize and prepare for the model one or several sequence(s).
-
- Args:
- raw_audio (`np.ndarray`, `list[float]`, `list[np.ndarray]`, `list[list[float]]`):
- The sequence or batch of sequences to be processed. Each sequence can be a numpy array, a list of float
- values, a list of numpy arrays or a list of list of float values. The numpy array must be of shape
- `(num_samples,)` for mono audio (`feature_size = 1`), or `(2, num_samples)` for stereo audio
- (`feature_size = 2`).
- padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `True`):
- Select a strategy to pad the returned sequences (according to the model's padding side and padding
- index) among:
-
- - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
- sequence if provided).
- - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
- acceptable input length for the model if that argument is not provided.
- - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
- lengths).
- truncation (`bool`, *optional*, defaults to `False`):
- Activates truncation to cut input sequences longer than `max_length` to `max_length`.
- max_length (`int`, *optional*):
- Maximum length of the returned list and optionally padding length (see above).
- return_tensors (`str` or [`~utils.TensorType`], *optional*):
- If set, will return tensors instead of list of python integers. Acceptable values are:
-
- - `'pt'`: Return PyTorch `torch.Tensor` objects.
- - `'np'`: Return Numpy `np.ndarray` objects.
- sampling_rate (`int`, *optional*):
- The sampling rate at which the `audio` input was sampled. It is strongly recommended to pass
- `sampling_rate` at the forward call to prevent silent errors.
- """
- if sampling_rate is not None:
- if sampling_rate != self.sampling_rate:
- raise ValueError(
- f"The model corresponding to this feature extractor: {self} was trained using a sampling rate of"
- f" {self.sampling_rate}. Please make sure that the provided audio input was sampled with"
- f" {self.sampling_rate} and not {sampling_rate}."
- )
- else:
- logger.warning(
- f"It is strongly recommended to pass the `sampling_rate` argument to `{self.__class__.__name__}()`. "
- "Failing to do so can result in silent errors that might be hard to debug."
- )
-
- if padding and truncation:
- raise ValueError("Both padding and truncation were set. Make sure you only set one.")
- elif padding is None:
- # by default let's pad the inputs
- padding = True
-
- is_batched = bool(
- isinstance(raw_audio, (list, tuple)) and (isinstance(raw_audio[0], (np.ndarray, tuple, list)))
- )
-
- if is_batched:
- raw_audio = [np.asarray(audio, dtype=np.float32).T for audio in raw_audio]
- elif not is_batched and not isinstance(raw_audio, np.ndarray):
- raw_audio = np.asarray(raw_audio, dtype=np.float32)
- elif isinstance(raw_audio, np.ndarray) and raw_audio.dtype is np.dtype(np.float64):
- raw_audio = raw_audio.astype(np.float32)
-
- # always return batch
- if not is_batched:
- raw_audio = [np.asarray(raw_audio).T]
-
- # verify inputs are valid
- for idx, example in enumerate(raw_audio):
- if example.ndim > 2:
- raise ValueError(f"Expected input shape (channels, length) but got shape {example.shape}")
- if self.feature_size == 1 and example.ndim != 1:
- raise ValueError(f"Expected mono audio but example has {example.shape[-1]} channels")
- if self.feature_size == 2 and example.shape[-1] != 2:
- raise ValueError(f"Expected stereo audio but example has {example.shape[-1]} channels")
-
- padded_inputs = None
- input_values = BatchFeature({"input_values": raw_audio})
- if self.chunk_stride is not None and self.chunk_length is not None and max_length is None:
- if truncation:
- max_length = min(array.shape[0] for array in raw_audio)
- nb_step = int(np.floor(max_length / self.chunk_stride))
- max_length = (nb_step - 1) * self.chunk_stride + self.chunk_length
- elif padding:
- max_length = max(array.shape[0] for array in raw_audio)
- nb_step = int(np.ceil(max_length / self.chunk_stride))
- max_length = (nb_step - 1) * self.chunk_stride + self.chunk_length
- padding = "max_length"
- else:
- padded_inputs = input_values
-
- # normal padding on batch
- if padded_inputs is None:
- padded_inputs = self.pad(
- input_values,
- max_length=max_length,
- truncation=truncation,
- padding=padding,
- return_attention_mask=padding,
- )
- if padding:
- padded_inputs["padding_mask"] = padded_inputs.pop("attention_mask")
+"""Backwards-compatibility shim: re-exports the legacy ``EncodecFeatureExtractor`` name as a
+deprecated alias of [`EncodecAudioProcessor`]. Importing or instantiating the alias emits a
+``FutureWarning``; the alias is removed in transformers v5.15 (see ADR 0002).
+"""
- input_values = []
- for example in padded_inputs.pop("input_values"):
- if self.feature_size == 1:
- example = example[..., None]
- input_values.append(example.T)
+from ...audio_processing_base import make_legacy_audio_processor_alias
+from .audio_processing_encodec import EncodecAudioProcessor
- padded_inputs["input_values"] = input_values
- if return_tensors is not None:
- padded_inputs = padded_inputs.convert_to_tensors(return_tensors)
- return padded_inputs
+EncodecFeatureExtractor = make_legacy_audio_processor_alias(EncodecAudioProcessor, "EncodecFeatureExtractor")
__all__ = ["EncodecFeatureExtractor"]
diff --git a/src/transformers/models/gemma3n/__init__.py b/src/transformers/models/gemma3n/__init__.py
index 229e91827036..dc98eaff53a3 100644
--- a/src/transformers/models/gemma3n/__init__.py
+++ b/src/transformers/models/gemma3n/__init__.py
@@ -18,6 +18,8 @@
if TYPE_CHECKING:
+ from .audio_processing_gemma3n import *
+ from .audio_processing_numpy_gemma3n import *
from .configuration_gemma3n import *
from .feature_extraction_gemma3n import *
from .modeling_gemma3n import *
diff --git a/src/transformers/models/gemma3n/audio_processing_gemma3n.py b/src/transformers/models/gemma3n/audio_processing_gemma3n.py
new file mode 100644
index 000000000000..6c3089744b58
--- /dev/null
+++ b/src/transformers/models/gemma3n/audio_processing_gemma3n.py
@@ -0,0 +1,35 @@
+# Copyright 2025 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+
+from ...audio_processing_backends import TorchAudioBackend
+from .audio_processing_numpy_gemma3n import Gemma3nAudioProcessorMixin
+
+
+class Gemma3nAudioProcessor(Gemma3nAudioProcessorMixin, TorchAudioBackend):
+ """Torch sibling of [`Gemma3nAudioProcessorNumpy`]. USM-style unfold-based STFT framed
+ at `win_length + 1` samples with HTK-flavor preemphasis, driven by `spectrogram_config`."""
+
+ def _normalize_magnitude(self, features, *, spectrogram_config, **kwargs):
+ result = super()._normalize_magnitude(features, spectrogram_config=spectrogram_config, **kwargs)
+ # stats cast to float32 BEFORE subtracting (legacy rounding, unlike the numpy sibling)
+ if self.per_bin_mean is not None:
+ result = result - self.per_bin_mean.to(device=result.device, dtype=result.dtype)
+ if self.per_bin_stddev is not None:
+ result = result / self.per_bin_stddev.to(device=result.device, dtype=result.dtype)
+ return result.to(torch.float32)
+
+
+__all__ = ["Gemma3nAudioProcessor"]
diff --git a/src/transformers/models/gemma3n/audio_processing_numpy_gemma3n.py b/src/transformers/models/gemma3n/audio_processing_numpy_gemma3n.py
new file mode 100644
index 000000000000..9f692afbdb57
--- /dev/null
+++ b/src/transformers/models/gemma3n/audio_processing_numpy_gemma3n.py
@@ -0,0 +1,96 @@
+# Copyright 2026 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from dataclasses import replace
+
+import numpy as np
+
+from ...audio_processing_backends import NumpyAudioBackend
+from ...audio_utils import MelScaleConfig, SpectrogramConfig, StftConfig
+
+
+class Gemma3nAudioProcessorMixin:
+ """Gemma3n audio logic shared by the numpy and torch siblings; the USM-style pipeline
+ is fully described by `spectrogram_config`."""
+
+ sampling_rate = 16000
+ force_mono = True
+ max_length = 480000 # 30 seconds
+ truncation = True
+ pad_to_multiple_of = 128
+ # Gemma3n-specific kwargs, folded into config/arrays by `_set_attributes`
+ preemphasis_htk_flavor = True
+ per_bin_mean = None
+ per_bin_stddev = None
+
+ spectrogram_config = SpectrogramConfig(
+ stft_config=StftConfig(
+ n_fft=1024,
+ win_length=512,
+ hop_length=160,
+ power=1.0,
+ center=False,
+ window_fn="hann_window_f32",
+ frame_extension=1,
+ fft_dtype="float64",
+ ),
+ mel_scale_config=MelScaleConfig(
+ n_mels=128,
+ f_min=125.0,
+ f_max=7600.0,
+ mel_scale="htk",
+ matmul_order="features_first",
+ ),
+ mel_floor=1e-5,
+ log_mode="log",
+ preemphasis=0.97,
+ preemphasis_mode="htk_per_frame",
+ computation_dtype="float64",
+ )
+
+ def _set_attributes(self, **kwargs):
+ super()._set_attributes(**kwargs)
+ if not self.preemphasis_htk_flavor and self.spectrogram_config.preemphasis_mode == "htk_per_frame":
+ self.spectrogram_config = replace(self.spectrogram_config, preemphasis_mode="per_frame")
+ n_mels = self.spectrogram_config.mel_scale_config.n_mels
+ if self.per_bin_mean is not None:
+ self.per_bin_mean = self._as_backend_array(np.asarray(self.per_bin_mean)).reshape(1, n_mels)
+ if self.per_bin_stddev is not None:
+ self.per_bin_stddev = self._as_backend_array(np.asarray(self.per_bin_stddev)).reshape(1, n_mels)
+
+ def _get_features_lengths(self, audio_lengths, spectrogram_config, include_center_frame=False):
+ """Extended-frame count for the mask width; per-utterance validity is
+ ``ceil(L / hop)`` (legacy strided-sample-mask semantics)."""
+ stft_cfg = spectrogram_config.stft_config
+ if include_center_frame:
+ frame_size = stft_cfg.win_length + 1
+ return (audio_lengths - frame_size) // stft_cfg.hop_length + 1
+ return (audio_lengths + stft_cfg.hop_length - 1) // stft_cfg.hop_length
+
+
+class Gemma3nAudioProcessorNumpy(Gemma3nAudioProcessorMixin, NumpyAudioBackend):
+ """NumPy sibling of [`Gemma3nAudioProcessor`], bit-exact with the legacy
+ `Gemma3nAudioFeatureExtractor`."""
+
+ def _normalize_magnitude(self, features, *, spectrogram_config, **kwargs):
+ result = super()._normalize_magnitude(features, spectrogram_config=spectrogram_config, **kwargs)
+ # float64 stats promote the result before the final float32 cast (legacy rounding)
+ if self.per_bin_mean is not None:
+ result = result - self.per_bin_mean
+ if self.per_bin_stddev is not None:
+ result = result / self.per_bin_stddev
+ return result.astype(np.float32)
+
+
+__all__ = ["Gemma3nAudioProcessorNumpy"]
diff --git a/src/transformers/models/gemma3n/feature_extraction_gemma3n.py b/src/transformers/models/gemma3n/feature_extraction_gemma3n.py
index e2b24fb1f19f..6761b1633964 100644
--- a/src/transformers/models/gemma3n/feature_extraction_gemma3n.py
+++ b/src/transformers/models/gemma3n/feature_extraction_gemma3n.py
@@ -1,333 +1,20 @@
-# Copyright 2025 Google LLC
+# Copyright 2026 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import math
-from collections.abc import Sequence
-
-import numpy as np
-
-from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
-from ...feature_extraction_utils import BatchFeature
-from ...utils import PaddingStrategy, TensorType, logging
-
-
-logger = logging.get_logger(__name__)
-
-
-def create_fb_matrix(
- n_freqs: int,
- f_min: float,
- f_max: float,
- n_mels: int,
- sample_rate: int,
- fft_length: int,
- norm: str | None = None,
-) -> np.ndarray:
- r"""Create a frequency bin conversion matrix (NumPy version).
-
- Args:
- n_freqs (int): Number of frequencies to highlight/apply
- f_min (float): Minimum frequency (Hz)
- f_max (float): Maximum frequency (Hz)
- n_mels (int): Number of mel filterbanks
- sample_rate (int): Sample rate of the audio waveform
- fft_length (int): FFT length
- norm (Optional[str]): If 'slaney', divide the triangular mel weights by
- the width of the mel band (area normalization). (Default: ``None``)
-
- Returns:
- np.ndarray: Triangular filter banks (fb matrix) of size (``n_freqs``,
- ``n_mels``)
- meaning number of frequencies to highlight/apply to x the number of
- filterbanks.
- Each column is a filterbank so that assuming there is a matrix A of
- size (..., ``n_freqs``), the applied result would be
- ``A @ create_fb_matrix_numpy(A.shape[-1], ...)``.
- """
-
- if norm is not None and norm != "slaney":
- raise ValueError("norm must be one of None or 'slaney'")
-
- # freq bins
- all_freqs = np.arange(n_freqs, dtype=np.float32) * (sample_rate / fft_length)
-
- # calculate mel freq bins
- # hertz to mel(f) is 2595. * math.log10(1. + (f / 700.))
- m_min = 2595.0 * math.log10(1.0 + (f_min / 700.0))
- m_max = 2595.0 * math.log10(1.0 + (f_max / 700.0))
- m_pts = np.linspace(m_min, m_max, n_mels + 2)
- # mel to hertz(mel) is 700. * (10**(mel / 2595.) - 1.)
- f_pts = 700.0 * (10 ** (m_pts / 2595.0) - 1.0)
- # calculate difference between each mel point and each stft freq point in Hz
- f_diff = f_pts[1:] - f_pts[:-1] # (n_mels + 1)
- slopes = np.expand_dims(f_pts, 0) - np.expand_dims(all_freqs, 1) # (n_freqs, n_mels + 2)
- # create overlapping triangles
- zero = np.zeros(1, dtype=np.float32)
- down_slopes = (-1.0 * slopes[:, :-2]) / f_diff[:-1] # (n_freqs, n_mels)
- up_slopes = slopes[:, 2:] / f_diff[1:] # (n_freqs, n_mels)
- fb = np.maximum(zero, np.minimum(down_slopes, up_slopes))
-
- if norm is not None and norm == "slaney":
- # Slaney-style mel is scaled to be approx constant energy per channel
- enorm = 2.0 / (f_pts[2 : n_mels + 2] - f_pts[:n_mels])
- fb *= np.expand_dims(enorm, 0)
-
- return fb
-
-
-def _unfold(array: np.ndarray, dimension: int, size: int, step: int) -> np.ndarray:
- """A basic NumPy equivalent of PyTorch's unfold for 2D arrays along the last dim."""
- if array.ndim != 2:
- raise ValueError("This unfold implementation currently supports 2D arrays (batch, time).")
- if dimension != -1 and dimension != array.ndim - 1:
- raise ValueError("This unfold implementation only supports unfolding the last dimension.")
-
- batch_size, original_length = array.shape
- num_frames = (original_length - size) // step + 1
-
- if num_frames <= 0:
- return np.zeros((batch_size, 0, size), dtype=array.dtype)
-
- output_shape = (batch_size, num_frames, size)
- output_strides = (array.strides[0], array.strides[1] * step, array.strides[1])
-
- return np.lib.stride_tricks.as_strided(array, shape=output_shape, strides=output_strides)
-
-
-class Gemma3nAudioFeatureExtractor(SequenceFeatureExtractor):
- """An audio feature extractor Universal Speech Models https://huggingface.co/papers/2303.01037.
-
- Args:
- feature_size (`int`, *optional*, defaults to 128):
- The feature dimension of the extracted features.
- sampling_rate (`int`, *optional*, defaults to 16000):
- The sampling rate at which the audio files should be digitalized expressed in hertz (Hz).
- padding_value (`float`, *optional*, defaults to 0.0):
- Padding value used to pad the audio. Should correspond to silences.
- return_attention_mask (`bool`, *optional*, defaults to `True`):
- Whether to return the attention mask for the generated MEL spectrograms.
- frame_length_ms (`float`, *optional*, defaults to 32.0):
- The length of a frame in milliseconds.
- hop_length_ms (`float`, *optional*, defaults to 10.0):
- Length of the overlapping windows for the STFT used to obtain the Mel Frequency coefficients.
- min_frequency (`float`, *optional*, defaults to 125.0):
- The minimum frequency (in Hz) for the Mel filterbank.
- max_frequency (`float`, *optional*, defaults to 7600.0):
- The maximum frequency (in Hz) for the Mel filterbank.
- preemphasis (`float`, *optional*, defaults to 0.97):
- The preemphasis coefficient.
- preemphasis_htk_flavor (`bool`, *optional*, defaults to `True`):
- Whether to use HTK-style preemphasis.
- fft_overdrive (`bool`, *optional*, defaults to `True`):
- Whether to use FFT overdrive.
- dither (`float`, *optional*, defaults to 0.0):
- Adds dithering. In other words, adds a small Gaussian noise to each frame.
- E.g. use 0.0001 to add dithering with a normal distribution centered
- around 0.0 with standard deviation 0.0001 (assuming [-1,+1] range of raw_speech).
- The value 0.0 means no dithering.
- Dithering has similar effect as `spectrogram(mel_floor=...)`. It reduces
- the high log_mel_fbank values for signals with hard-zero sections,
- when VAD cutoff is present in the signal.
- input_scale_factor (`float`, *optional*, defaults to 1.0):
- Scaling factor applied to the input waveform.
- mel_floor (`float`, *optional*, defaults to 1e-05):
- Minimum value for Mel spectrograms to avoid log(0).
- per_bin_mean (`Optional[Sequence[float]]`, *optional*):
- Mean values for per-bin normalization.
- per_bin_stddev (`Optional[Sequence[float]]`, *optional*):
- Standard deviation values for per-bin normalization.
- """
-
- model_input_names = ["input_features", "input_features_mask"]
-
- def __init__(
- self,
- feature_size: int = 128,
- sampling_rate: int = 16_000,
- padding_value: float = 0.0,
- return_attention_mask: bool = True,
- frame_length_ms: float = 32.0,
- hop_length_ms: float = 10.0,
- min_frequency: float = 125.0,
- max_frequency: float = 7600.0,
- preemphasis: float = 0.97,
- preemphasis_htk_flavor: bool = True,
- fft_overdrive: bool = True,
- dither: float = 0.0,
- input_scale_factor: float = 1.0,
- mel_floor: float = 1e-5,
- per_bin_mean: Sequence[float] | None = None,
- per_bin_stddev: Sequence[float] | None = None,
- **kwargs,
- ):
- super().__init__(
- feature_size=feature_size,
- sampling_rate=sampling_rate,
- padding_value=padding_value,
- return_attention_mask=return_attention_mask,
- **kwargs,
- )
-
- self.min_frequency = min_frequency
- self.max_frequency = max_frequency
- self.preemphasis = preemphasis
- self.preemphasis_htk_flavor = preemphasis_htk_flavor
- self.fft_overdrive = fft_overdrive
- self.dither = dither
- self.input_scale_factor = input_scale_factor
- self.frame_length = int(round(sampling_rate * frame_length_ms / 1000.0))
- self.hop_length = int(round(sampling_rate * hop_length_ms / 1000.0))
- self.mel_floor = np.array(mel_floor, dtype=np.float64)
-
- fft_length = 2 ** math.ceil(math.log2(self.frame_length))
- if self.fft_overdrive:
- fft_length *= 2
- self.fft_length = fft_length
-
- hann_arange = np.arange(self.frame_length, dtype=np.float32)
- window = 0.5 * (1 - np.cos(2 * np.pi * hann_arange / self.frame_length))
- self.window = window.astype(np.float32)
-
- self.mel_filters = create_fb_matrix(
- n_freqs=self.fft_length // 2 + 1,
- f_min=min_frequency,
- f_max=max_frequency,
- n_mels=feature_size,
- sample_rate=self.sampling_rate,
- norm=None,
- fft_length=fft_length,
- )
-
- if per_bin_mean is not None:
- self.per_bin_mean = np.array(per_bin_mean).reshape(1, 1, feature_size)
- else:
- self.per_bin_mean = None
-
- if per_bin_stddev is not None:
- self.per_bin_stddev = np.array(per_bin_stddev).reshape(1, 1, feature_size)
- else:
- self.per_bin_stddev = None
-
- def _extract_spectrogram(self, waveform: np.ndarray, attention_mask: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
- """"""
- if waveform.ndim == 1: # If single waveform, add batch dimension
- waveform = np.expand_dims(waveform, axis=0)
-
- if self.dither > 0.0:
- waveform = waveform + self.dither * np.random.randn(*waveform.shape).astype(waveform.dtype)
-
- if self.input_scale_factor != 1.0:
- waveform = waveform * self.input_scale_factor
-
- frame_size_for_unfold = self.frame_length + 1
-
- # NumPy equivalent of unfold for [B, NumFrames, frame_size_for_unfold]
- frames_to_process = _unfold(waveform, dimension=-1, size=frame_size_for_unfold, step=self.hop_length)
-
- if self.preemphasis > 0.0:
- if self.preemphasis_htk_flavor:
- first_in_frame = frames_to_process[..., :1] * (1.0 - self.preemphasis)
- rest_in_frame = frames_to_process[..., 1:-1] - self.preemphasis * frames_to_process[..., :-2]
- frames = np.concatenate([first_in_frame, rest_in_frame], axis=-1)
- else:
- frames = frames_to_process[..., 1:] - self.preemphasis * frames_to_process[..., :-1]
- else:
- frames = frames_to_process[..., :-1]
-
- frames = frames * self.window # Broadcasting window
- stft = np.fft.rfft(frames, n=self.fft_length, axis=-1)
-
- magnitude_spec = np.abs(stft)
-
- mel_spec = np.matmul(magnitude_spec, self.mel_filters)
- log_mel_spec = np.log(np.maximum(mel_spec, self.mel_floor))
-
- if self.per_bin_mean is not None:
- log_mel_spec = log_mel_spec - self.per_bin_mean # Broadcasting
-
- if self.per_bin_stddev is not None:
- log_mel_spec = log_mel_spec / self.per_bin_stddev # Broadcasting
-
- mel_spectrogram = log_mel_spec.squeeze(0)
- mask = attention_mask[:: self.hop_length].astype(bool)
- # TODO: The filtered mask is always exactly 3 elements longer than the mel_spectrogram. Why???
- return mel_spectrogram, mask[: mel_spectrogram.shape[0]]
-
- def __call__(
- self,
- raw_speech: np.ndarray | list[float] | list[np.ndarray] | list[list[float]],
- padding: bool | str | PaddingStrategy = "longest",
- max_length: int | None = 480_000,
- truncation: bool = True,
- pad_to_multiple_of: int | None = 128,
- return_tensors: str | TensorType | None = None,
- return_attention_mask: bool | None = True,
- **kwargs,
- ) -> BatchFeature:
- """Creates a batch of MEL spectrograms from the provided raw speech.
-
- This implementation uses a different algorithm for windowing and preemphasis compared to the built-in
- `transformers.audio_utils.spectrogram()` function that _will_ result in different outputs. Consider this
- carefully when selecting an audio feature extractor, especially with pre-trained models.
-
- Args:
- raw_speech:
- The audio for which MEL spectrograms are created.
- padding (`Union[bool, str, PaddingStrategy]`, *optional*, defaults to `"longest"`):
- The padding strategy to use for batches of audio with different lengths.
- max_length (`int`, *optional*, defaults to 480000):
- If provided, defines the maximum length of the audio to allow. Audio longer than this will be
- truncated if `truncation=True`.
- truncation (`bool`, *optional*, defaults to `True`):
- Whether or not to truncate audio above `max_length`.
- pad_to_multiple_of (`int`, *optional*, defaults to 128):
- When padding, pad to a multiple of this value. The default value is defined for optimal TPU support.
- return_tensors (`Union[str, TensorType]`, *optional*, defaults to `None`):
- The type of tensors to return (e.g., NumPy, or Torch).
- return_attention_mask (`bool`, *optional*, defaults to `True`):
- Whether to return the attention mask for the generated MEL spectrograms.
- """
-
- is_batched_numpy = isinstance(raw_speech, np.ndarray) and len(raw_speech.shape) > 1
- is_batched_sequence = isinstance(raw_speech, Sequence) and isinstance(raw_speech[0], (np.ndarray, Sequence))
- is_batched = is_batched_numpy or is_batched_sequence
-
- # Always return a batch
- if not is_batched:
- raw_speech = [raw_speech]
- raw_speech = [np.asarray([rs]).T for rs in raw_speech]
+"""Backwards-compatibility shim: re-exports the legacy ``Gemma3nAudioFeatureExtractor`` name as a
+deprecated alias of [`Gemma3nAudioProcessor`]. Importing or instantiating the alias emits a
+``FutureWarning``; the alias is removed in transformers v5.15 (see ADR 0002).
+"""
- batched_speech = self.pad(
- BatchFeature({"input_features": raw_speech}),
- padding=padding,
- max_length=max_length,
- truncation=truncation,
- pad_to_multiple_of=pad_to_multiple_of,
- return_attention_mask=return_attention_mask,
- )
+from ...audio_processing_base import make_legacy_audio_processor_alias
+from .audio_processing_gemma3n import Gemma3nAudioProcessor
- prepared_speech = []
- prepared_speech_mask = []
- for speech, mask in zip(batched_speech.input_features, batched_speech.attention_mask):
- speech, mask = self._extract_spectrogram(speech.T, mask)
- prepared_speech.append(speech.astype(np.float32))
- prepared_speech_mask.append(mask)
- return BatchFeature(
- {"input_features": prepared_speech, "input_features_mask": prepared_speech_mask},
- tensor_type=return_tensors,
- )
+Gemma3nAudioFeatureExtractor = make_legacy_audio_processor_alias(Gemma3nAudioProcessor, "Gemma3nAudioFeatureExtractor")
__all__ = ["Gemma3nAudioFeatureExtractor"]
diff --git a/src/transformers/models/gemma4/__init__.py b/src/transformers/models/gemma4/__init__.py
index d108443c16cb..04ca55315112 100644
--- a/src/transformers/models/gemma4/__init__.py
+++ b/src/transformers/models/gemma4/__init__.py
@@ -19,6 +19,8 @@
if TYPE_CHECKING:
+ from .audio_processing_gemma4 import *
+ from .audio_processing_numpy_gemma4 import *
from .configuration_gemma4 import *
from .feature_extraction_gemma4 import *
from .image_processing_gemma4 import *
diff --git a/src/transformers/models/gemma4/audio_processing_gemma4.py b/src/transformers/models/gemma4/audio_processing_gemma4.py
new file mode 100644
index 000000000000..7e290e415d10
--- /dev/null
+++ b/src/transformers/models/gemma4/audio_processing_gemma4.py
@@ -0,0 +1,45 @@
+# Copyright 2026 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+
+from ...audio_processing_backends import TorchAudioBackend
+from .audio_processing_numpy_gemma4 import Gemma4AudioProcessorMixin
+
+
+class Gemma4AudioProcessor(Gemma4AudioProcessorMixin, TorchAudioBackend):
+ """Torch sibling of [`Gemma4AudioProcessorNumpy`]. See the mixin for the pipeline."""
+
+ def _apply_mel_scale(self, features, *, spectrogram_config, **kwargs):
+ # cast the float64 filters to the float32 features dtype (legacy torch behavior)
+ mel_filters = self.mel_filters.to(device=features.device, dtype=features.dtype)
+ return torch.matmul(features.transpose(-2, -1), mel_filters)
+
+ def _postprocess_output(self, output, audio_ranges=None, **kwargs):
+ if audio_ranges is None or "audio_features" not in output:
+ return output
+ features = output["audio_features"]
+ # same arithmetic as the numpy sibling; per-backend only for the device move
+ if self.per_bin_mean is not None:
+ features = features - self.per_bin_mean.to(device=features.device, dtype=features.dtype)
+ if self.per_bin_stddev is not None:
+ features = features / self.per_bin_stddev.to(device=features.device, dtype=features.dtype)
+ mask = output.get("audio_features_mask")
+ if mask is not None:
+ features = features * mask.to(features.dtype).unsqueeze(-1)
+ output["audio_features"] = features
+ return output
+
+
+__all__ = ["Gemma4AudioProcessor"]
diff --git a/src/transformers/models/gemma4/audio_processing_numpy_gemma4.py b/src/transformers/models/gemma4/audio_processing_numpy_gemma4.py
new file mode 100644
index 000000000000..a0e14e49efb6
--- /dev/null
+++ b/src/transformers/models/gemma4/audio_processing_numpy_gemma4.py
@@ -0,0 +1,144 @@
+# Copyright 2026 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+from dataclasses import replace
+
+import numpy as np
+
+from ...audio_processing_backends import NumpyAudioBackend
+from ...audio_utils import MelScaleConfig, SpectrogramConfig, StftConfig
+
+
+def _gemma4_frame_length_ms_to_win_length(value, config_dict):
+ sr = config_dict.get("sampling_rate") or 16000
+ spec = config_dict.setdefault("spectrogram_config", {})
+ stft = spec.setdefault("stft_config", {})
+ stft.setdefault("win_length", int(round(sr * value / 1000.0)))
+
+
+def _gemma4_hop_length_ms_to_hop_length(value, config_dict):
+ sr = config_dict.get("sampling_rate") or 16000
+ spec = config_dict.setdefault("spectrogram_config", {})
+ stft = spec.setdefault("stft_config", {})
+ stft.setdefault("hop_length", int(round(sr * value / 1000.0)))
+
+
+class Gemma4AudioProcessorMixin:
+ """Gemma4 audio logic shared by the numpy and torch siblings: USM-style mel extractor
+ (https://huggingface.co/papers/2303.01037), fully described by `spectrogram_config`."""
+
+ sampling_rate = 16000
+ force_mono = True
+ padding = "longest"
+ padding_value = 0.0
+ max_length = 480_000
+ truncation = True
+ pad_to_multiple_of = 128
+
+ # Gemma4-specific kwargs, folded into `spectrogram_config` by `_set_attributes`
+ preemphasis_htk_flavor: bool = True
+ fft_overdrive: bool = False
+ dither: float = 0.0
+ input_scale_factor: float = 1.0
+ per_bin_mean = None
+ per_bin_stddev = None
+
+ legacy_field_mapping = {
+ "feature_size": "spectrogram_config.mel_scale_config.n_mels",
+ "frame_length_ms": _gemma4_frame_length_ms_to_win_length,
+ "hop_length_ms": _gemma4_hop_length_ms_to_hop_length,
+ "min_frequency": "spectrogram_config.mel_scale_config.f_min",
+ "max_frequency": "spectrogram_config.mel_scale_config.f_max",
+ }
+
+ # `n_fft` is 2 ** ceil(log2(win_length)); `_maybe_rebuild_for_win_length` recomputes it
+ # for non-default `win_length`/`fft_overdrive`. The base's "left"-center length formula
+ # matches this framing, so no `_get_features_lengths` override is needed.
+ spectrogram_config = SpectrogramConfig(
+ stft_config=StftConfig(
+ n_fft=512,
+ win_length=320,
+ hop_length=160,
+ window_fn="hann_window_f32",
+ power=1.0,
+ center="left",
+ frame_extension=1,
+ fft_dtype="native",
+ ),
+ mel_scale_config=MelScaleConfig(
+ n_mels=128,
+ f_min=0.0,
+ f_max=8000.0,
+ mel_scale="htk",
+ matmul_order="features_first",
+ ),
+ preemphasis=0.0,
+ preemphasis_mode="htk_per_frame",
+ mel_floor=0.0, # no clamp; the log guard is pre_log_offset
+ pre_log_offset=1e-3,
+ log_mode="log",
+ # float64 mel filters, kept float64 for the numpy matmul; torch casts to float32 at apply time
+ computation_dtype="float64",
+ )
+
+ def _set_attributes(self, **kwargs):
+ super()._set_attributes(**kwargs)
+ updates = {}
+ if not self.preemphasis_htk_flavor and self.spectrogram_config.preemphasis_mode == "htk_per_frame":
+ updates["preemphasis_mode"] = "per_frame"
+ if self.input_scale_factor != 1.0 and self.spectrogram_config.waveform_scale is None:
+ updates["waveform_scale"] = self.input_scale_factor
+ if updates:
+ self.spectrogram_config = replace(self.spectrogram_config, **updates)
+ self._maybe_rebuild_for_win_length()
+ n_mels = self.spectrogram_config.mel_scale_config.n_mels
+ if self.per_bin_mean is not None:
+ self.per_bin_mean = self._as_backend_array(np.asarray(self.per_bin_mean)).reshape(1, 1, n_mels)
+ if self.per_bin_stddev is not None:
+ self.per_bin_stddev = self._as_backend_array(np.asarray(self.per_bin_stddev)).reshape(1, 1, n_mels)
+
+ def _maybe_rebuild_for_win_length(self):
+ stft_cfg = self.spectrogram_config.stft_config
+ expected_n_fft = 2 ** math.ceil(math.log2(stft_cfg.win_length))
+ if self.fft_overdrive:
+ expected_n_fft *= 2
+ if stft_cfg.n_fft != expected_n_fft:
+ self.spectrogram_config = replace(
+ self.spectrogram_config,
+ stft_config=replace(stft_cfg, n_fft=expected_n_fft),
+ )
+ self.mel_filters = self._mel_filter_bank(self.spectrogram_config)
+
+
+class Gemma4AudioProcessorNumpy(Gemma4AudioProcessorMixin, NumpyAudioBackend):
+ """NumPy sibling of [`Gemma4AudioProcessor`], bit-exact with the legacy extractor."""
+
+ def _postprocess_output(self, output, audio_ranges=None, **kwargs):
+ if audio_ranges is None or "audio_features" not in output:
+ return output
+ features = output["audio_features"]
+ # cast the float64 stats down BEFORE subtracting (legacy rounding, unlike gemma3n)
+ if self.per_bin_mean is not None:
+ features = features - self.per_bin_mean.astype(features.dtype)
+ if self.per_bin_stddev is not None:
+ features = features / self.per_bin_stddev.astype(features.dtype)
+ mask = output.get("audio_features_mask")
+ if mask is not None:
+ features = features * mask.astype(features.dtype)[..., None]
+ output["audio_features"] = features
+ return output
+
+
+__all__ = ["Gemma4AudioProcessorNumpy"]
diff --git a/src/transformers/models/gemma4/feature_extraction_gemma4.py b/src/transformers/models/gemma4/feature_extraction_gemma4.py
index 38382e8a85cb..ceb5ba1c25a4 100644
--- a/src/transformers/models/gemma4/feature_extraction_gemma4.py
+++ b/src/transformers/models/gemma4/feature_extraction_gemma4.py
@@ -1,298 +1,20 @@
-# Copyright 2026 Google LLC
+# Copyright 2026 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import math
-import warnings
-from collections.abc import Sequence
-
-import numpy as np
-
-from ...audio_utils import mel_filter_bank, window_function
-from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
-from ...feature_extraction_utils import BatchFeature
-from ...utils import PaddingStrategy, TensorType, logging
-
-
-logger = logging.get_logger(__name__)
-
-
-def _unfold(array: np.ndarray, dimension: int, size: int, step: int) -> np.ndarray:
- """A basic NumPy equivalent of PyTorch's unfold for 2D arrays along the last dim."""
- if array.ndim != 2:
- raise ValueError("This unfold implementation currently supports 2D arrays (batch, time).")
- if dimension != -1 and dimension != array.ndim - 1:
- raise ValueError("This unfold implementation only supports unfolding the last dimension.")
-
- batch_size, original_length = array.shape
- num_frames = (original_length - size) // step + 1
-
- if num_frames <= 0:
- return np.zeros((batch_size, 0, size), dtype=array.dtype)
-
- output_shape = (batch_size, num_frames, size)
- output_strides = (array.strides[0], array.strides[1] * step, array.strides[1])
-
- return np.lib.stride_tricks.as_strided(array, shape=output_shape, strides=output_strides)
-
-
-class Gemma4AudioFeatureExtractor(SequenceFeatureExtractor):
- """An audio feature extractor Universal Speech Models https://huggingface.co/papers/2303.01037.
-
- Args:
- feature_size (`int`, *optional*, defaults to 128):
- The feature dimension of the extracted features.
- sampling_rate (`int`, *optional*, defaults to 16000):
- The sampling rate at which the audio files should be digitalized expressed in hertz (Hz).
- padding_value (`float`, *optional*, defaults to 0.0):
- Padding value used to pad the audio. Should correspond to silences.
- return_attention_mask (`bool`, *optional*, defaults to `True`):
- Whether to return the attention mask for the generated MEL spectrograms.
- frame_length_ms (`float`, *optional*, defaults to 20.0):
- The length of a frame in milliseconds.
- hop_length_ms (`float`, *optional*, defaults to 10.0):
- Length of the overlapping windows for the STFT used to obtain the Mel Frequency coefficients.
- min_frequency (`float`, *optional*, defaults to 0.0):
- The minimum frequency (in Hz) for the Mel filterbank.
- max_frequency (`float`, *optional*, defaults to 8000.0):
- The maximum frequency (in Hz) for the Mel filterbank.
- preemphasis (`float`, *optional*, defaults to 0.0):
- The preemphasis coefficient.
- preemphasis_htk_flavor (`bool`, *optional*, defaults to `True`):
- Whether to use HTK-style preemphasis.
- fft_overdrive (`bool`, *optional*, defaults to `False`):
- Whether to use FFT overdrive.
- dither (`float`, *optional*, defaults to 0.0):
- Adds dithering. In other words, adds a small Gaussian noise to each frame.
- E.g. use 0.0001 to add dithering with a normal distribution centered
- around 0.0 with standard deviation 0.0001 (assuming [-1,+1] range of raw_speech).
- The value 0.0 means no dithering.
- Dithering has similar effect as `spectrogram(mel_floor=...)`. It reduces
- the high log_mel_fbank values for signals with hard-zero sections,
- when VAD cutoff is present in the signal.
- input_scale_factor (`float`, *optional*, defaults to 1.0):
- Scaling factor applied to the input waveform.
- mel_floor (`float`, *optional*, defaults to 0.001):
- Minimum value for Mel spectrograms to avoid log(0).
- per_bin_mean (`Optional[Sequence[float]]`, *optional*):
- Mean values for per-bin normalization.
- per_bin_stddev (`Optional[Sequence[float]]`, *optional*):
- Standard deviation values for per-bin normalization.
- """
-
- model_input_names = ["input_features", "input_features_mask"]
-
- def __init__(
- self,
- feature_size: int = 128,
- sampling_rate: int = 16_000,
- padding_value: float = 0.0,
- return_attention_mask: bool = True,
- frame_length_ms: float = 20.0,
- hop_length_ms: float = 10.0,
- min_frequency: float = 0.0,
- max_frequency: float = 8000.0,
- preemphasis: float = 0.0,
- preemphasis_htk_flavor: bool = True,
- fft_overdrive: bool = False,
- dither: float = 0.0,
- input_scale_factor: float = 1.0,
- mel_floor: float = 1e-3,
- per_bin_mean: Sequence[float] | None = None,
- per_bin_stddev: Sequence[float] | None = None,
- **kwargs,
- ):
- super().__init__(
- feature_size=feature_size,
- sampling_rate=sampling_rate,
- padding_value=padding_value,
- return_attention_mask=return_attention_mask,
- **kwargs,
- )
-
- self.min_frequency = min_frequency
- self.max_frequency = max_frequency
- self.preemphasis = preemphasis
- self.preemphasis_htk_flavor = preemphasis_htk_flavor
- self.fft_overdrive = fft_overdrive
- self.dither = dither
- self.input_scale_factor = input_scale_factor
- self.frame_length = int(round(sampling_rate * frame_length_ms / 1000.0))
- self.hop_length = int(round(sampling_rate * hop_length_ms / 1000.0))
- self.mel_floor = np.array(mel_floor, dtype=np.float64)
-
- fft_length = 2 ** math.ceil(math.log2(self.frame_length))
- if self.fft_overdrive:
- fft_length *= 2
- self.fft_length = fft_length
-
- # Use periodic Hann window, matching sl.STFT default (signal.hann_window)
- # For even frame_length: window[n] = 0.5 - 0.5 * cos(2*pi*n / frame_length)
- self.window = window_function(self.frame_length).astype(np.float32)
-
- # Use HuggingFace's mel_filter_bank for compatibility.
- # Suppress the expected warning about all-zero upper mel filters;
- # with fft_length=512 (257 bins) and 128 mel filters the uppermost
- # triangular filter falls between frequency bins, which is harmless.
- with warnings.catch_warnings():
- warnings.simplefilter("ignore")
- self.mel_filters = mel_filter_bank(
- num_frequency_bins=self.fft_length // 2 + 1,
- num_mel_filters=feature_size,
- min_frequency=min_frequency,
- max_frequency=max_frequency,
- sampling_rate=self.sampling_rate,
- norm=None,
- mel_scale="htk",
- )
-
- if per_bin_mean is not None:
- self.per_bin_mean = np.array(per_bin_mean).reshape(1, 1, feature_size)
- else:
- self.per_bin_mean = None
-
- if per_bin_stddev is not None:
- self.per_bin_stddev = np.array(per_bin_stddev).reshape(1, 1, feature_size)
- else:
- self.per_bin_stddev = None
-
- def _extract_spectrogram(self, waveform: np.ndarray, attention_mask: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
- """"""
- if waveform.ndim == 1: # If single waveform, add batch dimension
- waveform = np.expand_dims(waveform, axis=0)
-
- if self.dither > 0.0:
- waveform = waveform + self.dither * np.random.randn(*waveform.shape).astype(waveform.dtype)
-
- if self.input_scale_factor != 1.0:
- waveform = waveform * self.input_scale_factor
-
- # Semicausal time padding: prepend frame_length // 2 zeros so that the
- # first STFT frame is centered at t=0, matching sl.STFT(time_padding='semicausal').
- pad_left = self.frame_length // 2
- waveform = np.pad(waveform, ((0, 0), (pad_left, 0)), mode="constant")
- attention_mask = np.pad(attention_mask, (pad_left, 0), mode="constant", constant_values=0)
-
- frame_size_for_unfold = self.frame_length + 1
-
- # NumPy equivalent of unfold for [B, NumFrames, frame_size_for_unfold]
- frames_to_process = _unfold(waveform, dimension=-1, size=frame_size_for_unfold, step=self.hop_length)
-
- if self.preemphasis > 0.0:
- if self.preemphasis_htk_flavor:
- first_in_frame = frames_to_process[..., :1] * (1.0 - self.preemphasis)
- rest_in_frame = frames_to_process[..., 1:-1] - self.preemphasis * frames_to_process[..., :-2]
- frames = np.concatenate([first_in_frame, rest_in_frame], axis=-1)
- else:
- frames = frames_to_process[..., 1:] - self.preemphasis * frames_to_process[..., :-1]
- else:
- frames = frames_to_process[..., :-1]
-
- # Apply window, then RFFT. np.fft.rfft with n=fft_length implicitly
- # right-pads frames to fft_length.
- frames = frames * self.window # Broadcasting window
- stft = np.fft.rfft(frames, n=self.fft_length, axis=-1)
-
- magnitude_spec = np.abs(stft)
-
- mel_spec = np.matmul(magnitude_spec, self.mel_filters)
- log_mel_spec = np.log(mel_spec + self.mel_floor)
-
- if self.per_bin_mean is not None:
- log_mel_spec = log_mel_spec - self.per_bin_mean # Broadcasting
-
- if self.per_bin_stddev is not None:
- log_mel_spec = log_mel_spec / self.per_bin_stddev # Broadcasting
-
- mel_spectrogram = log_mel_spec.squeeze(0)
- num_mel_frames = mel_spectrogram.shape[0]
-
- # Build a frame-aware mask: a mel frame is valid only when every sample
- # in its analysis window [i*hop, i*hop + frame_size - 1] is real audio.
- # We check this by looking at the last sample of each frame's window.
- frame_end_indices = np.arange(num_mel_frames) * self.hop_length + frame_size_for_unfold - 1
- mask = attention_mask[frame_end_indices].astype(bool)
- return mel_spectrogram, mask
-
- def __call__(
- self,
- raw_speech: np.ndarray | list[float] | list[np.ndarray] | list[list[float]],
- padding: bool | str | PaddingStrategy = "longest",
- max_length: int | None = 480_000,
- truncation: bool = True,
- pad_to_multiple_of: int | None = 128,
- return_tensors: str | TensorType | None = None,
- return_attention_mask: bool | None = True,
- **kwargs,
- ) -> BatchFeature:
- """Creates a batch of MEL spectrograms from the provided raw speech.
-
- This implementation uses a different algorithm for windowing and preemphasis compared to the built-in
- `transformers.audio_utils.spectrogram()` function that _will_ result in different outputs. Consider this
- carefully when selecting an audio feature extractor, especially with pre-trained models.
-
- Args:
- raw_speech:
- The audio for which MEL spectrograms are created.
- padding (`Union[bool, str, PaddingStrategy]`, *optional*, defaults to `"longest"`):
- The padding strategy to use for batches of audio with different lengths.
- max_length (`int`, *optional*, defaults to 480000):
- If provided, defines the maximum length of the audio to allow. Audio longer than this will be
- truncated if `truncation=True`.
- truncation (`bool`, *optional*, defaults to `True`):
- Whether or not to truncate audio above `max_length`.
- pad_to_multiple_of (`int`, *optional*, defaults to 128):
- When padding, pad to a multiple of this value. The default value is defined for optimal TPU support.
- return_tensors (`Union[str, TensorType]`, *optional*, defaults to `None`):
- The type of tensors to return (e.g., NumPy, or Torch).
- return_attention_mask (`bool`, *optional*, defaults to `True`):
- Whether to return the attention mask for the generated MEL spectrograms.
- """
-
- is_batched_numpy = isinstance(raw_speech, np.ndarray) and len(raw_speech.shape) > 1
- is_batched_sequence = isinstance(raw_speech, Sequence) and isinstance(raw_speech[0], (np.ndarray, Sequence))
- is_batched = is_batched_numpy or is_batched_sequence
-
- if is_batched:
- raw_speech = [np.asarray([rs]).T for rs in raw_speech]
- elif not is_batched and not isinstance(raw_speech, np.ndarray):
- raw_speech = np.asarray(raw_speech)
-
- if not is_batched: # always return a batch
- raw_speech = [np.asarray([raw_speech])]
-
- batched_speech = self.pad(
- BatchFeature({"input_features": raw_speech}),
- padding=padding,
- max_length=max_length,
- truncation=truncation,
- pad_to_multiple_of=pad_to_multiple_of,
- return_attention_mask=return_attention_mask,
- )
+"""Backwards-compatibility shim: re-exports the legacy ``Gemma4AudioFeatureExtractor`` name as a
+deprecated alias of [`Gemma4AudioProcessor`]. Importing or instantiating the alias emits a
+``FutureWarning``; the alias is removed in transformers v5.15 (see ADR 0002).
+"""
- prepared_speech = []
- prepared_speech_mask = []
- for speech, mask in zip(batched_speech.input_features, batched_speech.attention_mask):
- speech, mask = self._extract_spectrogram(speech.T, mask)
- prepared_speech.append(speech.astype(np.float32))
- prepared_speech_mask.append(mask)
+from ...audio_processing_base import make_legacy_audio_processor_alias
+from .audio_processing_gemma4 import Gemma4AudioProcessor
- prepared_speech = [speech * mask[..., None] for speech, mask in zip(prepared_speech, prepared_speech_mask)]
- return BatchFeature(
- {"input_features": prepared_speech, "input_features_mask": prepared_speech_mask},
- tensor_type=return_tensors,
- )
+Gemma4AudioFeatureExtractor = make_legacy_audio_processor_alias(Gemma4AudioProcessor, "Gemma4AudioFeatureExtractor")
__all__ = ["Gemma4AudioFeatureExtractor"]
diff --git a/src/transformers/models/gemma4_unified/__init__.py b/src/transformers/models/gemma4_unified/__init__.py
index 25bf6e2ee80a..38b35ead011c 100644
--- a/src/transformers/models/gemma4_unified/__init__.py
+++ b/src/transformers/models/gemma4_unified/__init__.py
@@ -19,6 +19,8 @@
if TYPE_CHECKING:
+ from .audio_processing_gemma4_unified import *
+ from .audio_processing_numpy_gemma4_unified import *
from .configuration_gemma4_unified import *
from .feature_extraction_gemma4_unified import *
from .image_processing_gemma4_unified import *
diff --git a/src/transformers/models/gemma4_unified/audio_processing_gemma4_unified.py b/src/transformers/models/gemma4_unified/audio_processing_gemma4_unified.py
new file mode 100644
index 000000000000..5f6656baf999
--- /dev/null
+++ b/src/transformers/models/gemma4_unified/audio_processing_gemma4_unified.py
@@ -0,0 +1,66 @@
+# Copyright 2026 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+
+from ...audio_processing_backends import TorchAudioBackend
+from .audio_processing_numpy_gemma4_unified import Gemma4UnifiedAudioProcessorNumpy
+
+
+class Gemma4UnifiedAudioProcessor(TorchAudioBackend):
+ """Torch sibling of [`Gemma4UnifiedAudioProcessorNumpy`]. Encoder-free audio processor
+ that chunks raw 16 kHz waveforms into fixed-length frames:
+
+ 1. Each waveform is zero-padded to a multiple of ``audio_samples_per_token`` samples
+ (the padded tail stays inside the last frame — it never creates an extra one).
+ 2. The waveform is reshaped to ``(num_tokens, audio_samples_per_token)``; each frame
+ of raw samples becomes one audio soft token (640 samples = 40 ms at 16 kHz).
+ 3. Frames are padded across the batch at the token level; the mask marks every token
+ of a waveform valid, including the final partially-padded frame.
+
+ Unlike the standard Gemma4 audio processor there is no mel spectrogram stage."""
+
+ sampling_rate = 16000
+ force_mono = True
+ padding = "longest"
+ padding_value = 0.0
+
+ # Encoder-free raw-waveform chunking: no STFT/mel stage; see the numpy sibling.
+ do_extract_spectrogram = True
+ do_batch_spectrogram = False
+
+ audio_samples_per_token = Gemma4UnifiedAudioProcessorNumpy.audio_samples_per_token
+ legacy_field_mapping = Gemma4UnifiedAudioProcessorNumpy.legacy_field_mapping
+
+ def __init__(self, audio_samples_per_token: int | None = None, **kwargs):
+ super().__init__(**kwargs)
+ if audio_samples_per_token is not None:
+ self.audio_samples_per_token = audio_samples_per_token
+
+ def extract_spectrogram(self, audio, **kwargs):
+ # Not a spectrogram: chunk each raw waveform into (num_tokens, samples_per_token)
+ # frames. Each frame becomes one audio soft token.
+ return [self._chunk_waveform(waveform) for waveform in audio]
+
+ def _chunk_waveform(self, waveform):
+ """Chunk a 1-D waveform into fixed-length frames of `audio_samples_per_token`
+ samples, zero-padding the tail so the last (partial) frame is kept."""
+ pad_len = (-waveform.shape[-1]) % self.audio_samples_per_token
+ if pad_len:
+ waveform = torch.nn.functional.pad(waveform, (0, pad_len))
+ num_tokens = waveform.shape[-1] // self.audio_samples_per_token
+ return waveform.reshape(num_tokens, self.audio_samples_per_token).to(torch.float32)
+
+
+__all__ = ["Gemma4UnifiedAudioProcessor"]
diff --git a/src/transformers/models/gemma4_unified/audio_processing_numpy_gemma4_unified.py b/src/transformers/models/gemma4_unified/audio_processing_numpy_gemma4_unified.py
new file mode 100644
index 000000000000..003dcdd9de3c
--- /dev/null
+++ b/src/transformers/models/gemma4_unified/audio_processing_numpy_gemma4_unified.py
@@ -0,0 +1,69 @@
+# Copyright 2026 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from ...audio_processing_backends import NumpyAudioBackend
+
+
+def _gemma4_unified_feature_size_to_samples_per_token(value, config_dict):
+ # Legacy configs carry the frame size both as `feature_size` and
+ # `audio_samples_per_token`; keep the modern key if it is already present.
+ config_dict.setdefault("audio_samples_per_token", value)
+
+
+class Gemma4UnifiedAudioProcessorNumpy(NumpyAudioBackend):
+ """NumPy sibling of [`Gemma4UnifiedAudioProcessor`]. Bit-exact to the torch sibling
+ (ADR 0001): the pipeline is pure chunking with no floating-point arithmetic. See
+ [`Gemma4UnifiedAudioProcessor`] for the full pipeline description."""
+
+ sampling_rate = 16000
+ force_mono = True
+ padding = "longest"
+ padding_value = 0.0
+
+ # Encoder-free raw-waveform chunking: there is no STFT/mel stage. Features are
+ # extracted per waveform (`extract_spectrogram` is fully overridden, as sanctioned
+ # for non-spectrogram models) and padded at the token level, matching the legacy
+ # extractor's feature-level `pad()`.
+ do_extract_spectrogram = True
+ do_batch_spectrogram = False
+
+ audio_samples_per_token = 640
+
+ legacy_field_mapping = {
+ "feature_size": _gemma4_unified_feature_size_to_samples_per_token,
+ }
+
+ def __init__(self, audio_samples_per_token: int | None = None, **kwargs):
+ super().__init__(**kwargs)
+ if audio_samples_per_token is not None:
+ self.audio_samples_per_token = audio_samples_per_token
+
+ def extract_spectrogram(self, audio, **kwargs):
+ # Not a spectrogram: chunk each raw waveform into (num_tokens, samples_per_token)
+ # frames. Each frame becomes one audio soft token.
+ return [self._chunk_waveform(waveform) for waveform in audio]
+
+ def _chunk_waveform(self, waveform):
+ """Chunk a 1-D waveform into fixed-length frames of `audio_samples_per_token`
+ samples, zero-padding the tail so the last (partial) frame is kept."""
+ pad_len = (-waveform.shape[-1]) % self.audio_samples_per_token
+ if pad_len:
+ waveform = np.pad(waveform, (0, pad_len))
+ num_tokens = waveform.shape[-1] // self.audio_samples_per_token
+ return waveform.reshape(num_tokens, self.audio_samples_per_token).astype(np.float32)
+
+
+__all__ = ["Gemma4UnifiedAudioProcessorNumpy"]
diff --git a/src/transformers/models/gemma4_unified/feature_extraction_gemma4_unified.py b/src/transformers/models/gemma4_unified/feature_extraction_gemma4_unified.py
index 8bea68b86b1e..85d8c38095eb 100644
--- a/src/transformers/models/gemma4_unified/feature_extraction_gemma4_unified.py
+++ b/src/transformers/models/gemma4_unified/feature_extraction_gemma4_unified.py
@@ -1,151 +1,22 @@
-# Copyright 2026 the HuggingFace Team. All rights reserved.
+# Copyright 2026 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+"""Backwards-compatibility shim: re-exports the legacy ``Gemma4UnifiedAudioFeatureExtractor`` name
+as a deprecated alias of [`Gemma4UnifiedAudioProcessor`]. Importing or instantiating the alias
+emits a ``FutureWarning``; the alias is removed in transformers v5.15 (see ADR 0002).
+"""
+from ...audio_processing_base import make_legacy_audio_processor_alias
+from .audio_processing_gemma4_unified import Gemma4UnifiedAudioProcessor
-import numpy as np
-from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
-from ...image_processing_utils import BatchFeature
-from ...utils import (
- TensorType,
- is_torch_available,
+Gemma4UnifiedAudioFeatureExtractor = make_legacy_audio_processor_alias(
+ Gemma4UnifiedAudioProcessor, "Gemma4UnifiedAudioFeatureExtractor"
)
-if is_torch_available():
- import torch
-
-
-class Gemma4UnifiedAudioFeatureExtractor(SequenceFeatureExtractor):
- """Encoder-free audio feature extractor that chunks raw waveform into frames.
-
- Unlike the standard Gemma4 audio feature extractor which computes mel spectrograms,
- this unified version simply chunks raw 16 kHz audio into fixed-length frames
- of `audio_samples_per_token` samples each. Each frame becomes a single audio
- soft token with the raw waveform samples as its features.
-
- Args:
- feature_size (`int`, *optional*, defaults to 640):
- The feature dimension of the extracted features (samples per token).
- sampling_rate (`int`, *optional*, defaults to 16000):
- The sampling rate at which the audio files should be digitalized expressed in hertz (Hz).
- padding_value (`float`, *optional*, defaults to 0.0):
- Padding value used to pad the audio.
- audio_samples_per_token (`int`, *optional*, defaults to 640):
- Number of raw audio samples per output token. At 16 kHz, 640 samples = 40ms.
- """
-
- model_input_names = ["input_features", "input_features_mask"]
-
- def __init__(
- self,
- feature_size: int = 640,
- sampling_rate: int = 16_000,
- padding_value: float = 0.0,
- audio_samples_per_token: int = 640,
- **kwargs,
- ):
- super().__init__(
- feature_size=feature_size,
- sampling_rate=sampling_rate,
- padding_value=padding_value,
- **kwargs,
- )
- self.audio_samples_per_token = audio_samples_per_token
-
- def _extract_waveform_features(
- self,
- waveform: np.ndarray,
- ) -> tuple[np.ndarray, np.ndarray]:
- """Chunk a raw waveform into fixed-length frames.
-
- Each frame of `audio_samples_per_token` samples becomes one audio soft token.
- The waveform is zero-padded to be evenly divisible by the frame size.
-
- Args:
- waveform: 1-D array of raw audio samples.
-
- Returns:
- features: (num_tokens, audio_samples_per_token) array of waveform frames.
- mask: (num_tokens,) boolean array, True for all valid tokens.
- """
- # Pad waveform to be evenly divisible by samples_per_token
- pad_len = (-len(waveform)) % self.audio_samples_per_token
- if pad_len:
- waveform = np.pad(waveform, (0, pad_len))
-
- num_tokens = len(waveform) // self.audio_samples_per_token
- features = waveform.reshape(num_tokens, self.audio_samples_per_token).astype(np.float32)
-
- # All tokens are valid (padding is within the last frame, not creating extra frames)
- mask = np.ones(num_tokens, dtype=bool)
- return features, mask
-
- def __call__(
- self,
- raw_speech: np.ndarray | list[float] | list[np.ndarray] | list[list[float]],
- padding: bool | str = "longest",
- max_length: int | None = None,
- truncation: bool = True,
- return_tensors: str | TensorType | None = None,
- **kwargs,
- ) -> BatchFeature:
- """Chunk raw audio waveforms into fixed-length frames for the unified model.
-
- Args:
- raw_speech:
- The raw audio waveform(s) to process.
- padding (`str`, *optional*, defaults to `"longest"`):
- Padding strategy for batches with different lengths.
- max_length (`int`, *optional*):
- Maximum number of tokens to produce per audio.
- truncation (`bool`, *optional*, defaults to `True`):
- Whether to truncate audio above `max_length` tokens.
- return_tensors (`str`, *optional*):
- The type of tensors to return.
- """
- # Normalize input to list of 1-D arrays
- if isinstance(raw_speech, np.ndarray) and raw_speech.ndim == 1:
- raw_speech = [raw_speech]
- elif not isinstance(raw_speech, (list, tuple)):
- raw_speech = [np.asarray(raw_speech)]
- else:
- raw_speech = [np.asarray(s) for s in raw_speech]
-
- # Extract features for each waveform
- all_features = [{"input_features": self._extract_waveform_features(waveform)[0]} for waveform in raw_speech]
-
- # Delegate padding and truncation to the parent class
- padded_inputs = self.pad(
- all_features,
- padding=padding,
- max_length=max_length,
- truncation=truncation and max_length is not None,
- return_attention_mask=True,
- return_tensors=return_tensors,
- )
-
- # Rename attention_mask → input_features_mask.
- # pad() produces int32 (0/1); downstream code expects a boolean mask for indexing.
- mask = padded_inputs.pop("attention_mask")
- if is_torch_available() and isinstance(mask, torch.Tensor):
- mask = mask.bool()
- else:
- mask = np.asarray(mask, dtype=bool)
- padded_inputs["input_features_mask"] = mask
-
- return padded_inputs
-
-
__all__ = ["Gemma4UnifiedAudioFeatureExtractor"]
diff --git a/src/transformers/models/granite_speech/audio_processing_granite_speech.py b/src/transformers/models/granite_speech/audio_processing_granite_speech.py
new file mode 100644
index 000000000000..cec423fd5936
--- /dev/null
+++ b/src/transformers/models/granite_speech/audio_processing_granite_speech.py
@@ -0,0 +1,79 @@
+# Copyright 2025 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+
+import torch
+
+from ...audio_processing_backends import TorchAudioBackend
+from ...audio_utils import MelScaleConfig, SpectrogramConfig, StftConfig
+
+
+class GraniteSpeechAudioProcessor(TorchAudioBackend):
+ sampling_rate = 16000
+ force_mono = True
+ return_padding_mask = False
+ do_extract_spectrogram = True
+ projector_window_size = 15
+ projector_downsample_rate = 5
+
+ # Native pipeline, bit-equal to the upstream FE's `torchaudio.transforms.MelSpectrogram`
+ # + log10 + Whisper-style max-clip/rescale (ADR 0004 post-log fields).
+ spectrogram_config = SpectrogramConfig(
+ stft_config=StftConfig(
+ n_fft=512,
+ win_length=400,
+ hop_length=160,
+ power=2.0,
+ ),
+ mel_scale_config=MelScaleConfig(n_mels=80),
+ log_mode="log10",
+ mel_floor=1e-10,
+ clip_max_offset=8.0,
+ post_log_shift=4.0,
+ post_log_scale=0.25,
+ )
+
+ def extract_spectrogram(self, audio, **kwargs):
+ logmel = super().extract_spectrogram(audio, **kwargs).transpose(-1, -2) # (batch, time, n_mels)
+ # Remove last frame if odd, then stack frame pairs
+ if logmel.shape[1] % 2 == 1:
+ logmel = logmel[:, :-1]
+ return logmel.reshape(logmel.shape[0], -1, 2 * logmel.shape[-1])
+
+ def _postprocess_output(self, output, audio_ranges=None, **kwargs):
+ hop_length = self.spectrogram_config.stft_config.hop_length
+
+ # Compute audio_embed_sizes from original audio lengths
+ effective_window_size = self.projector_window_size // self.projector_downsample_rate
+ audio_embed_sizes = []
+ for start, end in audio_ranges:
+ raw_length = end - start
+ mel_length = raw_length // hop_length + 1
+ encoder_length = mel_length // 2
+ nblocks = math.ceil(encoder_length / self.projector_window_size)
+ projector_length = nblocks * effective_window_size
+ audio_embed_sizes.append(projector_length)
+
+ # Build input_features_mask matching the FE
+ input_features_mask = torch.arange(max(audio_embed_sizes)).view(1, -1) < torch.tensor(
+ audio_embed_sizes
+ ).view(-1, 1)
+
+ output["audio_embed_sizes"] = audio_embed_sizes
+ output["audio_features_mask"] = input_features_mask
+ return output
+
+
+__all__ = ["GraniteSpeechAudioProcessor"]
diff --git a/src/transformers/models/granite_speech/feature_extraction_granite_speech.py b/src/transformers/models/granite_speech/feature_extraction_granite_speech.py
index 0d60bdc81e07..ff290e15835a 100644
--- a/src/transformers/models/granite_speech/feature_extraction_granite_speech.py
+++ b/src/transformers/models/granite_speech/feature_extraction_granite_speech.py
@@ -1,195 +1,20 @@
-# Copyright 2025 The HuggingFace Inc. team.
+# Copyright 2026 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Feature extractor class for Granite Speech."""
-
-import math
-from collections.abc import Sequence
-
-import numpy as np
-
-from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
-from ...feature_extraction_utils import BatchFeature
-from ...tokenization_utils_base import AudioInput
-from ...utils import is_torch_available, is_torchaudio_available, logging
-from ...utils.import_utils import requires_backends
-
-
-logger = logging.get_logger(__name__)
-
-if is_torch_available():
- import torch
-
-if is_torchaudio_available():
- import torchaudio
-
-
-class GraniteSpeechFeatureExtractor(SequenceFeatureExtractor):
- model_input_names = ["input_features"]
-
- def __init__(
- self,
- sampling_rate: int = 16000,
- n_fft: int = 512,
- win_length: int = 400,
- hop_length: int = 160,
- n_mels: int = 80,
- projector_window_size: int = 15,
- projector_downsample_rate: int = 5,
- padding_value: float = 0.0,
- **kwargs,
- ):
- # Saving a processor will save `feature_size` which conflict with n-mels
- # While both are the same concept, the naming is different so we pop `feature_size`
- # before calling `super`
- feature_size = n_mels if kwargs.get("feature_size") is None else kwargs.pop("feature_size")
- super().__init__(
- feature_size=feature_size,
- sampling_rate=sampling_rate,
- padding_value=padding_value,
- **kwargs,
- )
- self.sampling_rate = sampling_rate
- self.melspec_kwargs = {
- "sample_rate": sampling_rate,
- "n_fft": n_fft,
- "win_length": win_length,
- "hop_length": hop_length,
- "n_mels": n_mels,
- }
- requires_backends(self, ["torchaudio"])
- self.mel_filters = torchaudio.transforms.MelSpectrogram(**self.melspec_kwargs)
- self.projector_window_size = projector_window_size
- self.projector_downsample_rate = projector_downsample_rate
-
- def __call__(
- self,
- audios: AudioInput,
- device: str | None = "cpu",
- ) -> BatchFeature:
- requires_backends(self, ["torchaudio"])
-
- speech_inputs = {}
- batched_audio, audio_lengths = self._get_audios_and_audio_lengths(audios)
- speech_inputs["input_features"] = self._extract_mel_spectrograms(
- batched_audio,
- device=device,
- )
- audio_embed_sizes = self._get_num_audio_features(audio_lengths)
- speech_inputs["audio_embed_sizes"] = audio_embed_sizes
- # TODO (@alex-jw-brooks): Currently input_features_mask is not
- # a great name, because input_features and input_features_mask
- # have different shapes (before/after the projector).
- #
- # We should align this with other multimodal models, e.g,. llava
- # and qwen2audio and refactor this to ensure input_feature_mask
- # has the same dimensionality as input_features, or compute it in
- # the model based on the audio embedding sizes (since we do not
- # have an attention mask for the audio features to infer padding from).
- speech_inputs["input_features_mask"] = torch.arange(max(audio_embed_sizes)).view(1, -1) < torch.tensor(
- audio_embed_sizes
- ).view(-1, 1)
- return BatchFeature(data=speech_inputs)
-
- def _extract_mel_spectrograms(self, audio: "torch.Tensor", device="cpu"):
- """
- Compute the Mel features to be passed to the conformer encoder.
- """
- requires_backends(self, ["torchaudio"])
- if device is not None:
- melspec = self.mel_filters.to(device)
- audio = audio.to(device)
- else:
- melspec = self.mel_filters
-
- bsz = audio.shape[0]
- with torch.no_grad():
- # Compute mel features
- mel = melspec(audio.float())
- logmel = mel.transpose(-1, -2).clip_(min=1e-10).log10_()
- mx = logmel.amax(dim=(-2, -1), keepdim=True)
- logmel = torch.maximum(logmel, mx - 8.0).div_(4).add_(1)
- # remove last frame if odd
- if logmel.shape[1] % 2 == 1:
- logmel = logmel[:, :-1]
-
- # stacking and skipping by 2
- audio = logmel.reshape(bsz, -1, 2 * logmel.shape[-1])
-
- return audio
-
- def _get_num_audio_features(self, audio_lengths: Sequence[int]) -> Sequence[int]:
- """
- Gets the (variable length) number of features (i.e., projector output) for the sequences
- being considered.
-
- Args:
- audio_lengths (`Sequence[int]`):
- Sequence of one or more raw audio lengths.
- """
- hop_length = self.melspec_kwargs["hop_length"]
- effective_window_size = self.projector_window_size // self.projector_downsample_rate
-
- projector_lengths = []
- for raw_length in audio_lengths:
- # mel sequence length computation
- mel_length = raw_length // hop_length + 1
- # encoder frame takes two mel features
- encoder_length = mel_length // 2
- nblocks = math.ceil(encoder_length / self.projector_window_size)
- # projector output length
- projector_length = nblocks * effective_window_size
- projector_lengths.append(projector_length)
-
- return projector_lengths
-
- def _get_audios_and_audio_lengths(self, audios: AudioInput) -> Sequence["torch.Tensor", Sequence[int]]:
- """
- Coerces audio inputs to torch tensors and extracts audio lengths prior to stacking.
-
- Args:
- audios (`AudioInput`):
- Audio sequence, numpy array, or torch tensor.
- """
- requires_backends(self, ["torch"])
-
- # Coerce to PyTorch tensors if we have numpy arrays, since
- # currently we have a dependency on torch/torchaudio anyway
- if isinstance(audios, np.ndarray):
- audios = torch.from_numpy(audios)
- elif isinstance(audios, Sequence) and isinstance(audios[0], np.ndarray):
- audios = [torch.from_numpy(arr) for arr in audios]
-
- if isinstance(audios, torch.Tensor):
- if audios.ndim == 1:
- audios = audios.unsqueeze(0)
- if not torch.is_floating_point(audios):
- raise ValueError("Invalid audio provided. Audio should be a floating point between 0 and 1")
+"""Backwards-compatibility shim: re-exports the legacy ``GraniteSpeechFeatureExtractor`` name as a
+deprecated alias of [`GraniteSpeechAudioProcessor`]. Importing or instantiating the alias emits a
+``FutureWarning``; the alias is removed in transformers v5.15 (see ADR 0002).
+"""
- if audios.shape[0] > 1:
- logger.warning("Audio samples are already collated; assuming they all have the same length")
- lengths = [audios.shape[-1]] * audios.shape[0]
- return audios, lengths
+from ...audio_processing_base import make_legacy_audio_processor_alias
+from .audio_processing_granite_speech import GraniteSpeechAudioProcessor
- elif isinstance(audios, Sequence) and isinstance(audios[0], torch.Tensor):
- if not torch.is_floating_point(audios[0]):
- raise ValueError("Invalid audio provided. Audio should be a floating point between 0 and 1")
- lengths = [audio.shape[-1] for audio in audios]
- audios = [audio.squeeze(0) for audio in audios]
- audios = torch.nn.utils.rnn.pad_sequence(audios, batch_first=True, padding_value=0.0)
- return audios, lengths
- raise TypeError("Invalid audio provided. Audio should be a one or more torch tensors or numpy arrays")
+GraniteSpeechFeatureExtractor = make_legacy_audio_processor_alias(GraniteSpeechAudioProcessor, "GraniteSpeechFeatureExtractor")
__all__ = ["GraniteSpeechFeatureExtractor"]
diff --git a/src/transformers/models/inkling/__init__.py b/src/transformers/models/inkling/__init__.py
index 90fa10984598..2283a82100be 100644
--- a/src/transformers/models/inkling/__init__.py
+++ b/src/transformers/models/inkling/__init__.py
@@ -19,6 +19,7 @@
if TYPE_CHECKING:
+ from .audio_processing_inkling import *
from .configuration_inkling import *
from .feature_extraction_inkling import *
from .image_processing_inkling import *
diff --git a/src/transformers/models/inkling/audio_processing_inkling.py b/src/transformers/models/inkling/audio_processing_inkling.py
new file mode 100644
index 000000000000..6a66b0d148ca
--- /dev/null
+++ b/src/transformers/models/inkling/audio_processing_inkling.py
@@ -0,0 +1,95 @@
+# Copyright 2026 the HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+
+import torch
+
+from ...audio_processing_backends import TorchAudioBackend
+from ...audio_utils import MelScaleConfig, SpectrogramConfig, StftConfig
+
+
+class InklingAudioProcessor(TorchAudioBackend):
+ """Audio processor for Inkling.
+
+ Produces log10 mel-filterbank features (mel energies in log10 space). Uses the base
+ `_standard_mel_banks` (slaney norm + slaney mel scale — no librosa), a magnitude (not power)
+ spectrogram, and Inkling's fixed framing: `center=False` with a left pad of `n_fft - hop`
+ and a right pad up to a multiple of `hop`. Downstream dMel quantization is done by
+ `InklingProcessor`, not here.
+ """
+
+ sampling_rate = 16000
+ force_mono = True
+ model_input_names = ["input_features", "input_features_mask"]
+ spectrogram_config = SpectrogramConfig(
+ stft_config=StftConfig(
+ n_fft=1600,
+ hop_length=800,
+ win_length=1600,
+ window_fn="hann_window",
+ power=1.0,
+ center=False,
+ periodic=True,
+ ),
+ mel_scale_config=MelScaleConfig(
+ n_mels=80,
+ f_min=0.0,
+ f_max=8000.0,
+ norm="slaney",
+ mel_scale="slaney",
+ ),
+ log_mode="log10",
+ mel_floor=1e-10,
+ )
+
+ def _stft(self, audio, *, spectrogram_config, audio_ranges=None, **kwargs):
+ # Inkling's fixed framing: left-pad (n_fft - hop) and right-pad up to a hop multiple, center=False.
+ stft_cfg = spectrogram_config.stft_config
+ hop, n_fft = stft_cfg.hop_length, stft_cfg.n_fft
+ right_pad = math.ceil(audio.shape[-1] / hop) * hop - audio.shape[-1]
+ left_pad = max(n_fft - hop, 0)
+ audio = torch.nn.functional.pad(audio, (left_pad, right_pad))
+ return super()._stft(audio, spectrogram_config=spectrogram_config, **kwargs)
+
+ def _compute_magnitudes(self, stft_out, power, spectrogram_config=None):
+ magnitudes = torch.view_as_real(stft_out)
+ magnitudes = magnitudes.pow(2).sum(-1).clamp_min(1e-10).sqrt()
+ if power != 1.0:
+ magnitudes = magnitudes.pow(power)
+ return magnitudes
+
+ def _normalize_magnitude(self, features, *, spectrogram_config, **kwargs):
+ # Pointwise log10 (ADR 0005), then transpose to (batch, frames, mels).
+ features = features.clamp_min(spectrogram_config.mel_floor).log10()
+ return features.transpose(1, 2)
+
+ def _get_features_lengths(self, audio_lengths, spectrogram_config, include_center_frame=False):
+ # Inkling emits ceil(audio_length / hop) frames (its right-pad rounds up to a hop multiple).
+ hop = spectrogram_config.stft_config.hop_length
+ return (audio_lengths + hop - 1) // hop
+
+ def _postprocess_output(self, output, audio_ranges=None, feature_ranges=None, **kwargs):
+ # No normalization; zero padded frames and emit the legacy keys the model consumes.
+ # The mask is named `input_features_mask` so it doesn't collide with a text `attention_mask`.
+ features = output.pop("audio_features")
+ mask = output.pop("audio_features_mask", None)
+ if mask is not None:
+ features = features * mask.unsqueeze(-1).to(features.dtype)
+ output["input_features_mask"] = mask
+ output["input_features"] = features
+ return output
+
+
+__all__ = ["InklingAudioProcessor"]
diff --git a/src/transformers/models/inkling/feature_extraction_inkling.py b/src/transformers/models/inkling/feature_extraction_inkling.py
index 257ac4a00d3e..637fc60e3620 100644
--- a/src/transformers/models/inkling/feature_extraction_inkling.py
+++ b/src/transformers/models/inkling/feature_extraction_inkling.py
@@ -5,244 +5,16 @@
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import math
-
-import numpy as np
-import torch
-import torch.nn.functional as F
-
-from ...audio_utils import mel_filter_bank
-from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
-from ...feature_extraction_utils import BatchFeature
-from ...utils import PaddingStrategy, TensorType, logging
-from ...utils.import_utils import requires
-
-
-logger = logging.get_logger(__name__)
-
-
-def _to_exact_int(value: float, name: str, tolerance: float = 1e-6) -> int:
- rounded = round(value)
- if abs(value - rounded) > tolerance:
- raise ValueError(f"{name} must resolve to an integer sample count, got {value}")
- return int(rounded)
-
-
-@requires(backends=("torch",))
-class InklingFeatureExtractor(SequenceFeatureExtractor):
- r"""
- Constructs a TML audio feature extractor, which converts raw audio waveforms into log-mel spectrogram
- features (mel filterbank energies in log10 space). The quantization of these features into discrete
- dMel bins is performed downstream by [`InklingProcessor`].
-
- This feature extractor inherits from [`~feature_extraction_sequence_utils.SequenceFeatureExtractor`]
- which contains most of the main methods. Users should refer to this superclass for more information
- regarding those methods.
-
- Args:
- feature_size (`int`, *optional*, defaults to 80):
- The feature dimension of the extracted features, i.e. the number of mel filterbanks.
- sampling_rate (`int`, *optional*, defaults to 16000):
- The sampling rate at which the audio files should be digitized, expressed in hertz (Hz).
- padding_value (`float`, *optional*, defaults to 0.0):
- The value used to pad the log-mel spectrograms to the same length in a batch.
- audio_token_duration_s (`float`, *optional*, defaults to 0.05):
- Duration, in seconds, represented by a single audio token, i.e. the STFT hop length.
- window_size_multiplier (`float`, *optional*, defaults to 2.0):
- Multiplier applied to `audio_token_duration_s` to obtain the STFT window length.
- n_fft (`int`, *optional*):
- FFT size. Defaults to the window length (`audio_token_duration_s * window_size_multiplier *
- sampling_rate`) when not provided.
- """
-
- model_input_names = ["input_features", "input_features_mask"]
-
- def __init__(
- self,
- feature_size: int = 80,
- sampling_rate: int = 16_000,
- padding_value: float = 0.0,
- audio_token_duration_s: float = 0.05,
- window_size_multiplier: float = 2.0,
- n_fft: int | None = None,
- **kwargs,
- ):
- super().__init__(
- feature_size=feature_size,
- sampling_rate=sampling_rate,
- padding_value=padding_value,
- **kwargs,
- )
- self.audio_token_duration_s = audio_token_duration_s
- self.window_size_multiplier = window_size_multiplier
-
- self.hop_length = _to_exact_int(
- audio_token_duration_s * sampling_rate, "audio_token_duration_s * sampling_rate"
- )
- self.window_size = _to_exact_int(
- audio_token_duration_s * window_size_multiplier * sampling_rate,
- "audio_token_duration_s * window_size_multiplier * sampling_rate",
- )
- self.n_fft = n_fft or self.window_size
- if self.hop_length <= 0 or self.window_size <= 0 or self.n_fft <= 0:
- raise ValueError("hop_length, window_size, and n_fft must all be positive")
-
- # Precomputed once at init, mirrors e.g. WhisperFeatureExtractor.mel_filters.
- self.window = torch.hann_window(self.window_size, periodic=True, dtype=torch.float32)
- # `mel_filter_bank` returns `(num_frequency_bins, feature_size)`; transpose to
- # `(feature_size, num_frequency_bins)` so it left-multiplies the magnitude spectrogram.
- mel_filters = mel_filter_bank(
- num_frequency_bins=self.n_fft // 2 + 1,
- num_mel_filters=feature_size,
- min_frequency=0.0,
- max_frequency=sampling_rate / 2.0,
- sampling_rate=sampling_rate,
- norm="slaney",
- mel_scale="slaney",
- )
- self.mel_filters = torch.from_numpy(np.ascontiguousarray(mel_filters.T, dtype=np.float32))
-
- def _torch_extract_fbank_features(self, waveform: torch.Tensor, device: str = "cpu") -> torch.Tensor:
- right_pad = math.ceil(waveform.shape[-1] / self.hop_length) * self.hop_length - waveform.shape[-1]
- left_pad = max(self.n_fft - self.hop_length, 0)
- waveform = F.pad(waveform, (left_pad, right_pad))
-
- stft = torch.stft(
- waveform,
- self.n_fft,
- hop_length=self.hop_length,
- win_length=self.window_size,
- window=self.window.to(device),
- center=False,
- return_complex=True,
- )
- magnitudes = torch.view_as_real(stft)
- magnitudes = magnitudes.pow(2).sum(-1).clamp_min(1e-10).sqrt()
-
- mel_filters = self.mel_filters.to(device)
- mel_spec = mel_filters @ magnitudes
- mel_spec = mel_spec.clamp_min(1e-10).log10()
-
- # (batch_size, feature_size, num_frames) -> (batch_size, num_frames, feature_size)
- return mel_spec.transpose(1, 2)
-
- def __call__(
- self,
- raw_speech: np.ndarray | list[float] | list[np.ndarray] | list[list[float]],
- sampling_rate: int | None = None,
- padding: bool | str | PaddingStrategy = True,
- max_length: int | None = None,
- truncation: bool = False,
- pad_to_multiple_of: int | None = None,
- return_attention_mask: bool | None = True,
- return_tensors: str | TensorType | None = None,
- device: str | None = "cpu",
- **kwargs,
- ) -> BatchFeature:
- """
- Extract log-mel spectrogram features from one or several audio clip(s).
-
- Args:
- raw_speech (`np.ndarray`, `list[float]`, `list[np.ndarray]`, `list[list[float]]`):
- The sequence or batch of sequences to be padded. Each sequence can be a numpy array, a list
- of float values, a list of numpy arrays or a list of list of float values. Must be mono
- channel audio at `self.sampling_rate`, not stereo, i.e. single float per timestep. Decoding
- and resampling of raw audio (bytes / paths / URLs) is handled upstream by the processor's
- `apply_chat_template`, not here.
- sampling_rate (`int`, *optional*):
- The sampling rate of `raw_speech`, used only to validate against `self.sampling_rate`.
- device (`str`, *optional*, defaults to `"cpu"`):
- The device on which the log-mel spectrogram is computed in `_torch_extract_fbank_features`.
- """
- if sampling_rate is not None:
- if sampling_rate != self.sampling_rate:
- raise ValueError(
- f"The model corresponding to this feature extractor was trained using a sampling "
- f"rate of {self.sampling_rate}. Please make sure that the provided audio input "
- f"was sampled with {self.sampling_rate} and not {sampling_rate}."
- )
- else:
- logger.warning_once(
- "It is strongly recommended to pass the `sampling_rate` argument to this function. "
- "Failing to do so can result in silent errors that might be hard to debug."
- )
-
- cls_name = self.__class__.__name__
-
- def _to_mono(clip: "np.ndarray | torch.Tensor | list") -> torch.Tensor:
- tensor = clip if isinstance(clip, torch.Tensor) else torch.as_tensor(np.asarray(clip))
- tensor = tensor.to(torch.float32)
- if tensor.ndim == 2:
- logger.warning_once(
- f"Only mono-channel audio is supported for input to {cls_name}. "
- "Taking the mean over the channel (last) axis to convert to mono."
- )
- tensor = tensor.mean(dim=-1)
- elif tensor.ndim != 1:
- raise ValueError(
- f"Each audio clip must be 1-D (mono) or 2-D (multichannel), got shape {tuple(tensor.shape)}."
- )
- return tensor
-
- if isinstance(raw_speech, np.ndarray):
- raw_speech = torch.from_numpy(raw_speech)
- if isinstance(raw_speech, torch.Tensor):
- # A single array is one clip: 1-D mono or 2-D multichannel (never a batch).
- if raw_speech.ndim > 2:
- raise ValueError(
- f"A single array input must be 1-D (mono) or 2-D (multichannel); got {raw_speech.ndim} dims. "
- "Pass a list of arrays for a batch of clips."
- )
- clips = [raw_speech]
- elif isinstance(raw_speech, (list, tuple)):
- if len(raw_speech) == 0:
- raise ValueError("Received an empty audio input.")
- # A flat list of scalars is a single mono clip; a list of arrays/lists is a batch of clips.
- if isinstance(raw_speech[0], (int, float, np.integer, np.floating)):
- clips = [raw_speech]
- else:
- clips = list(raw_speech)
- else:
- raise TypeError(f"Unsupported audio input type for {cls_name}: {type(raw_speech)}")
-
- raw_speech = [_to_mono(clip)[:, None] for clip in clips]
-
- # Stack and pad the raw waveforms to the longest clip in the batch, then extract the log-mel
- # spectrogram on the batched audio in a single `torch.stft` pass (mirrors Parakeet).
- audio_lengths = [len(speech) for speech in raw_speech]
- batched_speech = BatchFeature({"input_features": raw_speech, "audio_lengths": audio_lengths})
- padded_inputs = self.pad(
- batched_speech,
- padding=padding,
- max_length=max_length,
- truncation=truncation,
- pad_to_multiple_of=pad_to_multiple_of,
- return_tensors="pt",
- )
- input_waveforms = padded_inputs.input_features.squeeze(-1) # (batch_size, num_samples)
+"""Backwards-compatibility shim: re-exports the legacy ``InklingFeatureExtractor`` name as a
+deprecated alias of [`InklingAudioProcessor`]. Importing or instantiating the alias emits a
+``FutureWarning``; the alias is removed in transformers v5.15 (see ADR 0002).
+"""
- input_features = self._torch_extract_fbank_features(input_waveforms, device) # (batch_size, T, feature_size)
+from ...audio_processing_base import make_legacy_audio_processor_alias
+from .audio_processing_inkling import InklingAudioProcessor
- # Number of valid frames per clip == ceil(audio_length / hop_length); everything beyond it is
- # padding, which we zero out so it carries `padding_value`.
- num_frames = torch.div(
- padded_inputs.audio_lengths + self.hop_length - 1, self.hop_length, rounding_mode="floor"
- )
- input_features_mask = torch.arange(input_features.shape[1], device=device)[None, :] < num_frames[:, None]
- input_features = input_features * input_features_mask.unsqueeze(-1)
- data = {"input_features": input_features}
- if return_attention_mask:
- # Named `input_features_mask` (not `attention_mask`) so it does not collide with the text
- # `attention_mask` when the processor merges audio and text inputs.
- data["input_features_mask"] = input_features_mask
- return BatchFeature(data=data, tensor_type=return_tensors)
+InklingFeatureExtractor = make_legacy_audio_processor_alias(InklingAudioProcessor, "InklingFeatureExtractor")
__all__ = ["InklingFeatureExtractor"]
diff --git a/src/transformers/models/kyutai_speech_to_text/__init__.py b/src/transformers/models/kyutai_speech_to_text/__init__.py
index 5823883c6cb8..9418f465b4fb 100644
--- a/src/transformers/models/kyutai_speech_to_text/__init__.py
+++ b/src/transformers/models/kyutai_speech_to_text/__init__.py
@@ -18,6 +18,8 @@
if TYPE_CHECKING:
+ from .audio_processing_kyutai_speech_to_text import *
+ from .audio_processing_numpy_kyutai_speech_to_text import *
from .configuration_kyutai_speech_to_text import *
from .feature_extraction_kyutai_speech_to_text import *
from .modeling_kyutai_speech_to_text import *
diff --git a/src/transformers/models/kyutai_speech_to_text/audio_processing_kyutai_speech_to_text.py b/src/transformers/models/kyutai_speech_to_text/audio_processing_kyutai_speech_to_text.py
new file mode 100644
index 000000000000..2bb328d91e14
--- /dev/null
+++ b/src/transformers/models/kyutai_speech_to_text/audio_processing_kyutai_speech_to_text.py
@@ -0,0 +1,43 @@
+# Copyright 2025 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+
+from ...audio_processing_backends import TorchAudioBackend
+
+
+class KyutaiSpeechToTextAudioProcessor(TorchAudioBackend):
+ sampling_rate = 24000
+ force_mono = True
+ add_channel_dim = True
+ audio_delay_seconds = 2.5
+ audio_silence_prefix_seconds = 1.0
+
+ def _postprocess_output(self, output, **kwargs):
+ # Add silence prefix (left) and delay (right) padding
+ pad_left = int(self.audio_silence_prefix_seconds * self.sampling_rate)
+ pad_right = int((self.audio_delay_seconds + 1.0) * self.sampling_rate)
+
+ if pad_left > 0 or pad_right > 0:
+ output["audio_values"] = torch.nn.functional.pad(
+ output["audio_values"], (pad_left, pad_right), mode="constant", value=0.0,
+ )
+ output["audio_values_mask"] = torch.nn.functional.pad(
+ output["audio_values_mask"], (pad_left, pad_right), mode="constant", value=0,
+ )
+
+ return output
+
+
+__all__ = ["KyutaiSpeechToTextAudioProcessor"]
diff --git a/src/transformers/models/kyutai_speech_to_text/audio_processing_numpy_kyutai_speech_to_text.py b/src/transformers/models/kyutai_speech_to_text/audio_processing_numpy_kyutai_speech_to_text.py
new file mode 100644
index 000000000000..bb012a803bd3
--- /dev/null
+++ b/src/transformers/models/kyutai_speech_to_text/audio_processing_numpy_kyutai_speech_to_text.py
@@ -0,0 +1,46 @@
+# Copyright 2026 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from ...audio_processing_backends import NumpyAudioBackend
+
+
+class KyutaiSpeechToTextAudioProcessorNumpy(NumpyAudioBackend):
+ """NumPy sibling of [`KyutaiSpeechToTextAudioProcessor`]. Raw-audio passthrough with
+ a silence prefix / delay suffix applied in `_postprocess_output` (ADR 0001)."""
+
+ sampling_rate = 24000
+ force_mono = True
+ add_channel_dim = True
+ audio_delay_seconds = 2.5
+ audio_silence_prefix_seconds = 1.0
+
+ def _postprocess_output(self, output, **kwargs):
+ # Add silence prefix (left) and delay (right) padding
+ pad_left = int(self.audio_silence_prefix_seconds * self.sampling_rate)
+ pad_right = int((self.audio_delay_seconds + 1.0) * self.sampling_rate)
+
+ if pad_left > 0 or pad_right > 0:
+ output["audio_values"] = np.pad(
+ output["audio_values"], [(0, 0), (0, 0), (pad_left, pad_right)], mode="constant", constant_values=0.0,
+ )
+ output["audio_values_mask"] = np.pad(
+ output["audio_values_mask"], [(0, 0), (pad_left, pad_right)], mode="constant", constant_values=0,
+ )
+
+ return output
+
+
+__all__ = ["KyutaiSpeechToTextAudioProcessorNumpy"]
diff --git a/src/transformers/models/kyutai_speech_to_text/feature_extraction_kyutai_speech_to_text.py b/src/transformers/models/kyutai_speech_to_text/feature_extraction_kyutai_speech_to_text.py
index b472473a19e5..bc8c04081ec6 100644
--- a/src/transformers/models/kyutai_speech_to_text/feature_extraction_kyutai_speech_to_text.py
+++ b/src/transformers/models/kyutai_speech_to_text/feature_extraction_kyutai_speech_to_text.py
@@ -1,234 +1,20 @@
-# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
-# This file was automatically generated from src/transformers/models/kyutai_speech_to_text/modular_kyutai_speech_to_text.py.
-# Do NOT edit this file manually as any edits will be overwritten by the generation of
-# the file from the modular. If any change should be done, please apply the change to the
-# modular_kyutai_speech_to_text.py file directly. One of our CI enforces this.
-# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
-# Copyright 2025 Kyutai and The HuggingFace Inc. team. All rights reserved.
+# Copyright 2026 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-import numpy as np
-
-from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
-from ...feature_extraction_utils import BatchFeature
-from ...utils import PaddingStrategy, TensorType, logging
-
-
-logger = logging.get_logger(__name__)
-
-
-class KyutaiSpeechToTextFeatureExtractor(SequenceFeatureExtractor):
- r"""
- Constructs an KyutaiSpeechToText feature extractor.
-
- This feature extractor inherits from [`~feature_extraction_sequence_utils.SequenceFeatureExtractor`] which contains
- most of the main methods. Users should refer to this superclass for more information regarding those methods.
-
- Args:
- feature_size (`int`, *optional*, defaults to 1):
- The feature dimension of the extracted features. Use 1 for mono, 2 for stereo.
- sampling_rate (`int`, *optional*, defaults to 24000):
- The sampling rate at which the audio waveform should be digitalized expressed in hertz (Hz).
- padding_value (`float`, *optional*, defaults to 0.0):
- The value that is used to fill the padding values.
- chunk_length_s (`float`, *optional*):
- If defined the audio is pre-processed into chunks of lengths `chunk_length_s` and then encoded.
- overlap (`float`, *optional*):
- Defines the overlap between each chunk. It is used to compute the `chunk_stride` using the following
- formulae : `int((1.0 - self.overlap) * self.chunk_length)`.
- audio_delay_seconds (`float`, *optional*, defaults to 0.0):
- The delay in seconds to add after the audio (right padding).
- audio_silence_prefix_seconds (`float`, *optional*, defaults to 0.0):
- The silence prefix in seconds to add before the audio (left padding).
- """
-
- model_input_names = ["input_values", "padding_mask"]
-
- def __init__(
- self,
- feature_size: int = 1,
- sampling_rate: int = 24000,
- padding_value: float = 0.0,
- chunk_length_s: float | None = None,
- overlap: float | None = None,
- audio_delay_seconds: float | None = 0.0,
- audio_silence_prefix_seconds: float | None = 0.0,
- **kwargs,
- ):
- super().__init__(feature_size=feature_size, sampling_rate=sampling_rate, padding_value=padding_value, **kwargs)
- self.chunk_length_s = chunk_length_s
- self.overlap = overlap
- self.audio_delay_seconds = audio_delay_seconds
- self.audio_silence_prefix_seconds = audio_silence_prefix_seconds
-
- # This is a property because you might want to change the chunk_length_s on the fly
- @property
- def chunk_length(self) -> int | None:
- if self.chunk_length_s is None:
- return None
- else:
- return int(self.chunk_length_s * self.sampling_rate)
-
- # This is a property because you might want to change the chunk_length_s on the fly
- @property
- def chunk_stride(self) -> int | None:
- if self.chunk_length_s is None or self.overlap is None:
- return None
- else:
- return max(1, int((1.0 - self.overlap) * self.chunk_length))
-
- def __call__(
- self,
- raw_audio: np.ndarray | list[float] | list[np.ndarray] | list[list[float]],
- padding: bool | str | PaddingStrategy | None = None,
- truncation: bool | None = False,
- max_length: int | None = None,
- return_tensors: str | TensorType | None = None,
- sampling_rate: int | None = None,
- ) -> BatchFeature:
- """
- Main method to featurize and prepare for the model one or several sequence(s).
-
- Args:
- raw_audio (`np.ndarray`, `list[float]`, `list[np.ndarray]`, `list[list[float]]`):
- The sequence or batch of sequences to be processed. Each sequence can be a numpy array, a list of float
- values, a list of numpy arrays or a list of list of float values. The numpy array must be of shape
- `(num_samples,)` for mono audio (`feature_size = 1`), or `(2, num_samples)` for stereo audio
- (`feature_size = 2`).
- padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `True`):
- Select a strategy to pad the returned sequences (according to the model's padding side and padding
- index) among:
-
- - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
- sequence if provided).
- - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
- acceptable input length for the model if that argument is not provided.
- - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
- lengths).
- truncation (`bool`, *optional*, defaults to `False`):
- Activates truncation to cut input sequences longer than `max_length` to `max_length`.
- max_length (`int`, *optional*):
- Maximum length of the returned list and optionally padding length (see above).
- return_tensors (`str` or [`~utils.TensorType`], *optional*):
- If set, will return tensors instead of list of python integers. Acceptable values are:
-
- - `'pt'`: Return PyTorch `torch.Tensor` objects.
- - `'np'`: Return Numpy `np.ndarray` objects.
- sampling_rate (`int`, *optional*):
- The sampling rate at which the `audio` input was sampled. It is strongly recommended to pass
- `sampling_rate` at the forward call to prevent silent errors.
- """
- if sampling_rate is not None:
- if sampling_rate != self.sampling_rate:
- raise ValueError(
- f"The model corresponding to this feature extractor: {self} was trained using a sampling rate of"
- f" {self.sampling_rate}. Please make sure that the provided audio input was sampled with"
- f" {self.sampling_rate} and not {sampling_rate}."
- )
- else:
- logger.warning(
- f"It is strongly recommended to pass the `sampling_rate` argument to `{self.__class__.__name__}()`. "
- "Failing to do so can result in silent errors that might be hard to debug."
- )
-
- if padding and truncation:
- raise ValueError("Both padding and truncation were set. Make sure you only set one.")
- elif padding is None:
- # by default let's pad the inputs
- padding = True
-
- is_batched = bool(
- isinstance(raw_audio, (list, tuple)) and (isinstance(raw_audio[0], (np.ndarray, tuple, list)))
- )
-
- if is_batched:
- raw_audio = [np.asarray(audio, dtype=np.float32).T for audio in raw_audio]
- elif not is_batched and not isinstance(raw_audio, np.ndarray):
- raw_audio = np.asarray(raw_audio, dtype=np.float32)
- elif isinstance(raw_audio, np.ndarray) and raw_audio.dtype is np.dtype(np.float64):
- raw_audio = raw_audio.astype(np.float32)
-
- # always return batch
- if not is_batched:
- raw_audio = [np.asarray(raw_audio).T]
-
- # verify inputs are valid
- for idx, example in enumerate(raw_audio):
- if example.ndim > 2:
- raise ValueError(f"Expected input shape (channels, length) but got shape {example.shape}")
- if self.feature_size == 1 and example.ndim != 1:
- raise ValueError(f"Expected mono audio but example has {example.shape[-1]} channels")
- if self.feature_size == 2 and example.shape[-1] != 2:
- raise ValueError(f"Expected stereo audio but example has {example.shape[-1]} channels")
-
- padded_inputs = None
- input_values = BatchFeature({"input_values": raw_audio})
- if self.chunk_stride is not None and self.chunk_length is not None and max_length is None:
- if truncation:
- max_length = min(array.shape[0] for array in raw_audio)
- nb_step = int(np.floor(max_length / self.chunk_stride))
- max_length = (nb_step - 1) * self.chunk_stride + self.chunk_length
- elif padding:
- max_length = max(array.shape[0] for array in raw_audio)
- nb_step = int(np.ceil(max_length / self.chunk_stride))
- max_length = (nb_step - 1) * self.chunk_stride + self.chunk_length
- padding = "max_length"
- else:
- padded_inputs = input_values
-
- # normal padding on batch
- if padded_inputs is None:
- padded_inputs = self.pad(
- input_values,
- max_length=max_length,
- truncation=truncation,
- padding=padding,
- return_attention_mask=padding,
- )
-
- if padding:
- padded_inputs["padding_mask"] = padded_inputs.pop("attention_mask")
-
- # now let's pad left and right
- pad_left = int(self.audio_silence_prefix_seconds * self.sampling_rate)
- pad_right = int((self.audio_delay_seconds + 1.0) * self.sampling_rate)
- padded_inputs["input_values"] = np.pad(
- padded_inputs["input_values"],
- ((0, 0), (pad_left, pad_right)),
- mode="constant",
- constant_values=0.0,
- )
- if padding:
- padded_inputs["padding_mask"] = np.pad(
- padded_inputs["padding_mask"],
- ((0, 0), (pad_left, pad_right)),
- mode="constant",
- constant_values=0,
- )
+"""Backwards-compatibility shim: re-exports the legacy ``KyutaiSpeechToTextFeatureExtractor`` name as a
+deprecated alias of [`KyutaiSpeechToTextAudioProcessor`]. Importing or instantiating the alias emits a
+``FutureWarning``; the alias is removed in transformers v5.15 (see ADR 0002).
+"""
- input_values = []
- for example in padded_inputs.pop("input_values"):
- if self.feature_size == 1:
- example = example[..., None]
- input_values.append(example.T)
+from ...audio_processing_base import make_legacy_audio_processor_alias
+from .audio_processing_kyutai_speech_to_text import KyutaiSpeechToTextAudioProcessor
- padded_inputs["input_values"] = input_values
- if return_tensors is not None:
- padded_inputs = padded_inputs.convert_to_tensors(return_tensors)
- return padded_inputs
+KyutaiSpeechToTextFeatureExtractor = make_legacy_audio_processor_alias(KyutaiSpeechToTextAudioProcessor, "KyutaiSpeechToTextFeatureExtractor")
__all__ = ["KyutaiSpeechToTextFeatureExtractor"]
diff --git a/src/transformers/models/kyutai_speech_to_text/modular_kyutai_speech_to_text.py b/src/transformers/models/kyutai_speech_to_text/modular_kyutai_speech_to_text.py
index af59b595b3d1..3a89f8c6141a 100644
--- a/src/transformers/models/kyutai_speech_to_text/modular_kyutai_speech_to_text.py
+++ b/src/transformers/models/kyutai_speech_to_text/modular_kyutai_speech_to_text.py
@@ -25,7 +25,7 @@
from ...modeling_utils import PreTrainedModel
from ...utils import PaddingStrategy, TensorType, logging
from ..auto import AutoModel
-from ..encodec.feature_extraction_encodec import EncodecFeatureExtractor
+from ..encodec.audio_processing_encodec import EncodecAudioProcessor
from ..llama.modeling_llama import LlamaForCausalLM
from ..mimi.modeling_mimi import MimiConv1dPaddingCache
from ..moshi.modeling_moshi import MoshiModel, MoshiPreTrainedModel
@@ -34,7 +34,7 @@
logger = logging.get_logger(__name__)
-class KyutaiSpeechToTextFeatureExtractor(EncodecFeatureExtractor):
+class KyutaiSpeechToTextFeatureExtractor(EncodecAudioProcessor):
r"""
Constructs an KyutaiSpeechToText feature extractor.
diff --git a/src/transformers/models/lasr/audio_processing_lasr.py b/src/transformers/models/lasr/audio_processing_lasr.py
new file mode 100644
index 000000000000..1da4b14113c4
--- /dev/null
+++ b/src/transformers/models/lasr/audio_processing_lasr.py
@@ -0,0 +1,48 @@
+# Copyright 2025 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ...audio_processing_backends import TorchAudioBackend
+from ...audio_utils import MelScaleConfig, SpectrogramConfig, StftConfig
+
+
+class LasrAudioProcessor(TorchAudioBackend):
+ sampling_rate = 16000
+ force_mono = True
+ spectrogram_config = SpectrogramConfig(
+ stft_config=StftConfig(
+ n_fft=512,
+ hop_length=160,
+ win_length=400,
+ power=2.0,
+ center=False,
+ periodic=False,
+ left_align_fft=True,
+ ),
+ mel_scale_config=MelScaleConfig(
+ n_mels=128,
+ f_min=125.0,
+ f_max=7500.0,
+ mel_scale="kaldi",
+ triangularize_in_mel_space=True,
+ bands_to_zero=1,
+ computation_dtype="float64",
+ matmul_order="features_first",
+ ),
+ log_mode="log",
+ mel_floor=1e-5,
+ computation_dtype="float64",
+ )
+
+
+__all__ = ["LasrAudioProcessor"]
diff --git a/src/transformers/models/lasr/feature_extraction_lasr.py b/src/transformers/models/lasr/feature_extraction_lasr.py
index 7cf1822ee40d..29b5205e0348 100644
--- a/src/transformers/models/lasr/feature_extraction_lasr.py
+++ b/src/transformers/models/lasr/feature_extraction_lasr.py
@@ -1,275 +1,20 @@
-# Copyright 2025 The HuggingFace Inc. team and Google LLC. All rights reserved.
+# Copyright 2026 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import numpy as np
-import torch
-
-from ...audio_utils import hertz_to_mel
-from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
-from ...feature_extraction_utils import BatchFeature
-from ...utils import TensorType, logging
-from ...utils.import_utils import requires
-
-
-logger = logging.get_logger(__name__)
-
-
-# TODO: @eustlb, we should be able to remove this and use mel_filter_bank from audio_utils
-def linear_to_mel_weight_matrix(
- num_mel_bins: int,
- num_spectrogram_bins: int,
- sample_rate: float,
- lower_edge_hertz: float,
- upper_edge_hertz: float,
- dtype,
-) -> np.ndarray:
- """NumPy-port of the JAX mel weight matrix logic."""
- # We use float64 for precision, matching the JAX implementation.
- internal_dtype = np.float64
-
- # HTK excludes the spectrogram DC bin.
- bands_to_zero = 1
- nyquist_hertz = sample_rate / 2.0
- linear_frequencies = np.linspace(0.0, nyquist_hertz, num_spectrogram_bins, dtype=internal_dtype)[bands_to_zero:]
- spectrogram_bins_mel = hertz_to_mel(linear_frequencies, mel_scale="kaldi")[:, np.newaxis]
-
- edges = np.linspace(
- hertz_to_mel(lower_edge_hertz, mel_scale="kaldi"),
- hertz_to_mel(upper_edge_hertz, mel_scale="kaldi"),
- num_mel_bins + 2,
- dtype=internal_dtype,
- )
-
- lower_edge_mel, center_mel, upper_edge_mel = (
- edges[:-2][np.newaxis, :],
- edges[1:-1][np.newaxis, :],
- edges[2:][np.newaxis, :],
- )
-
- lower_slopes = (spectrogram_bins_mel - lower_edge_mel) / (center_mel - lower_edge_mel)
- upper_slopes = (upper_edge_mel - spectrogram_bins_mel) / (upper_edge_mel - center_mel)
- mel_weights_matrix = np.maximum(0.0, np.minimum(lower_slopes, upper_slopes))
- return np.pad(mel_weights_matrix, [[bands_to_zero, 0], [0, 0]]).astype(dtype)
-
-
-@requires(backends=("torch",))
-class LasrFeatureExtractor(SequenceFeatureExtractor):
- r"""
- Constructs a LASR feature extractor.
-
- This feature extractor inherits from [`~feature_extraction_sequence_utils.SequenceFeatureExtractor`] which contains
- most of the main methods. Users should refer to this superclass for more information regarding those methods.
-
- This class extracts mel-filter bank features from raw speech using a custom numpy implementation of the `Short Time
- Fourier Transform` which should match pytorch's `torch.stft` equivalent.
-
- Args:
- feature_size (`int`, *optional*, defaults to 128):
- The feature dimension of the extracted features.
- sampling_rate (`int`, *optional*, defaults to 16000):
- The sampling rate at which the audio files should be digitalized expressed in hertz (Hz).
- hop_length (`int`, *optional*, defaults to 160):
- Length of the overlapping windows for the STFT used to obtain the Mel Frequency coefficients.
- n_fft (`int`, *optional*, defaults to 512):
- Size of the Fourier transform.
- win_length (`int`, *optional*, defaults to 400):
- The window length for the STFT computation.
- padding_value (`float`, *optional*, defaults to 0.0):
- Padding value used to pad the audio. Should correspond to silences.
- """
-
- model_input_names = ["input_features", "attention_mask"]
-
- def __init__(
- self,
- feature_size=128,
- sampling_rate=16000,
- hop_length=160,
- n_fft=512,
- win_length=400,
- padding_value=0.0,
- **kwargs,
- ):
- super().__init__(feature_size=feature_size, sampling_rate=sampling_rate, padding_value=padding_value, **kwargs)
-
- self.hop_length = hop_length
- self.n_fft = n_fft
- self.win_length = win_length
- self.mel_filters = torch.from_numpy(
- linear_to_mel_weight_matrix(
- num_mel_bins=feature_size,
- num_spectrogram_bins=n_fft // 2 + 1,
- sample_rate=sampling_rate,
- lower_edge_hertz=125.0,
- upper_edge_hertz=7500.0,
- dtype=np.float64,
- )
- )
-
- def _torch_extract_fbank_features(self, waveform, device="cpu"):
- # spectrogram
- window = torch.hann_window(self.win_length, periodic=False, device=device, dtype=torch.float64)
- waveform = waveform.to(torch.float64)
-
- # TODO: @eustlb, to be standardized
- # here we cannot use directly torch.stft because every fft frame is padded with zeros
- # due to unfold then rfft, while torch.stft unfolds with the number of fft points
- frames = waveform.unfold(-1, self.win_length, self.hop_length)
- stft = torch.fft.rfft(window * frames, n=self.n_fft)
- power_spec = torch.abs(stft) ** 2
-
- # log mel spectrogram
- mel_filters = self.mel_filters.to(device)
- mel_spec = torch.clamp(power_spec @ mel_filters, min=1e-5)
- mel_spec = torch.log(mel_spec)
-
- return mel_spec
-
- def __call__(
- self,
- raw_speech: np.ndarray | list[float] | list[np.ndarray] | list[list[float]],
- truncation: bool = False,
- pad_to_multiple_of: int | None = None,
- return_tensors: str | TensorType | None = None,
- return_attention_mask: bool | None = None,
- padding: str | None = "longest",
- max_length: int | None = None,
- sampling_rate: int | None = None,
- do_normalize: bool | None = None,
- device: str | None = "cpu",
- return_token_timestamps: bool | None = None,
- **kwargs,
- ) -> BatchFeature:
- """
- Main method to featurize and prepare for the model one or several sequence(s). Implementation uses PyTorch for
- the STFT computation if available, otherwise a slower NumPy based one.
-
- Args:
- raw_speech (`np.ndarray`, `list[float]`, `list[np.ndarray]`, `list[list[float]]`):
- The sequence or batch of sequences to be padded. Each sequence can be a numpy array, a list of float
- values, a list of numpy arrays or a list of list of float values. Must be mono channel audio, not
- stereo, i.e. single float per timestep.
- truncation (`bool`, *optional*, default to `True`):
- Activates truncation to cut input sequences longer than *max_length* to *max_length*.
- pad_to_multiple_of (`int`, *optional*, defaults to None):
- If set will pad the sequence to a multiple of the provided value.
-
- This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability
- `>= 7.5` (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128.
- return_attention_mask (`bool`, *optional*):
- Whether to return the attention mask. If left to the default, will return the attention mask according
- to the specific feature_extractor's default.
-
- [What are attention masks?](../glossary#attention-mask)
-
-
-
- For Parakeet models, `attention_mask` should always be passed for batched inference, to avoid subtle
- bugs.
-
-
-
- return_tensors (`str` or [`~utils.TensorType`], *optional*):
- If set, will return tensors instead of list of python integers. Acceptable values are:
-
- - `'tf'`: Return TensorFlow `tf.constant` objects.
- - `'pt'`: Return PyTorch `torch.Tensor` objects.
- - `'np'`: Return Numpy `np.ndarray` objects.
- sampling_rate (`int`, *optional*):
- The sampling rate at which the `raw_speech` input was sampled. It is strongly recommended to pass
- `sampling_rate` at the forward call to prevent silent errors and allow automatic speech recognition
- pipeline.
- padding_value (`float`, *optional*, defaults to 0.0):
- The value that is used to fill the padding values / vectors.
- do_normalize (`bool`, *optional*, defaults to `False`):
- Whether or not to zero-mean unit-variance normalize the input. Normalizing can help to significantly
- improve the performance of the model.
- device (`str`, *optional*, defaults to `'cpu'`):
- Specifies the device for computation of the log-mel spectrogram of audio signals in the
- `_torch_extract_fbank_features` method. (e.g., "cpu", "cuda")
- return_token_timestamps (`bool`, *optional*, defaults to `None`):
- Deprecated. Use `return_attention_mask` instead from which the number of frames can be inferred.
-
- Whether or not to return the number of frames of the input raw_speech.
- These num_frames can be used by the model to compute word level timestamps.
- """
- if sampling_rate is not None:
- if sampling_rate != self.sampling_rate:
- raise ValueError(
- f"The model corresponding to this feature extractor: {self.__class__.__name__} was trained using a"
- f" sampling rate of {self.sampling_rate}. Please make sure that the provided `raw_speech` input"
- f" was sampled with {self.sampling_rate} and not {sampling_rate}."
- )
- else:
- logger.warning(
- f"It is strongly recommended to pass the `sampling_rate` argument to `{self.__class__.__name__}()`. "
- "Failing to do so can result in silent errors that might be hard to debug."
- )
-
- # Convert to torch tensor
- if isinstance(raw_speech, np.ndarray):
- raw_speech = torch.tensor(raw_speech)
- elif isinstance(raw_speech, (list, tuple)):
- if isinstance(raw_speech[0], (list, np.ndarray)):
- raw_speech = [torch.tensor(speech) for speech in raw_speech]
- else: # list[float]
- raw_speech = torch.tensor(raw_speech)
-
- is_batched_torch = isinstance(raw_speech, torch.Tensor) and len(raw_speech.shape) > 1
- if is_batched_torch and len(raw_speech.shape) > 2:
- logger.warning(
- f"Only mono-channel audio is supported for input to {self.__class__.__name__}. "
- "We will take the mean of the channels to convert to mono."
- )
- raw_speech = raw_speech.mean(-1)
-
- is_batched_sequence = isinstance(raw_speech, (list, tuple))
- if is_batched_sequence:
- for speech in raw_speech:
- if len(speech.shape) > 1:
- logger.warning(
- f"Only mono-channel audio is supported for input to {self.__class__.__name__}. "
- "We will take the mean of the channels to convert to mono."
- )
- speech = speech.mean(-1)
-
- if is_batched_torch or is_batched_sequence:
- raw_speech = [speech[:, None].to(torch.float32) for speech in raw_speech]
- else:
- raw_speech = [raw_speech[:, None].to(torch.float32)]
+"""Backwards-compatibility shim: re-exports the legacy ``LasrFeatureExtractor`` name as a
+deprecated alias of [`LasrAudioProcessor`]. Importing or instantiating the alias emits a
+``FutureWarning``; the alias is removed in transformers v5.15 (see ADR 0002).
+"""
- batched_speech = BatchFeature({"input_features": raw_speech})
- padded_inputs = self.pad(
- batched_speech,
- padding=padding,
- max_length=max_length,
- truncation=truncation,
- pad_to_multiple_of=pad_to_multiple_of,
- return_attention_mask=return_attention_mask,
- return_tensors="pt",
- )
- input_features = padded_inputs.input_features.squeeze(-1)
- input_features = self._torch_extract_fbank_features(input_features, device)
- data = {
- "input_features": input_features.to(torch.float32),
- }
+from ...audio_processing_base import make_legacy_audio_processor_alias
+from .audio_processing_lasr import LasrAudioProcessor
- if return_attention_mask:
- attention_mask = padded_inputs.attention_mask[:, self.win_length - 1 :: self.hop_length]
- data["attention_mask"] = attention_mask.to(torch.bool)
- return BatchFeature(data=data, tensor_type=return_tensors)
+LasrFeatureExtractor = make_legacy_audio_processor_alias(LasrAudioProcessor, "LasrFeatureExtractor")
__all__ = ["LasrFeatureExtractor"]
diff --git a/src/transformers/models/markuplm/feature_extraction_markuplm.py b/src/transformers/models/markuplm/feature_extraction_markuplm.py
index b049fef2c191..43357653bfce 100644
--- a/src/transformers/models/markuplm/feature_extraction_markuplm.py
+++ b/src/transformers/models/markuplm/feature_extraction_markuplm.py
@@ -17,8 +17,9 @@
import html
-from ...feature_extraction_utils import BatchFeature, FeatureExtractionMixin
-from ...utils import is_bs4_available, logging, requires_backends
+from ...feature_extraction_utils import BatchFeature
+from ...preprocessing_base import PreprocessingMixin
+from ...utils import FEATURE_EXTRACTOR_NAME, is_bs4_available, logging, requires_backends
if is_bs4_available():
@@ -29,16 +30,25 @@
logger = logging.get_logger(__name__)
-class MarkupLMFeatureExtractor(FeatureExtractionMixin):
+class MarkupLMFeatureExtractor(PreprocessingMixin):
r"""
Constructs a MarkupLM feature extractor. This can be used to get a list of nodes and corresponding xpaths from HTML
strings.
- This feature extractor inherits from [`~feature_extraction_utils.PreTrainedFeatureExtractor`] which contains most
- of the main methods. Users should refer to this superclass for more information regarding those methods.
+ This feature extractor inherits from [`~preprocessing_base.PreprocessingMixin`] which contains most of the main
+ methods. Users should refer to this superclass for more information regarding those methods.
"""
+ # MarkupLM extracts nodes/xpaths from HTML rather than audio or images, so it builds directly on
+ # `PreprocessingMixin` and carries the feature-extractor identity itself (config filename, config
+ # type key and auto class are unchanged, so existing checkpoints keep loading).
+ _config_name = FEATURE_EXTRACTOR_NAME
+ _type_key = "feature_extractor_type"
+ _nested_config_keys = ["feature_extractor"]
+ _auto_class_default = "AutoFeatureExtractor"
+ _file_type_label = "feature extractor"
+
def __init__(self, **kwargs):
requires_backends(self, ["bs4"])
super().__init__(**kwargs)
diff --git a/src/transformers/models/musicgen_melody/__init__.py b/src/transformers/models/musicgen_melody/__init__.py
index 51456aac76b0..2e4385fa00ae 100644
--- a/src/transformers/models/musicgen_melody/__init__.py
+++ b/src/transformers/models/musicgen_melody/__init__.py
@@ -19,6 +19,7 @@
if TYPE_CHECKING:
from .configuration_musicgen_melody import *
+ from .feature_extraction_musicgen_melody import *
from .modeling_musicgen_melody import *
else:
import sys
diff --git a/src/transformers/models/musicgen_melody/audio_processing_musicgen_melody.py b/src/transformers/models/musicgen_melody/audio_processing_musicgen_melody.py
new file mode 100644
index 000000000000..43f0acf128b8
--- /dev/null
+++ b/src/transformers/models/musicgen_melody/audio_processing_musicgen_melody.py
@@ -0,0 +1,81 @@
+# Copyright 2025 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ...audio_processing_backends import TorchAudioBackend
+from ...utils.import_utils import requires
+
+
+class MusicgenMelodyAudioProcessor(TorchAudioBackend):
+ sampling_rate = 32000
+ force_mono = True
+ do_extract_spectrogram = True
+ return_padding_mask = False
+ n_fft = 16384
+ hop_length = 4096
+ n_chroma = 12
+ chunk_length = 30
+
+ @requires(backends=("librosa", "torch"))
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+ import librosa
+ import torch
+
+ self.chroma_filters = torch.from_numpy(
+ librosa.filters.chroma(sr=self.sampling_rate, n_fft=self.n_fft, tuning=0, n_chroma=self.n_chroma)
+ ).float()
+
+ def extract_spectrogram(self, audio, **kwargs):
+ import torch
+ import torchaudio
+
+ waveform = audio # Already a batched tensor from _to_batch
+ device = waveform.device
+ batch_size = waveform.shape[0]
+
+ # Pad if too short for FFT
+ if waveform.shape[-1] < self.n_fft:
+ pad = self.n_fft - waveform.shape[-1]
+ rest = 0 if pad % 2 == 0 else 1
+ waveform = torch.nn.functional.pad(waveform, (pad // 2, pad // 2 + rest), "constant", 0)
+
+ # Add channel dim for spectrogram: (batch, 1, length)
+ waveform = waveform.unsqueeze(1)
+
+ # Power spectrogram (normalized)
+ spec_transform = torchaudio.transforms.Spectrogram(
+ n_fft=self.n_fft, win_length=self.n_fft, hop_length=self.hop_length,
+ power=2, center=True, pad=0, normalized=True,
+ ).to(device)
+ spec = spec_transform(waveform).squeeze(1)
+
+ # Chroma features
+ chroma_filters = self.chroma_filters.to(device)
+ raw_chroma = torch.einsum("cf, ...ft->...ct", chroma_filters, spec)
+
+ # Normalize with inf norm
+ norm_chroma = torch.nn.functional.normalize(raw_chroma, p=float("inf"), dim=-2, eps=1e-6)
+
+ # Transpose: (batch, chroma, frames) -> (batch, frames, chroma)
+ norm_chroma = norm_chroma.transpose(1, 2)
+
+ # One-hot encoding: argmax along chroma dim
+ idx = norm_chroma.argmax(-1, keepdim=True)
+ norm_chroma[:] = 0
+ norm_chroma.scatter_(dim=-1, index=idx, value=1)
+
+ return norm_chroma
+
+
+__all__ = ["MusicgenMelodyAudioProcessor"]
diff --git a/src/transformers/models/musicgen_melody/feature_extraction_musicgen_melody.py b/src/transformers/models/musicgen_melody/feature_extraction_musicgen_melody.py
index 1811fa11e630..dbff71fe9d93 100644
--- a/src/transformers/models/musicgen_melody/feature_extraction_musicgen_melody.py
+++ b/src/transformers/models/musicgen_melody/feature_extraction_musicgen_melody.py
@@ -1,334 +1,20 @@
-# Copyright 2024 Meta AI and The HuggingFace Inc. team. All rights reserved.
+# Copyright 2026 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""
-Feature extractor class for Musicgen Melody
+"""Backwards-compatibility shim: re-exports the legacy ``MusicgenMelodyFeatureExtractor`` name as a
+deprecated alias of [`MusicgenMelodyAudioProcessor`]. Importing or instantiating the alias emits a
+``FutureWarning``; the alias is removed in transformers v5.15 (see ADR 0002).
"""
-import copy
-from typing import Any
-
-import numpy as np
-
-from ...audio_utils import chroma_filter_bank
-from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
-from ...feature_extraction_utils import BatchFeature
-from ...utils import TensorType, is_torch_available, is_torchaudio_available, logging
-from ...utils.import_utils import requires
-
-
-if is_torch_available():
- import torch
-
-if is_torchaudio_available():
- import torchaudio
-
-logger = logging.get_logger(__name__)
-
-
-@requires(backends=("torchaudio",))
-class MusicgenMelodyFeatureExtractor(SequenceFeatureExtractor):
- r"""
- Constructs a MusicgenMelody feature extractor.
-
- This feature extractor inherits from [`~feature_extraction_sequence_utils.SequenceFeatureExtractor`] which contains
- most of the main methods. Users should refer to this superclass for more information regarding those methods.
-
- This class extracts chroma features from audio processed by [Demucs](https://github.com/adefossez/demucs/tree/main) or
- directly from raw audio waveform.
-
- Args:
- feature_size (`int`, *optional*, defaults to 12):
- The feature dimension of the extracted features.
- sampling_rate (`int`, *optional*, defaults to 32000):
- The sampling rate at which the audio files should be digitalized expressed in hertz (Hz).
- hop_length (`int`, *optional*, defaults to 4096):
- Length of the overlapping windows for the STFT used to obtain the Mel Frequency coefficients.
- chunk_length (`int`, *optional*, defaults to 30):
- The maximum number of chunks of `sampling_rate` samples used to trim and pad longer or shorter audio
- sequences.
- n_fft (`int`, *optional*, defaults to 16384):
- Size of the Fourier transform.
- num_chroma (`int`, *optional*, defaults to 12):
- Number of chroma bins to use.
- padding_value (`float`, *optional*, defaults to 0.0):
- Padding value used to pad the audio.
- return_attention_mask (`bool`, *optional*, defaults to `False`):
- Whether to return the attention mask. Can be overwritten when calling the feature extractor.
-
- [What are attention masks?](../glossary#attention-mask)
-
-
-
- For Whisper models, `attention_mask` should always be passed for batched inference, to avoid subtle
- bugs.
-
-
- stem_indices (`list[int]`, *optional*, defaults to `[3, 2]`):
- Stem channels to extract if demucs outputs are passed.
- """
-
- model_input_names = ["input_features"]
-
- def __init__(
- self,
- feature_size=12,
- sampling_rate=32000,
- hop_length=4096,
- chunk_length=30,
- n_fft=16384,
- num_chroma=12,
- padding_value=0.0,
- return_attention_mask=False, # pad inputs to max length with silence token (zero) and no attention mask
- stem_indices=[3, 2],
- **kwargs,
- ):
- super().__init__(
- feature_size=feature_size,
- sampling_rate=sampling_rate,
- padding_value=padding_value,
- return_attention_mask=return_attention_mask,
- **kwargs,
- )
- self.n_fft = n_fft
- self.hop_length = hop_length
- self.chunk_length = chunk_length
- self.n_samples = chunk_length * sampling_rate
- self.sampling_rate = sampling_rate
- self.chroma_filters = torch.from_numpy(
- chroma_filter_bank(sampling_rate=sampling_rate, num_frequency_bins=n_fft, tuning=0, num_chroma=num_chroma)
- ).float()
- self.spectrogram = torchaudio.transforms.Spectrogram(
- n_fft=n_fft, win_length=n_fft, hop_length=hop_length, power=2, center=True, pad=0, normalized=True
- )
- self.stem_indices = stem_indices
-
- def _torch_extract_fbank_features(self, waveform: torch.Tensor) -> torch.Tensor:
- """
- Compute the chroma spectrogram of the provided audio using the torchaudio spectrogram implementation and the librosa chroma features.
- """
-
- # if wav length is not long enough, pad it
- wav_length = waveform.shape[-1]
- if wav_length < self.n_fft:
- pad = self.n_fft - wav_length
- rest = 0 if pad % 2 == 0 else 1
- waveform = torch.nn.functional.pad(waveform, (pad // 2, pad // 2 + rest), "constant", 0)
-
- # squeeze alongside channel dimension
- spec = self.spectrogram(waveform).squeeze(1)
-
- # sum along the frequency dimension
- raw_chroma = torch.einsum("cf, ...ft->...ct", self.chroma_filters, spec)
-
- # normalise with max value
- norm_chroma = torch.nn.functional.normalize(raw_chroma, p=float("inf"), dim=-2, eps=1e-6)
-
- # transpose time and chroma dimension -> (batch, time, chroma)
- norm_chroma = norm_chroma.transpose(1, 2)
-
- # replace max value alongside chroma dimension with 1 and replace the rest with 0
- idx = norm_chroma.argmax(-1, keepdim=True)
- norm_chroma[:] = 0
- norm_chroma.scatter_(dim=-1, index=idx, value=1)
-
- return norm_chroma
-
- def _extract_stem_indices(self, audio, sampling_rate=None):
- """
- Extracts stems from the output of the [Demucs](https://github.com/adefossez/demucs/tree/main) audio separation model,
- then converts to mono-channel and resample to the feature extractor sampling rate.
-
- Args:
- audio (`torch.Tensor` of shape `(batch_size, num_stems, channel_size, audio_length)`):
- The output of the Demucs model to be processed.
- sampling_rate (`int`, *optional*):
- Demucs sampling rate. If not specified, defaults to `44000`.
- """
- sampling_rate = 44000 if sampling_rate is None else sampling_rate
-
- # extract "vocals" and "others" sources from audio encoder (demucs) output
- # [batch_size, num_stems, channel_size, audio_length]
- wav = audio[:, torch.tensor(self.stem_indices)]
-
- # merge extracted stems to single waveform
- wav = wav.sum(1)
-
- # convert to mono-channel waveform
- wav = wav.mean(dim=1, keepdim=True)
-
- # resample to model sampling rate
- # not equivalent to julius.resample
- if sampling_rate != self.sampling_rate:
- wav = torchaudio.functional.resample(
- wav, sampling_rate, self.sampling_rate, rolloff=0.945, lowpass_filter_width=24
- )
-
- # [batch_size, 1, audio_length] -> [batch_size, audio_length]
- wav = wav.squeeze(1)
-
- return wav
-
- def __call__(
- self,
- audio: np.ndarray | list[float] | list[np.ndarray] | list[list[float]],
- truncation: bool = True,
- pad_to_multiple_of: int | None = None,
- return_tensors: str | TensorType | None = None,
- return_attention_mask: bool | None = None,
- padding: str | None = True,
- max_length: int | None = None,
- sampling_rate: int | None = None,
- **kwargs,
- ) -> BatchFeature:
- """
- Main method to featurize and prepare for the model one or several sequence(s).
-
- Args:
- audio (`torch.Tensor`, `np.ndarray`, `list[float]`, `list[np.ndarray]`, `list[torch.Tensor]`, `list[list[float]]`):
- The sequence or batch of sequences to be padded. Each sequence can be a torch tensor, a numpy array, a list of float
- values, a list of numpy arrays, a list of torch tensors, or a list of list of float values.
- If `audio` is the output of Demucs, it has to be a torch tensor of shape `(batch_size, num_stems, channel_size, audio_length)`.
- Otherwise, it must be mono or stereo channel audio.
- truncation (`bool`, *optional*, default to `True`):
- Activates truncation to cut input sequences longer than *max_length* to *max_length*.
- pad_to_multiple_of (`int`, *optional*, defaults to None):
- If set will pad the sequence to a multiple of the provided value.
-
- This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability
- `>= 7.5` (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128.
- return_tensors (`str` or [`~utils.TensorType`], *optional*):
- If set, will return tensors instead of list of python integers. Acceptable values are:
-
- - `'pt'`: Return PyTorch `torch.Tensor` objects.
- - `'np'`: Return Numpy `np.ndarray` objects.
- return_attention_mask (`bool`, *optional*):
- Whether to return the attention mask. If left to the default, will return the attention mask according
- to the specific feature_extractor's default.
-
- [What are attention masks?](../glossary#attention-mask)
-
-
- For Musicgen Melody models, audio `attention_mask` is not necessary.
-
-
- padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `True`):
- Select a strategy to pad the returned sequences (according to the model's padding side and padding
- index) among:
-
- - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
- sequence if provided).
- - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
- acceptable input length for the model if that argument is not provided.
- - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
- lengths).
- max_length (`int`, *optional*):
- Maximum length of the returned list and optionally padding length (see above).
- sampling_rate (`int`, *optional*):
- The sampling rate at which the `audio` input was sampled. It is strongly recommended to pass
- `sampling_rate` at the forward call to prevent silent errors.
- Note that if `audio` is the output of Demucs, `sampling_rate` must be the sampling rate at which Demucs operates.
- """
-
- if sampling_rate is None:
- logger.warning_once(
- f"It is strongly recommended to pass the `sampling_rate` argument to `{self.__class__.__name__}()`. "
- "Failing to do so can result in silent errors that might be hard to debug."
- )
-
- if isinstance(audio, torch.Tensor) and len(audio.shape) == 4:
- logger.warning_once(
- "`audio` is a 4-dimensional torch tensor and has thus been recognized as the output of `Demucs`. "
- "If this is not the case, make sure to read Musicgen Melody docstrings and "
- "to correct `audio` to get the right behaviour."
- "Link to the docstrings: https://huggingface.co/docs/transformers/main/en/model_doc/musicgen_melody"
- )
- audio = self._extract_stem_indices(audio, sampling_rate=sampling_rate)
- elif sampling_rate is not None and sampling_rate != self.sampling_rate:
- audio = torchaudio.functional.resample(
- audio, sampling_rate, self.sampling_rate, rolloff=0.945, lowpass_filter_width=24
- )
-
- is_batched = isinstance(audio, (np.ndarray, torch.Tensor)) and len(audio.shape) > 1
- is_batched = is_batched or (
- isinstance(audio, (list, tuple)) and (isinstance(audio[0], (torch.Tensor, np.ndarray, tuple, list)))
- )
-
- if is_batched and not isinstance(audio[0], torch.Tensor):
- audio = [torch.tensor(speech, dtype=torch.float32).unsqueeze(-1) for speech in audio]
- elif is_batched:
- audio = [speech.unsqueeze(-1) for speech in audio]
- elif not is_batched and not isinstance(audio, torch.Tensor):
- audio = torch.tensor(audio, dtype=torch.float32).unsqueeze(-1)
-
- if isinstance(audio[0], torch.Tensor) and audio[0].dtype is torch.float64:
- audio = [speech.to(torch.float32) for speech in audio]
-
- # always return batch
- if not is_batched:
- audio = [audio]
-
- if len(audio[0].shape) == 3:
- logger.warning_once(
- "`audio` has been detected as a batch of stereo signals. Will be convert to mono signals. "
- "If this is an undesired behaviour, make sure to read Musicgen Melody docstrings and "
- "to correct `audio` to get the right behaviour."
- "Link to the docstrings: https://huggingface.co/docs/transformers/main/en/model_doc/musicgen_melody"
- )
- # convert to mono-channel waveform
- audio = [stereo.mean(dim=0) for stereo in audio]
-
- batched_speech = BatchFeature({"input_features": audio})
-
- padded_inputs = self.pad(
- batched_speech,
- padding=padding,
- max_length=max_length if max_length else self.n_samples,
- truncation=truncation,
- pad_to_multiple_of=pad_to_multiple_of,
- return_attention_mask=return_attention_mask,
- return_tensors="pt",
- )
-
- input_features = self._torch_extract_fbank_features(padded_inputs["input_features"].squeeze(-1))
-
- padded_inputs["input_features"] = input_features
-
- if return_attention_mask:
- # rescale from raw audio length to spectrogram length
- padded_inputs["attention_mask"] = padded_inputs["attention_mask"][:, :: self.hop_length]
-
- if return_tensors is not None:
- padded_inputs = padded_inputs.convert_to_tensors(return_tensors)
+from ...audio_processing_base import make_legacy_audio_processor_alias
+from .audio_processing_musicgen_melody import MusicgenMelodyAudioProcessor
- return padded_inputs
- def to_dict(self) -> dict[str, Any]:
- """
- Serializes this instance to a Python dictionary. Returns:
- `dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance.
- """
- output = copy.deepcopy(self.__dict__)
- output["feature_extractor_type"] = self.__class__.__name__
- if "mel_filters" in output:
- del output["mel_filters"]
- if "window" in output:
- del output["window"]
- if "chroma_filters" in output:
- del output["chroma_filters"]
- if "spectrogram" in output:
- del output["spectrogram"]
- return output
+MusicgenMelodyFeatureExtractor = make_legacy_audio_processor_alias(MusicgenMelodyAudioProcessor, "MusicgenMelodyFeatureExtractor")
__all__ = ["MusicgenMelodyFeatureExtractor"]
diff --git a/src/transformers/models/nemotron_asr_streaming/__init__.py b/src/transformers/models/nemotron_asr_streaming/__init__.py
index 76b9368298f5..65c3612f68c5 100644
--- a/src/transformers/models/nemotron_asr_streaming/__init__.py
+++ b/src/transformers/models/nemotron_asr_streaming/__init__.py
@@ -18,6 +18,7 @@
if TYPE_CHECKING:
+ from .audio_processing_nemotron_asr_streaming import *
from .configuration_nemotron_asr_streaming import *
from .feature_extraction_nemotron_asr_streaming import *
from .modeling_nemotron_asr_streaming import *
diff --git a/src/transformers/models/nemotron_asr_streaming/audio_processing_nemotron_asr_streaming.py b/src/transformers/models/nemotron_asr_streaming/audio_processing_nemotron_asr_streaming.py
new file mode 100644
index 000000000000..71839f4440d3
--- /dev/null
+++ b/src/transformers/models/nemotron_asr_streaming/audio_processing_nemotron_asr_streaming.py
@@ -0,0 +1,45 @@
+# Copyright 2026 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ..parakeet.audio_processing_parakeet import ParakeetAudioProcessor
+
+
+class NemotronAsrStreamingAudioProcessor(ParakeetAudioProcessor):
+ """Audio processor for NemotronAsrStreaming.
+
+ The STFT + mel + preemphasis + log pipeline is identical to Parakeet's
+ (`n_fft=512`, `hop_length=160`, `win_length=400`, `power=2.0`,
+ `pad_mode="constant"`, `periodic=False`, slaney mel, `preemphasis=0.97`,
+ `log_mode="log"`, `mel_floor=2**-24`), including the librosa-bit-exact
+ `_standard_mel_banks`. Unlike Parakeet, NemotronAsrStreaming never applies
+ per-utterance mean/variance normalization — it only zeroes the padded frames
+ and emits the legacy output keys the model consumes (`input_features` /
+ `attention_mask`).
+ """
+
+ model_input_names = ["input_features", "attention_mask"]
+
+ def _postprocess_output(self, output, audio_ranges=None, feature_ranges=None, **kwargs):
+ # No per-utterance mean/var normalization (unlike Parakeet's CMVN). Zero the padded
+ # frames via the mask, then rename to the legacy keys the model's forward expects.
+ features = output.pop("audio_features")
+ mask = output.pop("audio_features_mask", None)
+ if mask is not None:
+ features = features * mask.unsqueeze(-1).to(features.dtype)
+ output["attention_mask"] = mask
+ output["input_features"] = features
+ return output
+
+
+__all__ = ["NemotronAsrStreamingAudioProcessor"]
diff --git a/src/transformers/models/nemotron_asr_streaming/feature_extraction_nemotron_asr_streaming.py b/src/transformers/models/nemotron_asr_streaming/feature_extraction_nemotron_asr_streaming.py
index 2b1dbc96ef9f..172e15e70294 100644
--- a/src/transformers/models/nemotron_asr_streaming/feature_extraction_nemotron_asr_streaming.py
+++ b/src/transformers/models/nemotron_asr_streaming/feature_extraction_nemotron_asr_streaming.py
@@ -1,270 +1,22 @@
-# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
-# This file was automatically generated from src/transformers/models/nemotron_asr_streaming/modular_nemotron_asr_streaming.py.
-# Do NOT edit this file manually as any edits will be overwritten by the generation of
-# the file from the modular. If any change should be done, please apply the change to the
-# modular_nemotron_asr_streaming.py file directly. One of our CI enforces this.
-# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
-# Copyright 2026 The HuggingFace Inc. team. All rights reserved.
+# Copyright 2026 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-import numpy as np
-import torch
-
-from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
-from ...feature_extraction_utils import BatchFeature
-from ...utils import TensorType, is_librosa_available, logging
-from ...utils.import_utils import requires
-
-
-if is_librosa_available():
- import librosa
-
-
-logger = logging.get_logger(__name__)
-LOG_ZERO_GUARD_VALUE = 2**-24
-
-
-@requires(backends=("torch", "librosa"))
-class NemotronAsrStreamingFeatureExtractor(SequenceFeatureExtractor):
- r"""
- Constructs a NemotronAsrStreaming feature extractor.
-
- This feature extractor inherits from [`~feature_extraction_sequence_utils.SequenceFeatureExtractor`] which contains
- most of the main methods. Users should refer to this superclass for more information regarding those methods.
-
- This class extracts mel-filter bank features from raw speech using a custom numpy implementation of the `Short Time
- Fourier Transform` which should match pytorch's `torch.stft` equivalent.
-
- Args:
- feature_size (`int`, *optional*, defaults to 80):
- The feature dimension of the extracted features.
- sampling_rate (`int`, *optional*, defaults to 16000):
- The sampling rate at which the audio files should be digitalized expressed in hertz (Hz).
- hop_length (`int`, *optional*, defaults to 160):
- Length of the overlapping windows for the STFT used to obtain the Mel Frequency coefficients.
- n_fft (`int`, *optional*, defaults to 512):
- Size of the Fourier transform.
- win_length (`int`, *optional*, defaults to 400):
- The window length for the STFT computation.
- preemphasis (`float`, *optional*, defaults to 0.97):
- A preemphasis filter coefficient. 0.0 means no preemphasis filter.
- padding_value (`float`, *optional*, defaults to 0.0):
- Padding value used to pad the audio. Should correspond to silences.
- """
-
- model_input_names = ["input_features", "attention_mask"]
-
- def __init__(
- self,
- feature_size=80,
- sampling_rate=16000,
- hop_length=160,
- n_fft=512,
- win_length=400,
- preemphasis=0.97,
- padding_value=0.0,
- **kwargs,
- ):
- super().__init__(feature_size=feature_size, sampling_rate=sampling_rate, padding_value=padding_value, **kwargs)
-
- self.hop_length = hop_length
- self.n_fft = n_fft
- self.win_length = win_length
- self.preemphasis = preemphasis
-
- # TODO: @eustlb, for now we use librosa to compute the mel filters
- # indeed mel_filter_bank uses np.float64 (while librosa uses np.float32), giving numerical differences
- # self.mel_filters = mel_filter_bank(
- # num_frequency_bins=n_fft // 2 + 1,
- # num_mel_filters=feature_size,
- # min_frequency=0.0,
- # max_frequency=sampling_rate / 2,
- # sampling_rate=sampling_rate,
- # norm="slaney",
- # mel_scale="slaney",
- # )
- mel_filters = librosa.filters.mel(
- sr=sampling_rate, n_fft=n_fft, n_mels=feature_size, fmin=0.0, fmax=sampling_rate / 2, norm="slaney"
- )
- self.mel_filters = torch.from_numpy(mel_filters).to(torch.float32)
-
- def _torch_extract_fbank_features(self, waveform, device="cpu", center=True):
- window = torch.hann_window(self.win_length, periodic=False, device=device)
- stft = torch.stft(
- waveform,
- self.n_fft,
- hop_length=self.hop_length,
- win_length=self.win_length,
- window=window,
- return_complex=True,
- pad_mode="constant",
- center=center,
- )
- magnitudes = torch.view_as_real(stft)
- magnitudes = torch.sqrt(magnitudes.pow(2).sum(-1))
- magnitudes = magnitudes.pow(2)
-
- # log mel spectrogram
- mel_filters = self.mel_filters.to(device)
- mel_spec = mel_filters @ magnitudes
- mel_spec = torch.log(mel_spec + LOG_ZERO_GUARD_VALUE)
-
- # (batch_size, num_mel_filters, num_frames) -> (batch_size, num_frames, num_mel_filters)
- mel_spec = mel_spec.permute(0, 2, 1)
-
- return mel_spec
-
- def __call__(
- self,
- raw_speech: "np.ndarray | list[float] | list[np.ndarray] | list[list[float]]",
- truncation: bool = False,
- pad_to_multiple_of: int | None = None,
- return_tensors: "str | TensorType | None" = None,
- return_attention_mask: bool | None = None,
- padding: str | None = "longest",
- max_length: int | None = None,
- sampling_rate: int | None = None,
- device: str | None = "cpu",
- return_token_timestamps: bool | None = None,
- center: bool = True,
- **kwargs,
- ) -> BatchFeature:
- """
- Main method to featurize and prepare for the model one or several sequence(s). Implementation uses PyTorch for
- the STFT computation if available, otherwise a slower NumPy based one.
-
- Args:
- raw_speech (`np.ndarray`, `list[float]`, `list[np.ndarray]`, `list[list[float]]`):
- The sequence or batch of sequences to be padded. Each sequence can be a numpy array, a list of float
- values, a list of numpy arrays or a list of list of float values. Must be mono channel audio, not
- stereo, i.e. single float per timestep.
- truncation (`bool`, *optional*, default to `True`):
- Activates truncation to cut input sequences longer than *max_length* to *max_length*.
- pad_to_multiple_of (`int`, *optional*, defaults to None):
- If set will pad the sequence to a multiple of the provided value.
- return_attention_mask (`bool`, *optional*):
- Whether to return the attention mask. If left to the default, will return the attention mask according
- to the specific feature_extractor's default.
- return_tensors (`str` or [`~utils.TensorType`], *optional*):
- If set, will return tensors instead of list of python integers. Acceptable values are:
-
- - `'tf'`: Return TensorFlow `tf.constant` objects.
- - `'pt'`: Return PyTorch `torch.Tensor` objects.
- - `'np'`: Return Numpy `np.ndarray` objects.
- sampling_rate (`int`, *optional*):
- The sampling rate at which the `raw_speech` input was sampled. It is strongly recommended to pass
- `sampling_rate` at the forward call to prevent silent errors and allow automatic speech recognition
- pipeline.
- device (`str`, *optional*, defaults to `'cpu'`):
- Specifies the device for computation of the log-mel spectrogram of audio signals in the
- `_torch_extract_fbank_features` method. (e.g., "cpu", "cuda")
- return_token_timestamps (`bool`, *optional*, defaults to `None`):
- Deprecated. Use `return_attention_mask` instead from which the number of frames can be inferred.
- center (`bool`, *optional*, defaults to `True`):
- Whether to pad the audio on both sides so STFT frames are centered (`torch.stft(center=True)`). Use
- `True` for offline extraction and for the first chunk of a streaming session. Use `False` for
- subsequent streaming chunks: feeding `audio[hop * frame - n_fft // 2 : ...]` with `center=False`
- reproduces, frame-for-frame, the features that a single `center=True` pass over the whole utterance
- would have produced for those frames.
- """
- if sampling_rate is not None:
- if sampling_rate != self.sampling_rate:
- raise ValueError(
- f"The model corresponding to this feature extractor: {self.__class__.__name__} was trained using a"
- f" sampling rate of {self.sampling_rate}. Please make sure that the provided `raw_speech` input"
- f" was sampled with {self.sampling_rate} and not {sampling_rate}."
- )
- else:
- logger.warning(
- f"It is strongly recommended to pass the `sampling_rate` argument to `{self.__class__.__name__}()`. "
- "Failing to do so can result in silent errors that might be hard to debug."
- )
-
- # Convert to torch tensor
- if isinstance(raw_speech, np.ndarray):
- raw_speech = torch.tensor(raw_speech)
- elif isinstance(raw_speech, (list, tuple)) and isinstance(raw_speech[0], np.ndarray):
- raw_speech = [torch.tensor(speech) for speech in raw_speech]
-
- is_batched_torch = isinstance(raw_speech, torch.Tensor) and len(raw_speech.shape) > 1
- if is_batched_torch and len(raw_speech.shape) > 2:
- logger.warning(
- f"Only mono-channel audio is supported for input to {self.__class__.__name__}. "
- "We will take the mean of the channels to convert to mono."
- )
- raw_speech = raw_speech.mean(-1)
-
- is_batched_sequence = isinstance(raw_speech, (list, tuple))
- if is_batched_sequence:
- for speech in raw_speech:
- if len(speech.shape) > 1:
- logger.warning(
- f"Only mono-channel audio is supported for input to {self.__class__.__name__}. "
- "We will take the mean of the channels to convert to mono."
- )
- speech = speech.mean(-1)
-
- if is_batched_torch or is_batched_sequence:
- raw_speech = [speech[:, None].to(torch.float32) for speech in raw_speech]
- else:
- raw_speech = [raw_speech[:, None].to(torch.float32)]
-
- audio_lengths = [len(speech) for speech in raw_speech]
- batched_speech = BatchFeature({"input_features": raw_speech, "audio_lengths": audio_lengths})
-
- padded_inputs = self.pad(
- batched_speech,
- padding=padding,
- max_length=max_length,
- truncation=truncation,
- pad_to_multiple_of=pad_to_multiple_of,
- return_tensors="pt",
- )
- input_features = padded_inputs.input_features.squeeze(-1)
-
- # preemphasis
- if self.preemphasis is not None:
- timemask = torch.arange(input_features.shape[1], device=input_features.device).unsqueeze(
- 0
- ) < padded_inputs.audio_lengths.unsqueeze(1)
- input_features = torch.cat(
- [input_features[:, :1], input_features[:, 1:] - self.preemphasis * input_features[:, :-1]], dim=1
- )
- input_features = input_features.masked_fill(~timemask, 0.0)
+"""Backwards-compatibility shim: re-exports the legacy ``NemotronAsrStreamingFeatureExtractor`` name
+as a deprecated alias of [`NemotronAsrStreamingAudioProcessor`]. Importing or instantiating the alias
+emits a ``FutureWarning``; the alias is removed in transformers v5.15 (see ADR 0002).
+"""
- input_features = self._torch_extract_fbank_features(input_features, device, center=center)
- if center:
- # `center=True` pads `n_fft // 2` on each side, so the number of valid frames is `floor(L / hop)`.
- features_lengths = torch.floor_divide(
- padded_inputs.audio_lengths + self.n_fft // 2 * 2 - self.n_fft, self.hop_length
- )
- else:
- # `center=False` does no padding: `floor((L - n_fft) / hop) + 1` frames.
- features_lengths = torch.floor_divide(padded_inputs.audio_lengths - self.n_fft, self.hop_length) + 1
- attention_mask = torch.arange(input_features.shape[1], device=device)[None, :] < features_lengths[:, None]
+from ...audio_processing_base import make_legacy_audio_processor_alias
+from .audio_processing_nemotron_asr_streaming import NemotronAsrStreamingAudioProcessor
- # NemotronAsrStreaming never normalizes the mel features
- input_features *= attention_mask.unsqueeze(-1)
- return BatchFeature(
- data={
- "input_features": input_features,
- "attention_mask": attention_mask,
- },
- tensor_type=return_tensors,
- )
+NemotronAsrStreamingFeatureExtractor = make_legacy_audio_processor_alias(
+ NemotronAsrStreamingAudioProcessor, "NemotronAsrStreamingFeatureExtractor"
+)
__all__ = ["NemotronAsrStreamingFeatureExtractor"]
diff --git a/src/transformers/models/nemotron_asr_streaming/modular_nemotron_asr_streaming.py b/src/transformers/models/nemotron_asr_streaming/modular_nemotron_asr_streaming.py
index c6682a5b16cb..521dde73000f 100644
--- a/src/transformers/models/nemotron_asr_streaming/modular_nemotron_asr_streaming.py
+++ b/src/transformers/models/nemotron_asr_streaming/modular_nemotron_asr_streaming.py
@@ -16,19 +16,16 @@
from collections.abc import Callable
from dataclasses import dataclass
-import numpy as np
import torch
from huggingface_hub.dataclasses import strict
from torch import nn
from ...cache_utils import Cache, DynamicCache
-from ...feature_extraction_utils import BatchFeature
from ...masking_utils import create_bidirectional_mask
from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling
from ...modeling_utils import ALL_ATTENTION_FUNCTIONS
from ...processing_utils import Unpack
from ...utils import (
- TensorType,
TransformersKwargs,
auto_docstring,
can_return_tuple,
@@ -40,7 +37,6 @@
from ..fastspeech2_conformer.modeling_fastspeech2_conformer import FastSpeech2ConformerConvolutionModule
from ..llama.modeling_llama import eager_attention_forward
from ..parakeet.configuration_parakeet import ParakeetEncoderConfig, ParakeetRNNTConfig
-from ..parakeet.feature_extraction_parakeet import ParakeetFeatureExtractor
from ..parakeet.modeling_parakeet import (
ParakeetEncoder,
ParakeetEncoderAttention,
@@ -62,8 +58,6 @@
)
-LOG_ZERO_GUARD_VALUE = 2**-24
-
logger = logging.get_logger(__name__)
@@ -171,174 +165,6 @@ class NemotronAsrStreamingConfig(ParakeetRNNTConfig):
blank_token_id: int = 1024
-class NemotronAsrStreamingFeatureExtractor(ParakeetFeatureExtractor):
- def _torch_extract_fbank_features(self, waveform, device="cpu", center=True):
- window = torch.hann_window(self.win_length, periodic=False, device=device)
- stft = torch.stft(
- waveform,
- self.n_fft,
- hop_length=self.hop_length,
- win_length=self.win_length,
- window=window,
- return_complex=True,
- pad_mode="constant",
- center=center,
- )
- magnitudes = torch.view_as_real(stft)
- magnitudes = torch.sqrt(magnitudes.pow(2).sum(-1))
- magnitudes = magnitudes.pow(2)
-
- # log mel spectrogram
- mel_filters = self.mel_filters.to(device)
- mel_spec = mel_filters @ magnitudes
- mel_spec = torch.log(mel_spec + LOG_ZERO_GUARD_VALUE)
-
- # (batch_size, num_mel_filters, num_frames) -> (batch_size, num_frames, num_mel_filters)
- mel_spec = mel_spec.permute(0, 2, 1)
-
- return mel_spec
-
- def __call__(
- self,
- raw_speech: "np.ndarray | list[float] | list[np.ndarray] | list[list[float]]",
- truncation: bool = False,
- pad_to_multiple_of: int | None = None,
- return_tensors: "str | TensorType | None" = None,
- return_attention_mask: bool | None = None,
- padding: str | None = "longest",
- max_length: int | None = None,
- sampling_rate: int | None = None,
- device: str | None = "cpu",
- return_token_timestamps: bool | None = None,
- center: bool = True,
- **kwargs,
- ) -> BatchFeature:
- """
- Main method to featurize and prepare for the model one or several sequence(s). Implementation uses PyTorch for
- the STFT computation if available, otherwise a slower NumPy based one.
-
- Args:
- raw_speech (`np.ndarray`, `list[float]`, `list[np.ndarray]`, `list[list[float]]`):
- The sequence or batch of sequences to be padded. Each sequence can be a numpy array, a list of float
- values, a list of numpy arrays or a list of list of float values. Must be mono channel audio, not
- stereo, i.e. single float per timestep.
- truncation (`bool`, *optional*, default to `True`):
- Activates truncation to cut input sequences longer than *max_length* to *max_length*.
- pad_to_multiple_of (`int`, *optional*, defaults to None):
- If set will pad the sequence to a multiple of the provided value.
- return_attention_mask (`bool`, *optional*):
- Whether to return the attention mask. If left to the default, will return the attention mask according
- to the specific feature_extractor's default.
- return_tensors (`str` or [`~utils.TensorType`], *optional*):
- If set, will return tensors instead of list of python integers. Acceptable values are:
-
- - `'tf'`: Return TensorFlow `tf.constant` objects.
- - `'pt'`: Return PyTorch `torch.Tensor` objects.
- - `'np'`: Return Numpy `np.ndarray` objects.
- sampling_rate (`int`, *optional*):
- The sampling rate at which the `raw_speech` input was sampled. It is strongly recommended to pass
- `sampling_rate` at the forward call to prevent silent errors and allow automatic speech recognition
- pipeline.
- device (`str`, *optional*, defaults to `'cpu'`):
- Specifies the device for computation of the log-mel spectrogram of audio signals in the
- `_torch_extract_fbank_features` method. (e.g., "cpu", "cuda")
- return_token_timestamps (`bool`, *optional*, defaults to `None`):
- Deprecated. Use `return_attention_mask` instead from which the number of frames can be inferred.
- center (`bool`, *optional*, defaults to `True`):
- Whether to pad the audio on both sides so STFT frames are centered (`torch.stft(center=True)`). Use
- `True` for offline extraction and for the first chunk of a streaming session. Use `False` for
- subsequent streaming chunks: feeding `audio[hop * frame - n_fft // 2 : ...]` with `center=False`
- reproduces, frame-for-frame, the features that a single `center=True` pass over the whole utterance
- would have produced for those frames.
- """
- if sampling_rate is not None:
- if sampling_rate != self.sampling_rate:
- raise ValueError(
- f"The model corresponding to this feature extractor: {self.__class__.__name__} was trained using a"
- f" sampling rate of {self.sampling_rate}. Please make sure that the provided `raw_speech` input"
- f" was sampled with {self.sampling_rate} and not {sampling_rate}."
- )
- else:
- logger.warning(
- f"It is strongly recommended to pass the `sampling_rate` argument to `{self.__class__.__name__}()`. "
- "Failing to do so can result in silent errors that might be hard to debug."
- )
-
- # Convert to torch tensor
- if isinstance(raw_speech, np.ndarray):
- raw_speech = torch.tensor(raw_speech)
- elif isinstance(raw_speech, (list, tuple)) and isinstance(raw_speech[0], np.ndarray):
- raw_speech = [torch.tensor(speech) for speech in raw_speech]
-
- is_batched_torch = isinstance(raw_speech, torch.Tensor) and len(raw_speech.shape) > 1
- if is_batched_torch and len(raw_speech.shape) > 2:
- logger.warning(
- f"Only mono-channel audio is supported for input to {self.__class__.__name__}. "
- "We will take the mean of the channels to convert to mono."
- )
- raw_speech = raw_speech.mean(-1)
-
- is_batched_sequence = isinstance(raw_speech, (list, tuple))
- if is_batched_sequence:
- for speech in raw_speech:
- if len(speech.shape) > 1:
- logger.warning(
- f"Only mono-channel audio is supported for input to {self.__class__.__name__}. "
- "We will take the mean of the channels to convert to mono."
- )
- speech = speech.mean(-1)
-
- if is_batched_torch or is_batched_sequence:
- raw_speech = [speech[:, None].to(torch.float32) for speech in raw_speech]
- else:
- raw_speech = [raw_speech[:, None].to(torch.float32)]
-
- audio_lengths = [len(speech) for speech in raw_speech]
- batched_speech = BatchFeature({"input_features": raw_speech, "audio_lengths": audio_lengths})
-
- padded_inputs = self.pad(
- batched_speech,
- padding=padding,
- max_length=max_length,
- truncation=truncation,
- pad_to_multiple_of=pad_to_multiple_of,
- return_tensors="pt",
- )
- input_features = padded_inputs.input_features.squeeze(-1)
-
- # preemphasis
- if self.preemphasis is not None:
- timemask = torch.arange(input_features.shape[1], device=input_features.device).unsqueeze(
- 0
- ) < padded_inputs.audio_lengths.unsqueeze(1)
- input_features = torch.cat(
- [input_features[:, :1], input_features[:, 1:] - self.preemphasis * input_features[:, :-1]], dim=1
- )
- input_features = input_features.masked_fill(~timemask, 0.0)
-
- input_features = self._torch_extract_fbank_features(input_features, device, center=center)
- if center:
- # `center=True` pads `n_fft // 2` on each side, so the number of valid frames is `floor(L / hop)`.
- features_lengths = torch.floor_divide(
- padded_inputs.audio_lengths + self.n_fft // 2 * 2 - self.n_fft, self.hop_length
- )
- else:
- # `center=False` does no padding: `floor((L - n_fft) / hop) + 1` frames.
- features_lengths = torch.floor_divide(padded_inputs.audio_lengths - self.n_fft, self.hop_length) + 1
- attention_mask = torch.arange(input_features.shape[1], device=device)[None, :] < features_lengths[:, None]
-
- # NemotronAsrStreaming never normalizes the mel features
- input_features *= attention_mask.unsqueeze(-1)
-
- return BatchFeature(
- data={
- "input_features": input_features,
- "attention_mask": attention_mask,
- },
- tensor_type=return_tensors,
- )
-
-
class NemotronAsrStreamingEncoderCausalConv1dCacheLayer(VoxtralRealtimeConv1dCacheLayer): ...
@@ -1069,7 +895,6 @@ def forward(
__all__ = [
"NemotronAsrStreamingConfig",
"NemotronAsrStreamingEncoderConfig",
- "NemotronAsrStreamingFeatureExtractor",
"NemotronAsrStreamingEncoderModelOutput",
"NemotronAsrStreamingRNNTOutput",
"NemotronAsrStreamingForRNNT",
diff --git a/src/transformers/models/parakeet/__init__.py b/src/transformers/models/parakeet/__init__.py
index e8bbfe7faf45..cc87a1e034e5 100644
--- a/src/transformers/models/parakeet/__init__.py
+++ b/src/transformers/models/parakeet/__init__.py
@@ -18,6 +18,8 @@
if TYPE_CHECKING:
+ from .audio_processing_numpy_parakeet import *
+ from .audio_processing_parakeet import *
from .configuration_parakeet import *
from .feature_extraction_parakeet import *
from .modeling_parakeet import *
diff --git a/src/transformers/models/parakeet/audio_processing_numpy_parakeet.py b/src/transformers/models/parakeet/audio_processing_numpy_parakeet.py
new file mode 100644
index 000000000000..b169b7d1c25e
--- /dev/null
+++ b/src/transformers/models/parakeet/audio_processing_numpy_parakeet.py
@@ -0,0 +1,84 @@
+# Copyright 2026 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from ...audio_processing_backends import NumpyAudioBackend
+from ...audio_utils import MelScaleConfig, SpectrogramConfig, StftConfig
+
+
+class ParakeetAudioProcessorNumpy(NumpyAudioBackend):
+ """NumPy sibling of [`ParakeetAudioProcessor`]. Bit-exact to the torch sibling within
+ the float32 noise floor (ADR 0001)."""
+
+ sampling_rate = 16000
+ force_mono = True
+ spectrogram_config = SpectrogramConfig(
+ stft_config=StftConfig(
+ n_fft=512,
+ hop_length=160,
+ win_length=400,
+ window_fn="hann_window",
+ power=2.0,
+ pad_mode="constant",
+ periodic=False,
+ ),
+ mel_scale_config=MelScaleConfig(
+ n_mels=80,
+ f_min=0.0,
+ norm="slaney",
+ mel_scale="slaney",
+ ),
+ preemphasis=0.97,
+ preemphasis_mode="waveform",
+ log_mode="log",
+ mel_floor=0.0, # base clamp is a no-op; the log guard is pre_log_offset
+ pre_log_offset=2**-24,
+ )
+
+ # The base numpy backend already builds librosa's per-band float32 filters and applies
+ # the mel matmul / magnitude / `log(x + pre_log_offset)` forms this model needs.
+
+ def _normalize_magnitude(self, features, *, spectrogram_config, **kwargs):
+ # Base handles the legacy `log(x + guard)` form via `pre_log_offset`;
+ # transpose to (batch, frames, mels).
+ features = super()._normalize_magnitude(features, spectrogram_config=spectrogram_config, **kwargs)
+ return np.transpose(features, axes=(0, 2, 1))
+
+ def _postprocess_output(self, output, audio_ranges=None, **kwargs):
+ if audio_ranges is None or "audio_features" not in output:
+ return output
+
+ features = output["audio_features"]
+ stft_cfg = self.spectrogram_config.stft_config
+ audio_lengths = np.asarray([end - start for start, end in audio_ranges])
+ features_lengths = np.floor_divide(
+ audio_lengths + stft_cfg.n_fft // 2 * 2 - stft_cfg.n_fft, stft_cfg.hop_length
+ )
+ attention_mask = np.arange(features.shape[1])[None, :] < features_lengths[:, None]
+ mask = np.expand_dims(attention_mask, axis=-1)
+ # NumPy promotes float32 / int64 → float64; cast lengths to the feature dtype to keep
+ # parity with torch (which preserves the floating dtype across float/int division).
+ features_lengths_f = features_lengths.astype(features.dtype)
+ mel_masked = features * mask
+ mean = np.expand_dims(mel_masked.sum(axis=1) / np.expand_dims(features_lengths_f, axis=-1), axis=1)
+ variance = ((mel_masked - mean) ** 2 * mask).sum(axis=1) / np.expand_dims(
+ features_lengths_f - 1, axis=-1
+ )
+ std = np.expand_dims(np.sqrt(variance), axis=1)
+ output["audio_features"] = (features - mean) / (std + 1e-5) * mask
+ return output
+
+
+__all__ = ["ParakeetAudioProcessorNumpy"]
diff --git a/src/transformers/models/parakeet/audio_processing_parakeet.py b/src/transformers/models/parakeet/audio_processing_parakeet.py
new file mode 100644
index 000000000000..d5b3a9ebd4d2
--- /dev/null
+++ b/src/transformers/models/parakeet/audio_processing_parakeet.py
@@ -0,0 +1,93 @@
+# Copyright 2025 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+
+from ...audio_processing_backends import TorchAudioBackend
+from ...audio_utils import _create_triangular_filter_bank, hertz_to_mel, mel_to_hertz
+from .audio_processing_numpy_parakeet import ParakeetAudioProcessorNumpy
+
+
+class ParakeetAudioProcessor(TorchAudioBackend):
+ sampling_rate = 16000
+ force_mono = True
+
+ spectrogram_config = ParakeetAudioProcessorNumpy.spectrogram_config
+
+ def _standard_mel_banks(self, num_mel_filters, num_frequency_bins, min_frequency,
+ max_frequency, sampling_rate, n_fft, mel_cfg, computation_dtype):
+ """Torch-native build of librosa's per-band float32 rounding pattern.
+
+ The legacy FE's filters are librosa's: triangular weights computed in float64,
+ cast to float32, then the slaney area-norm applied *after* that cast with a
+ second float32 rounding. The base torch leaf matches torchaudio instead
+ (float32-native ops), and a float64 build with the norm applied before the
+ final cast differs in the last ulp — only librosa's exact rounding order
+ reproduces the legacy filters bit-exactly. Torch ops only (float64 linspace /
+ exp match numpy's bitwise here); no numpy construction.
+ """
+ mel_min = hertz_to_mel(min_frequency, mel_scale=mel_cfg.mel_scale)
+ mel_max = hertz_to_mel(max_frequency, mel_scale=mel_cfg.mel_scale)
+ mel_freqs = torch.linspace(mel_min, mel_max, num_mel_filters + 2, dtype=torch.float64)
+ filter_freqs = mel_to_hertz(mel_freqs, mel_scale=mel_cfg.mel_scale)
+ fft_freqs = torch.linspace(0, sampling_rate // 2, num_frequency_bins, dtype=torch.float64)
+ mel_filters = _create_triangular_filter_bank(fft_freqs, filter_freqs).to(torch.float32)
+ if mel_cfg.norm == "slaney":
+ enorm = 2.0 / (filter_freqs[2 : num_mel_filters + 2] - filter_freqs[:num_mel_filters])
+ mel_filters = (mel_filters * enorm[None, :]).to(torch.float32)
+ return mel_filters
+
+ def _compute_magnitudes(self, stft_out, power, spectrogram_config=None):
+ import torch
+
+ magnitudes = torch.view_as_real(stft_out)
+ magnitudes = torch.sqrt(magnitudes.pow(2).sum(-1))
+ if power != 1.0:
+ magnitudes = magnitudes.pow(power)
+ return magnitudes
+
+ def _apply_mel_scale(self, features, *, spectrogram_config, **kwargs):
+ import torch
+
+ return torch.matmul(self.mel_filters.T, features)
+
+ def _normalize_magnitude(self, features, *, spectrogram_config, **kwargs):
+ # Base handles the legacy `log(x + guard)` form via `pre_log_offset`;
+ # transpose to (batch, frames, mels).
+ features = super()._normalize_magnitude(features, spectrogram_config=spectrogram_config, **kwargs)
+ return features.permute(0, 2, 1)
+
+ def _postprocess_output(self, output, audio_ranges=None, **kwargs):
+ import torch
+
+ if audio_ranges is None or "audio_features" not in output:
+ return output
+
+ features = output["audio_features"]
+ stft_cfg = self.spectrogram_config.stft_config
+ audio_lengths = torch.tensor([end - start for start, end in audio_ranges])
+ features_lengths = torch.floor_divide(
+ audio_lengths + stft_cfg.n_fft // 2 * 2 - stft_cfg.n_fft, stft_cfg.hop_length
+ )
+ attention_mask = torch.arange(features.shape[1])[None, :] < features_lengths[:, None]
+ mask = attention_mask.unsqueeze(-1)
+ mel_masked = features * mask
+ mean = (mel_masked.sum(dim=1) / features_lengths.unsqueeze(-1)).unsqueeze(1)
+ variance = ((mel_masked - mean) ** 2 * mask).sum(dim=1) / (features_lengths - 1).unsqueeze(-1)
+ std = torch.sqrt(variance).unsqueeze(1)
+ output["audio_features"] = (features - mean) / (std + 1e-5) * mask
+ return output
+
+
+__all__ = ["ParakeetAudioProcessor"]
diff --git a/src/transformers/models/parakeet/feature_extraction_parakeet.py b/src/transformers/models/parakeet/feature_extraction_parakeet.py
index c745d02c9629..430ae900f2b9 100644
--- a/src/transformers/models/parakeet/feature_extraction_parakeet.py
+++ b/src/transformers/models/parakeet/feature_extraction_parakeet.py
@@ -1,285 +1,20 @@
-# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
+# Copyright 2026 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import numpy as np
-import torch
-
-from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
-from ...feature_extraction_utils import BatchFeature
-from ...utils import TensorType, is_librosa_available, logging
-from ...utils.import_utils import requires
-
-
-if is_librosa_available():
- import librosa
-
-
-EPSILON = 1e-5
-LOG_ZERO_GUARD_VALUE = 2**-24
-
-
-logger = logging.get_logger(__name__)
-
-
-@requires(backends=("torch", "librosa"))
-class ParakeetFeatureExtractor(SequenceFeatureExtractor):
- r"""
- Constructs a Parakeet feature extractor.
-
- This feature extractor inherits from [`~feature_extraction_sequence_utils.SequenceFeatureExtractor`] which contains
- most of the main methods. Users should refer to this superclass for more information regarding those methods.
-
- This class extracts mel-filter bank features from raw speech using a custom numpy implementation of the `Short Time
- Fourier Transform` which should match pytorch's `torch.stft` equivalent.
-
- Args:
- feature_size (`int`, *optional*, defaults to 80):
- The feature dimension of the extracted features.
- sampling_rate (`int`, *optional*, defaults to 16000):
- The sampling rate at which the audio files should be digitalized expressed in hertz (Hz).
- hop_length (`int`, *optional*, defaults to 160):
- Length of the overlapping windows for the STFT used to obtain the Mel Frequency coefficients.
- n_fft (`int`, *optional*, defaults to 512):
- Size of the Fourier transform.
- win_length (`int`, *optional*, defaults to 400):
- The window length for the STFT computation.
- preemphasis (`float`, *optional*, defaults to 0.97):
- A preemphasis filter coefficient. 0.0 means no preemphasis filter.
- padding_value (`float`, *optional*, defaults to 0.0):
- Padding value used to pad the audio. Should correspond to silences.
- """
-
- model_input_names = ["input_features", "attention_mask"]
-
- def __init__(
- self,
- feature_size=80,
- sampling_rate=16000,
- hop_length=160,
- n_fft=512,
- win_length=400,
- preemphasis=0.97,
- padding_value=0.0,
- **kwargs,
- ):
- super().__init__(feature_size=feature_size, sampling_rate=sampling_rate, padding_value=padding_value, **kwargs)
-
- self.hop_length = hop_length
- self.n_fft = n_fft
- self.win_length = win_length
- self.preemphasis = preemphasis
-
- # TODO: @eustlb, for now we use librosa to compute the mel filters
- # indeed mel_filter_bank uses np.float64 (while librosa uses np.float32), giving numerical differences
- # self.mel_filters = mel_filter_bank(
- # num_frequency_bins=n_fft // 2 + 1,
- # num_mel_filters=feature_size,
- # min_frequency=0.0,
- # max_frequency=sampling_rate / 2,
- # sampling_rate=sampling_rate,
- # norm="slaney",
- # mel_scale="slaney",
- # )
- mel_filters = librosa.filters.mel(
- sr=sampling_rate, n_fft=n_fft, n_mels=feature_size, fmin=0.0, fmax=sampling_rate / 2, norm="slaney"
- )
- self.mel_filters = torch.from_numpy(mel_filters).to(torch.float32)
-
- def _torch_extract_fbank_features(self, waveform, device="cpu"):
- # spectrogram
- window = torch.hann_window(self.win_length, periodic=False, device=device)
- stft = torch.stft(
- waveform,
- self.n_fft,
- hop_length=self.hop_length,
- win_length=self.win_length,
- window=window,
- return_complex=True,
- pad_mode="constant",
- )
- # Let's math original implementation
- # magnitudes = torch.abs(stft) ** 2
- magnitudes = torch.view_as_real(stft)
- magnitudes = torch.sqrt(magnitudes.pow(2).sum(-1))
- magnitudes = magnitudes.pow(2)
-
- # log mel spectrogram
- mel_filters = self.mel_filters.to(device)
- mel_spec = mel_filters @ magnitudes
- mel_spec = torch.log(mel_spec + LOG_ZERO_GUARD_VALUE)
-
- # (batch_size, num_mel_filters, num_frames) -> (batch_size, num_frames, num_mel_filters)
- mel_spec = mel_spec.permute(0, 2, 1)
-
- return mel_spec
-
- def __call__(
- self,
- raw_speech: np.ndarray | list[float] | list[np.ndarray] | list[list[float]],
- truncation: bool = False,
- pad_to_multiple_of: int | None = None,
- return_tensors: str | TensorType | None = None,
- return_attention_mask: bool | None = None,
- padding: str | None = "longest",
- max_length: int | None = None,
- sampling_rate: int | None = None,
- do_normalize: bool | None = None,
- device: str | None = "cpu",
- return_token_timestamps: bool | None = None,
- **kwargs,
- ) -> BatchFeature:
- """
- Main method to featurize and prepare for the model one or several sequence(s). Implementation uses PyTorch for
- the STFT computation if available, otherwise a slower NumPy based one.
-
- Args:
- raw_speech (`np.ndarray`, `list[float]`, `list[np.ndarray]`, `list[list[float]]`):
- The sequence or batch of sequences to be padded. Each sequence can be a numpy array, a list of float
- values, a list of numpy arrays or a list of list of float values. Must be mono channel audio, not
- stereo, i.e. single float per timestep.
- truncation (`bool`, *optional*, default to `True`):
- Activates truncation to cut input sequences longer than *max_length* to *max_length*.
- pad_to_multiple_of (`int`, *optional*, defaults to None):
- If set will pad the sequence to a multiple of the provided value.
-
- This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability
- `>= 7.5` (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128.
- return_attention_mask (`bool`, *optional*):
- Whether to return the attention mask. If left to the default, will return the attention mask according
- to the specific feature_extractor's default.
-
- [What are attention masks?](../glossary#attention-mask)
-
-
-
- For Parakeet models, `attention_mask` should always be passed for batched inference, to avoid subtle
- bugs.
-
-
-
- return_tensors (`str` or [`~utils.TensorType`], *optional*):
- If set, will return tensors instead of list of python integers. Acceptable values are:
-
- - `'tf'`: Return TensorFlow `tf.constant` objects.
- - `'pt'`: Return PyTorch `torch.Tensor` objects.
- - `'np'`: Return Numpy `np.ndarray` objects.
- sampling_rate (`int`, *optional*):
- The sampling rate at which the `raw_speech` input was sampled. It is strongly recommended to pass
- `sampling_rate` at the forward call to prevent silent errors and allow automatic speech recognition
- pipeline.
- padding_value (`float`, *optional*, defaults to 0.0):
- The value that is used to fill the padding values / vectors.
- do_normalize (`bool`, *optional*, defaults to `False`):
- Whether or not to zero-mean unit-variance normalize the input. Normalizing can help to significantly
- improve the performance of the model.
- device (`str`, *optional*, defaults to `'cpu'`):
- Specifies the device for computation of the log-mel spectrogram of audio signals in the
- `_torch_extract_fbank_features` method. (e.g., "cpu", "cuda")
- return_token_timestamps (`bool`, *optional*, defaults to `None`):
- Deprecated. Use `return_attention_mask` instead from which the number of frames can be inferred.
-
- Whether or not to return the number of frames of the input raw_speech.
- These num_frames can be used by the model to compute word level timestamps.
- """
- if sampling_rate is not None:
- if sampling_rate != self.sampling_rate:
- raise ValueError(
- f"The model corresponding to this feature extractor: {self.__class__.__name__} was trained using a"
- f" sampling rate of {self.sampling_rate}. Please make sure that the provided `raw_speech` input"
- f" was sampled with {self.sampling_rate} and not {sampling_rate}."
- )
- else:
- logger.warning(
- f"It is strongly recommended to pass the `sampling_rate` argument to `{self.__class__.__name__}()`. "
- "Failing to do so can result in silent errors that might be hard to debug."
- )
-
- # Convert to torch tensor
- if isinstance(raw_speech, np.ndarray):
- raw_speech = torch.tensor(raw_speech)
- elif isinstance(raw_speech, (list, tuple)) and isinstance(raw_speech[0], np.ndarray):
- raw_speech = [torch.tensor(speech) for speech in raw_speech]
-
- is_batched_torch = isinstance(raw_speech, torch.Tensor) and len(raw_speech.shape) > 1
- if is_batched_torch and len(raw_speech.shape) > 2:
- logger.warning(
- f"Only mono-channel audio is supported for input to {self.__class__.__name__}. "
- "We will take the mean of the channels to convert to mono."
- )
- raw_speech = raw_speech.mean(-1)
-
- is_batched_sequence = isinstance(raw_speech, (list, tuple))
- if is_batched_sequence:
- for speech in raw_speech:
- if len(speech.shape) > 1:
- logger.warning(
- f"Only mono-channel audio is supported for input to {self.__class__.__name__}. "
- "We will take the mean of the channels to convert to mono."
- )
- speech = speech.mean(-1)
-
- if is_batched_torch or is_batched_sequence:
- raw_speech = [speech[:, None].to(torch.float32) for speech in raw_speech]
- else:
- raw_speech = [raw_speech[:, None].to(torch.float32)]
-
- audio_lengths = [len(speech) for speech in raw_speech]
- batched_speech = BatchFeature({"input_features": raw_speech, "audio_lengths": audio_lengths})
-
- padded_inputs = self.pad(
- batched_speech,
- padding=padding,
- max_length=max_length,
- truncation=truncation,
- pad_to_multiple_of=pad_to_multiple_of,
- return_tensors="pt",
- )
- input_features = padded_inputs.input_features.squeeze(-1)
-
- # preemphasis
- if self.preemphasis is not None:
- timemask = torch.arange(input_features.shape[1], device=input_features.device).unsqueeze(
- 0
- ) < padded_inputs.audio_lengths.unsqueeze(1)
- input_features = torch.cat(
- [input_features[:, :1], input_features[:, 1:] - self.preemphasis * input_features[:, :-1]], dim=1
- )
- input_features = input_features.masked_fill(~timemask, 0.0)
+"""Backwards-compatibility shim: re-exports the legacy ``ParakeetFeatureExtractor`` name as a
+deprecated alias of [`ParakeetAudioProcessor`]. Importing or instantiating the alias emits a
+``FutureWarning``; the alias is removed in transformers v5.15 (see ADR 0002).
+"""
- input_features = self._torch_extract_fbank_features(input_features, device)
- features_lengths = torch.floor_divide(
- padded_inputs.audio_lengths + self.n_fft // 2 * 2 - self.n_fft, self.hop_length
- )
- attention_mask = torch.arange(input_features.shape[1], device=device)[None, :] < features_lengths[:, None]
+from ...audio_processing_base import make_legacy_audio_processor_alias
+from .audio_processing_parakeet import ParakeetAudioProcessor
- # normalize mel features, ignoring padding
- mask = attention_mask.unsqueeze(-1)
- input_features_masked = input_features * mask
- mean = input_features_masked.sum(dim=1) / features_lengths.unsqueeze(-1)
- mean = mean.unsqueeze(1)
- variance = ((input_features_masked - mean) ** 2 * mask).sum(dim=1) / (features_lengths - 1).unsqueeze(-1)
- std = torch.sqrt(variance).unsqueeze(1)
- input_features = (input_features - mean) / (std + EPSILON)
- input_features *= mask
- return BatchFeature(
- data={
- "input_features": input_features,
- "attention_mask": attention_mask,
- },
- tensor_type=return_tensors,
- )
+ParakeetFeatureExtractor = make_legacy_audio_processor_alias(ParakeetAudioProcessor, "ParakeetFeatureExtractor")
__all__ = ["ParakeetFeatureExtractor"]
diff --git a/src/transformers/models/pe_audio/__init__.py b/src/transformers/models/pe_audio/__init__.py
index dead10e3f5de..ffd4a1455e2f 100644
--- a/src/transformers/models/pe_audio/__init__.py
+++ b/src/transformers/models/pe_audio/__init__.py
@@ -18,6 +18,8 @@
if TYPE_CHECKING:
+ from .audio_processing_numpy_pe_audio import *
+ from .audio_processing_pe_audio import *
from .configuration_pe_audio import *
from .feature_extraction_pe_audio import *
from .modeling_pe_audio import *
diff --git a/src/transformers/models/pe_audio/audio_processing_numpy_pe_audio.py b/src/transformers/models/pe_audio/audio_processing_numpy_pe_audio.py
new file mode 100644
index 000000000000..16bb58a1276e
--- /dev/null
+++ b/src/transformers/models/pe_audio/audio_processing_numpy_pe_audio.py
@@ -0,0 +1,25 @@
+# Copyright 2026 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ...audio_processing_backends import NumpyAudioBackend
+
+
+class PeAudioAudioProcessorNumpy(NumpyAudioBackend):
+ """NumPy sibling of [`PeAudioAudioProcessor`] (ADR 0001)."""
+
+ sampling_rate = 16000
+ force_mono = True
+
+
+__all__ = ["PeAudioAudioProcessorNumpy"]
diff --git a/src/transformers/models/pe_audio/audio_processing_pe_audio.py b/src/transformers/models/pe_audio/audio_processing_pe_audio.py
new file mode 100644
index 000000000000..71709f1c17a6
--- /dev/null
+++ b/src/transformers/models/pe_audio/audio_processing_pe_audio.py
@@ -0,0 +1,23 @@
+# Copyright 2025 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ...audio_processing_backends import TorchAudioBackend
+
+
+class PeAudioAudioProcessor(TorchAudioBackend):
+ sampling_rate = 16000
+ force_mono = True
+
+
+__all__ = ["PeAudioAudioProcessor"]
diff --git a/src/transformers/models/pe_audio/feature_extraction_pe_audio.py b/src/transformers/models/pe_audio/feature_extraction_pe_audio.py
index a7738d3089ac..c33ec8ebfbe7 100644
--- a/src/transformers/models/pe_audio/feature_extraction_pe_audio.py
+++ b/src/transformers/models/pe_audio/feature_extraction_pe_audio.py
@@ -1,160 +1,20 @@
-# Copyright 2025 the HuggingFace Inc. team. All rights reserved.
+# Copyright 2026 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import numpy as np
-
-from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
-from ...feature_extraction_utils import BatchFeature
-from ...processing_utils import load_audio
-from ...utils import PaddingStrategy, TensorType, logging
-
-
-logger = logging.get_logger(__name__)
-
-
-class PeAudioFeatureExtractor(SequenceFeatureExtractor):
- r"""
- Constructs a PeAudioFeatureExtractor feature extractor.
-
- This feature extractor inherits from [`~feature_extraction_sequence_utils.SequenceFeatureExtractor`] which contains
- most of the main methods. Users should refer to this superclass for more information regarding those methods.
-
- Args:
- feature_size (`int`, *optional*, defaults to 1):
- The feature dimension of the extracted features. Use 1 for mono, 2 for stereo.
- sampling_rate (`int`, *optional*, defaults to 48000):
- The sampling rate at which the audio waveform should be digitalized, expressed in hertz (Hz).
- padding_value (`float`, *optional*, defaults to 0.0):
- The value that is used for padding.
- hop_length (`int`, *optional*, defaults to 1920):
- Overlap length between successive windows.
- """
-
- model_input_names = ["input_values"]
-
- def __init__(
- self,
- feature_size: int = 1,
- sampling_rate: int = 48_000,
- padding_value: float = 0.0,
- hop_length: int = 1920,
- **kwargs,
- ):
- super().__init__(feature_size=feature_size, sampling_rate=sampling_rate, padding_value=padding_value, **kwargs)
- self.hop_length = hop_length
-
- def _reflect_pad(self, wav):
- if len(wav) % self.hop_length == 0:
- return wav
- p1d = (0, self.hop_length - (len(wav) % self.hop_length))
- return np.pad(wav, p1d, "reflect")
-
- def __call__(
- self,
- raw_audio: np.ndarray | list[float] | list[np.ndarray] | list[list[float]] | str | list[str],
- padding: bool | str | PaddingStrategy | None = None,
- truncation: bool | None = False,
- max_length: int | None = None,
- return_tensors: str | TensorType | None = None,
- sampling_rate: int | None = None,
- ) -> BatchFeature:
- from_file = False
- if isinstance(raw_audio, str):
- raw_audio = [raw_audio]
-
- if isinstance(raw_audio, (list, tuple)) and isinstance(raw_audio[0], str):
- loaded = []
- for audio_file in raw_audio:
- loaded.append(load_audio(audio_file, self.sampling_rate))
- raw_audio = loaded
- from_file = True
-
- if sampling_rate is not None:
- if sampling_rate != self.sampling_rate:
- raise ValueError(
- f"The model corresponding to this feature extractor: {self} was trained using a sampling rate of"
- f" {self.sampling_rate}. Please make sure that the provided audio input was sampled with"
- f" {self.sampling_rate} and not {sampling_rate}."
- )
- elif not from_file:
- logger.warning(
- f"It is strongly recommended to pass the `sampling_rate` argument to `{self.__class__.__name__}()`. "
- "Failing to do so can result in silent errors that might be hard to debug."
- )
-
- if padding and truncation:
- raise ValueError("Both padding and truncation were set. Make sure you only set one.")
- elif padding is None:
- # by default let's pad the inputs
- padding = True
-
- is_batched = bool(
- isinstance(raw_audio, (list, tuple)) and (isinstance(raw_audio[0], (np.ndarray, tuple, list)))
- )
-
- if is_batched:
- raw_audio = [np.asarray(audio, dtype=np.float32).T for audio in raw_audio]
- elif not is_batched and not isinstance(raw_audio, np.ndarray):
- raw_audio = np.asarray(raw_audio, dtype=np.float32)
- elif isinstance(raw_audio, np.ndarray) and raw_audio.dtype is np.dtype(np.float64):
- raw_audio = raw_audio.astype(np.float32)
-
- # always return batch
- if not is_batched:
- raw_audio = [np.asarray(raw_audio).T]
-
- if isinstance(raw_audio, list):
- raw_audio = [self._reflect_pad(x) for x in raw_audio]
- else:
- raw_audio = self._reflect_pad(raw_audio)
-
- # verify inputs are valid
- for example in raw_audio:
- if example.ndim > 2:
- raise ValueError(f"Expected input shape (channels, length) but got shape {example.shape}")
- if self.feature_size == 1 and example.ndim != 1:
- raise ValueError(f"Expected mono audio but example has {example.shape[-1]} channels")
- if self.feature_size == 2:
- raise ValueError("Stereo audio isn't supported for now")
-
- input_values = BatchFeature({"input_values": raw_audio})
-
- # normal padding on batch
- padded_inputs = self.pad(
- input_values,
- max_length=max_length,
- truncation=truncation,
- padding=padding,
- return_attention_mask=padding,
- pad_to_multiple_of=self.hop_length,
- )
- if padding:
- padded_inputs["padding_mask"] = padded_inputs.pop("attention_mask")
- if padding:
- padded_inputs.input_values = padded_inputs.input_values[:, np.newaxis, :]
+"""Backwards-compatibility shim: re-exports the legacy ``PeAudioFeatureExtractor`` name as a
+deprecated alias of [`PeAudioAudioProcessor`]. Importing or instantiating the alias emits a
+``FutureWarning``; the alias is removed in transformers v5.15 (see ADR 0002).
+"""
- input_values = []
- for example in padded_inputs.pop("input_values"):
- if self.feature_size == 1:
- example = example[..., None]
- input_values.append(example.T)
+from ...audio_processing_base import make_legacy_audio_processor_alias
+from .audio_processing_pe_audio import PeAudioAudioProcessor
- padded_inputs["input_values"] = input_values
- if return_tensors is not None:
- padded_inputs = padded_inputs.convert_to_tensors(return_tensors)
- return padded_inputs
+PeAudioFeatureExtractor = make_legacy_audio_processor_alias(PeAudioAudioProcessor, "PeAudioFeatureExtractor")
__all__ = ["PeAudioFeatureExtractor"]
diff --git a/src/transformers/models/phi4_multimodal/audio_processing_phi4_multimodal.py b/src/transformers/models/phi4_multimodal/audio_processing_phi4_multimodal.py
new file mode 100644
index 000000000000..e8664b1e64e2
--- /dev/null
+++ b/src/transformers/models/phi4_multimodal/audio_processing_phi4_multimodal.py
@@ -0,0 +1,111 @@
+# Copyright 2025 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+
+from ...audio_processing_backends import TorchAudioBackend
+from ...audio_utils import MelScaleConfig, SpectrogramConfig, StftConfig
+
+
+class Phi4MultimodalAudioProcessor(TorchAudioBackend):
+ sampling_rate = 16000
+ force_mono = True
+ audio_compression_rate = 8
+ audio_downsample_rate = 1
+ audio_feat_stride = 1
+ spectrogram_config = SpectrogramConfig(
+ stft_config=StftConfig(
+ n_fft=512,
+ win_length=400,
+ hop_length=160,
+ window_fn="hamming_window",
+ periodic=False,
+ center=False,
+ power=2.0,
+ window_dtype="float64",
+ ),
+ preemphasis=0.97,
+ mel_scale_config=MelScaleConfig(
+ n_mels=80,
+ f_min=0,
+ f_max=7690,
+ mel_scale="kaldi",
+ triangularize_in_mel_space=True,
+ matmul_order="features_first",
+ # The legacy FE builds the kaldi bank in float64 numpy then casts to float32;
+ # a float64 build (cast back to the default float by the base dispatcher)
+ # reproduces those filters bit-exactly, unlike the default float32 kaldi path.
+ computation_dtype="float64",
+ ),
+ mel_floor=1.0,
+ log_mode="log",
+ )
+
+ def _apply_frame_processing(self, frames, *, spectrogram_config, audio_ranges=None, **kwargs):
+ # Mask frames that overlap the boundary between real audio and padding
+ stft_cfg = spectrogram_config.stft_config
+ win_length = stft_cfg.win_length or stft_cfg.n_fft
+ hop_length = stft_cfg.hop_length or win_length // 2
+ batch_size = frames.shape[0]
+
+ if audio_ranges is not None and batch_size > 1:
+ audio_lengths_t = torch.tensor([end - start for start, end in audio_ranges])
+ to_mask_idxs = torch.arange(batch_size)[audio_lengths_t != audio_lengths_t.max()]
+ if to_mask_idxs.numel() > 0:
+ frames = frames.clone()
+ down = (audio_lengths_t[to_mask_idxs] - win_length) // hop_length + 1
+ up = audio_lengths_t[to_mask_idxs] // hop_length - 1
+ offset = down.min()
+ max_idx = up.max()
+
+ mask_range = torch.arange(max_idx - offset).expand(to_mask_idxs.shape[0], -1)
+ mask = ((down - offset).unsqueeze(1) <= mask_range) & (mask_range < (up - offset).unsqueeze(1))
+ mask = mask.unsqueeze(-1).expand(-1, -1, win_length)
+
+ masked_frames = frames[to_mask_idxs, offset:max_idx].masked_fill_(mask, 0)
+ frames[to_mask_idxs, offset:max_idx] = masked_frames
+
+ frames_prev = torch.roll(frames, 1, dims=-1)
+ frames_prev[..., 0] = frames_prev[..., 1]
+ return (frames - spectrogram_config.preemphasis * frames_prev) * 32768
+
+ def _window_and_fft(self, frames, window, frame_length, n_fft, stft_cfg, audio_dtype=None):
+ frames = frames * window
+ if frame_length < n_fft:
+ frames = torch.nn.functional.pad(frames, (0, n_fft - frame_length))
+ # Cast to complex64 before abs() to match the FE's precision path
+ spec = torch.fft.rfft(frames, n=n_fft).to(torch.complex64)
+ if stft_cfg.normalized:
+ spec = spec / window.pow(2.0).sum().sqrt()
+ return spec.transpose(-2, -1)
+
+ def _compute_audio_embed_size(self, audio_frames):
+ integer = audio_frames // self.audio_compression_rate
+ remainder = audio_frames % self.audio_compression_rate
+ result = integer + (remainder > 0).to(integer.dtype)
+
+ integer = result // self.audio_downsample_rate
+ remainder = result % self.audio_downsample_rate
+ result = integer + (remainder > 0).to(integer.dtype)
+
+ return result
+
+ def _postprocess_output(self, output, **kwargs):
+ feature_lengths = output["audio_features_mask"].sum(dim=-1)
+ feature_lengths = feature_lengths * self.audio_feat_stride
+ output["audio_embed_sizes"] = self._compute_audio_embed_size(feature_lengths)
+ return output
+
+
+__all__ = ["Phi4MultimodalAudioProcessor"]
diff --git a/src/transformers/models/phi4_multimodal/feature_extraction_phi4_multimodal.py b/src/transformers/models/phi4_multimodal/feature_extraction_phi4_multimodal.py
index 9ce98251e50e..08b8342c5546 100644
--- a/src/transformers/models/phi4_multimodal/feature_extraction_phi4_multimodal.py
+++ b/src/transformers/models/phi4_multimodal/feature_extraction_phi4_multimodal.py
@@ -1,281 +1,20 @@
-# Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved.
+# Copyright 2026 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-Processor class for Phi4Multimodal
+"""Backwards-compatibility shim: re-exports the legacy ``Phi4MultimodalFeatureExtractor`` name as a
+deprecated alias of [`Phi4MultimodalAudioProcessor`]. Importing or instantiating the alias emits a
+``FutureWarning``; the alias is removed in transformers v5.15 (see ADR 0002).
"""
-import numpy as np
-
-from ...audio_utils import AudioInput, mel_filter_bank
-from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
-from ...image_processing_utils import BatchFeature
-from ...utils import TensorType, is_torch_available, logging
-
-
-if is_torch_available():
- import torch
-
-
-logger = logging.get_logger(__name__)
-
-
-class Phi4MultimodalFeatureExtractor(SequenceFeatureExtractor):
- model_input_names = ["audio_input_features", "audio_embed_sizes", "audio_attention_mask"]
-
- def __init__(
- self,
- feature_size: int = 80,
- sampling_rate: int = 16000,
- hop_length: int = 160,
- n_fft: int = 512,
- win_length: int = 400,
- preemphasis: float = 0.97,
- padding_value: float = 0.0,
- audio_compression_rate: int = 8,
- audio_downsample_rate: int = 1,
- audio_feat_stride: int = 1,
- mel_min_frequency: float = 0,
- mel_max_frequency: float = 7690,
- **kwargs,
- ):
- super().__init__(feature_size=feature_size, sampling_rate=sampling_rate, padding_value=padding_value, **kwargs)
-
- self.hop_length = hop_length
- self.n_fft = n_fft
- self.win_length = win_length
- self.preemphasis = preemphasis
- self.padding_value = padding_value
- self.audio_compression_rate = audio_compression_rate
- self.audio_downsample_rate = audio_downsample_rate
- self.audio_feat_stride = audio_feat_stride
-
- self.mel_filters = mel_filter_bank(
- num_frequency_bins=self.n_fft // 2 + 1,
- num_mel_filters=self.feature_size,
- min_frequency=mel_min_frequency,
- max_frequency=mel_max_frequency,
- sampling_rate=self.sampling_rate,
- triangularize_in_mel_space=True,
- mel_scale="kaldi",
- )
-
- def __call__(
- self,
- raw_speech: AudioInput,
- sampling_rate: int | None = None,
- pad_to_multiple_of: int | None = None,
- padding: str | None = "longest",
- max_length: int | None = None,
- truncation: bool = False,
- return_tensors: str | TensorType | None = None,
- return_attention_mask: bool | None = True,
- device: str | None = "cpu",
- **kwargs,
- ) -> BatchFeature:
- """
- Main method to featurize and prepare for the model one or several audio sequence(s). Implementation uses PyTorch for
- the STFT computation if available, otherwise a slower NumPy based one.
-
- Args:
- raw_speech (`np.ndarray`, `torch.Tensor`, `list[np.ndarray]`, `list[torch.Tensor]`):
- The sequence or batch of sequences to be processed. Each sequence can be a numpy array or PyTorch tensor.
- For batched inputs, sequences can be a list of numpy arrays or PyTorch tensors, or a single numpy array or
- PyTorch tensor with first dimension being the batch size.
- sampling_rate (`int`, *optional*):
- The sampling rate at which the `raw_speech` input was sampled. It is strongly recommended to pass
- `sampling_rate` at the forward call to prevent silent errors.
- pad_to_multiple_of (`int`, *optional*, defaults to None):
- If set will pad the sequence to a multiple of the provided value.
- padding (`str`, *optional*, defaults to "longest"):
- Padding strategy. Can be "longest" to pad to the longest sequence in the batch, or a specific length.
- max_length (`int`, *optional*):
- Maximum length of the returned list and optionally padding length.
- truncation (`bool`, *optional*, defaults to False):
- Activates truncation to cut input sequences longer than *max_length* to *max_length*.
- return_tensors (`str` or [`~utils.TensorType`], *optional*):
- If set, will return tensors instead of numpy arrays. Acceptable values are:
- - `'pt'`: Return PyTorch `torch.Tensor` objects.
- - `'np'`: Return Numpy `np.ndarray` objects.
- return_attention_mask (`bool`, *optional*, defaults to `True`):
- Whether to return the extracted audio input features' attention mask.
- device (`str`, *optional*, defaults to "cpu"):
- Specifies the device for computation of the audio features. (e.g., "cpu", "cuda")
-
- Returns:
- [`BatchFeature`]: A [`BatchFeature`] with the following fields:
- - **audio_input_features** -- Audio features extracted from the raw audio input, shape (batch_size, max_feature_length, feature_size).
- - **audio_lengths** -- Length of each audio sample in the batch, shape (batch_size,).
- - **audio_attention_mask** -- Attention mask for the audio input, shape (batch_size, max_feature_length).
- If `return_tensors` is not specified, the fields will be PyTorch tensors if PyTorch is available, otherwise NumPy arrays.
- """
- if sampling_rate is not None:
- if sampling_rate != self.sampling_rate:
- raise ValueError(
- f"The model corresponding to this feature extractor: {self.__class__.__name__} was trained using a"
- f" sampling rate of {self.sampling_rate}. Please make sure that the provided `raw_speech` input"
- f" was sampled with {self.sampling_rate} and not {sampling_rate}."
- )
- else:
- logger.warning(
- f"It is strongly recommended to pass the `sampling_rate` argument to `{self.__class__.__name__}()`. "
- "Failing to do so can result in silent errors that might be hard to debug."
- )
-
- # Convert to torch tensor
- if isinstance(raw_speech, np.ndarray):
- raw_speech = torch.tensor(raw_speech)
- elif isinstance(raw_speech, (list, tuple)) and isinstance(raw_speech[0], np.ndarray):
- raw_speech = [torch.tensor(speech) for speech in raw_speech]
-
- is_batched_torch = isinstance(raw_speech, torch.Tensor) and len(raw_speech.shape) > 1
- if is_batched_torch and len(raw_speech.shape) > 2:
- logger.warning(
- f"Only mono-channel audio is supported for input to {self.__class__.__name__}. "
- "We will take the mean of the channels to convert to mono."
- )
- raw_speech = raw_speech.mean(-1)
-
- is_batched_sequence = isinstance(raw_speech, (list, tuple))
- if is_batched_sequence:
- for speech in raw_speech:
- if len(speech.shape) > 1:
- logger.warning(
- f"Only mono-channel audio is supported for input to {self.__class__.__name__}. "
- "We will take the mean of the channels to convert to mono."
- )
- speech = speech.mean(-1)
-
- if is_batched_torch or is_batched_sequence:
- raw_speech = [speech[:, None].to(torch.float32) for speech in raw_speech]
- else:
- raw_speech = [raw_speech[:, None].to(torch.float32)]
-
- audio_lengths = [len(speech) for speech in raw_speech]
-
- # convert into correct format for padding
- batched_speech = BatchFeature(data={"audio_input_features": raw_speech, "audio_lengths": audio_lengths})
- padded_inputs = self.pad(
- batched_speech,
- padding=padding,
- max_length=max_length,
- truncation=truncation,
- pad_to_multiple_of=pad_to_multiple_of,
- return_tensors="pt",
- )
- input_features = padded_inputs.audio_input_features.squeeze(-1)
- audio_lengths = padded_inputs.audio_lengths
-
- input_features = self._torch_extract_fbank_features(input_features, audio_lengths, device)
-
- feature_lengths = (audio_lengths - self.win_length) // self.hop_length + 1
- feature_lengths = feature_lengths * self.audio_feat_stride
- audio_embed_sizes = self._compute_audio_embed_size(feature_lengths)
-
- feature_attention_mask = (
- torch.arange(0, feature_lengths.max()) if is_torch_available() else np.arange(0, feature_lengths.max())
- )
- feature_attention_mask = (
- feature_attention_mask[None, :] < feature_lengths[:, None] if len(feature_lengths) > 1 else None
- )
-
- data = {
- "audio_input_features": input_features,
- "audio_embed_sizes": audio_embed_sizes,
- }
- if feature_attention_mask is not None and return_attention_mask:
- data["audio_attention_mask"] = feature_attention_mask
-
- return BatchFeature(data=data, tensor_type=return_tensors)
-
- # TODO; @eustlb, move this to audio_utils in a general spectogram_batch function that handles torch and numpy
- def _torch_extract_fbank_features(
- self, waveform: "torch.FloatTensor", audio_lengths: "torch.Tensor", device: str = "cpu"
- ) -> "torch.FloatTensor":
- """
- Compute the log mel-scaled spectrogram of batched waveforms using PyTorch's FFT implementation.
-
- Args:
- waveform (torch.FloatTensor` of shape `(batch_size, max_audio_length)`):
- The batched waveforms.
- audio_lengths (`torch.Tensor` of shape `(batch_size,)`):
- The lengths of the waveforms along the max_audio_length dimension.
- device (`str`, *optional*, defaults to "cpu"):
- The device to run the computation on. (e.g., "cpu", "cuda")
-
- Returns:
- `torch.FloatTensor` of shape `(batch_size, max_feature_length, feature_size)`:
- The log mel-scaled spectrogram of the batched waveforms.
- """
- fft_window = torch.hamming_window(self.win_length, periodic=False, device=device, dtype=torch.float64)
-
- # batched implementation
- batch_size = waveform.shape[0]
- frames = waveform.unfold(-1, self.win_length, self.hop_length)
-
- # ---
- # the unbatched (and unpaded) original implementation skips last few audio values that can't be included in a frame
- # we need to ensure that the corresponding frames for the padded input also mask these values
- if batch_size > 1:
- frames = frames.clone()
- # concerned batch indices
- to_mask_batch_idxs = torch.arange(batch_size)[audio_lengths != audio_lengths.max()]
- if to_mask_batch_idxs.numel() > 0:
- batch_idxs_down = (audio_lengths[to_mask_batch_idxs] - self.win_length) // self.hop_length + 1
- batch_idxs_up = (audio_lengths[to_mask_batch_idxs] // self.hop_length) - 1
- offset_idx = batch_idxs_down.min()
- max_idx = batch_idxs_up.max()
-
- mask = torch.arange(max_idx - offset_idx, device=device).expand(to_mask_batch_idxs.shape[0], -1)
- mask = ((batch_idxs_down - offset_idx).unsqueeze(1) <= mask) & (
- mask < (batch_idxs_up - offset_idx).unsqueeze(1)
- )
- mask = mask.unsqueeze(-1).expand(-1, -1, self.win_length)
- masked_frames = frames[to_mask_batch_idxs, offset_idx:max_idx].masked_fill_(mask, 0)
- frames[to_mask_batch_idxs, offset_idx:max_idx] = masked_frames
- # ---
-
- # apply pre-emphasis first order filter on fft windows
- frames_prev = torch.roll(frames, 1, dims=-1)
- frames_prev[:, :, 0] = frames_prev[:, :, 1]
- frames = (frames - self.preemphasis * frames_prev) * 32768
-
- # apply fft
- S = torch.fft.rfft(fft_window * frames.view(-1, self.win_length), n=self.n_fft, dim=1)
- S = S.view(frames.shape[0], -1, S.shape[-1])
- S = S.to(torch.complex64)
-
- spec = torch.abs(S)
- spec_power = spec**2
-
- # apply triangular mel filter bank
- mel_filters = torch.from_numpy(self.mel_filters).to(device, torch.float32)
- log_spec = torch.clamp(spec_power @ mel_filters, min=1.0)
- log_spec = torch.log(log_spec)
-
- return log_spec
-
- def _compute_audio_embed_size(self, audio_frames):
- integer = audio_frames // self.audio_compression_rate
- remainder = audio_frames % self.audio_compression_rate
- result = integer + (remainder > 0).to(integer.dtype)
+from ...audio_processing_base import make_legacy_audio_processor_alias
+from .audio_processing_phi4_multimodal import Phi4MultimodalAudioProcessor
- integer = result // self.audio_downsample_rate
- remainder = result % self.audio_downsample_rate
- result = integer + (remainder > 0).to(integer.dtype) # qformer compression
- return result
+Phi4MultimodalFeatureExtractor = make_legacy_audio_processor_alias(Phi4MultimodalAudioProcessor, "Phi4MultimodalFeatureExtractor")
__all__ = ["Phi4MultimodalFeatureExtractor"]
diff --git a/src/transformers/models/pop2piano/__init__.py b/src/transformers/models/pop2piano/__init__.py
index cbbfcbf157b5..e9eb13f28be2 100644
--- a/src/transformers/models/pop2piano/__init__.py
+++ b/src/transformers/models/pop2piano/__init__.py
@@ -18,7 +18,10 @@
if TYPE_CHECKING:
+ from .audio_processing_numpy_pop2piano import *
+ from .audio_processing_pop2piano import *
from .configuration_pop2piano import *
+ from .feature_extraction_pop2piano import *
from .modeling_pop2piano import *
else:
import sys
diff --git a/src/transformers/models/pop2piano/audio_processing_numpy_pop2piano.py b/src/transformers/models/pop2piano/audio_processing_numpy_pop2piano.py
new file mode 100644
index 000000000000..bb393c805c1f
--- /dev/null
+++ b/src/transformers/models/pop2piano/audio_processing_numpy_pop2piano.py
@@ -0,0 +1,37 @@
+# Copyright 2026 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# NOTE: Full Pop2Piano feature extraction requires the Essentia library for
+# beat detection (RhythmExtractor2013) and scipy for beat interpolation.
+# This audio processor provides the basic mel spectrogram configuration but
+# does not implement the complete beat-aligned segmentation pipeline.
+
+from ...audio_processing_backends import NumpyAudioBackend
+from ...audio_utils import MelScaleConfig, SpectrogramConfig, StftConfig
+
+
+class Pop2PianoAudioProcessorNumpy(NumpyAudioBackend):
+ """NumPy sibling of [`Pop2PianoAudioProcessor`]. Pure-config: log10 mel spectrogram
+ pipeline shared by both backends via the base class hooks (ADR 0001)."""
+
+ sampling_rate = 22050
+ force_mono = True
+ spectrogram_config = SpectrogramConfig(
+ stft_config=StftConfig(n_fft=4096, hop_length=1024, power=2.0),
+ mel_scale_config=MelScaleConfig(n_mels=512, f_min=10.0, mel_scale="htk"),
+ log_mode="log10",
+ )
+
+
+__all__ = ["Pop2PianoAudioProcessorNumpy"]
diff --git a/src/transformers/models/pop2piano/audio_processing_pop2piano.py b/src/transformers/models/pop2piano/audio_processing_pop2piano.py
new file mode 100644
index 000000000000..abdc264de139
--- /dev/null
+++ b/src/transformers/models/pop2piano/audio_processing_pop2piano.py
@@ -0,0 +1,31 @@
+# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# NOTE: Full Pop2Piano feature extraction requires the Essentia library for
+# beat detection (RhythmExtractor2013) and scipy for beat interpolation.
+# This audio processor provides the basic mel spectrogram configuration but
+# does not implement the complete beat-aligned segmentation pipeline.
+
+from ...audio_processing_backends import TorchAudioBackend
+from .audio_processing_numpy_pop2piano import Pop2PianoAudioProcessorNumpy
+
+
+class Pop2PianoAudioProcessor(TorchAudioBackend):
+ sampling_rate = 22050
+ force_mono = True
+
+ spectrogram_config = Pop2PianoAudioProcessorNumpy.spectrogram_config
+
+
+__all__ = ["Pop2PianoAudioProcessor"]
diff --git a/src/transformers/models/pop2piano/feature_extraction_pop2piano.py b/src/transformers/models/pop2piano/feature_extraction_pop2piano.py
index 4e770fcb1b71..7171159cd1e1 100644
--- a/src/transformers/models/pop2piano/feature_extraction_pop2piano.py
+++ b/src/transformers/models/pop2piano/feature_extraction_pop2piano.py
@@ -1,452 +1,20 @@
-# Copyright 2023 The HuggingFace Inc. team.
+# Copyright 2026 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Feature extractor class for Pop2Piano"""
-
-import warnings
-
-import numpy
-import numpy as np
-
-from ...audio_utils import mel_filter_bank, spectrogram
-from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
-from ...feature_extraction_utils import BatchFeature
-from ...utils import (
- TensorType,
- is_essentia_available,
- is_librosa_available,
- is_scipy_available,
- logging,
- requires_backends,
-)
-from ...utils.import_utils import requires
-
-
-if is_essentia_available():
- import essentia.standard
-
-if is_librosa_available():
- import librosa
-
-if is_scipy_available():
- import scipy
-
-
-logger = logging.get_logger(__name__)
-
-
-@requires(backends=("essentia", "librosa", "scipy", "torch"))
-class Pop2PianoFeatureExtractor(SequenceFeatureExtractor):
- r"""
- Constructs a Pop2Piano feature extractor.
-
- This feature extractor inherits from [`~feature_extraction_sequence_utils.SequenceFeatureExtractor`] which contains
- most of the main methods. Users should refer to this superclass for more information regarding those methods.
-
- This class extracts rhythm and preprocesses the audio before it is passed to the model. First the audio is passed
- to `RhythmExtractor2013` algorithm which extracts the beat_times, beat positions and estimates their confidence as
- well as tempo in bpm, then beat_times is interpolated and to get beatsteps. Later we calculate
- extrapolated_beatsteps from it to be used in tokenizer. On the other hand audio is resampled to self.sampling_rate
- and preprocessed and then log mel spectogram is computed from that to be used in our transformer model.
-
- Args:
- sampling_rate (`int`, *optional*, defaults to 22050):
- Target Sampling rate of audio signal. It's the sampling rate that we forward to the model.
- padding_value (`int`, *optional*, defaults to 0):
- Padding value used to pad the audio. Should correspond to silences.
- window_size (`int`, *optional*, defaults to 4096):
- Length of the window in samples to which the Fourier transform is applied.
- hop_length (`int`, *optional*, defaults to 1024):
- Step size between each window of the waveform, in samples.
- min_frequency (`float`, *optional*, defaults to 10.0):
- Lowest frequency that will be used in the log-mel spectrogram.
- feature_size (`int`, *optional*, defaults to 512):
- The feature dimension of the extracted features.
- num_bars (`int`, *optional*, defaults to 2):
- Determines interval between each sequence.
- """
-
- model_input_names = ["input_features", "beatsteps", "extrapolated_beatstep"]
-
- def __init__(
- self,
- sampling_rate: int = 22050,
- padding_value: int = 0,
- window_size: int = 4096,
- hop_length: int = 1024,
- min_frequency: float = 10.0,
- feature_size: int = 512,
- num_bars: int = 2,
- **kwargs,
- ):
- super().__init__(
- feature_size=feature_size,
- sampling_rate=sampling_rate,
- padding_value=padding_value,
- **kwargs,
- )
- self.sampling_rate = sampling_rate
- self.padding_value = padding_value
- self.window_size = window_size
- self.hop_length = hop_length
- self.min_frequency = min_frequency
- self.feature_size = feature_size
- self.num_bars = num_bars
- self.mel_filters = mel_filter_bank(
- num_frequency_bins=(self.window_size // 2) + 1,
- num_mel_filters=self.feature_size,
- min_frequency=self.min_frequency,
- max_frequency=float(self.sampling_rate // 2),
- sampling_rate=self.sampling_rate,
- norm=None,
- mel_scale="htk",
- )
-
- def mel_spectrogram(self, sequence: np.ndarray):
- """
- Generates MelSpectrogram.
-
- Args:
- sequence (`numpy.ndarray`):
- The sequence of which the mel-spectrogram will be computed.
- """
- mel_specs = []
- for seq in sequence:
- window = np.hanning(self.window_size + 1)[:-1]
- mel_specs.append(
- spectrogram(
- waveform=seq,
- window=window,
- frame_length=self.window_size,
- hop_length=self.hop_length,
- power=2.0,
- mel_filters=self.mel_filters,
- )
- )
- mel_specs = np.array(mel_specs)
-
- return mel_specs
-
- def extract_rhythm(self, audio: np.ndarray):
- """
- This algorithm(`RhythmExtractor2013`) extracts the beat positions and estimates their confidence as well as
- tempo in bpm for an audio signal. For more information please visit
- https://essentia.upf.edu/reference/std_RhythmExtractor2013.html .
-
- Args:
- audio(`numpy.ndarray`):
- raw audio waveform which is passed to the Rhythm Extractor.
- """
- requires_backends(self, ["essentia"])
- essentia_tracker = essentia.standard.RhythmExtractor2013(method="multifeature")
- bpm, beat_times, confidence, estimates, essentia_beat_intervals = essentia_tracker(audio)
-
- return bpm, beat_times, confidence, estimates, essentia_beat_intervals
-
- def interpolate_beat_times(
- self, beat_times: numpy.ndarray, steps_per_beat: numpy.ndarray, n_extend: numpy.ndarray
- ):
- """
- This method takes beat_times and then interpolates that using `scipy.interpolate.interp1d` and the output is
- then used to convert raw audio to log-mel-spectrogram.
-
- Args:
- beat_times (`numpy.ndarray`):
- beat_times is passed into `scipy.interpolate.interp1d` for processing.
- steps_per_beat (`int`):
- used as an parameter to control the interpolation.
- n_extend (`int`):
- used as an parameter to control the interpolation.
- """
-
- requires_backends(self, ["scipy"])
- beat_times_function = scipy.interpolate.interp1d(
- np.arange(beat_times.size),
- beat_times,
- bounds_error=False,
- fill_value="extrapolate",
- )
-
- ext_beats = beat_times_function(
- np.linspace(0, beat_times.size + n_extend - 1, beat_times.size * steps_per_beat + n_extend)
- )
-
- return ext_beats
-
- def preprocess_mel(self, audio: np.ndarray, beatstep: np.ndarray):
- """
- Preprocessing for log-mel-spectrogram
-
- Args:
- audio (`numpy.ndarray` of shape `(audio_length, )` ):
- Raw audio waveform to be processed.
- beatstep (`numpy.ndarray`):
- Interpolated values of the raw audio. If beatstep[0] is greater than 0.0, then it will be shifted by
- the value at beatstep[0].
- """
-
- if audio is not None and len(audio.shape) != 1:
- raise ValueError(
- f"Expected `audio` to be a single channel audio input of shape `(n, )` but found shape {audio.shape}."
- )
- if beatstep[0] > 0.0:
- beatstep = beatstep - beatstep[0]
-
- num_steps = self.num_bars * 4
- num_target_steps = len(beatstep)
- extrapolated_beatstep = self.interpolate_beat_times(
- beat_times=beatstep, steps_per_beat=1, n_extend=(self.num_bars + 1) * 4 + 1
- )
-
- sample_indices = []
- max_feature_length = 0
- for i in range(0, num_target_steps, num_steps):
- start_idx = i
- end_idx = min(i + num_steps, num_target_steps)
- start_sample = int(extrapolated_beatstep[start_idx] * self.sampling_rate)
- end_sample = int(extrapolated_beatstep[end_idx] * self.sampling_rate)
- sample_indices.append((start_sample, end_sample))
- max_feature_length = max(max_feature_length, end_sample - start_sample)
- padded_batch = []
- for start_sample, end_sample in sample_indices:
- feature = audio[start_sample:end_sample]
- padded_feature = np.pad(
- feature,
- ((0, max_feature_length - feature.shape[0]),),
- "constant",
- constant_values=0,
- )
- padded_batch.append(padded_feature)
-
- padded_batch = np.asarray(padded_batch)
- return padded_batch, extrapolated_beatstep
-
- def _pad(self, features: np.ndarray, add_zero_line=True):
- features_shapes = [each_feature.shape for each_feature in features]
- attention_masks, padded_features = [], []
- for i, each_feature in enumerate(features):
- # To pad "input_features".
- if len(each_feature.shape) == 3:
- features_pad_value = max([*zip(*features_shapes)][1]) - features_shapes[i][1]
- attention_mask = np.ones(features_shapes[i][:2], dtype=np.int64)
- feature_padding = ((0, 0), (0, features_pad_value), (0, 0))
- attention_mask_padding = (feature_padding[0], feature_padding[1])
-
- # To pad "beatsteps" and "extrapolated_beatstep".
- else:
- each_feature = each_feature.reshape(1, -1)
- features_pad_value = max([*zip(*features_shapes)][0]) - features_shapes[i][0]
- attention_mask = np.ones(features_shapes[i], dtype=np.int64).reshape(1, -1)
- feature_padding = attention_mask_padding = ((0, 0), (0, features_pad_value))
-
- each_padded_feature = np.pad(each_feature, feature_padding, "constant", constant_values=self.padding_value)
- attention_mask = np.pad(
- attention_mask, attention_mask_padding, "constant", constant_values=self.padding_value
- )
-
- if add_zero_line:
- # if it is batched then we separate each examples using zero array
- zero_array_len = max([*zip(*features_shapes)][1])
-
- # we concatenate the zero array line here
- each_padded_feature = np.concatenate(
- [each_padded_feature, np.zeros([1, zero_array_len, self.feature_size])], axis=0
- )
- attention_mask = np.concatenate(
- [attention_mask, np.zeros([1, zero_array_len], dtype=attention_mask.dtype)], axis=0
- )
-
- padded_features.append(each_padded_feature)
- attention_masks.append(attention_mask)
-
- padded_features = np.concatenate(padded_features, axis=0).astype(np.float32)
- attention_masks = np.concatenate(attention_masks, axis=0).astype(np.int64)
-
- return padded_features, attention_masks
-
- def pad(
- self,
- inputs: BatchFeature,
- is_batched: bool,
- return_attention_mask: bool,
- return_tensors: str | TensorType | None = None,
- ):
- """
- Pads the inputs to same length and returns attention_mask.
-
- Args:
- inputs (`BatchFeature`):
- Processed audio features.
- is_batched (`bool`):
- Whether inputs are batched or not.
- return_attention_mask (`bool`):
- Whether to return attention mask or not.
- return_tensors (`str` or [`~utils.TensorType`], *optional*):
- If set, will return tensors instead of list of python integers. Acceptable values are:
- - `'pt'`: Return PyTorch `torch.Tensor` objects.
- - `'np'`: Return Numpy `np.ndarray` objects.
- If nothing is specified, it will return list of `np.ndarray` arrays.
- Return:
- `BatchFeature` with attention_mask, attention_mask_beatsteps and attention_mask_extrapolated_beatstep added
- to it:
- - **attention_mask** numpy.ndarray of shape `(batch_size, max_input_features_seq_length)` --
- Example :
- 1, 1, 1, 0, 0 (audio 1, also here it is padded to max length of 5 that's why there are 2 zeros at
- the end indicating they are padded)
-
- 0, 0, 0, 0, 0 (zero pad to separate audio 1 and 2)
-
- 1, 1, 1, 1, 1 (audio 2)
-
- 0, 0, 0, 0, 0 (zero pad to separate audio 2 and 3)
-
- 1, 1, 1, 1, 1 (audio 3)
- - **attention_mask_beatsteps** numpy.ndarray of shape `(batch_size, max_beatsteps_seq_length)`
- - **attention_mask_extrapolated_beatstep** numpy.ndarray of shape `(batch_size,
- max_extrapolated_beatstep_seq_length)`
- """
-
- processed_features_dict = {}
- for feature_name, feature_value in inputs.items():
- if feature_name == "input_features":
- padded_feature_values, attention_mask = self._pad(feature_value, add_zero_line=True)
- processed_features_dict[feature_name] = padded_feature_values
- if return_attention_mask:
- processed_features_dict["attention_mask"] = attention_mask
- else:
- padded_feature_values, attention_mask = self._pad(feature_value, add_zero_line=False)
- processed_features_dict[feature_name] = padded_feature_values
- if return_attention_mask:
- processed_features_dict[f"attention_mask_{feature_name}"] = attention_mask
-
- # If we are processing only one example, we should remove the zero array line since we don't need it to
- # separate examples from each other.
- if not is_batched and not return_attention_mask:
- processed_features_dict["input_features"] = processed_features_dict["input_features"][:-1, ...]
-
- outputs = BatchFeature(processed_features_dict, tensor_type=return_tensors)
-
- return outputs
-
- def __call__(
- self,
- audio: np.ndarray | list[float] | list[np.ndarray] | list[list[float]],
- sampling_rate: int | list[int],
- steps_per_beat: int = 2,
- resample: bool | None = True,
- return_attention_mask: bool | None = False,
- return_tensors: str | TensorType | None = None,
- **kwargs,
- ) -> BatchFeature:
- """
- Main method to featurize and prepare for the model.
-
- Args:
- audio (`np.ndarray`, `List`):
- The audio or batch of audio to be processed. Each audio can be a numpy array, a list of float values, a
- list of numpy arrays or a list of list of float values.
- sampling_rate (`int`):
- The sampling rate at which the `audio` input was sampled. It is strongly recommended to pass
- `sampling_rate` at the forward call to prevent silent errors.
- steps_per_beat (`int`, *optional*, defaults to 2):
- This is used in interpolating `beat_times`.
- resample (`bool`, *optional*, defaults to `True`):
- Determines whether to resample the audio to `sampling_rate` or not before processing. Must be True
- during inference.
- return_attention_mask (`bool` *optional*, defaults to `False`):
- Denotes if attention_mask for input_features, beatsteps and extrapolated_beatstep will be given as
- output or not. Automatically set to True for batched inputs.
- return_tensors (`str` or [`~utils.TensorType`], *optional*):
- If set, will return tensors instead of list of python integers. Acceptable values are:
- - `'pt'`: Return PyTorch `torch.Tensor` objects.
- - `'np'`: Return Numpy `np.ndarray` objects.
- If nothing is specified, it will return list of `np.ndarray` arrays.
- """
-
- requires_backends(self, ["librosa"])
- is_batched = isinstance(audio, (list, tuple)) and isinstance(audio[0], (np.ndarray, tuple, list))
- if is_batched:
- # This enables the user to process files of different sampling_rate at same time
- if not isinstance(sampling_rate, list):
- raise ValueError(
- "Please give sampling_rate of each audio separately when you are passing multiple raw_audios at the same time. "
- f"Received {sampling_rate}, expected [audio_1_sr, ..., audio_n_sr]."
- )
- return_attention_mask = True if return_attention_mask is None else return_attention_mask
- else:
- audio = [audio]
- sampling_rate = [sampling_rate]
- return_attention_mask = False if return_attention_mask is None else return_attention_mask
-
- batch_input_features, batch_beatsteps, batch_ext_beatstep = [], [], []
- for single_raw_audio, single_sampling_rate in zip(audio, sampling_rate):
- bpm, beat_times, confidence, estimates, essentia_beat_intervals = self.extract_rhythm(
- audio=single_raw_audio
- )
- beatsteps = self.interpolate_beat_times(beat_times=beat_times, steps_per_beat=steps_per_beat, n_extend=1)
-
- if self.sampling_rate != single_sampling_rate and self.sampling_rate is not None:
- if resample:
- # Change sampling_rate to self.sampling_rate
- single_raw_audio = librosa.core.resample(
- single_raw_audio,
- orig_sr=single_sampling_rate,
- target_sr=self.sampling_rate,
- res_type="kaiser_best",
- )
- else:
- warnings.warn(
- f"The sampling_rate of the provided audio is different from the target sampling_rate "
- f"of the Feature Extractor, {self.sampling_rate} vs {single_sampling_rate}. "
- f"In these cases it is recommended to use `resample=True` in the `__call__` method to "
- f"get the optimal behaviour."
- )
-
- single_sampling_rate = self.sampling_rate
- start_sample = int(beatsteps[0] * single_sampling_rate)
- end_sample = int(beatsteps[-1] * single_sampling_rate)
-
- input_features, extrapolated_beatstep = self.preprocess_mel(
- single_raw_audio[start_sample:end_sample], beatsteps - beatsteps[0]
- )
-
- mel_specs = self.mel_spectrogram(input_features.astype(np.float32))
-
- # apply np.log to get log mel-spectrograms
- log_mel_specs = np.log(np.clip(mel_specs, a_min=1e-6, a_max=None))
-
- input_features = np.transpose(log_mel_specs, (0, -1, -2))
-
- batch_input_features.append(input_features)
- batch_beatsteps.append(beatsteps)
- batch_ext_beatstep.append(extrapolated_beatstep)
+"""Backwards-compatibility shim: re-exports the legacy ``Pop2PianoFeatureExtractor`` name as a
+deprecated alias of [`Pop2PianoAudioProcessor`]. Importing or instantiating the alias emits a
+``FutureWarning``; the alias is removed in transformers v5.15 (see ADR 0002).
+"""
- output = BatchFeature(
- {
- "input_features": batch_input_features,
- "beatsteps": batch_beatsteps,
- "extrapolated_beatstep": batch_ext_beatstep,
- }
- )
+from ...audio_processing_base import make_legacy_audio_processor_alias
+from .audio_processing_pop2piano import Pop2PianoAudioProcessor
- output = self.pad(
- output,
- is_batched=is_batched,
- return_attention_mask=return_attention_mask,
- return_tensors=return_tensors,
- )
- return output
+Pop2PianoFeatureExtractor = make_legacy_audio_processor_alias(Pop2PianoAudioProcessor, "Pop2PianoFeatureExtractor")
__all__ = ["Pop2PianoFeatureExtractor"]
diff --git a/src/transformers/models/qwen3_asr/__init__.py b/src/transformers/models/qwen3_asr/__init__.py
index 19df31aaf924..c7790b94d107 100644
--- a/src/transformers/models/qwen3_asr/__init__.py
+++ b/src/transformers/models/qwen3_asr/__init__.py
@@ -18,6 +18,8 @@
if TYPE_CHECKING:
+ from .audio_processing_numpy_qwen3_asr import *
+ from .audio_processing_qwen3_asr import *
from .configuration_qwen3_asr import *
from .feature_extraction_qwen3_asr import *
from .modeling_qwen3_asr import *
diff --git a/src/transformers/models/qwen3_asr/audio_processing_numpy_qwen3_asr.py b/src/transformers/models/qwen3_asr/audio_processing_numpy_qwen3_asr.py
new file mode 100644
index 000000000000..3919f6e3509c
--- /dev/null
+++ b/src/transformers/models/qwen3_asr/audio_processing_numpy_qwen3_asr.py
@@ -0,0 +1,124 @@
+# Copyright 2026 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from ...audio_processing_backends import NumpyAudioBackend
+from ...audio_utils import MelScaleConfig, SpectrogramConfig, StftConfig
+from ...processing_utils import AudioKwargs
+
+
+def _qwen3_asr_chunk_length_to_max_length(value, config_dict):
+ # Legacy Qwen3 ASR hub configs store `chunk_length=30` (seconds); the new API uses
+ # `max_length` in samples. Translate using the sampling rate carried by the pass-through
+ # `sampling_rate` key.
+ sampling_rate = config_dict.get("sampling_rate") or 16000
+ config_dict.setdefault("max_length", value * sampling_rate)
+
+
+class Qwen3ASRAudioKwargs(AudioKwargs, total=False):
+ n_window: int | None
+
+
+class Qwen3ASRAudioProcessorNumpy(NumpyAudioBackend):
+ """NumPy sibling of [`Qwen3ASRAudioProcessor`]. Required to produce bit-exact outputs
+ against the torch sibling (ADR 0001).
+
+ Whisper-style 128-bin log-mel features with three Qwen3-ASR-specific twists:
+
+ - clips shorter than ``min_length`` samples are zero-padded up to it (and counted as
+ valid in the padding mask, matching the original Qwen3-ASR library),
+ - the padding mask lives on the mel-frame axis (sample mask strided by ``hop_length``),
+ - the mel time axis (features and mask) is right-padded to a multiple of
+ ``2 * n_window`` frames, as required by ``Qwen3ASREncoder``'s chunked attention.
+ """
+
+ sampling_rate = 16000
+ force_mono = True
+ padding = "max_length"
+ max_length = 480000 # 30 seconds at 16000 Hz
+ min_length = 8000
+ n_window = 50
+ valid_kwargs = Qwen3ASRAudioKwargs
+
+ spectrogram_config = SpectrogramConfig(
+ stft_config=StftConfig(
+ n_fft=400,
+ hop_length=160,
+ power=2.0,
+ ),
+ mel_scale_config=MelScaleConfig(
+ n_mels=128,
+ mel_scale="slaney",
+ norm="slaney",
+ computation_dtype="float64",
+ ),
+ log_mode="log10",
+ clip_max_offset=8.0,
+ post_log_shift=4.0,
+ post_log_scale=0.25,
+ )
+
+ legacy_field_mapping = {
+ "feature_size": "spectrogram_config.mel_scale_config.n_mels",
+ "chunk_length": _qwen3_asr_chunk_length_to_max_length,
+ "n_samples": "max_length",
+ "nb_max_frames": None,
+ }
+
+ def _process_audio(self, audio_el):
+ audio_el = super()._process_audio(audio_el)
+ if self.min_length and audio_el.shape[-1] < self.min_length:
+ audio_el = self._pad_single(audio_el, self.min_length)
+ return audio_el
+
+ def _extract_spectrogram(self, audio, *, spectrogram_config, **kwargs):
+ features = super()._extract_spectrogram(audio, spectrogram_config=spectrogram_config, **kwargs)
+ # Drop the trailing center frame *before* the mel projection, like the legacy FE
+ # (`stft[..., :-1]`); see the NOTE on `spectrogram_config`.
+ return features[..., :-1]
+
+ def _apply_mel_scale(self, features, *, spectrogram_config, **kwargs):
+ # `filters_first` matmul order with mel_floor clamp, matching the torch sibling.
+ return np.maximum(spectrogram_config.mel_floor, np.matmul(self.mel_filters.T, features))
+
+ def _get_features_lengths(self, audio_lengths, spectrogram_config, include_center_frame=False):
+ hop_length = spectrogram_config.stft_config.hop_length
+ if include_center_frame:
+ # Mask width over the padded batch: the legacy FE strides the sample-level mask
+ # by hop_length and trims the tail column when it doesn't divide evenly, i.e.
+ # padded_length // hop_length — the feature width after `skip_last_frame`.
+ return audio_lengths // hop_length
+ # Per-utterance valid frames: strided sample-mask indices 0, hop, 2*hop, ... below
+ # the valid length, i.e. ceil(length / hop_length).
+ return (audio_lengths + hop_length - 1) // hop_length
+
+ def _postprocess_output(self, output, audio_ranges=None, n_window=None, **kwargs):
+ # Right-pad the mel time axis (features and mask) to a multiple of `2 * n_window`
+ # (needed by `Qwen3ASREncoder`). `n_window=0` disables this padding.
+ if n_window is None:
+ n_window = self.n_window
+ multiple = 2 * n_window if n_window else 0
+ if multiple > 1:
+ features = output["audio_features"]
+ remainder = features.shape[-1] % multiple
+ if remainder:
+ padded_length = features.shape[-1] + multiple - remainder
+ output["audio_features"] = self._pad_single(features, padded_length)
+ if "audio_features_mask" in output:
+ output["audio_features_mask"] = self._pad_single(output["audio_features_mask"], padded_length)
+ return output
+
+
+__all__ = ["Qwen3ASRAudioProcessorNumpy"]
diff --git a/src/transformers/models/qwen3_asr/audio_processing_qwen3_asr.py b/src/transformers/models/qwen3_asr/audio_processing_qwen3_asr.py
new file mode 100644
index 000000000000..87376edaf235
--- /dev/null
+++ b/src/transformers/models/qwen3_asr/audio_processing_qwen3_asr.py
@@ -0,0 +1,79 @@
+# Copyright 2026 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+
+from ...audio_processing_backends import TorchAudioBackend
+from .audio_processing_numpy_qwen3_asr import Qwen3ASRAudioKwargs, Qwen3ASRAudioProcessorNumpy
+
+
+class Qwen3ASRAudioProcessor(TorchAudioBackend):
+
+ sampling_rate = 16000
+ force_mono = True
+ padding = "max_length"
+ max_length = 480000 # 30 seconds at 16000 Hz
+ min_length = 8000
+ n_window = 50
+ valid_kwargs = Qwen3ASRAudioKwargs
+
+ spectrogram_config = Qwen3ASRAudioProcessorNumpy.spectrogram_config
+ legacy_field_mapping = Qwen3ASRAudioProcessorNumpy.legacy_field_mapping
+
+ def _process_audio(self, audio_el):
+ audio_el = super()._process_audio(audio_el)
+ if self.min_length and audio_el.shape[-1] < self.min_length:
+ audio_el = self._pad_single(audio_el, self.min_length)
+ return audio_el
+
+ def _extract_spectrogram(self, audio, *, spectrogram_config, **kwargs):
+ features = super()._extract_spectrogram(audio, spectrogram_config=spectrogram_config, **kwargs)
+ # Drop the trailing center frame *before* the mel projection, like the legacy FE
+ # (`stft[..., :-1]`): the mel matmul is shape-sensitive at 1 ulp (BLAS blocking),
+ # so dropping it post-mel via `skip_last_frame` would not be bit-exact.
+ return features[..., :-1]
+
+ def _apply_mel_scale(self, features, *, spectrogram_config, **kwargs):
+ mel_filters = self.mel_filters.to(device=features.device)
+ return torch.clamp(torch.matmul(mel_filters.T, features), min=spectrogram_config.mel_floor)
+
+ def _get_features_lengths(self, audio_lengths, spectrogram_config, include_center_frame=False):
+ hop_length = spectrogram_config.stft_config.hop_length
+ if include_center_frame:
+ # Mask width over the padded batch: the legacy FE strides the sample-level mask
+ # by hop_length and trims the tail column when it doesn't divide evenly, i.e.
+ # padded_length // hop_length — the feature width after `skip_last_frame`.
+ return audio_lengths // hop_length
+ # Per-utterance valid frames: strided sample-mask indices 0, hop, 2*hop, ... below
+ # the valid length, i.e. ceil(length / hop_length).
+ return (audio_lengths + hop_length - 1) // hop_length
+
+ def _postprocess_output(self, output, audio_ranges=None, n_window=None, **kwargs):
+ # Right-pad the mel time axis (features and mask) to a multiple of `2 * n_window`
+ # (needed by `Qwen3ASREncoder`). `n_window=0` disables this padding.
+ if n_window is None:
+ n_window = self.n_window
+ multiple = 2 * n_window if n_window else 0
+ if multiple > 1:
+ features = output["audio_features"]
+ remainder = features.shape[-1] % multiple
+ if remainder:
+ padded_length = features.shape[-1] + multiple - remainder
+ output["audio_features"] = self._pad_single(features, padded_length)
+ if "audio_features_mask" in output:
+ output["audio_features_mask"] = self._pad_single(output["audio_features_mask"], padded_length)
+ return output
+
+
+__all__ = ["Qwen3ASRAudioProcessor"]
diff --git a/src/transformers/models/qwen3_asr/feature_extraction_qwen3_asr.py b/src/transformers/models/qwen3_asr/feature_extraction_qwen3_asr.py
index 6f22a8c8f4ff..6faf02b67f21 100644
--- a/src/transformers/models/qwen3_asr/feature_extraction_qwen3_asr.py
+++ b/src/transformers/models/qwen3_asr/feature_extraction_qwen3_asr.py
@@ -1,239 +1,20 @@
-# Copyright 2026 The HuggingFace Inc. team. All rights reserved.
+# Copyright 2026 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import numpy as np
-
-from ...audio_utils import mel_filter_bank
-from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
-from ...feature_extraction_utils import BatchFeature
-from ...utils import logging
-from ...utils.import_utils import is_torch_available, requires
-
-
-if is_torch_available():
- import torch
-
-logger = logging.get_logger(__name__)
-
-
-@requires(backends=("torch",))
-class Qwen3ASRFeatureExtractor(SequenceFeatureExtractor):
- r"""
- Constructs a Qwen3 ASR feature extractor.
-
- Extracts 128-bin log-mel features from raw speech, then right-pads the mel time axis to a multiple of ``2 * n_window``.
-
- Args:
- feature_size (`int`, *optional*, defaults to 128):
- Number of mel filter banks.
- sampling_rate (`int`, *optional*, defaults to 16000):
- Audio sampling rate in Hz.
- hop_length (`int`, *optional*, defaults to 160):
- Length of the overlapping windows for the STFT used to obtain the Mel Frequency coefficients.
- chunk_length (`int`, *optional*, defaults to 30):
- Maximum audio length (in seconds) used to trim/pad when ``padding="max_length"``.
- n_fft (`int`, *optional*, defaults to 400):
- Size of the Fourier transform.
- padding_value (`float`, *optional*, defaults to 0.0):
- Padding value used to pad the raw audio.
- dither (`float`, *optional*, defaults to 0.0):
- If non-zero, adds Gaussian noise (`std = dither`) to each STFT frame.
- return_attention_mask (`bool`, *optional*, defaults to `True`):
- Whether to return the attention mask corresponding to the padded mel frames.
- n_window (`int`, *optional*, defaults to 50):
- Half the mel-frame chunk size used for padding. The log-mel time axis is right-padded to a
- multiple of ``2 * n_window``.
- min_length (`int`, *optional*, defaults to 8000):
- Minimum number of samples for each audio clip. Clips shorter than this are zero-padded, matching the
- original Qwen3-ASR library behaviour.
- """
-
- model_input_names = ["input_features"]
-
- def __init__(
- self,
- feature_size=128,
- sampling_rate=16000,
- hop_length=160,
- chunk_length=30,
- n_fft=400,
- padding_value=0.0,
- dither=0.0,
- return_attention_mask=True,
- n_window=50,
- min_length=8000,
- **kwargs,
- ):
- super().__init__(
- feature_size=feature_size,
- sampling_rate=sampling_rate,
- padding_value=padding_value,
- return_attention_mask=return_attention_mask,
- **kwargs,
- )
- self.n_fft = n_fft
- self.hop_length = hop_length
- self.min_length = min_length
- self.chunk_length = chunk_length
- self.n_samples = chunk_length * sampling_rate
- self.nb_max_frames = self.n_samples // hop_length
- self.sampling_rate = sampling_rate
- self.dither = dither
- self.n_window = n_window
- self.mel_filters = mel_filter_bank(
- num_frequency_bins=1 + n_fft // 2,
- num_mel_filters=feature_size,
- min_frequency=0.0,
- max_frequency=8000.0,
- sampling_rate=sampling_rate,
- norm="slaney",
- mel_scale="slaney",
- )
-
- def _torch_extract_fbank_features(self, waveform: np.ndarray, device: str = "cpu") -> np.ndarray:
- """Compute log-mel spectrograms using PyTorch's (optionally GPU-accelerated) STFT."""
- waveform = torch.from_numpy(waveform).to(device, torch.float32)
- window = torch.hann_window(self.n_fft, device=device)
-
- if self.dither != 0.0:
- waveform += self.dither * torch.randn(waveform.shape, dtype=waveform.dtype, device=waveform.device)
-
- stft = torch.stft(waveform, self.n_fft, self.hop_length, window=window, return_complex=True)
- magnitudes = stft[..., :-1].abs() ** 2
-
- mel_filters = torch.from_numpy(self.mel_filters).to(device, torch.float32)
- mel_spec = mel_filters.T @ magnitudes
-
- log_spec = torch.clamp(mel_spec, min=1e-10).log10()
- if waveform.dim() == 2:
- max_val = log_spec.max(dim=2, keepdim=True)[0].max(dim=1, keepdim=True)[0]
- log_spec = torch.maximum(log_spec, max_val - 8.0)
- else:
- log_spec = torch.maximum(log_spec, log_spec.max() - 8.0)
- log_spec = (log_spec + 4.0) / 4.0
- if device != "cpu":
- log_spec = log_spec.detach().cpu()
- return log_spec.numpy()
-
- def __call__(
- self,
- raw_speech: np.ndarray | list[float] | list[np.ndarray] | list[list[float]],
- truncation: bool = False,
- pad_to_multiple_of: int | None = None,
- return_tensors: str | None = "pt",
- return_attention_mask: bool | None = None,
- padding: str | None = "max_length",
- max_length: int | None = None,
- sampling_rate: int | None = None,
- n_window: int | None = None,
- device: str | None = "cpu",
- **kwargs,
- ) -> BatchFeature:
- r"""
- Prepare log-mel features from one or several audio sequences.
-
- Args:
- raw_speech (`np.ndarray`, `list[float]`, `list[np.ndarray]`, `list[list[float]]`):
- The sequence or batch of sequences to be padded. Mono-channel audio only.
- pad_to_multiple_of (`int`, *optional*):
- If set, pads the raw audio to a multiple of this value (in samples). Separate from
- ``n_window``, which applies to the mel-frame axis.
- n_window (`int`, *optional*):
- Override the instance's ``n_window`` for this call. The mel axis is padded to a multiple
- of ``2 * n_window``. Set to ``0`` to skip mel-axis padding entirely.
- device (`str`, *optional*, defaults to `"cpu"`):
- Device used to compute the log-mel spectrogram.
- """
- if sampling_rate is not None:
- if sampling_rate != self.sampling_rate:
- raise ValueError(
- f"The model corresponding to this feature extractor: {self.__class__.__name__} was trained using a"
- f" sampling rate of {self.sampling_rate}. Please make sure that the provided `raw_speech` input"
- f" was sampled with {self.sampling_rate} and not {sampling_rate}."
- )
- else:
- logger.warning(
- f"It is strongly recommended to pass the `sampling_rate` argument to `{self.__class__.__name__}()`. "
- "Failing to do so can result in silent errors that might be hard to debug."
- )
-
- is_batched_numpy = isinstance(raw_speech, np.ndarray) and len(raw_speech.shape) > 1
- if is_batched_numpy and len(raw_speech.shape) > 2:
- raise ValueError(f"Only mono-channel audio is supported for input to {self}")
- is_batched = is_batched_numpy or (
- isinstance(raw_speech, (list, tuple)) and (isinstance(raw_speech[0], (np.ndarray, tuple, list)))
- )
-
- if is_batched:
- raw_speech = [np.asarray([speech], dtype=np.float32).T for speech in raw_speech]
- elif not is_batched and not isinstance(raw_speech, np.ndarray):
- raw_speech = np.asarray(raw_speech, dtype=np.float32)
- elif isinstance(raw_speech, np.ndarray) and raw_speech.dtype is np.dtype(np.float64):
- raw_speech = raw_speech.astype(np.float32)
-
- # always return batch
- if not is_batched:
- raw_speech = [np.asarray([raw_speech]).T]
-
- # Zero-pad clips shorter than min_length before batching, matching the original Qwen3-ASR library:
- # https://github.com/QwenLM/Qwen3-ASR/blob/c17a131fe028b2e428b6e80a33d30bb4fa57b8df/qwen_asr/inference/utils.py#L322
- # NOTE: as original, do not adjust padding/attention masks (hurts performance on AMI)
- if self.min_length > 0:
- raw_speech = [
- np.pad(s, ((0, self.min_length - s.shape[0]), (0, 0))) if s.shape[0] < self.min_length else s
- for s in raw_speech
- ]
-
- batched_speech = BatchFeature({"input_features": raw_speech})
-
- padded_inputs = self.pad(
- batched_speech,
- padding=padding,
- max_length=max_length if max_length else self.n_samples,
- truncation=truncation,
- pad_to_multiple_of=pad_to_multiple_of,
- return_attention_mask=return_attention_mask,
- )
-
- input_features = padded_inputs["input_features"].transpose(2, 0, 1)
- input_features = self._torch_extract_fbank_features(input_features[0], device)
- padded_inputs["input_features"] = input_features
-
- # Rescale raw-sample attention mask to mel-frame resolution.
- rescaled_attention_mask = padded_inputs["attention_mask"][:, :: self.hop_length]
- if padded_inputs["attention_mask"].shape[1] % self.hop_length != 0:
- rescaled_attention_mask = rescaled_attention_mask[:, :-1]
- padded_inputs["attention_mask"] = rescaled_attention_mask
-
- # Right-pad the mel time axis to a multiple of `2 * n_window` (needed by `Qwen3ASREncoder`).
- if n_window is None:
- n_window = self.n_window
- multiple = n_window * 2
- if multiple and multiple > 1:
- remainder = padded_inputs["input_features"].shape[-1] % multiple
- pad = (multiple - remainder) if remainder else 0
- if pad:
- padded_inputs["input_features"] = np.pad(padded_inputs["input_features"], [(0, 0), (0, 0), (0, pad)])
- padded_inputs["attention_mask"] = np.pad(padded_inputs["attention_mask"], [(0, 0), (0, pad)])
+"""Backwards-compatibility shim: re-exports the legacy ``Qwen3ASRFeatureExtractor`` name as a
+deprecated alias of [`Qwen3ASRAudioProcessor`]. Importing or instantiating the alias emits a
+``FutureWarning``; the alias is removed in transformers v5.15 (see ADR 0002).
+"""
- if not return_attention_mask:
- padded_inputs.pop("attention_mask", None)
+from ...audio_processing_base import make_legacy_audio_processor_alias
+from .audio_processing_qwen3_asr import Qwen3ASRAudioProcessor
- if return_tensors is not None:
- padded_inputs = padded_inputs.convert_to_tensors(return_tensors)
- return padded_inputs
+Qwen3ASRFeatureExtractor = make_legacy_audio_processor_alias(Qwen3ASRAudioProcessor, "Qwen3ASRFeatureExtractor")
__all__ = ["Qwen3ASRFeatureExtractor"]
diff --git a/src/transformers/models/seamless_m4t/__init__.py b/src/transformers/models/seamless_m4t/__init__.py
index 27c08de501c1..f3704726fa17 100644
--- a/src/transformers/models/seamless_m4t/__init__.py
+++ b/src/transformers/models/seamless_m4t/__init__.py
@@ -18,6 +18,8 @@
if TYPE_CHECKING:
+ from .audio_processing_numpy_seamless_m4t import *
+ from .audio_processing_seamless_m4t import *
from .configuration_seamless_m4t import *
from .feature_extraction_seamless_m4t import *
from .modeling_seamless_m4t import *
diff --git a/src/transformers/models/seamless_m4t/audio_processing_numpy_seamless_m4t.py b/src/transformers/models/seamless_m4t/audio_processing_numpy_seamless_m4t.py
new file mode 100644
index 000000000000..c8931d35791e
--- /dev/null
+++ b/src/transformers/models/seamless_m4t/audio_processing_numpy_seamless_m4t.py
@@ -0,0 +1,97 @@
+# Copyright 2026 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from ...audio_processing_backends import NumpyAudioBackend
+from ...audio_utils import MelScaleConfig, SpectrogramConfig, StftConfig
+
+
+class SeamlessM4tAudioProcessorNumpy(NumpyAudioBackend):
+ """NumPy sibling of [`SeamlessM4tAudioProcessor`]. Per-utterance mean/var normalization
+ plus stride concatenation reshape, matching the legacy `SeamlessM4TFeatureExtractor`."""
+
+ sampling_rate = 16000
+ force_mono = True
+ do_batch_spectrogram = False
+ stride = 2
+ pad_to_multiple_of = 2 # Align feature padding to stride
+
+ spectrogram_config = SpectrogramConfig(
+ stft_config=StftConfig(
+ n_fft=512,
+ win_length=400,
+ hop_length=160,
+ window_fn="povey",
+ power=2.0,
+ center=False,
+ periodic=False,
+ ),
+ mel_scale_config=MelScaleConfig(
+ n_mels=80,
+ f_min=20.0,
+ f_max=8000.0,
+ mel_scale="kaldi",
+ triangularize_in_mel_space=True,
+ ),
+ log_mode="log",
+ preemphasis=0.97,
+ remove_dc_offset=True,
+ mel_floor=1.192092955078125e-07,
+ computation_dtype="float64",
+ )
+ waveform_scale = 32768.0
+
+ def extract_spectrogram(self, audio, **kwargs):
+ # Per-waveform fbank extraction returning (time, n_mels)
+ features = []
+ for waveform in audio:
+ waveform = np.squeeze(waveform) * self.waveform_scale
+ f = super().extract_spectrogram([waveform], spectrogram_config=self.spectrogram_config)
+ features.append(f[0].T)
+ return features
+
+ def _postprocess_features(self, features, feature_lengths):
+ # Per-utterance mean/variance normalization (before padding)
+ normalized = []
+ for f in features:
+ mean = np.expand_dims(f.mean(axis=0), 0)
+ var = np.expand_dims(f.var(axis=0, ddof=1), 0)
+ normalized.append((f - mean) / np.sqrt(var + 1e-7))
+ return normalized
+
+ def _postprocess_output(self, output, feature_ranges=None, **kwargs):
+ features = output["audio_features"] # (batch, num_frames, num_channels)
+ batch_size, num_frames, num_channels = features.shape
+
+ # Stride concatenation
+ remainder = num_frames % self.stride
+ if remainder != 0:
+ features = features[:, :num_frames - remainder, :]
+ num_frames = num_frames - remainder
+
+ output["audio_features"] = features.reshape(batch_size, num_frames // self.stride, num_channels * self.stride)
+
+ # Adjust mask for stride
+ if "audio_features_mask" in output:
+ mask = output["audio_features_mask"]
+ if remainder != 0:
+ mask = mask[:, :num_frames]
+ indices = np.arange(0, num_frames)
+ output["audio_features_mask"] = mask[:, indices % self.stride == 1]
+
+ return output
+
+
+__all__ = ["SeamlessM4tAudioProcessorNumpy"]
diff --git a/src/transformers/models/seamless_m4t/audio_processing_seamless_m4t.py b/src/transformers/models/seamless_m4t/audio_processing_seamless_m4t.py
new file mode 100644
index 000000000000..4a6053ff57cb
--- /dev/null
+++ b/src/transformers/models/seamless_m4t/audio_processing_seamless_m4t.py
@@ -0,0 +1,90 @@
+# Copyright 2025 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+import torch
+
+from ...audio_processing_backends import TorchAudioBackend
+from .audio_processing_numpy_seamless_m4t import SeamlessM4tAudioProcessorNumpy
+
+
+class SeamlessM4tAudioProcessor(TorchAudioBackend):
+ sampling_rate = 16000
+ force_mono = True
+ do_batch_spectrogram = False
+ stride = 2
+ pad_to_multiple_of = 2 # Align feature padding to stride
+
+
+ spectrogram_config = SeamlessM4tAudioProcessorNumpy.spectrogram_config
+ waveform_scale = 32768.0
+
+ def extract_spectrogram(self, audio, **kwargs):
+ # Per-waveform fbank extraction returning (time, n_mels)
+ features = []
+ for waveform in audio:
+ waveform = waveform.squeeze() * self.waveform_scale
+ f = super().extract_spectrogram([waveform], spectrogram_config=self.spectrogram_config)
+ features.append(f[0].transpose(-2, -1))
+ return features
+
+ # Mel filters: the base dispatcher resolves the top-level `computation_dtype="float64"`
+ # into a float64 torch-native kaldi-exact build, which is bit-identical to the legacy
+ # FE's float64 numpy build (the float32 mel-space cancellation that motivated the old
+ # numpy-built override only appears in the default float32 kaldi path).
+
+ def _window_and_fft(self, frames, window, frame_length, n_fft, stft_cfg, audio_dtype=None):
+ spec = super()._window_and_fft(frames, window, frame_length, n_fft, stft_cfg, audio_dtype=audio_dtype)
+ # The legacy FE stores FFT frames in a complex64 buffer before taking float64
+ # magnitudes (`np.abs(spectrogram, dtype=np.float64) ** power`); quantize then upcast
+ # so `_compute_magnitudes` sees the same values.
+ return spec.to(torch.complex64).to(torch.complex128)
+
+ def _postprocess_features(self, features, feature_lengths):
+ # Per-utterance mean/variance normalization (before padding). Computed in numpy to stay
+ # bit-exact with the legacy FE: numpy reductions use pairwise summation, whose
+ # accumulation order differs from torch's float32 `mean`/`var`. The legacy features are
+ # F-contiguous (a `.T` view of the (n_mels, time) spectrogram) and numpy's accumulation
+ # order depends on memory layout, so match it with `asfortranarray`.
+ normalized = []
+ for f in features:
+ x = np.asfortranarray(f.detach().cpu().numpy())
+ x = (x - np.expand_dims(x.mean(0), 0)) / np.sqrt(np.expand_dims(x.var(0, ddof=1), 0) + 1e-7)
+ normalized.append(torch.from_numpy(x))
+ return normalized
+
+ def _postprocess_output(self, output, feature_ranges=None, **kwargs):
+ features = output["audio_features"] # (batch, num_frames, num_channels)
+ batch_size, num_frames, num_channels = features.shape
+
+ # Stride concatenation
+ remainder = num_frames % self.stride
+ if remainder != 0:
+ features = features[:, :num_frames - remainder, :]
+ num_frames = num_frames - remainder
+
+ output["audio_features"] = features.reshape(batch_size, num_frames // self.stride, num_channels * self.stride)
+
+ # Adjust mask for stride
+ if "audio_features_mask" in output:
+ mask = output["audio_features_mask"]
+ if remainder != 0:
+ mask = mask[:, :num_frames]
+ indices = torch.arange(0, num_frames)
+ output["audio_features_mask"] = mask[:, indices % self.stride == 1]
+
+ return output
+
+
+__all__ = ["SeamlessM4tAudioProcessor"]
diff --git a/src/transformers/models/seamless_m4t/feature_extraction_seamless_m4t.py b/src/transformers/models/seamless_m4t/feature_extraction_seamless_m4t.py
index ad7999747e81..bb1fb55027ab 100644
--- a/src/transformers/models/seamless_m4t/feature_extraction_seamless_m4t.py
+++ b/src/transformers/models/seamless_m4t/feature_extraction_seamless_m4t.py
@@ -1,304 +1,20 @@
-# Copyright 2023 The HuggingFace Inc. team.
+# Copyright 2026 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""
-Feature extractor class for SeamlessM4T
+"""Backwards-compatibility shim: re-exports the legacy ``SeamlessM4TFeatureExtractor`` name as a
+deprecated alias of [`SeamlessM4tAudioProcessor`]. Importing or instantiating the alias emits a
+``FutureWarning``; the alias is removed in transformers v5.15 (see ADR 0002).
"""
-import numpy as np
-
-from ...utils import is_torch_available
-
-
-if is_torch_available():
- import torch
-
-from ...audio_utils import mel_filter_bank, spectrogram, window_function
-from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
-from ...feature_extraction_utils import BatchFeature
-from ...utils import PaddingStrategy, TensorType, logging
-
-
-logger = logging.get_logger(__name__)
-
-
-class SeamlessM4TFeatureExtractor(SequenceFeatureExtractor):
- r"""
- Constructs a SeamlessM4T feature extractor.
-
- This feature extractor inherits from [`SequenceFeatureExtractor`] which contains most of the main methods. Users
- should refer to this superclass for more information regarding those methods.
-
- This class extracts mel-filter bank features from raw speech.
-
- Args:
- feature_size (`int`, *optional*, defaults to 80):
- The feature dimension of the extracted features.
- sampling_rate (`int`, *optional*, defaults to 16000):
- The sampling rate at which the audio files should be digitalized expressed in hertz (Hz).
- num_mel_bins (`int`, *optional*, defaults to 80):
- Number of Mel-frequency bins.
- padding_value (`float`, *optional*, defaults to 0.0):
- The value that is used to fill the padding vectors.
- stride (`int`, *optional*, defaults to 2):
- Stride used to reshape audios from shape (batch_size,num_frames,num_mel_bins) to
- (batch_size,num_frames//stride,num_mel_bins*stride).
- """
-
- model_input_names = ["input_features", "attention_mask"]
-
- def __init__(
- self,
- feature_size=80,
- sampling_rate=16000,
- num_mel_bins=80,
- padding_value=0.0,
- stride=2,
- **kwargs,
- ):
- self.num_mel_bins = num_mel_bins
- self.return_attention_mask = True
- self.stride = stride
-
- mel_filters = mel_filter_bank(
- num_frequency_bins=257,
- num_mel_filters=self.num_mel_bins,
- min_frequency=20,
- max_frequency=sampling_rate // 2,
- sampling_rate=sampling_rate,
- norm=None,
- mel_scale="kaldi",
- triangularize_in_mel_space=True,
- )
-
- self.mel_filters = mel_filters
- self.window = window_function(400, "povey", periodic=False)
-
- super().__init__(feature_size=feature_size, sampling_rate=sampling_rate, padding_value=padding_value, **kwargs)
-
- @staticmethod
- # Copied from transformers.models.wav2vec2.feature_extraction_wav2vec2.Wav2Vec2FeatureExtractor.zero_mean_unit_var_norm
- def zero_mean_unit_var_norm(
- input_values: list[np.ndarray], attention_mask: list[np.ndarray], padding_value: float = 0.0
- ) -> list[np.ndarray]:
- """
- Every array in the list is normalized to have zero mean and unit variance
- """
- if attention_mask is not None:
- attention_mask = np.array(attention_mask, np.int32)
- normed_input_values = []
-
- for vector, length in zip(input_values, attention_mask.sum(-1)):
- normed_slice = (vector - vector[:length].mean()) / np.sqrt(vector[:length].var() + 1e-7)
- if length < normed_slice.shape[0]:
- normed_slice[length:] = padding_value
-
- normed_input_values.append(normed_slice)
- else:
- normed_input_values = [(x - x.mean()) / np.sqrt(x.var() + 1e-7) for x in input_values]
-
- return normed_input_values
-
- def _extract_fbank_features(
- self,
- waveform: np.ndarray,
- ) -> np.ndarray:
- """
- Get mel-filter bank features using Numpy method to mimic Kaldi.
- """
- # by default, it extracts the left channel if stereo
- if len(waveform.shape) == 2:
- waveform = waveform[0]
-
- waveform = np.squeeze(waveform) * (2**15) # Kaldi compliance: 16-bit signed integers
- features = spectrogram(
- waveform,
- self.window,
- frame_length=400,
- hop_length=160,
- fft_length=512,
- power=2.0,
- center=False,
- preemphasis=0.97,
- mel_filters=self.mel_filters,
- log_mel="log",
- mel_floor=1.192092955078125e-07,
- remove_dc_offset=True,
- ).T
- return features
-
- def __call__(
- self,
- raw_speech: np.ndarray | list[float] | list[np.ndarray] | list[list[float]],
- padding: bool | str | PaddingStrategy = True,
- pad_to_multiple_of: int | None = 2,
- max_length: int | None = None,
- truncation: bool = False,
- return_tensors: str | TensorType | None = None,
- sampling_rate: int | None = None,
- return_attention_mask: bool | None = None,
- do_normalize_per_mel_bins: bool | None = True,
- **kwargs,
- ) -> BatchFeature:
- """
- Main method to featurize and prepare for the model one or several sequence(s).
-
- Args:
- raw_speech (`np.ndarray`, `torch.Tensor`, `list[float]`, `list[np.ndarray]`, `list[torch.Tensor]`,
- `list[list[float]]`, `list[list[list[float]]]`):
- The sequence or batch of sequences to be padded. Each sequence can be a numpy array,
- a torch tensor, a list of float values, a list of numpy arrays, a list of torch tensors,
- a list of list of float values or a list of a list of list of float values.
- If `raw_speech` is a one-dimensional `np.ndarray`, `torch.Tensor` or a `list[float]`, `raw_speech` is
- considered a single-channel, single-sample sound. In all other cases, the first dimension of
- `raw_speech`, whether from an `np.ndarray`, a `torch.Tensor` or a `list[...]`,
- corresponds to the number of samples in the batch, and the number of channels
- (i.e. mono or stereo character) is derived from the other dimensions
- (1D -> single-channel waveform batches; 2D-> stereo-channel waveform batches).
- padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `True`):
- Select a strategy to pad the returned sequences (according to the model's padding side and padding
- index) among:
-
- - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
- sequence if provided).
- - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
- acceptable input length for the model if that argument is not provided.
- - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
- lengths).
- pad_to_multiple_of (`int`, *optional*, defaults to 2):
- If set will pad the sequence to a multiple of the provided value.
-
- This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability
- `>= 7.5` (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128.
- max_length (`int`, *optional*):
- Maximum length of the returned list and optionally padding length (see above).
- truncation (`bool`):
- Activates truncation to cut input sequences longer than *max_length* to *max_length*.
- return_attention_mask (`bool`, *optional*):
- Whether to return the attention mask. If left to the default, will return the attention mask according
- to the specific feature_extractor's default.
-
- [What are attention masks?](../glossary#attention-mask)
-
-
-
- For SeamlessM4T models, `attention_mask` should always be passed for batched inference, to avoid subtle
- bugs.
-
-
-
- return_tensors (`str` or [`~utils.TensorType`], *optional*):
- If set, will return tensors instead of list of python integers. Acceptable values are:
-
- - `'pt'`: Return PyTorch `torch.Tensor` objects.
- - `'np'`: Return Numpy `np.ndarray` objects.
- sampling_rate (`int`, *optional*):
- The sampling rate at which the `raw_speech` input was sampled. It is strongly recommended to pass
- `sampling_rate` at the forward call to prevent silent errors.
- do_normalize_per_mel_bins (`bool`, *optional*, defaults to `True`):
- Whether or not to zero-mean unit-variance normalize the input per mel-channel.
- kwargs (*optional*):
- Remaining dictionary of keyword arguments that will be passed to the tokenizer or the feature
- extractor.
- """
- if sampling_rate is not None:
- if sampling_rate != self.sampling_rate:
- raise ValueError(
- f"The model corresponding to this feature extractor: {self} was trained using a sampling rate of"
- f" {self.sampling_rate}. Please make sure that the provided `raw_speech` input was sampled with"
- f" {self.sampling_rate} and not {sampling_rate}."
- )
- else:
- logger.warning(
- f"It is strongly recommended to pass the `sampling_rate` argument to `{self.__class__.__name__}()`. "
- "Failing to do so can result in silent errors that might be hard to debug."
- )
-
- return_attention_mask = (
- return_attention_mask if return_attention_mask is not None else self.return_attention_mask
- )
-
- is_batched_numpy = isinstance(raw_speech, np.ndarray) and len(raw_speech.shape) > 1
- if is_batched_numpy and len(raw_speech.shape) > 3:
- raise ValueError(f"Only mono-channel or stereo-channel audio is supported for input to {self}")
-
- acceptable_types = (
- (torch.Tensor, np.ndarray, tuple, list) if is_torch_available() else (np.ndarray, tuple, list)
- )
- is_batched = is_batched_numpy or (
- isinstance(raw_speech, (list, tuple)) and (isinstance(raw_speech[0], acceptable_types))
- )
-
- if is_batched:
- raw_speech = [np.asarray(speech, dtype=np.float32) for speech in raw_speech]
- elif not is_batched and not isinstance(raw_speech, np.ndarray):
- raw_speech = np.asarray(raw_speech, dtype=np.float32)
- elif isinstance(raw_speech, np.ndarray) and raw_speech.dtype is np.dtype(np.float64):
- raw_speech = raw_speech.astype(np.float32)
-
- # always return batch
- if not is_batched:
- raw_speech = [raw_speech]
-
- # extract fbank features
- features = [self._extract_fbank_features(waveform) for waveform in raw_speech]
-
- if do_normalize_per_mel_bins:
- # torch defaults to ddof=1, and numpy defaults to ddof=0
- features = [
- (x - np.expand_dims(x.mean(0), 0)) / np.sqrt(np.expand_dims(x.var(0, ddof=1), 0) + 1e-7)
- for x in features
- ]
-
- # convert into correct format for padding
- encoded_inputs = BatchFeature({"input_features": features})
-
- padded_inputs = self.pad(
- encoded_inputs,
- padding=padding,
- max_length=max_length,
- truncation=truncation,
- pad_to_multiple_of=pad_to_multiple_of,
- return_attention_mask=True,
- return_tensors="np",
- )
-
- # SeamlessM4T needs to process extracted features
- input_features = padded_inputs.get("input_features")
- attention_mask = padded_inputs.pop("attention_mask")
-
- batch_size, num_frames, num_channels = input_features.shape
-
- remainder = num_frames % self.stride
- if remainder != 0:
- input_features = input_features[:, : num_frames - remainder, :]
- attention_mask = attention_mask[:, : num_frames - remainder]
-
- input_features = np.reshape(
- input_features, (batch_size, num_frames // self.stride, num_channels * self.stride)
- )
-
- indices = np.arange(0, num_frames - remainder)
- attention_mask = attention_mask[:, indices % self.stride == 1]
-
- padded_inputs["input_features"] = input_features
- if return_attention_mask:
- padded_inputs["attention_mask"] = attention_mask
+from ...audio_processing_base import make_legacy_audio_processor_alias
+from .audio_processing_seamless_m4t import SeamlessM4tAudioProcessor
- if return_tensors is not None:
- padded_inputs = padded_inputs.convert_to_tensors(return_tensors)
- return padded_inputs
+SeamlessM4TFeatureExtractor = make_legacy_audio_processor_alias(SeamlessM4tAudioProcessor, "SeamlessM4TFeatureExtractor")
__all__ = ["SeamlessM4TFeatureExtractor"]
diff --git a/src/transformers/models/speech_to_text/__init__.py b/src/transformers/models/speech_to_text/__init__.py
index b4dce9e2cc61..c15448fd6614 100644
--- a/src/transformers/models/speech_to_text/__init__.py
+++ b/src/transformers/models/speech_to_text/__init__.py
@@ -18,6 +18,8 @@
if TYPE_CHECKING:
+ from .audio_processing_numpy_speech_to_text import *
+ from .audio_processing_speech_to_text import *
from .configuration_speech_to_text import *
from .feature_extraction_speech_to_text import *
from .modeling_speech_to_text import *
diff --git a/src/transformers/models/speech_to_text/audio_processing_numpy_speech_to_text.py b/src/transformers/models/speech_to_text/audio_processing_numpy_speech_to_text.py
new file mode 100644
index 000000000000..49f097c7eb8e
--- /dev/null
+++ b/src/transformers/models/speech_to_text/audio_processing_numpy_speech_to_text.py
@@ -0,0 +1,95 @@
+# Copyright 2026 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from ...audio_processing_backends import NumpyAudioBackend
+from ...audio_utils import MelScaleConfig, SpectrogramConfig, StftConfig
+
+
+class SpeechToTextAudioProcessorNumpy(NumpyAudioBackend):
+ """NumPy sibling of [`SpeechToTextAudioProcessor`]. Per-waveform kaldi fbank features
+ followed by per-utterance CMVN on the padded batch (ADR 0001)."""
+
+ sampling_rate = 16000
+ force_mono = True
+ do_batch_spectrogram = False
+
+ spectrogram_config = SpectrogramConfig(
+ stft_config=StftConfig(
+ n_fft=512,
+ win_length=400,
+ hop_length=160,
+ window_fn="povey",
+ power=2.0,
+ center=False,
+ periodic=False,
+ left_align_fft=True,
+ ),
+ mel_scale_config=MelScaleConfig(
+ n_mels=80,
+ f_min=20.0,
+ f_max=8000.0,
+ mel_scale="kaldi",
+ triangularize_in_mel_space=True,
+ ),
+ log_mode="log",
+ preemphasis=0.97,
+ remove_dc_offset=True,
+ mel_floor=1.192092955078125e-07,
+ waveform_scale=32768.0,
+ )
+
+ def __init__(self, normalize_means=True, normalize_vars=True, **kwargs):
+ super().__init__(**kwargs)
+ self.normalize_means = normalize_means
+ self.normalize_vars = normalize_vars
+
+ def _extract_fbank_features(self, waveform):
+ """Extract log-mel filterbank features for a single waveform."""
+ # The `_kaldi_fbank` bridge bypasses the base pipeline (which would apply
+ # `spectrogram_config.waveform_scale` itself), so scale manually here.
+ waveform = waveform * self.spectrogram_config.waveform_scale
+ return self._kaldi_fbank(waveform, num_mel_bins=80)
+
+ def extract_spectrogram(self, audio, **kwargs):
+ # Per-waveform fbank extraction returning (time, n_mels)
+ return [self._extract_fbank_features(waveform) for waveform in audio]
+
+ @staticmethod
+ def utterance_cmvn(x, input_length, normalize_means=True, normalize_vars=True, padding_value=0.0):
+ if normalize_means:
+ mean = x[:input_length].mean(axis=0)
+ x = np.subtract(x, mean)
+ if normalize_vars:
+ std = x[:input_length].std(axis=0)
+ x = np.divide(x, std)
+ if input_length < x.shape[0]:
+ x[input_length:] = padding_value
+ return x.astype(np.float32)
+
+ def _postprocess_output(self, output, feature_ranges=None, **kwargs):
+ # Apply utterance CMVN normalization on the padded, stacked features
+ features = output["audio_features"] # (batch, time, n_mels)
+ normalized = []
+ for i, (start, end) in enumerate(feature_ranges):
+ length = end - start
+ normalized.append(
+ self.utterance_cmvn(features[i], length, self.normalize_means, self.normalize_vars, self.padding_value)
+ )
+ output["audio_features"] = np.stack(normalized)
+ return output
+
+
+__all__ = ["SpeechToTextAudioProcessorNumpy"]
diff --git a/src/transformers/models/speech_to_text/audio_processing_speech_to_text.py b/src/transformers/models/speech_to_text/audio_processing_speech_to_text.py
new file mode 100644
index 000000000000..42246fba42b8
--- /dev/null
+++ b/src/transformers/models/speech_to_text/audio_processing_speech_to_text.py
@@ -0,0 +1,75 @@
+# Copyright 2025 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+import torch
+
+from ...audio_processing_backends import TorchAudioBackend
+from .audio_processing_numpy_speech_to_text import SpeechToTextAudioProcessorNumpy
+
+
+class SpeechToTextAudioProcessor(TorchAudioBackend):
+ """Torch sibling of [`SpeechToTextAudioProcessorNumpy`]. Per-waveform kaldi fbank features
+ followed by per-utterance CMVN on the padded batch."""
+
+ sampling_rate = 16000
+ force_mono = True
+ do_batch_spectrogram = False
+
+
+ spectrogram_config = SpeechToTextAudioProcessorNumpy.spectrogram_config
+
+ def __init__(self, normalize_means=True, normalize_vars=True, **kwargs):
+ super().__init__(**kwargs)
+ self.normalize_means = normalize_means
+ self.normalize_vars = normalize_vars
+
+ def extract_spectrogram(self, audio, **kwargs):
+ # Native kaldi-exact pipeline (bit-equal to `torchaudio.compliance.kaldi.fbank`),
+ # transposed to kaldi's (time, n_mels) orientation expected downstream.
+ features = super().extract_spectrogram(audio, **kwargs)
+ return [f.transpose(-2, -1) for f in features]
+
+ @staticmethod
+ def utterance_cmvn(x, input_length, normalize_means=True, normalize_vars=True, padding_value=0.0):
+ # CMVN is computed in numpy to stay bit-exact with the legacy feature extractor
+ # and the numpy sibling: numpy reductions use pairwise summation, whose
+ # accumulation order differs from torch's `mean`/`std` (~1e-5 drift in float32).
+ x = x.detach().cpu().numpy()
+ if normalize_means:
+ mean = x[:input_length].mean(axis=0)
+ x = np.subtract(x, mean)
+ if normalize_vars:
+ std = x[:input_length].std(axis=0)
+ x = np.divide(x, std)
+ if input_length < x.shape[0]:
+ if not (normalize_means or normalize_vars):
+ x = x.copy() # don't mutate the caller's tensor through the numpy view
+ x[input_length:] = padding_value
+ return torch.from_numpy(x.astype(np.float32))
+
+ def _postprocess_output(self, output, feature_ranges=None, **kwargs):
+ # Apply utterance CMVN normalization on the padded, stacked features
+ features = output["audio_features"] # (batch, time, n_mels)
+ normalized = []
+ for i, (start, end) in enumerate(feature_ranges):
+ length = end - start
+ normalized.append(
+ self.utterance_cmvn(features[i], length, self.normalize_means, self.normalize_vars, self.padding_value)
+ )
+ output["audio_features"] = torch.stack(normalized)
+ return output
+
+
+__all__ = ["SpeechToTextAudioProcessor"]
diff --git a/src/transformers/models/speech_to_text/feature_extraction_speech_to_text.py b/src/transformers/models/speech_to_text/feature_extraction_speech_to_text.py
index 9685e9be0134..ae960fcf1628 100644
--- a/src/transformers/models/speech_to_text/feature_extraction_speech_to_text.py
+++ b/src/transformers/models/speech_to_text/feature_extraction_speech_to_text.py
@@ -1,311 +1,20 @@
-# Copyright 2021 The HuggingFace Inc. team.
+# Copyright 2026 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""
-Feature extractor class for Speech2Text
+"""Backwards-compatibility shim: re-exports the legacy ``Speech2TextFeatureExtractor`` name as a
+deprecated alias of [`SpeechToTextAudioProcessor`]. Importing or instantiating the alias emits a
+``FutureWarning``; the alias is removed in transformers v5.15 (see ADR 0002).
"""
-import numpy as np
-
-from ...audio_utils import mel_filter_bank, spectrogram, window_function
-from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
-from ...feature_extraction_utils import BatchFeature
-from ...utils import PaddingStrategy, TensorType, is_speech_available, logging
-
-
-if is_speech_available():
- import torch
- import torchaudio.compliance.kaldi as ta_kaldi
-
-logger = logging.get_logger(__name__)
-
-
-class Speech2TextFeatureExtractor(SequenceFeatureExtractor):
- r"""
- Constructs a Speech2Text feature extractor.
-
- This feature extractor inherits from [`Speech2TextFeatureExtractor`] which contains most of the main methods. Users
- should refer to this superclass for more information regarding those methods.
-
- This class extracts mel-filter bank features from raw speech using TorchAudio if installed or using numpy
- otherwise, and applies utterance-level cepstral mean and variance normalization to the extracted features.
-
- Args:
- feature_size (`int`, *optional*, defaults to 80):
- The feature dimension of the extracted features.
- sampling_rate (`int`, *optional*, defaults to 16000):
- The sampling rate at which the audio files should be digitalized expressed in hertz (Hz).
- num_mel_bins (`int`, *optional*, defaults to 80):
- Number of Mel-frequency bins.
- padding_value (`float`, *optional*, defaults to 0.0):
- The value that is used to fill the padding vectors.
- dither (`float`, *optional*, defaults to 0.0):
- Adds dithering. In other words, adds a small Gaussian noise to each frame.
- E.g. use 4.0 to add dithering with a normal distribution centered
- around 0.0 with standard deviation 4.0 (assuming [-32k,+32k] range of kaldi waveform).
- The value 0.0 means no dithering.
- Dithering has similar effect as `mel_floor`. It reduces the high log_mel_fbank
- values for signals with hard-zero sections, when VAD cutoff is present in the signal.
- do_ceptral_normalize (`bool`, *optional*, defaults to `True`):
- Whether or not to apply utterance-level cepstral mean and variance normalization to extracted features.
- normalize_means (`bool`, *optional*, defaults to `True`):
- Whether or not to zero-mean normalize the extracted features.
- normalize_vars (`bool`, *optional*, defaults to `True`):
- Whether or not to unit-variance normalize the extracted features.
- """
-
- model_input_names = ["input_features", "attention_mask"]
-
- def __init__(
- self,
- feature_size=80,
- sampling_rate=16000,
- num_mel_bins=80,
- padding_value=0.0,
- dither=0.0,
- do_ceptral_normalize=True,
- normalize_means=True,
- normalize_vars=True,
- **kwargs,
- ):
- super().__init__(feature_size=feature_size, sampling_rate=sampling_rate, padding_value=padding_value, **kwargs)
- self.num_mel_bins = num_mel_bins
- self.dither = dither
- self.do_ceptral_normalize = do_ceptral_normalize
- self.normalize_means = normalize_means
- self.normalize_vars = normalize_vars
- self.return_attention_mask = True
-
- if not is_speech_available():
- mel_filters = mel_filter_bank(
- num_frequency_bins=257,
- num_mel_filters=self.num_mel_bins,
- min_frequency=20,
- max_frequency=sampling_rate // 2,
- sampling_rate=sampling_rate,
- norm=None,
- mel_scale="kaldi",
- triangularize_in_mel_space=True,
- )
-
- self.mel_filters = mel_filters
- self.window = window_function(400, "povey", periodic=False)
-
- def _extract_fbank_features(
- self,
- waveform: np.ndarray,
- ) -> np.ndarray:
- """
- Get mel-filter bank features using TorchAudio. Note that TorchAudio requires 16-bit signed integers as inputs
- and hence the waveform should not be normalized before feature extraction.
- """
- waveform = waveform * (2**15) # Kaldi compliance: 16-bit signed integers
- if is_speech_available():
- waveform = torch.from_numpy(waveform).unsqueeze(0)
- features = ta_kaldi.fbank(
- waveform,
- dither=self.dither,
- num_mel_bins=self.num_mel_bins,
- sample_frequency=self.sampling_rate,
- )
- features = features.numpy()
- else:
- waveform = np.squeeze(waveform)
- features = spectrogram(
- waveform,
- self.window,
- frame_length=400,
- hop_length=160,
- fft_length=512,
- power=2.0,
- center=False,
- dither=self.dither,
- preemphasis=0.97,
- mel_filters=self.mel_filters,
- log_mel="log",
- mel_floor=1.192092955078125e-07,
- remove_dc_offset=True,
- ).T
- return features
-
- @staticmethod
- def utterance_cmvn(
- x: np.ndarray,
- input_length: int,
- normalize_means: bool | None = True,
- normalize_vars: bool | None = True,
- padding_value: float = 0.0,
- ) -> np.ndarray:
- # make sure we normalize float32 arrays
- if normalize_means:
- mean = x[:input_length].mean(axis=0)
- x = np.subtract(x, mean)
- if normalize_vars:
- std = x[:input_length].std(axis=0)
- x = np.divide(x, std)
-
- if input_length < x.shape[0]:
- x[input_length:] = padding_value
-
- # make sure array is in float32
- x = x.astype(np.float32)
-
- return x
-
- def normalize(
- self, input_features: list[np.ndarray], attention_mask: np.ndarray | None = None
- ) -> list[np.ndarray]:
- lengths = attention_mask.sum(-1) if attention_mask is not None else [x.shape[0] for x in input_features]
- return [
- self.utterance_cmvn(x, n, self.normalize_means, self.normalize_vars, self.padding_value)
- for x, n in zip(input_features, lengths)
- ]
-
- def __call__(
- self,
- raw_speech: np.ndarray | list[float] | list[np.ndarray] | list[list[float]],
- padding: bool | str | PaddingStrategy = False,
- max_length: int | None = None,
- truncation: bool = False,
- pad_to_multiple_of: int | None = None,
- return_tensors: str | TensorType | None = None,
- sampling_rate: int | None = None,
- return_attention_mask: bool | None = None,
- **kwargs,
- ) -> BatchFeature:
- """
- Main method to featurize and prepare for the model one or several sequence(s).
-
- Args:
- raw_speech (`np.ndarray`, `list[float]`, `list[np.ndarray]`, `list[list[float]]`):
- The sequence or batch of sequences to be padded. Each sequence can be a numpy array, a list of float
- values, a list of numpy arrays or a list of list of float values. Must be mono channel audio, not
- stereo, i.e. single float per timestep.
- padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `True`):
- Select a strategy to pad the returned sequences (according to the model's padding side and padding
- index) among:
-
- - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
- sequence if provided).
- - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
- acceptable input length for the model if that argument is not provided.
- - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
- lengths).
- max_length (`int`, *optional*):
- Maximum length of the returned list and optionally padding length (see above).
- truncation (`bool`):
- Activates truncation to cut input sequences longer than *max_length* to *max_length*.
- pad_to_multiple_of (`int`, *optional*):
- If set will pad the sequence to a multiple of the provided value.
-
- This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability
- `>= 7.5` (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128.
- return_attention_mask (`bool`, *optional*):
- Whether to return the attention mask. If left to the default, will return the attention mask according
- to the specific feature_extractor's default.
-
- [What are attention masks?](../glossary#attention-mask)
-
-
-
- For Speech2TextTransformer models, `attention_mask` should always be passed for batched inference, to
- avoid subtle bugs.
-
-
-
- return_tensors (`str` or [`~utils.TensorType`], *optional*):
- If set, will return tensors instead of list of python integers. Acceptable values are:
-
- - `'pt'`: Return PyTorch `torch.Tensor` objects.
- - `'np'`: Return Numpy `np.ndarray` objects.
- sampling_rate (`int`, *optional*):
- The sampling rate at which the `raw_speech` input was sampled. It is strongly recommended to pass
- `sampling_rate` at the forward call to prevent silent errors.
- padding_value (`float`, *optional*, defaults to 0.0):
- The value that is used to fill the padding values / vectors.
- """
-
- if sampling_rate is not None:
- if sampling_rate != self.sampling_rate:
- raise ValueError(
- f"The model corresponding to this feature extractor: {self} was trained using a sampling rate of"
- f" {self.sampling_rate}. Please make sure that the provided `raw_speech` input was sampled with"
- f" {self.sampling_rate} and not {sampling_rate}."
- )
- else:
- logger.warning(
- f"It is strongly recommended to pass the `sampling_rate` argument to `{self.__class__.__name__}()`. "
- "Failing to do so can result in silent errors that might be hard to debug."
- )
-
- is_batched_numpy = isinstance(raw_speech, np.ndarray) and len(raw_speech.shape) > 1
- if is_batched_numpy and len(raw_speech.shape) > 2:
- raise ValueError(f"Only mono-channel audio is supported for input to {self}")
- is_batched = is_batched_numpy or (
- isinstance(raw_speech, (list, tuple)) and (isinstance(raw_speech[0], (np.ndarray, tuple, list)))
- )
-
- if is_batched:
- raw_speech = [np.asarray(speech, dtype=np.float32) for speech in raw_speech]
- elif not is_batched and not isinstance(raw_speech, np.ndarray):
- raw_speech = np.asarray(raw_speech, dtype=np.float32)
- elif isinstance(raw_speech, np.ndarray) and raw_speech.dtype is np.dtype(np.float64):
- raw_speech = raw_speech.astype(np.float32)
-
- # always return batch
- if not is_batched:
- raw_speech = [raw_speech]
-
- # extract fbank features
- features = [self._extract_fbank_features(waveform) for waveform in raw_speech]
-
- # convert into correct format for padding
- encoded_inputs = BatchFeature({"input_features": features})
-
- padded_inputs = self.pad(
- encoded_inputs,
- padding=padding,
- max_length=max_length,
- truncation=truncation,
- pad_to_multiple_of=pad_to_multiple_of,
- return_attention_mask=return_attention_mask,
- **kwargs,
- )
-
- # make sure list is in array format
- input_features = padded_inputs.get("input_features")
- if isinstance(input_features[0], list):
- padded_inputs["input_features"] = [np.asarray(feature, dtype=np.float32) for feature in input_features]
-
- attention_mask = padded_inputs.get("attention_mask")
- if attention_mask is not None:
- padded_inputs["attention_mask"] = [np.asarray(array, dtype=np.int32) for array in attention_mask]
-
- # Utterance-level cepstral mean and variance normalization
- if self.do_ceptral_normalize:
- attention_mask = (
- np.array(attention_mask, dtype=np.int32)
- if self._get_padding_strategies(padding, max_length=max_length) is not PaddingStrategy.DO_NOT_PAD
- else None
- )
- padded_inputs["input_features"] = self.normalize(
- padded_inputs["input_features"], attention_mask=attention_mask
- )
+from ...audio_processing_base import make_legacy_audio_processor_alias
+from .audio_processing_speech_to_text import SpeechToTextAudioProcessor
- if return_tensors is not None:
- padded_inputs = padded_inputs.convert_to_tensors(return_tensors)
- return padded_inputs
+Speech2TextFeatureExtractor = make_legacy_audio_processor_alias(SpeechToTextAudioProcessor, "Speech2TextFeatureExtractor")
__all__ = ["Speech2TextFeatureExtractor"]
diff --git a/src/transformers/models/speecht5/__init__.py b/src/transformers/models/speecht5/__init__.py
index 52fee59ae9b9..aac7e28ddc3e 100644
--- a/src/transformers/models/speecht5/__init__.py
+++ b/src/transformers/models/speecht5/__init__.py
@@ -18,6 +18,8 @@
if TYPE_CHECKING:
+ from .audio_processing_numpy_speecht5 import *
+ from .audio_processing_speecht5 import *
from .configuration_speecht5 import *
from .feature_extraction_speecht5 import *
from .modeling_speecht5 import *
diff --git a/src/transformers/models/speecht5/audio_processing_numpy_speecht5.py b/src/transformers/models/speecht5/audio_processing_numpy_speecht5.py
new file mode 100644
index 000000000000..e76a910cdbbb
--- /dev/null
+++ b/src/transformers/models/speecht5/audio_processing_numpy_speecht5.py
@@ -0,0 +1,26 @@
+# Copyright 2026 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ...audio_processing_backends import NumpyAudioBackend
+
+
+class SpeechT5AudioProcessorNumpy(NumpyAudioBackend):
+ """NumPy sibling of [`SpeechT5AudioProcessor`]. Bit-exact to the torch sibling within
+ the float32 noise floor (ADR 0001)."""
+
+ sampling_rate = 16000
+ force_mono = True
+
+
+__all__ = ["SpeechT5AudioProcessorNumpy"]
diff --git a/src/transformers/models/speecht5/audio_processing_speecht5.py b/src/transformers/models/speecht5/audio_processing_speecht5.py
new file mode 100644
index 000000000000..3259ac112f34
--- /dev/null
+++ b/src/transformers/models/speecht5/audio_processing_speecht5.py
@@ -0,0 +1,23 @@
+# Copyright 2025 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ...audio_processing_backends import TorchAudioBackend
+
+
+class SpeechT5AudioProcessor(TorchAudioBackend):
+ sampling_rate = 16000
+ force_mono = True
+
+
+__all__ = ["SpeechT5AudioProcessor"]
diff --git a/src/transformers/models/speecht5/feature_extraction_speecht5.py b/src/transformers/models/speecht5/feature_extraction_speecht5.py
index 5b9ca2e1f954..c98875c86f46 100644
--- a/src/transformers/models/speecht5/feature_extraction_speecht5.py
+++ b/src/transformers/models/speecht5/feature_extraction_speecht5.py
@@ -1,374 +1,20 @@
-# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
+# Copyright 2026 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Feature extractor class for SpeechT5."""
-
-from typing import Any
-
-import numpy as np
-
-from ...audio_utils import mel_filter_bank, optimal_fft_length, spectrogram, window_function
-from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
-from ...feature_extraction_utils import BatchFeature
-from ...utils import PaddingStrategy, TensorType, logging
-
-
-logger = logging.get_logger(__name__)
-
-
-class SpeechT5FeatureExtractor(SequenceFeatureExtractor):
- r"""
- Constructs a SpeechT5 feature extractor.
-
- This class can pre-process a raw speech signal by (optionally) normalizing to zero-mean unit-variance, for use by
- the SpeechT5 speech encoder prenet.
-
- This class can also extract log-mel filter bank features from raw speech, for use by the SpeechT5 speech decoder
- prenet.
-
- This feature extractor inherits from [`~feature_extraction_sequence_utils.SequenceFeatureExtractor`] which contains
- most of the main methods. Users should refer to this superclass for more information regarding those methods.
-
- Args:
- feature_size (`int`, *optional*, defaults to 1):
- The feature dimension of the extracted features.
- sampling_rate (`int`, *optional*, defaults to 16000):
- The sampling rate at which the audio files should be digitalized expressed in hertz (Hz).
- padding_value (`float`, *optional*, defaults to 0.0):
- The value that is used to fill the padding values.
- do_normalize (`bool`, *optional*, defaults to `False`):
- Whether or not to zero-mean unit-variance normalize the input. Normalizing can help to significantly
- improve the performance for some models.
- num_mel_bins (`int`, *optional*, defaults to 80):
- The number of mel-frequency bins in the extracted spectrogram features.
- hop_length (`int`, *optional*, defaults to 16):
- Number of ms between windows. Otherwise referred to as "shift" in many papers.
- win_length (`int`, *optional*, defaults to 64):
- Number of ms per window.
- win_function (`str`, *optional*, defaults to `"hann_window"`):
- Name for the window function used for windowing, must be accessible via `torch.{win_function}`
- fmin (`float`, *optional*, defaults to 80):
- Minimum mel frequency in Hz.
- fmax (`float`, *optional*, defaults to 7600):
- Maximum mel frequency in Hz.
- mel_floor (`float`, *optional*, defaults to 1e-10):
- Minimum value of mel frequency banks..
- return_attention_mask (`bool`, *optional*, defaults to `True`):
- Whether or not [`~SpeechT5FeatureExtractor.__call__`] should return `attention_mask`.
- """
-
- model_input_names = ["input_values", "attention_mask"]
-
- def __init__(
- self,
- feature_size: int = 1,
- sampling_rate: int = 16000,
- padding_value: float = 0.0,
- do_normalize: bool = False,
- num_mel_bins: int = 80,
- hop_length: int = 16,
- win_length: int = 64,
- win_function: str = "hann_window",
- fmin: float = 80,
- fmax: float = 7600,
- mel_floor: float = 1e-10,
- return_attention_mask: bool = True,
- **kwargs,
- ):
- super().__init__(feature_size=feature_size, sampling_rate=sampling_rate, padding_value=padding_value, **kwargs)
- self.do_normalize = do_normalize
- self.return_attention_mask = return_attention_mask
-
- self.num_mel_bins = num_mel_bins
- self.hop_length = hop_length
- self.win_length = win_length
- self.win_function = win_function
- self.fmin = fmin
- self.fmax = fmax
- self.mel_floor = mel_floor
-
- self.sample_size = win_length * sampling_rate // 1000
- self.sample_stride = hop_length * sampling_rate // 1000
- self.n_fft = optimal_fft_length(self.sample_size)
- self.n_freqs = (self.n_fft // 2) + 1
-
- self.window = window_function(window_length=self.sample_size, name=self.win_function, periodic=True)
-
- self.mel_filters = mel_filter_bank(
- num_frequency_bins=self.n_freqs,
- num_mel_filters=self.num_mel_bins,
- min_frequency=self.fmin,
- max_frequency=self.fmax,
- sampling_rate=self.sampling_rate,
- norm="slaney",
- mel_scale="slaney",
- )
-
- @staticmethod
- # Copied from transformers.models.wav2vec2.feature_extraction_wav2vec2.Wav2Vec2FeatureExtractor.zero_mean_unit_var_norm
- def zero_mean_unit_var_norm(
- input_values: list[np.ndarray], attention_mask: list[np.ndarray], padding_value: float = 0.0
- ) -> list[np.ndarray]:
- """
- Every array in the list is normalized to have zero mean and unit variance
- """
- if attention_mask is not None:
- attention_mask = np.array(attention_mask, np.int32)
- normed_input_values = []
-
- for vector, length in zip(input_values, attention_mask.sum(-1)):
- normed_slice = (vector - vector[:length].mean()) / np.sqrt(vector[:length].var() + 1e-7)
- if length < normed_slice.shape[0]:
- normed_slice[length:] = padding_value
-
- normed_input_values.append(normed_slice)
- else:
- normed_input_values = [(x - x.mean()) / np.sqrt(x.var() + 1e-7) for x in input_values]
-
- return normed_input_values
-
- def _extract_mel_features(
- self,
- one_waveform: np.ndarray,
- ) -> np.ndarray:
- """
- Extracts log-mel filterbank features for one waveform array (unbatched).
- """
- log_mel_spec = spectrogram(
- one_waveform,
- window=self.window,
- frame_length=self.sample_size,
- hop_length=self.sample_stride,
- fft_length=self.n_fft,
- mel_filters=self.mel_filters,
- mel_floor=self.mel_floor,
- log_mel="log10",
- )
- return log_mel_spec.T
-
- def __call__(
- self,
- audio: np.ndarray | list[float] | list[np.ndarray] | list[list[float]] | None = None,
- audio_target: np.ndarray | list[float] | list[np.ndarray] | list[list[float]] | None = None,
- padding: bool | str | PaddingStrategy = False,
- max_length: int | None = None,
- truncation: bool = False,
- pad_to_multiple_of: int | None = None,
- return_attention_mask: bool | None = None,
- return_tensors: str | TensorType | None = None,
- sampling_rate: int | None = None,
- **kwargs,
- ) -> BatchFeature:
- """
- Main method to featurize and prepare for the model one or several sequence(s).
-
- Pass in a value for `audio` to extract waveform features. Pass in a value for `audio_target` to extract log-mel
- spectrogram features.
-
- Args:
- audio (`np.ndarray`, `list[float]`, `list[np.ndarray]`, `list[list[float]]`, *optional*):
- The sequence or batch of sequences to be processed. Each sequence can be a numpy array, a list of float
- values, a list of numpy arrays or a list of list of float values. This outputs waveform features. Must
- be mono channel audio, not stereo, i.e. single float per timestep.
- audio_target (`np.ndarray`, `list[float]`, `list[np.ndarray]`, `list[list[float]]`, *optional*):
- The sequence or batch of sequences to be processed as targets. Each sequence can be a numpy array, a
- list of float values, a list of numpy arrays or a list of list of float values. This outputs log-mel
- spectrogram features.
- padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `False`):
- Select a strategy to pad the returned sequences (according to the model's padding side and padding
- index) among:
-
- - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
- sequence if provided).
- - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
- acceptable input length for the model if that argument is not provided.
- - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
- lengths).
- max_length (`int`, *optional*):
- Maximum length of the returned list and optionally padding length (see above).
- truncation (`bool`):
- Activates truncation to cut input sequences longer than *max_length* to *max_length*.
- pad_to_multiple_of (`int`, *optional*):
- If set will pad the sequence to a multiple of the provided value.
-
- This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability
- `>= 7.5` (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128.
- return_attention_mask (`bool`, *optional*):
- Whether to return the attention mask. If left to the default, will return the attention mask according
- to the specific feature_extractor's default.
-
- [What are attention masks?](../glossary#attention-mask)
-
- return_tensors (`str` or [`~utils.TensorType`], *optional*):
- If set, will return tensors instead of list of python integers. Acceptable values are:
-
- - `'pt'`: Return PyTorch `torch.Tensor` objects.
- - `'np'`: Return Numpy `np.ndarray` objects.
- sampling_rate (`int`, *optional*):
- The sampling rate at which the `audio` or `audio_target` input was sampled. It is strongly recommended
- to pass `sampling_rate` at the forward call to prevent silent errors.
- """
- if audio is None and audio_target is None:
- raise ValueError("You must provide either `audio` or `audio_target` values.")
-
- if sampling_rate is not None:
- if sampling_rate != self.sampling_rate:
- raise ValueError(
- f"The model corresponding to this feature extractor: {self} was trained using a sampling rate of"
- f" {self.sampling_rate}. Please make sure that the provided audio input was sampled with"
- f" {self.sampling_rate} and not {sampling_rate}."
- )
- else:
- logger.warning(
- f"It is strongly recommended to pass the `sampling_rate` argument to `{self.__class__.__name__}()`. "
- "Failing to do so can result in silent errors that might be hard to debug."
- )
-
- if audio is not None:
- inputs = self._process_audio(
- audio,
- False,
- padding,
- max_length,
- truncation,
- pad_to_multiple_of,
- return_attention_mask,
- return_tensors,
- **kwargs,
- )
- else:
- inputs = None
-
- if audio_target is not None:
- inputs_target = self._process_audio(
- audio_target,
- True,
- padding,
- max_length,
- truncation,
- pad_to_multiple_of,
- return_attention_mask,
- return_tensors,
- **kwargs,
- )
-
- if inputs is None:
- return inputs_target
- else:
- inputs["labels"] = inputs_target["input_values"]
- decoder_attention_mask = inputs_target.get("attention_mask")
- if decoder_attention_mask is not None:
- inputs["decoder_attention_mask"] = decoder_attention_mask
-
- return inputs
-
- def _process_audio(
- self,
- speech: np.ndarray | list[float] | list[np.ndarray] | list[list[float]],
- is_target: bool = False,
- padding: bool | str | PaddingStrategy = False,
- max_length: int | None = None,
- truncation: bool = False,
- pad_to_multiple_of: int | None = None,
- return_attention_mask: bool | None = None,
- return_tensors: str | TensorType | None = None,
- **kwargs,
- ) -> BatchFeature:
- is_batched_numpy = isinstance(speech, np.ndarray) and len(speech.shape) > 1
- if is_batched_numpy and len(speech.shape) > 2:
- raise ValueError(f"Only mono-channel audio is supported for input to {self}")
- is_batched = is_batched_numpy or (
- isinstance(speech, (list, tuple)) and (isinstance(speech[0], (np.ndarray, tuple, list)))
- )
-
- if is_batched:
- speech = [np.asarray(speech, dtype=np.float32) for speech in speech]
- elif not is_batched and not isinstance(speech, np.ndarray):
- speech = np.asarray(speech, dtype=np.float32)
- elif isinstance(speech, np.ndarray) and speech.dtype is np.dtype(np.float64):
- speech = speech.astype(np.float32)
-
- # always return batch
- if not is_batched:
- speech = [speech]
-
- # needed to make pad() work on spectrogram inputs
- feature_size_hack = self.feature_size
-
- # convert into correct format for padding
- if is_target:
- features = [self._extract_mel_features(waveform) for waveform in speech]
- encoded_inputs = BatchFeature({"input_values": features})
- self.feature_size = self.num_mel_bins
- else:
- encoded_inputs = BatchFeature({"input_values": speech})
-
- padded_inputs = self.pad(
- encoded_inputs,
- padding=padding,
- max_length=max_length,
- truncation=truncation,
- pad_to_multiple_of=pad_to_multiple_of,
- return_attention_mask=return_attention_mask,
- **kwargs,
- )
-
- self.feature_size = feature_size_hack
-
- # convert input values to correct format
- input_values = padded_inputs["input_values"]
- if not isinstance(input_values[0], np.ndarray):
- padded_inputs["input_values"] = [np.asarray(array, dtype=np.float32) for array in input_values]
- elif (
- not isinstance(input_values, np.ndarray)
- and isinstance(input_values[0], np.ndarray)
- and input_values[0].dtype is np.dtype(np.float64)
- ):
- padded_inputs["input_values"] = [array.astype(np.float32) for array in input_values]
- elif isinstance(input_values, np.ndarray) and input_values.dtype is np.dtype(np.float64):
- padded_inputs["input_values"] = input_values.astype(np.float32)
-
- # convert attention_mask to correct format
- attention_mask = padded_inputs.get("attention_mask")
- if attention_mask is not None:
- padded_inputs["attention_mask"] = [np.asarray(array, dtype=np.int32) for array in attention_mask]
-
- # zero-mean and unit-variance normalization
- if not is_target and self.do_normalize:
- attention_mask = (
- attention_mask
- if self._get_padding_strategies(padding, max_length=max_length) is not PaddingStrategy.DO_NOT_PAD
- else None
- )
- padded_inputs["input_values"] = self.zero_mean_unit_var_norm(
- padded_inputs["input_values"], attention_mask=attention_mask, padding_value=self.padding_value
- )
-
- if return_tensors is not None:
- padded_inputs = padded_inputs.convert_to_tensors(return_tensors)
-
- return padded_inputs
+"""Backwards-compatibility shim: re-exports the legacy ``SpeechT5FeatureExtractor`` name as a
+deprecated alias of [`SpeechT5AudioProcessor`]. Importing or instantiating the alias emits a
+``FutureWarning``; the alias is removed in transformers v5.15 (see ADR 0002).
+"""
- def to_dict(self) -> dict[str, Any]:
- output = super().to_dict()
+from ...audio_processing_base import make_legacy_audio_processor_alias
+from .audio_processing_speecht5 import SpeechT5AudioProcessor
- # Don't serialize these as they are derived from the other properties.
- names = ["window", "mel_filters", "sample_size", "sample_stride", "n_fft", "n_freqs"]
- for name in names:
- if name in output:
- del output[name]
- return output
+SpeechT5FeatureExtractor = make_legacy_audio_processor_alias(SpeechT5AudioProcessor, "SpeechT5FeatureExtractor")
__all__ = ["SpeechT5FeatureExtractor"]
diff --git a/src/transformers/models/univnet/__init__.py b/src/transformers/models/univnet/__init__.py
index 5d02d0dbe574..0ae4a2e8d01d 100644
--- a/src/transformers/models/univnet/__init__.py
+++ b/src/transformers/models/univnet/__init__.py
@@ -18,6 +18,8 @@
if TYPE_CHECKING:
+ from .audio_processing_numpy_univnet import *
+ from .audio_processing_univnet import *
from .configuration_univnet import *
from .feature_extraction_univnet import *
from .modeling_univnet import *
diff --git a/src/transformers/models/univnet/audio_processing_numpy_univnet.py b/src/transformers/models/univnet/audio_processing_numpy_univnet.py
new file mode 100644
index 000000000000..423a5f77e17f
--- /dev/null
+++ b/src/transformers/models/univnet/audio_processing_numpy_univnet.py
@@ -0,0 +1,92 @@
+# Copyright 2026 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from ...audio_processing_backends import NumpyAudioBackend
+from ...audio_utils import MelScaleConfig, SpectrogramConfig, StftConfig
+
+
+class UnivNetAudioProcessorNumpy(NumpyAudioBackend):
+ """NumPy sibling of [`UnivNetAudioProcessor`]. Reflect-padded STFT with mel_floor added
+ inside the magnitude sqrt, no mel-floor clamp, and a `(frames, n_mels)` output layout."""
+
+ sampling_rate = 24000
+ force_mono = True
+ mask_level = "audio"
+ mel_floor = 1e-9
+ compression_clip_val = 1e-5
+ compression_factor = 1.0
+ do_normalize = False
+ normalize_min = -11.512925148010254
+ normalize_max = 2.3143386840820312
+ max_length_s = 10
+ spectrogram_config = SpectrogramConfig(
+ stft_config=StftConfig(
+ n_fft=1024,
+ hop_length=256,
+ center=False,
+ window_fn="hann",
+ periodic=True,
+ power=1.0,
+ ),
+ mel_scale_config=MelScaleConfig(
+ n_mels=100,
+ f_min=0.0,
+ f_max=12000.0,
+ mel_scale="slaney",
+ norm="slaney",
+ ),
+ log_mode="log",
+ mel_floor=1e-5,
+ computation_dtype="float64",
+ )
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+ self.num_max_samples = self.max_length_s * self.sampling_rate
+
+ def _stft(self, audio, *, spectrogram_config, **kwargs):
+ # UnivNet uses reflect padding with (n_fft - hop_length) / 2 instead of center padding
+ stft_cfg = spectrogram_config.stft_config
+ pad_amount = int((stft_cfg.n_fft - stft_cfg.hop_length) / 2)
+ if audio.ndim > 1:
+ audio = np.pad(audio, ((0, 0), (pad_amount, pad_amount)), mode="reflect")
+ else:
+ audio = np.pad(audio, (pad_amount, pad_amount), mode="reflect")
+ return super()._stft(audio, spectrogram_config=spectrogram_config, **kwargs)
+
+ def _compute_magnitudes(self, stft_out, power, spectrogram_config=None):
+ # UnivNet adds mel_floor inside the sqrt: sqrt(real² + imag² + mel_floor)
+ return np.sqrt(np.real(stft_out) ** 2 + np.imag(stft_out) ** 2 + self.mel_floor)
+
+ def _apply_mel_scale(self, features, *, spectrogram_config, **kwargs):
+ # UnivNet applies mel filterbank without a floor
+ return np.matmul(self.mel_filters.T, features)
+
+ def _normalize_magnitude(self, features, *, spectrogram_config, **kwargs):
+ features = super()._normalize_magnitude(features, spectrogram_config=spectrogram_config, **kwargs)
+ if self.do_normalize:
+ features = 2 * ((features - self.normalize_min) / (self.normalize_max - self.normalize_min)) - 1
+ return features
+
+ def extract_spectrogram(self, audio, *, spectrogram_config, **kwargs):
+ features = super().extract_spectrogram(audio, spectrogram_config=spectrogram_config, **kwargs)
+ # Transpose from (..., n_mels, frames) to (..., frames, n_mels)
+ if isinstance(features, list):
+ return [np.swapaxes(f, -2, -1) for f in features]
+ return np.swapaxes(features, -2, -1)
+
+
+__all__ = ["UnivNetAudioProcessorNumpy"]
diff --git a/src/transformers/models/univnet/audio_processing_univnet.py b/src/transformers/models/univnet/audio_processing_univnet.py
new file mode 100644
index 000000000000..cb5463ac299e
--- /dev/null
+++ b/src/transformers/models/univnet/audio_processing_univnet.py
@@ -0,0 +1,85 @@
+# Copyright 2025 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+
+from ...audio_processing_backends import TorchAudioBackend
+from .audio_processing_numpy_univnet import UnivNetAudioProcessorNumpy
+
+
+class UnivNetAudioProcessor(TorchAudioBackend):
+ """Torch sibling of [`UnivNetAudioProcessorNumpy`]. Reflect-padded STFT with mel_floor
+ added inside the magnitude sqrt, no mel-floor clamp, and a `(frames, n_mels)` output layout."""
+
+ sampling_rate = 24000
+ force_mono = True
+ mask_level = "audio"
+ mel_floor = 1e-9
+ compression_clip_val = 1e-5
+ compression_factor = 1.0
+ do_normalize = False
+ normalize_min = -11.512925148010254
+ normalize_max = 2.3143386840820312
+ max_length_s = 10
+
+ spectrogram_config = UnivNetAudioProcessorNumpy.spectrogram_config
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+ self.num_max_samples = self.max_length_s * self.sampling_rate
+
+ def _stft(self, audio, *, spectrogram_config, **kwargs):
+ # UnivNet uses reflect padding with (n_fft - hop_length) / 2 instead of center padding
+ stft_cfg = spectrogram_config.stft_config
+ pad_amount = int((stft_cfg.n_fft - stft_cfg.hop_length) / 2)
+ # `torch.nn.functional.pad` reflects on the last dim by default; works for 1D or 2D.
+ audio = torch.nn.functional.pad(audio, (pad_amount, pad_amount), mode="reflect")
+ return super()._stft(audio, spectrogram_config=spectrogram_config, **kwargs)
+
+ # Mel filters: the base dispatcher resolves the top-level `computation_dtype="float64"`
+ # into float64 torch-native filters (matching the legacy FE's float64 numpy build
+ # within ~1e-16), kept float64 for the mel matmul below.
+
+ def _compute_magnitudes(self, stft_out, power, spectrogram_config=None):
+ # UnivNet adds mel_floor inside the sqrt: sqrt(real² + imag² + mel_floor).
+ # The legacy FE stores the STFT in a complex64 buffer and takes the sqrt in float32
+ # (mirrors the numpy sibling's complex64 cast). torch's float32 sqrt is not correctly
+ # rounded on all platforms, so round via a float64 sqrt (bit-identical to np.sqrt on
+ # float32 inputs), then promote back to float64 for the mel matmul the way numpy's
+ # float32 x float64 matmul promotion does.
+ stft_out = stft_out.to(torch.complex64)
+ presqrt = stft_out.real ** 2 + stft_out.imag ** 2 + self.mel_floor
+ return presqrt.double().sqrt().float().double()
+
+ def _apply_mel_scale(self, features, *, spectrogram_config, **kwargs):
+ # UnivNet applies mel filterbank without a floor.
+ # `mel_filters` is shape `(n_freq, n_mels)`; transposing gives `(n_mels, n_freq)`.
+ mel_filters = self.mel_filters.to(device=features.device, dtype=features.dtype)
+ return torch.matmul(mel_filters.T, features)
+
+ def _normalize_magnitude(self, features, *, spectrogram_config, **kwargs):
+ features = super()._normalize_magnitude(features, spectrogram_config=spectrogram_config, **kwargs)
+ if self.do_normalize:
+ features = 2 * ((features - self.normalize_min) / (self.normalize_max - self.normalize_min)) - 1
+ return features
+
+ def extract_spectrogram(self, audio, *, spectrogram_config, **kwargs):
+ features = super().extract_spectrogram(audio, spectrogram_config=spectrogram_config, **kwargs)
+ # Transpose from (..., n_mels, frames) to (..., frames, n_mels)
+ if isinstance(features, list):
+ return [f.transpose(-2, -1) for f in features]
+ return features.transpose(-2, -1)
+
+
+__all__ = ["UnivNetAudioProcessor"]
diff --git a/src/transformers/models/univnet/feature_extraction_univnet.py b/src/transformers/models/univnet/feature_extraction_univnet.py
index 84e9420a0f75..1a0e6d6de4fe 100644
--- a/src/transformers/models/univnet/feature_extraction_univnet.py
+++ b/src/transformers/models/univnet/feature_extraction_univnet.py
@@ -1,458 +1,20 @@
-# Copyright 2023 The HuggingFace Team. All rights reserved.
+# Copyright 2026 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Feature extractor class for UnivNetModel."""
-
-from typing import Any
-
-import numpy as np
-
-from ...audio_utils import mel_filter_bank, optimal_fft_length, spectrogram, window_function
-from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
-from ...feature_extraction_utils import BatchFeature
-from ...utils import PaddingStrategy, TensorType, logging
-
-
-logger = logging.get_logger(__name__)
-
-
-class UnivNetFeatureExtractor(SequenceFeatureExtractor):
- r"""
- Constructs a UnivNet feature extractor.
-
- This class extracts log-mel-filter bank features from raw speech using the short time Fourier Transform (STFT). The
- STFT implementation follows that of TacoTron 2 and Hifi-GAN.
-
- This feature extractor inherits from [`~feature_extraction_sequence_utils.SequenceFeatureExtractor`] which contains
- most of the main methods. Users should refer to this superclass for more information regarding those methods.
-
- Args:
- feature_size (`int`, *optional*, defaults to 1):
- The feature dimension of the extracted features.
- sampling_rate (`int`, *optional*, defaults to 24000):
- The sampling rate at which the audio files should be digitalized expressed in hertz (Hz).
- padding_value (`float`, *optional*, defaults to 0.0):
- The value to pad with when applying the padding strategy defined by the `padding` argument to
- [`UnivNetFeatureExtractor.__call__`]. Should correspond to audio silence. The `pad_end` argument to
- `__call__` will also use this padding value.
- do_normalize (`bool`, *optional*, defaults to `False`):
- Whether to perform Tacotron 2 normalization on the input. Normalizing can help to significantly improve the
- performance for some models.
- num_mel_bins (`int`, *optional*, defaults to 100):
- The number of mel-frequency bins in the extracted spectrogram features. This should match
- `UnivNetModel.config.num_mel_bins`.
- hop_length (`int`, *optional*, defaults to 256):
- The direct number of samples between sliding windows. Otherwise referred to as "shift" in many papers. Note
- that this is different from other audio feature extractors such as [`SpeechT5FeatureExtractor`] which take
- the `hop_length` in ms.
- win_length (`int`, *optional*, defaults to 1024):
- The direct number of samples for each sliding window. Note that this is different from other audio feature
- extractors such as [`SpeechT5FeatureExtractor`] which take the `win_length` in ms.
- win_function (`str`, *optional*, defaults to `"hann_window"`):
- Name for the window function used for windowing, must be accessible via `torch.{win_function}`
- filter_length (`int`, *optional*, defaults to 1024):
- The number of FFT components to use. If `None`, this is determined using
- `transformers.audio_utils.optimal_fft_length`.
- max_length_s (`int`, *optional*, defaults to 10):
- The maximum input length of the model in seconds. This is used to pad the audio.
- fmin (`float`, *optional*, defaults to 0.0):
- Minimum mel frequency in Hz.
- fmax (`float`, *optional*):
- Maximum mel frequency in Hz. If not set, defaults to `sampling_rate / 2`.
- mel_floor (`float`, *optional*, defaults to 1e-09):
- Minimum value of mel frequency banks. Note that the way [`UnivNetFeatureExtractor`] uses `mel_floor` is
- different than in [`transformers.audio_utils.spectrogram`].
- center (`bool`, *optional*, defaults to `False`):
- Whether to pad the waveform so that frame `t` is centered around time `t * hop_length`. If `False`, frame
- `t` will start at time `t * hop_length`.
- compression_factor (`float`, *optional*, defaults to 1.0):
- The multiplicative compression factor for dynamic range compression during spectral normalization.
- compression_clip_val (`float`, *optional*, defaults to 1e-05):
- The clip value applied to the waveform before applying dynamic range compression during spectral
- normalization.
- normalize_min (`float`, *optional*, defaults to -11.512925148010254):
- The min value used for Tacotron 2-style linear normalization. The default is the original value from the
- Tacotron 2 implementation.
- normalize_max (`float`, *optional*, defaults to 2.3143386840820312):
- The max value used for Tacotron 2-style linear normalization. The default is the original value from the
- Tacotron 2 implementation.
- model_in_channels (`int`, *optional*, defaults to 64):
- The number of input channels to the [`UnivNetModel`] model. This should match
- `UnivNetModel.config.model_in_channels`.
- pad_end_length (`int`, *optional*, defaults to 10):
- If padding the end of each waveform, the number of spectrogram frames worth of samples to append. The
- number of appended samples will be `pad_end_length * hop_length`.
- return_attention_mask (`bool`, *optional*, defaults to `True`):
- Whether or not [`~UnivNetFeatureExtractor.__call__`] should return `attention_mask`.
- """
-
- model_input_names = ["input_features", "noise_sequence", "padding_mask"]
-
- def __init__(
- self,
- feature_size: int = 1,
- sampling_rate: int = 24000,
- padding_value: float = 0.0,
- do_normalize: bool = False,
- num_mel_bins: int = 100,
- hop_length: int = 256,
- win_length: int = 1024,
- win_function: str = "hann_window",
- filter_length: int | None = 1024,
- max_length_s: int = 10,
- fmin: float = 0.0,
- fmax: float | None = None,
- mel_floor: float = 1e-9,
- center: bool = False,
- compression_factor: float = 1.0,
- compression_clip_val: float = 1e-5,
- normalize_min: float = -11.512925148010254,
- normalize_max: float = 2.3143386840820312,
- model_in_channels: int = 64,
- pad_end_length: int = 10,
- return_attention_mask=True,
- **kwargs,
- ):
- super().__init__(
- feature_size=feature_size,
- sampling_rate=sampling_rate,
- padding_value=padding_value,
- return_attention_mask=return_attention_mask,
- **kwargs,
- )
-
- self.do_normalize = do_normalize
-
- self.num_mel_bins = num_mel_bins
- self.hop_length = hop_length
- self.win_length = win_length
- self.win_function = win_function
- self.filter_length = filter_length
- self.fmin = fmin
- if fmax is None:
- # Follows the librosa.filters.mel implementation
- fmax = float(sampling_rate) / 2
- self.fmax = fmax
- self.mel_floor = mel_floor
-
- self.max_length_s = max_length_s
- self.num_max_samples = max_length_s * sampling_rate
-
- if self.filter_length is None:
- self.n_fft = optimal_fft_length(self.win_length)
- else:
- self.n_fft = self.filter_length
- self.n_freqs = (self.n_fft // 2) + 1
-
- self.window = window_function(window_length=self.win_length, name=self.win_function, periodic=True)
-
- self.mel_filters = mel_filter_bank(
- num_frequency_bins=self.n_freqs,
- num_mel_filters=self.num_mel_bins,
- min_frequency=self.fmin,
- max_frequency=self.fmax,
- sampling_rate=self.sampling_rate,
- norm="slaney",
- mel_scale="slaney",
- )
-
- self.center = center
- self.compression_factor = compression_factor
- self.compression_clip_val = compression_clip_val
- self.normalize_min = normalize_min
- self.normalize_max = normalize_max
- self.model_in_channels = model_in_channels
- self.pad_end_length = pad_end_length
-
- def normalize(self, spectrogram):
- return 2 * ((spectrogram - self.normalize_min) / (self.normalize_max - self.normalize_min)) - 1
-
- def denormalize(self, spectrogram):
- return self.normalize_min + (self.normalize_max - self.normalize_min) * ((spectrogram + 1) / 2)
-
- def mel_spectrogram(self, waveform: np.ndarray) -> np.ndarray:
- """
- Calculates log MEL spectrograms from a batch of waveforms. Note that the input waveform(s) will be padded by
- `int(self.n_fft - self.hop_length) / 2` on both sides using the `reflect` padding mode.
-
- Args:
- waveform (`np.ndarray` of shape `(length,)`):
- The input waveform. This must be a single real-valued, mono waveform.
-
- Returns:
- `numpy.ndarray`: Array containing a log-mel spectrogram of shape `(num_frames, num_mel_bins)`.
- """
- # Do custom padding based on the official MelGAN and Hifi-GAN implementations
- # See https://github.com/maum-ai/univnet/blob/9bb2b54838bb6d7ce767131cc7b8b61198bc7558/utils/stft.py#L84-L86
- waveform = np.pad(
- waveform,
- (int((self.n_fft - self.hop_length) / 2), int((self.n_fft - self.hop_length) / 2)),
- mode="reflect",
- )
-
- # Get the complex spectrogram.
- # Note: waveform must be unbatched currently due to the implementation of spectrogram(...).
- complex_spectrogram = spectrogram(
- waveform,
- window=self.window,
- frame_length=self.n_fft,
- hop_length=self.hop_length,
- fft_length=self.n_fft,
- power=None,
- center=self.center,
- mel_filters=None,
- mel_floor=None,
- )
-
- # Apply the MEL filter bank and MEL floor manually since UnivNet uses a slightly different implementation
- amplitude_spectrogram = np.sqrt(
- np.real(complex_spectrogram) ** 2 + np.imag(complex_spectrogram) ** 2 + self.mel_floor
- )
- mel_spectrogram = np.matmul(self.mel_filters.T, amplitude_spectrogram)
-
- # Perform spectral normalization to get the log mel spectrogram.
- log_mel_spectrogram = np.log(
- np.clip(mel_spectrogram, a_min=self.compression_clip_val, a_max=None) * self.compression_factor
- )
-
- # Return spectrogram with num_mel_bins last
- return log_mel_spectrogram.T
-
- def generate_noise(
- self,
- noise_length: int,
- generator: np.random.Generator | None = None,
- ) -> np.ndarray:
- """
- Generates a random noise sequence of standard Gaussian noise for use in the `noise_sequence` argument of
- [`UnivNetModel.forward`].
-
- Args:
- spectrogram_length (`int`):
- The length (dim 0) of the generated noise.
- model_in_channels (`int`, *optional*, defaults to `None`):
- The number of features (dim 1) of the generated noise. This should correspond to the
- `model_in_channels` of the [`UnivNetGan`] model. If not set, this will default to
- `self.config.model_in_channels`.
- generator (`numpy.random.Generator`, *optional*, defaults to `None`)
- An optional `numpy.random.Generator` random number generator to control noise generation. If not set, a
- new generator with fresh entropy will be created.
-
- Returns:
- `numpy.ndarray`: Array containing random standard Gaussian noise of shape `(noise_length,
- model_in_channels)`.
- """
- if generator is None:
- generator = np.random.default_rng()
-
- noise_shape = (noise_length, self.model_in_channels)
- noise = generator.standard_normal(noise_shape, dtype=np.float32)
-
- return noise
-
- def batch_decode(self, waveforms, waveform_lengths=None) -> list[np.ndarray]:
- r"""
- Removes padding from generated audio after running [`UnivNetModel.forward`]. This returns a ragged list of 1D
- audio waveform arrays and not a single tensor/array because in general the waveforms will have different
- lengths after removing padding.
-
- Args:
- waveforms (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
- The batched output waveforms from the [`UnivNetModel`].
- waveform_lengths (`torch.FloatTensor` of shape `(batch_size,)`, *optional*):
- The batched lengths of each waveform before padding.
-
- Returns:
- `list[np.ndarray]`: A ragged list of 1D waveform arrays with padding removed.
- """
- # Collapse the batched waveform tensor to a list of 1D audio waveforms
- waveforms = [waveform.detach().to(device="cpu", copy=True).numpy() for waveform in waveforms]
-
- if waveform_lengths is not None:
- waveforms = [waveform[: waveform_lengths[i]] for i, waveform in enumerate(waveforms)]
-
- return waveforms
-
- def __call__(
- self,
- raw_speech: np.ndarray | list[float] | list[np.ndarray] | list[list[float]],
- sampling_rate: int | None = None,
- padding: bool | str | PaddingStrategy = True,
- max_length: int | None = None,
- truncation: bool = True,
- pad_to_multiple_of: int | None = None,
- return_noise: bool = True,
- generator: np.random.Generator | None = None,
- pad_end: bool = False,
- pad_length: int | None = None,
- do_normalize: str | None = None,
- return_attention_mask: bool | None = None,
- return_tensors: str | TensorType | None = None,
- ) -> BatchFeature:
- """
- Main method to featurize and prepare for the model one or several sequence(s).
-
- Args:
- raw_speech (`np.ndarray`, `list[float]`, `list[np.ndarray]`, `list[list[float]]`):
- The sequence or batch of sequences to be padded. Each sequence can be a numpy array, a list of float
- values, a list of numpy arrays or a list of list of float values. Must be mono channel audio, not
- stereo, i.e. single float per timestep.
- sampling_rate (`int`, *optional*):
- The sampling rate at which the `raw_speech` input was sampled. It is strongly recommended to pass
- `sampling_rate` at the forward call to prevent silent errors and allow automatic speech recognition
- pipeline.
- padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `True`):
- Select a strategy to pad the input `raw_speech` waveforms (according to the model's padding side and
- padding index) among:
-
- - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
- sequence if provided).
- - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
- acceptable input length for the model if that argument is not provided.
- - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
- lengths).
-
- If `pad_end = True`, that padding will occur before the `padding` strategy is applied.
- max_length (`int`, *optional*):
- Maximum length of the returned list and optionally padding length (see above).
- truncation (`bool`, *optional*, defaults to `True`):
- Activates truncation to cut input sequences longer than `max_length` to `max_length`.
- pad_to_multiple_of (`int`, *optional*):
- If set will pad the sequence to a multiple of the provided value.
-
- This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability
- `>= 7.5` (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128.
- return_noise (`bool`, *optional*, defaults to `True`):
- Whether to generate and return a noise waveform for use in [`UnivNetModel.forward`].
- generator (`numpy.random.Generator`, *optional*, defaults to `None`):
- An optional `numpy.random.Generator` random number generator to use when generating noise.
- pad_end (`bool`, *optional*, defaults to `False`):
- Whether to pad the end of each waveform with silence. This can help reduce artifacts at the end of the
- generated audio sample; see https://github.com/seungwonpark/melgan/issues/8 for more details. This
- padding will be done before the padding strategy specified in `padding` is performed.
- pad_length (`int`, *optional*, defaults to `None`):
- If padding the end of each waveform, the length of the padding in spectrogram frames. If not set, this
- will default to `self.config.pad_end_length`.
- do_normalize (`bool`, *optional*):
- Whether to perform Tacotron 2 normalization on the input. Normalizing can help to significantly improve
- the performance for some models. If not set, this will default to `self.config.do_normalize`.
- return_attention_mask (`bool`, *optional*):
- Whether to return the attention mask. If left to the default, will return the attention mask according
- to the specific feature_extractor's default.
-
- [What are attention masks?](../glossary#attention-mask)
-
- return_tensors (`str` or [`~utils.TensorType`], *optional*):
- If set, will return tensors instead of list of python integers. Acceptable values are:
-
- - `'pt'`: Return PyTorch `torch.np.array` objects.
- - `'np'`: Return Numpy `np.ndarray` objects.
- """
- do_normalize = do_normalize if do_normalize is not None else self.do_normalize
-
- if sampling_rate is not None:
- if sampling_rate != self.sampling_rate:
- raise ValueError(
- f"The model corresponding to this feature extractor: {self.__class__.__name__} was trained using a"
- f" sampling rate of {self.sampling_rate}. Please make sure that the provided `raw_speech` input"
- f" was sampled with {self.sampling_rate} and not {sampling_rate}."
- )
- else:
- logger.warning(
- f"It is strongly recommended to pass the `sampling_rate` argument to `{self.__class__.__name__}()`. "
- "Failing to do so can result in silent errors that might be hard to debug."
- )
-
- is_batched_numpy = isinstance(raw_speech, np.ndarray) and len(raw_speech.shape) > 1
- if is_batched_numpy and len(raw_speech.shape) > 2:
- raise ValueError(f"Only mono-channel audio is supported for input to {self}")
- is_batched = is_batched_numpy or (
- isinstance(raw_speech, (list, tuple)) and (isinstance(raw_speech[0], (np.ndarray, tuple, list)))
- )
-
- if is_batched:
- raw_speech = [np.asarray(speech, dtype=np.float32) for speech in raw_speech]
- elif not is_batched and not isinstance(raw_speech, np.ndarray):
- raw_speech = np.asarray(raw_speech, dtype=np.float32)
- elif isinstance(raw_speech, np.ndarray) and raw_speech.dtype is np.dtype(np.float64):
- raw_speech = raw_speech.astype(np.float32)
-
- # always return batch
- if not is_batched:
- raw_speech = [np.asarray(raw_speech, dtype=np.float32)]
-
- # Pad end to reduce artifacts
- if pad_end:
- pad_length = pad_length if pad_length is not None else self.pad_end_length
- raw_speech = [
- np.pad(waveform, (0, pad_length * self.hop_length), constant_values=self.padding_value)
- for waveform in raw_speech
- ]
-
- batched_speech = BatchFeature({"input_features": raw_speech})
-
- padded_inputs = self.pad(
- batched_speech,
- padding=padding,
- max_length=max_length if max_length is not None else self.num_max_samples,
- truncation=truncation,
- pad_to_multiple_of=pad_to_multiple_of,
- return_attention_mask=return_attention_mask,
- )
-
- # make sure list is in array format
- # input_features = padded_inputs.get("input_features").transpose(2, 0, 1)
- input_features = padded_inputs.get("input_features")
-
- mel_spectrograms = [self.mel_spectrogram(waveform) for waveform in input_features]
-
- if isinstance(input_features[0], list):
- batched_speech["input_features"] = [np.asarray(mel, dtype=np.float32) for mel in mel_spectrograms]
- else:
- batched_speech["input_features"] = [mel.astype(np.float32) for mel in mel_spectrograms]
-
- # convert attention_mask to correct format
- attention_mask = padded_inputs.get("attention_mask")
- if attention_mask is not None:
- batched_speech["padding_mask"] = [np.asarray(array, dtype=np.int32) for array in attention_mask]
-
- if return_noise:
- noise = [
- self.generate_noise(spectrogram.shape[0], generator)
- for spectrogram in batched_speech["input_features"]
- ]
- batched_speech["noise_sequence"] = noise
-
- if do_normalize:
- batched_speech["input_features"] = [
- self.normalize(spectrogram) for spectrogram in batched_speech["input_features"]
- ]
-
- if return_tensors is not None:
- batched_speech = batched_speech.convert_to_tensors(return_tensors)
-
- return batched_speech
+"""Backwards-compatibility shim: re-exports the legacy ``UnivNetFeatureExtractor`` name as a
+deprecated alias of [`UnivNetAudioProcessor`]. Importing or instantiating the alias emits a
+``FutureWarning``; the alias is removed in transformers v5.15 (see ADR 0002).
+"""
- def to_dict(self) -> dict[str, Any]:
- output = super().to_dict()
+from ...audio_processing_base import make_legacy_audio_processor_alias
+from .audio_processing_univnet import UnivNetAudioProcessor
- # Don't serialize these as they are derived from the other properties.
- names = ["window", "mel_filters", "n_fft", "n_freqs", "num_max_samples"]
- for name in names:
- if name in output:
- del output[name]
- return output
+UnivNetFeatureExtractor = make_legacy_audio_processor_alias(UnivNetAudioProcessor, "UnivNetFeatureExtractor")
__all__ = ["UnivNetFeatureExtractor"]
diff --git a/src/transformers/models/vibevoice_acoustic_tokenizer/audio_processing_vibevoice_acoustic_tokenizer.py b/src/transformers/models/vibevoice_acoustic_tokenizer/audio_processing_vibevoice_acoustic_tokenizer.py
new file mode 100644
index 000000000000..ac8784b44fda
--- /dev/null
+++ b/src/transformers/models/vibevoice_acoustic_tokenizer/audio_processing_vibevoice_acoustic_tokenizer.py
@@ -0,0 +1,38 @@
+# Copyright 2025 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+
+from ...audio_processing_backends import TorchAudioBackend
+
+
+class VibevoiceAcousticTokenizerAudioProcessor(TorchAudioBackend):
+ sampling_rate = 24000
+ force_mono = True
+ add_channel_dim = True
+
+ target_dB_FS = -25
+ eps = 1e-6
+
+ def _process_audio(self, audio_el):
+ audio_el = super()._process_audio(audio_el)
+ rms = torch.sqrt(torch.mean(audio_el**2))
+ audio_el = audio_el * (10 ** (self.target_dB_FS / 20) / (rms + self.eps))
+ max_val = torch.max(torch.abs(audio_el))
+ if max_val > 1.0:
+ audio_el = audio_el / (max_val + self.eps)
+ return audio_el
+
+
+__all__ = ["VibevoiceAcousticTokenizerAudioProcessor"]
diff --git a/src/transformers/models/vibevoice_acoustic_tokenizer/feature_extraction_vibevoice_acoustic_tokenizer.py b/src/transformers/models/vibevoice_acoustic_tokenizer/feature_extraction_vibevoice_acoustic_tokenizer.py
index 859dc58e5873..9af23bf2e26b 100644
--- a/src/transformers/models/vibevoice_acoustic_tokenizer/feature_extraction_vibevoice_acoustic_tokenizer.py
+++ b/src/transformers/models/vibevoice_acoustic_tokenizer/feature_extraction_vibevoice_acoustic_tokenizer.py
@@ -1,151 +1,20 @@
-# Copyright 2026 The HuggingFace Inc. team. All rights reserved.
+# Copyright 2026 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from ...audio_utils import AudioInput, make_list_of_audio
-from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
-from ...feature_extraction_utils import BatchFeature
-from ...utils import PaddingStrategy, logging
-from ...utils.import_utils import is_torch_available, requires
-
-
-if is_torch_available():
- import torch
-
-logger = logging.get_logger(__name__)
-
-
-@requires(backends=("torch",))
-class VibeVoiceAcousticTokenizerFeatureExtractor(SequenceFeatureExtractor):
- r"""
- Constructs a VibeVoiceAcousticTokenizer feature extractor.
-
- Args:
- feature_size (`int`, *optional*, defaults to 1):
- The number of channels.
- sampling_rate (`int`, *optional*, defaults to 24000):
- The sampling rate at which the audio waveform should be digitalized, expressed in hertz (Hz).
- padding_value (`float`, *optional*, defaults to 0.0):
- The value that is used for padding.
- normalize_audio (`bool`, *optional*, defaults to `True`):
- Whether to normalize audio to a target dB FS.
- target_dB_FS (`float`, *optional*, defaults to -25):
- Target dB FS for normalization.
- eps (`float`, *optional*, defaults to 1e-06):
- A small value to avoid division by zero when normalizing.
-
- """
-
- model_input_names = ["input_values", "padding_mask"]
-
- def __init__(
- self,
- feature_size=1,
- sampling_rate=24000,
- padding_value=0.0,
- normalize_audio=True,
- target_dB_FS=-25,
- eps=1e-6,
- **kwargs,
- ):
- super().__init__(feature_size=feature_size, sampling_rate=sampling_rate, padding_value=padding_value, **kwargs)
-
- self.normalize_audio = normalize_audio
- self.target_dB_FS = target_dB_FS
- self.eps = eps
-
- def __call__(
- self,
- audio: AudioInput,
- sampling_rate: int | None = None,
- padding: bool | str | PaddingStrategy | None = True,
- pad_to_multiple_of: int | None = None,
- max_length: int | None = None,
- return_attention_mask: bool | None = True,
- return_tensors: str | None = "pt",
- **kwargs,
- ) -> BatchFeature:
- """
- Args:
- audio (`np.ndarray`, `torch.Tensor`, `list[np.ndarray]`, `list[torch.Tensor]`:
- The sequence or batch of sequences to be processed. Each sequence can be a numpy array, a torch tensor,
- a list of numpy arrays or a list of torch tensors.
- sampling_rate (`int`, *optional*):
- The sampling rate at which the `audio` input was sampled. It is strongly recommended to pass
- `sampling_rate` at the forward call to prevent silent errors.
- padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `True`):
- Select a strategy to pad the returned sequences (according to the model's padding side and padding
- index) among:
-
- - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
- sequence if provided).
- - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
- acceptable input length for the model if that argument is not provided.
- - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
- lengths).
- pad_to_multiple_of (`int`, *optional*):
- If set will pad the sequence to a multiple of the provided value.
-
- """
- if sampling_rate is not None:
- if sampling_rate != self.sampling_rate:
- raise ValueError(
- f"The model corresponding to this feature extractor: {self} was trained using a sampling rate of"
- f" {self.sampling_rate}. Please make sure that the provided audio input was sampled with"
- f" {self.sampling_rate} and not {sampling_rate}."
- )
- else:
- logger.warning(
- f"It is strongly recommended to pass the `sampling_rate` argument to `{self.__class__.__name__}()`. "
- "Failing to do so can result in silent errors that might be hard to debug."
- )
-
- if return_tensors != "pt":
- raise ValueError(f"{self.__class__.__name__} only supports `return_tensors='pt'`.")
-
- # Ensure batch of mono tensors
- audio = make_list_of_audio(audio)
- for idx, example in enumerate(audio):
- example = torch.tensor(example, dtype=torch.float32)
- if example.ndim != 1:
- raise ValueError(f"Audio should be mono, got shape: {example.shape}")
- audio[idx] = example
-
- if self.normalize_audio:
- for idx, example in enumerate(audio):
- rms = torch.sqrt(torch.mean(example**2))
- example *= 10 ** (self.target_dB_FS / 20) / (rms + self.eps)
- max_val = torch.max(torch.abs(example))
- if max_val > 1.0:
- example = example / (max_val + self.eps)
- audio[idx] = example
+"""Backwards-compatibility shim: re-exports the legacy ``VibeVoiceAcousticTokenizerFeatureExtractor`` name as a
+deprecated alias of [`VibevoiceAcousticTokenizerAudioProcessor`]. Importing or instantiating the alias emits a
+``FutureWarning``; the alias is removed in transformers v5.15 (see ADR 0002).
+"""
- output_values = BatchFeature({"input_values": audio})
- if padding or pad_to_multiple_of:
- output_values = self.pad(
- output_values,
- padding=padding,
- pad_to_multiple_of=pad_to_multiple_of,
- return_attention_mask=return_attention_mask,
- max_length=max_length,
- )
- if return_attention_mask:
- output_values["padding_mask"] = output_values.pop("attention_mask")
+from ...audio_processing_base import make_legacy_audio_processor_alias
+from .audio_processing_vibevoice_acoustic_tokenizer import VibevoiceAcousticTokenizerAudioProcessor
- # add channel dimension
- output_values["input_values"] = output_values["input_values"][:, None, :]
- return output_values
+VibeVoiceAcousticTokenizerFeatureExtractor = make_legacy_audio_processor_alias(VibevoiceAcousticTokenizerAudioProcessor, "VibeVoiceAcousticTokenizerFeatureExtractor")
__all__ = ["VibeVoiceAcousticTokenizerFeatureExtractor"]
diff --git a/src/transformers/models/voxtral_realtime/__init__.py b/src/transformers/models/voxtral_realtime/__init__.py
index 688fd7597bb7..9c38db2b8e99 100644
--- a/src/transformers/models/voxtral_realtime/__init__.py
+++ b/src/transformers/models/voxtral_realtime/__init__.py
@@ -19,6 +19,7 @@
if TYPE_CHECKING:
from .configuration_voxtral_realtime import *
+ from .feature_extraction_voxtral_realtime import *
from .modeling_voxtral_realtime import *
from .processing_voxtral_realtime import *
else:
diff --git a/src/transformers/models/voxtral_realtime/audio_processing_voxtral_realtime.py b/src/transformers/models/voxtral_realtime/audio_processing_voxtral_realtime.py
new file mode 100644
index 000000000000..1a7e54d88bfc
--- /dev/null
+++ b/src/transformers/models/voxtral_realtime/audio_processing_voxtral_realtime.py
@@ -0,0 +1,64 @@
+# Copyright 2025 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+
+from ...audio_processing_backends import TorchAudioBackend
+from ...audio_utils import MelScaleConfig, SpectrogramConfig, StftConfig
+
+
+class VoxtralRealtimeAudioProcessor(TorchAudioBackend):
+ sampling_rate = 16000
+ force_mono = True
+ spectrogram_config = SpectrogramConfig(
+ stft_config=StftConfig(
+ n_fft=400,
+ hop_length=160,
+ power=2.0,
+ ),
+ mel_scale_config=MelScaleConfig(
+ n_mels=128,
+ mel_scale="slaney",
+ norm="slaney",
+ computation_dtype="float64",
+ ),
+ log_mode="log10",
+ skip_last_frame=True,
+ )
+ global_log_mel_max = 1.5
+
+ def _apply_mel_scale(self, features, *, spectrogram_config, **kwargs):
+ mel_filters = self.mel_filters.to(device=features.device)
+ return torch.clamp(torch.matmul(mel_filters.T, features), min=spectrogram_config.mel_floor)
+
+ def _normalize_magnitude(self, features, *, spectrogram_config, **kwargs):
+ # Voxtral uses a *fixed* `global_log_mel_max` as the upper bound (rather than the
+ # per-utterance amax that the base `clip_max_offset` field expects), so we don't set
+ # the post-log fields on `spectrogram_config` and handle the whole rescale here.
+ features = super()._normalize_magnitude(features, spectrogram_config=spectrogram_config, **kwargs)
+ if self.global_log_mel_max is not None:
+ spec_max = torch.tensor(self.global_log_mel_max, device=features.device, dtype=features.dtype)
+ else:
+ spec_max = features.amax(dim=(-2, -1), keepdim=True)
+ features = torch.maximum(features, spec_max - 8.0)
+ features = (features + 4.0) / 4.0
+ return features
+
+ def _get_features_lengths(self, audio_lengths, spectrogram_config, include_center_frame=False):
+ stft_cfg = spectrogram_config.stft_config
+ win_length = stft_cfg.win_length or stft_cfg.n_fft
+ return (audio_lengths - win_length) // stft_cfg.hop_length + 1
+
+
+__all__ = ["VoxtralRealtimeAudioProcessor"]
diff --git a/src/transformers/models/voxtral_realtime/feature_extraction_voxtral_realtime.py b/src/transformers/models/voxtral_realtime/feature_extraction_voxtral_realtime.py
index 58355f3c0d7c..1ba03eb861c6 100644
--- a/src/transformers/models/voxtral_realtime/feature_extraction_voxtral_realtime.py
+++ b/src/transformers/models/voxtral_realtime/feature_extraction_voxtral_realtime.py
@@ -1,246 +1,20 @@
-# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
+# Copyright 2026 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import numpy as np
-import torch
-
-from ...audio_utils import mel_filter_bank
-from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
-from ...feature_extraction_utils import BatchFeature
-from ...utils import TensorType, logging
-from ...utils.import_utils import requires
-
-
-logger = logging.get_logger(__name__)
-
-
-@requires(backends=("torch",))
-class VoxtralRealtimeFeatureExtractor(SequenceFeatureExtractor):
- r"""
- Constructs a VOXTRAL_REALTIME feature extractor.
-
- This feature extractor inherits from [`~feature_extraction_sequence_utils.SequenceFeatureExtractor`] which contains
- most of the main methods. Users should refer to this superclass for more information regarding those methods.
-
- This class extracts mel-filter bank features from raw speech using a custom numpy implementation of the `Short Time
- Fourier Transform` which should match pytorch's `torch.stft` equivalent.
-
- Args:
- feature_size (`int`, *optional*, defaults to 128):
- The feature dimension of the extracted features.
- sampling_rate (`int`, *optional*, defaults to 16000):
- The sampling rate at which the audio files should be digitalized expressed in hertz (Hz).
- hop_length (`int`, *optional*, defaults to 160):
- Length of the overlapping windows for the STFT used to obtain the Mel Frequency coefficients.
- n_fft (`int`, *optional*, defaults to 512):
- Size of the Fourier transform.
- win_length (`int`, *optional*, defaults to 400):
- The window length for the STFT computation.
- padding_value (`float`, *optional*, defaults to 0.0):
- Padding value used to pad the audio. Should correspond to silences.
- """
-
- model_input_names = ["input_features", "attention_mask"]
-
- def __init__(
- self,
- feature_size=128,
- sampling_rate=16000,
- hop_length=160,
- n_fft=400,
- win_length=400,
- padding_value=0.0,
- global_log_mel_max=1.5,
- **kwargs,
- ):
- super().__init__(feature_size=feature_size, sampling_rate=sampling_rate, padding_value=padding_value, **kwargs)
-
- self.hop_length = hop_length
- self.n_fft = n_fft
- self.win_length = win_length
- self.mel_filters = mel_filter_bank(
- num_frequency_bins=1 + n_fft // 2,
- num_mel_filters=feature_size,
- min_frequency=0.0,
- max_frequency=8000.0,
- sampling_rate=sampling_rate,
- norm="slaney",
- mel_scale="slaney",
- )
- self.global_log_mel_max = global_log_mel_max
-
- def _torch_extract_fbank_features(self, waveform, device: str = "cpu", center: bool = True):
- window = torch.hann_window(self.n_fft, device=device)
- stft = torch.stft(waveform, self.n_fft, self.hop_length, window=window, return_complex=True, center=center)
- magnitudes = stft[..., :-1].abs() ** 2
-
- mel_filters = torch.from_numpy(self.mel_filters).to(device, torch.float32)
- mel_spec = mel_filters.T @ magnitudes
-
- log_spec = torch.clamp(mel_spec, min=1e-10).log10()
- if self.global_log_mel_max is not None:
- log_spec_max = torch.tensor(
- self.global_log_mel_max,
- device=log_spec.device,
- dtype=log_spec.dtype,
- )
- else:
- log_spec_max = log_spec.max()
-
- log_spec = torch.maximum(log_spec, log_spec_max - 8.0)
- log_spec = (log_spec + 4.0) / 4.0
- if device != "cpu":
- log_spec = log_spec.detach().cpu()
- return log_spec
-
- def __call__(
- self,
- raw_speech: np.ndarray | list[float] | list[np.ndarray] | list[list[float]],
- truncation: bool = False,
- pad_to_multiple_of: int | None = None,
- return_tensors: str | TensorType | None = None,
- return_attention_mask: bool | None = None,
- padding: str | None = "longest",
- max_length: int | None = None,
- sampling_rate: int | None = None,
- do_normalize: bool | None = None,
- device: str | None = "cpu",
- return_token_timestamps: bool | None = None,
- center: bool = True,
- **kwargs,
- ) -> BatchFeature:
- """
- Main method to featurize and prepare for the model one or several sequence(s). Implementation uses PyTorch for
- the STFT computation if available, otherwise a slower NumPy based one.
-
- Args:
- raw_speech (`np.ndarray`, `list[float]`, `list[np.ndarray]`, `list[list[float]]`):
- The sequence or batch of sequences to be padded. Each sequence can be a numpy array, a list of float
- values, a list of numpy arrays or a list of list of float values. Must be mono channel audio, not
- stereo, i.e. single float per timestep.
- truncation (`bool`, *optional*, default to `True`):
- Activates truncation to cut input sequences longer than *max_length* to *max_length*.
- pad_to_multiple_of (`int`, *optional*, defaults to None):
- If set will pad the sequence to a multiple of the provided value.
-
- This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability
- `>= 7.5` (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128.
- return_attention_mask (`bool`, *optional*):
- Whether to return the attention mask. If left to the default, will return the attention mask according
- to the specific feature_extractor's default.
-
- [What are attention masks?](../glossary#attention-mask)
-
-
-
- For Parakeet models, `attention_mask` should always be passed for batched inference, to avoid subtle
- bugs.
-
-
-
- return_tensors (`str` or [`~utils.TensorType`], *optional*):
- If set, will return tensors instead of list of python integers. Acceptable values are:
-
- - `'tf'`: Return TensorFlow `tf.constant` objects.
- - `'pt'`: Return PyTorch `torch.Tensor` objects.
- - `'np'`: Return Numpy `np.ndarray` objects.
- sampling_rate (`int`, *optional*):
- The sampling rate at which the `raw_speech` input was sampled. It is strongly recommended to pass
- `sampling_rate` at the forward call to prevent silent errors and allow automatic speech recognition
- pipeline.
- padding_value (`float`, *optional*, defaults to 0.0):
- The value that is used to fill the padding values / vectors.
- do_normalize (`bool`, *optional*, defaults to `False`):
- Whether or not to zero-mean unit-variance normalize the input. Normalizing can help to significantly
- improve the performance of the model.
- device (`str`, *optional*, defaults to `'cpu'`):
- Specifies the device for computation of the log-mel spectrogram of audio signals in the
- `_torch_extract_fbank_features` method. (e.g., "cpu", "cuda")
- return_token_timestamps (`bool`, *optional*, defaults to `None`):
- Deprecated. Use `return_attention_mask` instead from which the number of frames can be inferred.
-
- Whether or not to return the number of frames of the input raw_speech.
- These num_frames can be used by the model to compute word level timestamps.
- center (`bool`, *optional*, defaults to `True`):
- Whether to use centering for the STFT computation.
- """
- if sampling_rate is not None:
- if sampling_rate != self.sampling_rate:
- raise ValueError(
- f"The model corresponding to this feature extractor: {self.__class__.__name__} was trained using a"
- f" sampling rate of {self.sampling_rate}. Please make sure that the provided `raw_speech` input"
- f" was sampled with {self.sampling_rate} and not {sampling_rate}."
- )
- else:
- logger.warning(
- f"It is strongly recommended to pass the `sampling_rate` argument to `{self.__class__.__name__}()`. "
- "Failing to do so can result in silent errors that might be hard to debug."
- )
-
- # Convert to torch tensor
- if isinstance(raw_speech, np.ndarray):
- raw_speech = torch.tensor(raw_speech)
- elif isinstance(raw_speech, (list, tuple)):
- if isinstance(raw_speech[0], (list, np.ndarray)):
- raw_speech = [torch.tensor(speech) for speech in raw_speech]
- else: # list[float]
- raw_speech = torch.tensor(raw_speech)
-
- is_batched_torch = isinstance(raw_speech, torch.Tensor) and len(raw_speech.shape) > 1
- if is_batched_torch and len(raw_speech.shape) > 2:
- logger.warning(
- f"Only mono-channel audio is supported for input to {self.__class__.__name__}. "
- "We will take the mean of the channels to convert to mono."
- )
- raw_speech = raw_speech.mean(-1)
-
- is_batched_sequence = isinstance(raw_speech, (list, tuple))
- if is_batched_sequence:
- for speech in raw_speech:
- if len(speech.shape) > 1:
- logger.warning(
- f"Only mono-channel audio is supported for input to {self.__class__.__name__}. "
- "We will take the mean of the channels to convert to mono."
- )
- speech = speech.mean(-1)
-
- if is_batched_torch or is_batched_sequence:
- raw_speech = [speech[:, None].to(torch.float32) for speech in raw_speech]
- else:
- raw_speech = [raw_speech[:, None].to(torch.float32)]
+"""Backwards-compatibility shim: re-exports the legacy ``VoxtralRealtimeFeatureExtractor`` name as a
+deprecated alias of [`VoxtralRealtimeAudioProcessor`]. Importing or instantiating the alias emits a
+``FutureWarning``; the alias is removed in transformers v5.15 (see ADR 0002).
+"""
- batched_speech = BatchFeature({"input_features": raw_speech})
- padded_inputs = self.pad(
- batched_speech,
- padding=padding,
- max_length=max_length,
- truncation=truncation,
- pad_to_multiple_of=pad_to_multiple_of,
- return_attention_mask=return_attention_mask,
- return_tensors="pt",
- )
- input_features = padded_inputs.input_features.squeeze(-1)
- input_features = self._torch_extract_fbank_features(input_features, device, center)
- data = {
- "input_features": input_features.to(torch.float32),
- }
+from ...audio_processing_base import make_legacy_audio_processor_alias
+from .audio_processing_voxtral_realtime import VoxtralRealtimeAudioProcessor
- if return_attention_mask:
- attention_mask = padded_inputs.attention_mask[:, self.win_length - 1 :: self.hop_length]
- data["attention_mask"] = attention_mask.to(torch.bool)
- return BatchFeature(data=data, tensor_type=return_tensors)
+VoxtralRealtimeFeatureExtractor = make_legacy_audio_processor_alias(VoxtralRealtimeAudioProcessor, "VoxtralRealtimeFeatureExtractor")
__all__ = ["VoxtralRealtimeFeatureExtractor"]
diff --git a/src/transformers/models/wav2vec2/__init__.py b/src/transformers/models/wav2vec2/__init__.py
index aa3a5c4c82f8..1fde830bc834 100644
--- a/src/transformers/models/wav2vec2/__init__.py
+++ b/src/transformers/models/wav2vec2/__init__.py
@@ -18,6 +18,8 @@
if TYPE_CHECKING:
+ from .audio_processing_numpy_wav2vec2 import *
+ from .audio_processing_wav2vec2 import *
from .configuration_wav2vec2 import *
from .feature_extraction_wav2vec2 import *
from .modeling_wav2vec2 import *
diff --git a/src/transformers/models/wav2vec2/audio_processing_numpy_wav2vec2.py b/src/transformers/models/wav2vec2/audio_processing_numpy_wav2vec2.py
new file mode 100644
index 000000000000..476d3b3663ac
--- /dev/null
+++ b/src/transformers/models/wav2vec2/audio_processing_numpy_wav2vec2.py
@@ -0,0 +1,35 @@
+# Copyright 2026 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from ...audio_processing_backends import NumpyAudioBackend
+
+
+class Wav2Vec2AudioProcessorNumpy(NumpyAudioBackend):
+ """NumPy sibling of [`Wav2Vec2AudioProcessor`]. Bit-exact to the torch sibling within
+ the float32 noise floor (ADR 0001)."""
+
+ sampling_rate = 16000
+ force_mono = True
+ do_normalize = True
+
+ def _process_audio(self, audio_el):
+ audio_el = super()._process_audio(audio_el)
+ if self.do_normalize:
+ audio_el = (audio_el - audio_el.mean()) / np.sqrt(audio_el.var() + 1e-7)
+ return audio_el
+
+
+__all__ = ["Wav2Vec2AudioProcessorNumpy"]
diff --git a/src/transformers/models/wav2vec2/audio_processing_wav2vec2.py b/src/transformers/models/wav2vec2/audio_processing_wav2vec2.py
new file mode 100644
index 000000000000..ee3f64d1e499
--- /dev/null
+++ b/src/transformers/models/wav2vec2/audio_processing_wav2vec2.py
@@ -0,0 +1,34 @@
+# Copyright 2025 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+
+from ...audio_processing_backends import TorchAudioBackend
+
+
+class Wav2Vec2AudioProcessor(TorchAudioBackend):
+ sampling_rate = 16000
+ force_mono = True
+ do_normalize = True
+
+ def _process_audio(self, audio_el):
+ audio_el = super()._process_audio(audio_el)
+
+ if self.do_normalize:
+ audio_el = (audio_el - audio_el.mean()) / torch.sqrt(audio_el.var(correction=0) + 1e-7)
+
+ return audio_el
+
+
+__all__ = ["Wav2Vec2AudioProcessor"]
diff --git a/src/transformers/models/wav2vec2/feature_extraction_wav2vec2.py b/src/transformers/models/wav2vec2/feature_extraction_wav2vec2.py
index dea2f3af5b48..e0504ed580f4 100644
--- a/src/transformers/models/wav2vec2/feature_extraction_wav2vec2.py
+++ b/src/transformers/models/wav2vec2/feature_extraction_wav2vec2.py
@@ -1,239 +1,20 @@
-# Copyright 2021 The HuggingFace Inc. team.
+# Copyright 2026 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""
-Feature extractor class for Wav2Vec2
+"""Backwards-compatibility shim: re-exports the legacy ``Wav2Vec2FeatureExtractor`` name as a
+deprecated alias of [`Wav2Vec2AudioProcessor`]. Importing or instantiating the alias emits a
+``FutureWarning``; the alias is removed in transformers v5.15 (see ADR 0002).
"""
-import numpy as np
-
-from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
-from ...feature_extraction_utils import BatchFeature
-from ...utils import PaddingStrategy, TensorType, logging
-
-
-logger = logging.get_logger(__name__)
-
-
-class Wav2Vec2FeatureExtractor(SequenceFeatureExtractor):
- r"""
- Constructs a Wav2Vec2 feature extractor.
-
- This feature extractor inherits from [`~feature_extraction_sequence_utils.SequenceFeatureExtractor`] which contains
- most of the main methods. Users should refer to this superclass for more information regarding those methods.
-
- Args:
- feature_size (`int`, *optional*, defaults to 1):
- The feature dimension of the extracted features.
- sampling_rate (`int`, *optional*, defaults to 16000):
- The sampling rate at which the audio files should be digitalized expressed in hertz (Hz).
- padding_value (`float`, *optional*, defaults to 0.0):
- The value that is used to fill the padding values.
- do_normalize (`bool`, *optional*, defaults to `True`):
- Whether or not to zero-mean unit-variance normalize the input. Normalizing can help to significantly
- improve the performance for some models, *e.g.*,
- [wav2vec2-lv60](https://huggingface.co/models?search=lv60).
- return_attention_mask (`bool`, *optional*, defaults to `False`):
- Whether or not [`~Wav2Vec2FeatureExtractor.__call__`] should return `attention_mask`.
-
-
-
- Wav2Vec2 models that have set `config.feat_extract_norm == "group"`, such as
- [wav2vec2-base](https://huggingface.co/facebook/wav2vec2-base-960h), have **not** been trained using
- `attention_mask`. For such models, `input_values` should simply be padded with 0 and no `attention_mask`
- should be passed.
-
- For Wav2Vec2 models that have set `config.feat_extract_norm == "layer"`, such as
- [wav2vec2-lv60](https://huggingface.co/facebook/wav2vec2-large-960h-lv60-self), `attention_mask` should be
- passed for batched inference.
-
- """
-
- model_input_names = ["input_values", "attention_mask"]
-
- def __init__(
- self,
- feature_size=1,
- sampling_rate=16000,
- padding_value=0.0,
- return_attention_mask=False,
- do_normalize=True,
- **kwargs,
- ):
- super().__init__(feature_size=feature_size, sampling_rate=sampling_rate, padding_value=padding_value, **kwargs)
- self.return_attention_mask = return_attention_mask
- self.do_normalize = do_normalize
-
- @staticmethod
- def zero_mean_unit_var_norm(
- input_values: list[np.ndarray], attention_mask: list[np.ndarray], padding_value: float = 0.0
- ) -> list[np.ndarray]:
- """
- Every array in the list is normalized to have zero mean and unit variance
- """
- if attention_mask is not None:
- attention_mask = np.array(attention_mask, np.int32)
- normed_input_values = []
-
- for vector, length in zip(input_values, attention_mask.sum(-1)):
- normed_slice = (vector - vector[:length].mean()) / np.sqrt(vector[:length].var() + 1e-7)
- if length < normed_slice.shape[0]:
- normed_slice[length:] = padding_value
-
- normed_input_values.append(normed_slice)
- else:
- normed_input_values = [(x - x.mean()) / np.sqrt(x.var() + 1e-7) for x in input_values]
-
- return normed_input_values
-
- def __call__(
- self,
- raw_speech: np.ndarray | list[float] | list[np.ndarray] | list[list[float]],
- padding: bool | str | PaddingStrategy = False,
- max_length: int | None = None,
- truncation: bool = False,
- pad_to_multiple_of: int | None = None,
- return_attention_mask: bool | None = None,
- return_tensors: str | TensorType | None = None,
- sampling_rate: int | None = None,
- **kwargs,
- ) -> BatchFeature:
- """
- Main method to featurize and prepare for the model one or several sequence(s).
-
- Args:
- raw_speech (`np.ndarray`, `list[float]`, `list[np.ndarray]`, `list[list[float]]`):
- The sequence or batch of sequences to be padded. Each sequence can be a numpy array, a list of float
- values, a list of numpy arrays or a list of list of float values. Must be mono channel audio, not
- stereo, i.e. single float per timestep.
- padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `False`):
- Select a strategy to pad the returned sequences (according to the model's padding side and padding
- index) among:
-
- - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
- sequence if provided).
- - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
- acceptable input length for the model if that argument is not provided.
- - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
- lengths).
- max_length (`int`, *optional*):
- Maximum length of the returned list and optionally padding length (see above).
- truncation (`bool`):
- Activates truncation to cut input sequences longer than *max_length* to *max_length*.
- pad_to_multiple_of (`int`, *optional*):
- If set will pad the sequence to a multiple of the provided value.
-
- This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability
- `>= 7.5` (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128.
- return_attention_mask (`bool`, *optional*):
- Whether to return the attention mask. If left to the default, will return the attention mask according
- to the specific feature_extractor's default.
-
- [What are attention masks?](../glossary#attention-mask)
-
-
-
- Wav2Vec2 models that have set `config.feat_extract_norm == "group"`, such as
- [wav2vec2-base](https://huggingface.co/facebook/wav2vec2-base-960h), have **not** been trained using
- `attention_mask`. For such models, `input_values` should simply be padded with 0 and no
- `attention_mask` should be passed.
-
- For Wav2Vec2 models that have set `config.feat_extract_norm == "layer"`, such as
- [wav2vec2-lv60](https://huggingface.co/facebook/wav2vec2-large-960h-lv60-self), `attention_mask` should
- be passed for batched inference.
-
-
-
- return_tensors (`str` or [`~utils.TensorType`], *optional*):
- If set, will return tensors instead of list of python integers. Acceptable values are:
-
- - `'pt'`: Return PyTorch `torch.Tensor` objects.
- - `'np'`: Return Numpy `np.ndarray` objects.
- sampling_rate (`int`, *optional*):
- The sampling rate at which the `raw_speech` input was sampled. It is strongly recommended to pass
- `sampling_rate` at the forward call to prevent silent errors.
- padding_value (`float`, *optional*, defaults to 0.0):
- """
-
- if sampling_rate is not None:
- if sampling_rate != self.sampling_rate:
- raise ValueError(
- f"The model corresponding to this feature extractor: {self} was trained using a sampling rate of"
- f" {self.sampling_rate}. Please make sure that the provided `raw_speech` input was sampled with"
- f" {self.sampling_rate} and not {sampling_rate}."
- )
- else:
- logger.warning(
- f"It is strongly recommended to pass the `sampling_rate` argument to `{self.__class__.__name__}()`. "
- "Failing to do so can result in silent errors that might be hard to debug."
- )
-
- is_batched_numpy = isinstance(raw_speech, np.ndarray) and len(raw_speech.shape) > 1
- if is_batched_numpy and len(raw_speech.shape) > 2:
- raise ValueError(f"Only mono-channel audio is supported for input to {self}")
- is_batched = is_batched_numpy or (
- isinstance(raw_speech, (list, tuple)) and (isinstance(raw_speech[0], (np.ndarray, tuple, list)))
- )
-
- # always return batch
- if not is_batched:
- raw_speech = [raw_speech]
-
- # convert into correct format for padding
- encoded_inputs = BatchFeature({"input_values": raw_speech})
-
- padded_inputs = self.pad(
- encoded_inputs,
- padding=padding,
- max_length=max_length,
- truncation=truncation,
- pad_to_multiple_of=pad_to_multiple_of,
- return_attention_mask=return_attention_mask,
- )
-
- # convert input values to correct format
- input_values = padded_inputs["input_values"]
- if not isinstance(input_values[0], np.ndarray):
- padded_inputs["input_values"] = [np.asarray(array, dtype=np.float32) for array in input_values]
- elif (
- not isinstance(input_values, np.ndarray)
- and isinstance(input_values[0], np.ndarray)
- and input_values[0].dtype is np.dtype(np.float64)
- ):
- padded_inputs["input_values"] = [array.astype(np.float32) for array in input_values]
- elif isinstance(input_values, np.ndarray) and input_values.dtype is np.dtype(np.float64):
- padded_inputs["input_values"] = input_values.astype(np.float32)
-
- # convert attention_mask to correct format
- attention_mask = padded_inputs.get("attention_mask")
- if attention_mask is not None:
- padded_inputs["attention_mask"] = [np.asarray(array, dtype=np.int32) for array in attention_mask]
-
- # zero-mean and unit-variance normalization
- if self.do_normalize:
- attention_mask = (
- attention_mask
- if self._get_padding_strategies(padding, max_length=max_length) is not PaddingStrategy.DO_NOT_PAD
- else None
- )
- padded_inputs["input_values"] = self.zero_mean_unit_var_norm(
- padded_inputs["input_values"], attention_mask=attention_mask, padding_value=self.padding_value
- )
+from ...audio_processing_base import make_legacy_audio_processor_alias
+from .audio_processing_wav2vec2 import Wav2Vec2AudioProcessor
- if return_tensors is not None:
- padded_inputs = padded_inputs.convert_to_tensors(return_tensors)
- return padded_inputs
+Wav2Vec2FeatureExtractor = make_legacy_audio_processor_alias(Wav2Vec2AudioProcessor, "Wav2Vec2FeatureExtractor")
__all__ = ["Wav2Vec2FeatureExtractor"]
diff --git a/src/transformers/models/whisper/__init__.py b/src/transformers/models/whisper/__init__.py
index 50aec31b3e9f..23c457a3edf8 100644
--- a/src/transformers/models/whisper/__init__.py
+++ b/src/transformers/models/whisper/__init__.py
@@ -18,6 +18,8 @@
if TYPE_CHECKING:
+ from .audio_processing_numpy_whisper import *
+ from .audio_processing_whisper import *
from .configuration_whisper import *
from .feature_extraction_whisper import *
from .modeling_whisper import *
diff --git a/src/transformers/models/whisper/audio_processing_numpy_whisper.py b/src/transformers/models/whisper/audio_processing_numpy_whisper.py
new file mode 100644
index 000000000000..6afef2a068c9
--- /dev/null
+++ b/src/transformers/models/whisper/audio_processing_numpy_whisper.py
@@ -0,0 +1,67 @@
+# Copyright 2026 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from ...audio_processing_backends import NumpyAudioBackend
+from ...audio_utils import MelScaleConfig, SpectrogramConfig, StftConfig
+
+
+def _whisper_chunk_length_to_max_length(value, config_dict):
+ # Legacy Whisper hub configs store `chunk_length=30` (seconds); the new API uses `max_length`
+ # in samples. Translate using the sampling rate carried by the pass-through `sampling_rate` key.
+ sampling_rate = config_dict.get("sampling_rate") or 16000
+ config_dict.setdefault("max_length", value * sampling_rate)
+
+
+class WhisperAudioProcessorNumpy(NumpyAudioBackend):
+ """NumPy sibling of [`WhisperAudioProcessor`]. Required to produce bit-exact outputs
+ against the torch sibling (ADR 0001)."""
+
+ sampling_rate = 16000
+ force_mono = True
+ return_padding_mask = False
+ truncation = True
+ max_length = 480000 # 30 seconds at 16000 Hz
+ spectrogram_config = SpectrogramConfig(
+ stft_config=StftConfig(
+ n_fft=400,
+ hop_length=160,
+ power=2.0,
+ ),
+ mel_scale_config=MelScaleConfig(
+ n_mels=80,
+ mel_scale="slaney",
+ norm="slaney",
+ computation_dtype="float64",
+ ),
+ log_mode="log10",
+ skip_last_frame=True,
+ clip_max_offset=8.0,
+ post_log_shift=4.0,
+ post_log_scale=0.25,
+ )
+
+ legacy_field_mapping = {
+ "feature_size": "spectrogram_config.mel_scale_config.n_mels",
+ "chunk_length": _whisper_chunk_length_to_max_length,
+ "n_samples": "max_length",
+ }
+
+ def _apply_mel_scale(self, features, *, spectrogram_config, **kwargs):
+ # `filters_first` matmul order with mel_floor clamp, matching the torch sibling.
+ return np.maximum(spectrogram_config.mel_floor, np.matmul(self.mel_filters.T, features))
+
+
+__all__ = ["WhisperAudioProcessorNumpy"]
diff --git a/src/transformers/models/whisper/audio_processing_whisper.py b/src/transformers/models/whisper/audio_processing_whisper.py
new file mode 100644
index 000000000000..c455129872f1
--- /dev/null
+++ b/src/transformers/models/whisper/audio_processing_whisper.py
@@ -0,0 +1,36 @@
+# Copyright 2025 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+
+from ...audio_processing_backends import TorchAudioBackend
+from .audio_processing_numpy_whisper import WhisperAudioProcessorNumpy
+
+
+class WhisperAudioProcessor(TorchAudioBackend):
+ sampling_rate = 16000
+ force_mono = True
+ return_padding_mask = False
+ truncation = True
+ max_length = 480000 # 30 seconds at 16000 Hz
+
+ spectrogram_config = WhisperAudioProcessorNumpy.spectrogram_config
+ legacy_field_mapping = WhisperAudioProcessorNumpy.legacy_field_mapping
+
+ def _apply_mel_scale(self, features, *, spectrogram_config, **kwargs):
+ mel_filters = self.mel_filters.to(device=features.device)
+ return torch.clamp(torch.matmul(mel_filters.T, features), min=spectrogram_config.mel_floor)
+
+
+__all__ = ["WhisperAudioProcessor"]
diff --git a/src/transformers/models/whisper/feature_extraction_whisper.py b/src/transformers/models/whisper/feature_extraction_whisper.py
index 4151a3824dfd..39d7b5ad8b03 100644
--- a/src/transformers/models/whisper/feature_extraction_whisper.py
+++ b/src/transformers/models/whisper/feature_extraction_whisper.py
@@ -1,345 +1,20 @@
-# Copyright 2022 The HuggingFace Inc. team.
+# Copyright 2026 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""
-Feature extractor class for Whisper
+"""Backwards-compatibility shim: re-exports the legacy ``WhisperFeatureExtractor`` name as a
+deprecated alias of [`WhisperAudioProcessor`]. Importing or instantiating the alias emits a
+``FutureWarning``; the alias is removed in transformers v5.15 (see ADR 0002).
"""
-import numpy as np
-
-from ... import is_torch_available
-from ...audio_utils import mel_filter_bank, spectrogram, window_function
-from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
-from ...feature_extraction_utils import BatchFeature
-from ...utils import TensorType, logging
-
-
-if is_torch_available():
- import torch
-
-logger = logging.get_logger(__name__)
-
-
-class WhisperFeatureExtractor(SequenceFeatureExtractor):
- r"""
- Constructs a Whisper feature extractor.
-
- This feature extractor inherits from [`~feature_extraction_sequence_utils.SequenceFeatureExtractor`] which contains
- most of the main methods. Users should refer to this superclass for more information regarding those methods.
-
- This class extracts mel-filter bank features from raw speech using a custom numpy implementation of the `Short Time
- Fourier Transform` which should match pytorch's `torch.stft` equivalent.
-
- Args:
- feature_size (`int`, *optional*, defaults to 80):
- The feature dimension of the extracted features.
- sampling_rate (`int`, *optional*, defaults to 16000):
- The sampling rate at which the audio files should be digitalized expressed in hertz (Hz).
- hop_length (`int`, *optional*, defaults to 160):
- Length of the overlapping windows for the STFT used to obtain the Mel Frequency coefficients.
- chunk_length (`int`, *optional*, defaults to 30):
- The maximum number of chunks of `sampling_rate` samples used to trim and pad longer or shorter audio
- sequences.
- n_fft (`int`, *optional*, defaults to 400):
- Size of the Fourier transform.
- padding_value (`float`, *optional*, defaults to 0.0):
- Padding value used to pad the audio. Should correspond to silences.
- dither (`float`, *optional*, defaults to 0.0):
- Adds dithering. In other words, adds a small Gaussian noise to each frame.
- E.g. use 0.0001 to add dithering with a normal distribution centered
- around 0.0 with standard deviation 0.0001 (assuming [-1,+1] range of raw_speech).
- The value 0.0 means no dithering.
- Dithering has similar effect as `spectrogram(mel_floor=...)`. It reduces
- the high log_mel_fbank values for signals with hard-zero sections,
- when VAD cutoff is present in the signal.
- """
-
- model_input_names = ["input_features"]
-
- def __init__(
- self,
- feature_size=80,
- sampling_rate=16000,
- hop_length=160,
- chunk_length=30,
- n_fft=400,
- padding_value=0.0,
- dither=0.0,
- return_attention_mask=False, # pad inputs to max length with silence token (zero) and no attention mask
- **kwargs,
- ):
- super().__init__(
- feature_size=feature_size,
- sampling_rate=sampling_rate,
- padding_value=padding_value,
- return_attention_mask=return_attention_mask,
- **kwargs,
- )
- self.n_fft = n_fft
- self.hop_length = hop_length
- self.chunk_length = chunk_length
- self.n_samples = chunk_length * sampling_rate
- self.nb_max_frames = self.n_samples // hop_length
- self.sampling_rate = sampling_rate
- self.dither = dither
- self.mel_filters = mel_filter_bank(
- num_frequency_bins=1 + n_fft // 2,
- num_mel_filters=feature_size,
- min_frequency=0.0,
- max_frequency=8000.0,
- sampling_rate=sampling_rate,
- norm="slaney",
- mel_scale="slaney",
- )
-
- def _np_extract_fbank_features(self, waveform_batch: np.ndarray, device: str) -> np.ndarray:
- """
- Compute the log-mel spectrogram of the provided audio, gives similar results to Whisper's original torch
- implementation with 1e-5 tolerance.
- """
- if device != "cpu":
- raise ValueError(
- f"Got device `{device}` for feature extraction, but feature extraction on CUDA accelerator "
- "devices requires torch, which is not installed. Either set `device='cpu'`, or "
- "install torch according to the official instructions: https://pytorch.org/get-started/locally/"
- )
- log_spec_batch = []
- for waveform in waveform_batch:
- log_spec = spectrogram(
- waveform,
- window_function(self.n_fft, "hann"),
- frame_length=self.n_fft,
- hop_length=self.hop_length,
- power=2.0,
- dither=self.dither,
- mel_filters=self.mel_filters,
- log_mel="log10",
- )
- log_spec = log_spec[:, :-1]
- log_spec = np.maximum(log_spec, log_spec.max() - 8.0)
- log_spec = (log_spec + 4.0) / 4.0
- log_spec_batch.append(log_spec)
- log_spec_batch = np.array(log_spec_batch)
- return log_spec_batch
-
- def _torch_extract_fbank_features(self, waveform: np.ndarray, device: str = "cpu") -> np.ndarray:
- """
- Compute the log-mel spectrogram of the audio using PyTorch's GPU-accelerated STFT implementation with batching,
- yielding results similar to cpu computing with 1e-5 tolerance.
- """
- waveform = torch.from_numpy(waveform).to(device, torch.float32)
- window = torch.hann_window(self.n_fft, device=device)
-
- # Note: it would be better to dither the chunked waveform,
- # so overlapping signal does not get the same dithering.
- # But, chunking is happening inside pytorch, so it is here.
- if self.dither != 0.0:
- waveform += self.dither * torch.randn(waveform.shape, dtype=waveform.dtype, device=waveform.device)
-
- stft = torch.stft(waveform, self.n_fft, self.hop_length, window=window, return_complex=True)
- magnitudes = stft[..., :-1].abs() ** 2
-
- mel_filters = torch.from_numpy(self.mel_filters).to(device, torch.float32)
- mel_spec = mel_filters.T @ magnitudes
-
- log_spec = torch.clamp(mel_spec, min=1e-10).log10()
- if waveform.dim() == 2:
- max_val = log_spec.max(dim=2, keepdim=True)[0].max(dim=1, keepdim=True)[0]
- log_spec = torch.maximum(log_spec, max_val - 8.0)
- else:
- log_spec = torch.maximum(log_spec, log_spec.max() - 8.0)
- log_spec = (log_spec + 4.0) / 4.0
- if device != "cpu":
- log_spec = log_spec.detach().cpu()
- return log_spec.numpy()
-
- @staticmethod
- # Copied from transformers.models.wav2vec2.feature_extraction_wav2vec2.Wav2Vec2FeatureExtractor.zero_mean_unit_var_norm
- def zero_mean_unit_var_norm(
- input_values: list[np.ndarray], attention_mask: list[np.ndarray], padding_value: float = 0.0
- ) -> list[np.ndarray]:
- """
- Every array in the list is normalized to have zero mean and unit variance
- """
- if attention_mask is not None:
- attention_mask = np.array(attention_mask, np.int32)
- normed_input_values = []
-
- for vector, length in zip(input_values, attention_mask.sum(-1)):
- normed_slice = (vector - vector[:length].mean()) / np.sqrt(vector[:length].var() + 1e-7)
- if length < normed_slice.shape[0]:
- normed_slice[length:] = padding_value
-
- normed_input_values.append(normed_slice)
- else:
- normed_input_values = [(x - x.mean()) / np.sqrt(x.var() + 1e-7) for x in input_values]
-
- return normed_input_values
-
- def __call__(
- self,
- raw_speech: np.ndarray | list[float] | list[np.ndarray] | list[list[float]],
- truncation: bool = True,
- pad_to_multiple_of: int | None = None,
- return_tensors: str | TensorType | None = None,
- return_attention_mask: bool | None = None,
- padding: str | None = "max_length",
- max_length: int | None = None,
- sampling_rate: int | None = None,
- do_normalize: bool | None = None,
- device: str | None = "cpu",
- **kwargs,
- ) -> BatchFeature:
- """Main method to featurize and prepare for the model one or several sequence(s). Implementation uses PyTorch
- for the STFT computation if available, otherwise a slower NumPy based one.
-
- Args:
- raw_speech (`np.ndarray`, `list[float]`, `list[np.ndarray]`, `list[list[float]]`):
- The sequence or batch of sequences to be padded. Each sequence can be a numpy array, a list of float
- values, a list of numpy arrays or a list of list of float values. Must be mono channel audio, not
- stereo, i.e. single float per timestep.
- truncation (`bool`, *optional*, default to `True`):
- Activates truncation to cut input sequences longer than *max_length* to *max_length*.
- pad_to_multiple_of (`int`, *optional*, defaults to None):
- If set will pad the sequence to a multiple of the provided value.
-
- This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability
- `>= 7.5` (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128.
- return_tensors (`str` or [`~utils.TensorType`], *optional*):
- If set, will return tensors instead of list of python integers. Acceptable values are:
-
- - `'pt'`: Return PyTorch `torch.Tensor` objects.
- - `'np'`: Return Numpy `np.ndarray` objects.
- return_attention_mask (`bool`, *optional*):
- Whether to return the attention mask. If left to the default, will return the attention mask according
- to the specific feature_extractor's default.
-
- [What are attention masks?](../glossary#attention-mask)
-
-
-
- For Whisper models, `attention_mask` should always be passed for batched inference, to avoid subtle
- bugs.
-
-
- padding (`str` or [`~utils.PaddingStrategy`], *optional*, defaults to `'max_length'`):
- Activates and controls padding. Accepts the following values:
-
- - `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single sequence is
- provided).
- - `'max_length'` (default): Pad to a maximum length specified with the argument `max_length` or to the
- maximum acceptable input length for the model if that argument is not provided.
- - `'do_not_pad'`: No padding (i.e., can output a batch with sequences of different lengths).
- max_length (`int`, *optional*):
- Controls the maximum length to use by one of the truncation/padding parameters.
-
- If left unset or set to `None`, this will use the predefined model maximum length if a maximum length
- is required by one of the truncation/padding parameters. If the model has no specific maximum input
- length (like XLNet) truncation/padding to a maximum length will be deactivated.
- sampling_rate (`int`, *optional*):
- The sampling rate at which the `raw_speech` input was sampled. It is strongly recommended to pass
- `sampling_rate` at the forward call to prevent silent errors and allow automatic speech recognition
- pipeline.
- do_normalize (`bool`, *optional*, defaults to `False`):
- Whether or not to zero-mean unit-variance normalize the input. Normalizing can help to significantly
- improve the performance of the model.
- device (`str`, *optional*, defaults to `'cpu'`):
- Specifies the device for computation of the log-mel spectrogram of audio signals in the
- `_torch_extract_fbank_features` method. (e.g., "cpu", "cuda")
- **kwargs: Not supported by WhisperFeatureExtractor.__call__() and ignored.
- """
- if sampling_rate is not None:
- if sampling_rate != self.sampling_rate:
- raise ValueError(
- f"The model corresponding to this feature extractor: {self.__class__.__name__} was trained using a"
- f" sampling rate of {self.sampling_rate}. Please make sure that the provided `raw_speech` input"
- f" was sampled with {self.sampling_rate} and not {sampling_rate}."
- )
- else:
- logger.warning(
- f"It is strongly recommended to pass the `sampling_rate` argument to `{self.__class__.__name__}()`. "
- "Failing to do so can result in silent errors that might be hard to debug."
- )
-
- is_batched_numpy = isinstance(raw_speech, np.ndarray) and len(raw_speech.shape) > 1
- if is_batched_numpy and len(raw_speech.shape) > 2:
- raise ValueError(f"Only mono-channel audio is supported for input to {self}")
- is_batched = is_batched_numpy or (
- isinstance(raw_speech, (list, tuple)) and (isinstance(raw_speech[0], (np.ndarray, tuple, list)))
- )
-
- if is_batched:
- raw_speech = [np.asarray([speech], dtype=np.float32).T for speech in raw_speech]
- elif not is_batched and not isinstance(raw_speech, np.ndarray):
- raw_speech = np.asarray(raw_speech, dtype=np.float32)
- elif isinstance(raw_speech, np.ndarray) and raw_speech.dtype is np.dtype(np.float64):
- raw_speech = raw_speech.astype(np.float32)
-
- # always return batch
- if not is_batched:
- raw_speech = [np.asarray([raw_speech]).T]
-
- batched_speech = BatchFeature({"input_features": raw_speech})
-
- # convert into correct format for padding
-
- padded_inputs = self.pad(
- batched_speech,
- padding=padding,
- max_length=max_length if max_length else self.n_samples,
- truncation=truncation,
- pad_to_multiple_of=pad_to_multiple_of,
- return_attention_mask=return_attention_mask or do_normalize,
- )
-
- # zero-mean and unit-variance normalization
- if do_normalize:
- padded_inputs["input_features"] = self.zero_mean_unit_var_norm(
- padded_inputs["input_features"],
- attention_mask=padded_inputs["attention_mask"],
- padding_value=self.padding_value,
- )
- padded_inputs["input_features"] = np.stack(padded_inputs["input_features"], axis=0)
-
- # make sure list is in array format
- input_features = padded_inputs.get("input_features").transpose(2, 0, 1)
-
- extract_fbank_features = (
- self._torch_extract_fbank_features if is_torch_available() else self._np_extract_fbank_features
- )
- input_features = extract_fbank_features(input_features[0], device)
-
- if isinstance(input_features[0], list):
- padded_inputs["input_features"] = [np.asarray(feature, dtype=np.float32) for feature in input_features]
-
- else:
- padded_inputs["input_features"] = input_features
-
- if return_attention_mask:
- # rescale from sample (48000) to feature (3000)
- rescaled_attention_mask = padded_inputs["attention_mask"][:, :: self.hop_length]
-
- # The STFT computation produces L//hop_length + 1 frames, but we skip the last frame (see `_torch_extract_fbank_features`).
- # This means we need to trim the rescaled attention mask to match the actual number of frames (L//hop_length) when the input length
- # is not perfectly divisible by the hop length.
- if padded_inputs["attention_mask"].shape[1] % self.hop_length != 0:
- rescaled_attention_mask = rescaled_attention_mask[:, :-1]
- padded_inputs["attention_mask"] = rescaled_attention_mask
+from ...audio_processing_base import make_legacy_audio_processor_alias
+from .audio_processing_whisper import WhisperAudioProcessor
- if return_tensors is not None:
- padded_inputs = padded_inputs.convert_to_tensors(return_tensors)
- return padded_inputs
+WhisperFeatureExtractor = make_legacy_audio_processor_alias(WhisperAudioProcessor, "WhisperFeatureExtractor")
__all__ = ["WhisperFeatureExtractor"]
diff --git a/src/transformers/models/xcodec2/__init__.py b/src/transformers/models/xcodec2/__init__.py
index 6c5e6dc96822..04ee05efc1ff 100644
--- a/src/transformers/models/xcodec2/__init__.py
+++ b/src/transformers/models/xcodec2/__init__.py
@@ -18,6 +18,8 @@
if TYPE_CHECKING:
+ from .audio_processing_numpy_xcodec2 import *
+ from .audio_processing_xcodec2 import *
from .configuration_xcodec2 import *
from .feature_extraction_xcodec2 import *
from .modeling_xcodec2 import *
diff --git a/src/transformers/models/xcodec2/audio_processing_numpy_xcodec2.py b/src/transformers/models/xcodec2/audio_processing_numpy_xcodec2.py
new file mode 100644
index 000000000000..57d8383d3583
--- /dev/null
+++ b/src/transformers/models/xcodec2/audio_processing_numpy_xcodec2.py
@@ -0,0 +1,121 @@
+# Copyright 2026 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from ...audio_processing_backends import NumpyAudioBackend
+from ...audio_utils import MelScaleConfig, SpectrogramConfig, StftConfig
+
+
+class Xcodec2AudioProcessorNumpy(NumpyAudioBackend):
+ """NumPy sibling of [`Xcodec2AudioProcessor`]. Dual-output codec processor: raw padded
+ audio for the acoustic encoder (`audio_values`) plus per-utterance kaldi fbank features
+ for the semantic encoder (`audio_features`), computed in `_postprocess_output` from the
+ padded audio batch (matching the legacy `Xcodec2FeatureExtractor`)."""
+
+ sampling_rate = 16000
+ force_mono = True
+ add_channel_dim = True
+ padding_value = 0.0
+ # One acoustic-encoder frame = `hop_length` audio samples (product of downsampling ratios)
+ hop_length = 320
+ pad_to_multiple_of = 320
+ # Semantic features: pairs of consecutive fbank frames are concatenated (stride 2)
+ stride = 2
+ # Mel frames are padded with 1.0 (the legacy FE's `padding_value`), unlike the raw audio
+ feature_padding_value = 1.0
+ # Semantic features are derived from the padded audio in `_postprocess_output`, not via
+ # the base spectrogram path
+ do_extract_spectrogram = False
+
+ spectrogram_config = SpectrogramConfig(
+ stft_config=StftConfig(
+ n_fft=512,
+ win_length=400,
+ hop_length=160,
+ window_fn="povey",
+ power=2.0,
+ center=False,
+ periodic=False,
+ left_align_fft=True,
+ ),
+ mel_scale_config=MelScaleConfig(
+ n_mels=80,
+ f_min=20.0,
+ f_max=8000.0,
+ mel_scale="kaldi",
+ triangularize_in_mel_space=True,
+ ),
+ log_mode="log",
+ preemphasis=0.97,
+ remove_dc_offset=True,
+ mel_floor=1.192092955078125e-07,
+ waveform_scale=32768.0,
+ )
+
+ # Legacy hub configs describe the fbank geometry with flat keys that are fixed
+ # architecture constants already baked into `spectrogram_config` — drop them rather than
+ # letting the base mapping rebuild a partial (and wrong) nested config. `padding_value`
+ # in the legacy config is the *mel* padding value; the raw audio is padded with 0.0.
+ legacy_field_mapping = {
+ "feature_size": None,
+ "frame_length": None,
+ "frame_shift": None,
+ "num_mel_bins": None,
+ "hop_length": None,
+ "padding_value": "feature_padding_value",
+ }
+
+ def _process_audio(self, audio_el):
+ # The legacy FE appends one zero sample to every waveform before padding
+ audio_el = super()._process_audio(audio_el)
+ return np.pad(audio_el, (0, 1))
+
+ def _postprocess_output(self, output, audio_ranges=None, **kwargs):
+ audio_values = output["audio_values"] # (batch, 1, padded_length)
+ padded_length = audio_values.shape[-1]
+ half_hop = self.hop_length // 2
+
+ # Per-utterance fbank on the valid (hop-aligned) slice of the padded audio,
+ # normalized with per-utterance mean/variance before mel-frame padding
+ features = []
+ for i, (start, end) in enumerate(audio_ranges):
+ orig_length = end - start
+ valid_length = min((orig_length + self.hop_length - 1) // self.hop_length * self.hop_length, padded_length)
+ waveform = np.pad(audio_values[i, 0, :valid_length], (half_hop, half_hop))
+ f = self.extract_spectrogram([waveform], spectrogram_config=self.spectrogram_config)[0].T
+ f = (f - f.mean(axis=0)) / np.sqrt(f.var(axis=0, ddof=1) + 1e-7)
+ features.append(f)
+
+ # Pad mel frames to the longest utterance (aligned to `stride`) with `feature_padding_value`
+ frame_lengths = [f.shape[0] for f in features]
+ max_frames = max(frame_lengths)
+ if max_frames % self.stride:
+ max_frames += self.stride - max_frames % self.stride
+ batch = np.stack(
+ [
+ np.pad(f, ((0, max_frames - f.shape[0]), (0, 0)), constant_values=self.feature_padding_value)
+ for f in features
+ ]
+ )
+ mask = self._get_mask([(0, length) for length in frame_lengths], max_frames)
+
+ # Stride concatenation: (batch, frames, n_mels) -> (batch, frames // stride, n_mels * stride)
+ batch_size, num_frames, num_mel_bins = batch.shape
+ output["audio_features"] = batch.reshape(batch_size, num_frames // self.stride, num_mel_bins * self.stride)
+ output["audio_features_mask"] = mask.reshape(batch_size, num_frames // self.stride, self.stride).min(axis=-1)
+ return output
+
+
+__all__ = ["Xcodec2AudioProcessorNumpy"]
diff --git a/src/transformers/models/xcodec2/audio_processing_xcodec2.py b/src/transformers/models/xcodec2/audio_processing_xcodec2.py
new file mode 100644
index 000000000000..023e5c4b57f6
--- /dev/null
+++ b/src/transformers/models/xcodec2/audio_processing_xcodec2.py
@@ -0,0 +1,89 @@
+# Copyright 2026 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+
+from ...audio_processing_backends import TorchAudioBackend
+from .audio_processing_numpy_xcodec2 import Xcodec2AudioProcessorNumpy
+
+
+class Xcodec2AudioProcessor(TorchAudioBackend):
+ """Dual-output codec processor: raw padded audio for the acoustic encoder
+ (`audio_values`) plus per-utterance kaldi fbank features for the semantic encoder
+ (`audio_features`), computed in `_postprocess_output` from the padded audio batch.
+ The fbank path is bit-exact against `torchaudio.compliance.kaldi.fbank` as used by the
+ legacy `Xcodec2FeatureExtractor`."""
+
+ sampling_rate = 16000
+ force_mono = True
+ add_channel_dim = True
+ padding_value = 0.0
+ # One acoustic-encoder frame = `hop_length` audio samples (product of downsampling ratios)
+ hop_length = 320
+ pad_to_multiple_of = 320
+ # Semantic features: pairs of consecutive fbank frames are concatenated (stride 2)
+ stride = 2
+ # Mel frames are padded with 1.0 (the legacy FE's `padding_value`), unlike the raw audio
+ feature_padding_value = 1.0
+ # Semantic features are derived from the padded audio in `_postprocess_output`, not via
+ # the base spectrogram path
+ do_extract_spectrogram = False
+
+ spectrogram_config = Xcodec2AudioProcessorNumpy.spectrogram_config
+ legacy_field_mapping = Xcodec2AudioProcessorNumpy.legacy_field_mapping
+
+ def _process_audio(self, audio_el):
+ # The legacy FE appends one zero sample to every waveform before padding
+ audio_el = super()._process_audio(audio_el)
+ return torch.nn.functional.pad(audio_el, (0, 1))
+
+ def _postprocess_output(self, output, audio_ranges=None, **kwargs):
+ audio_values = output["audio_values"] # (batch, 1, padded_length)
+ padded_length = audio_values.shape[-1]
+ half_hop = self.hop_length // 2
+
+ # Per-utterance fbank on the valid (hop-aligned) slice of the padded audio,
+ # normalized with per-utterance mean/variance before mel-frame padding
+ features = []
+ for i, (start, end) in enumerate(audio_ranges):
+ orig_length = end - start
+ valid_length = min((orig_length + self.hop_length - 1) // self.hop_length * self.hop_length, padded_length)
+ waveform = torch.nn.functional.pad(audio_values[i, 0, :valid_length], (half_hop, half_hop))
+ f = self.extract_spectrogram([waveform], spectrogram_config=self.spectrogram_config)[0].transpose(-2, -1)
+ f = (f - f.mean(0)) / torch.sqrt(f.var(0, unbiased=True) + 1e-7)
+ features.append(f)
+
+ # Pad mel frames to the longest utterance (aligned to `stride`) with `feature_padding_value`
+ frame_lengths = [f.shape[0] for f in features]
+ max_frames = max(frame_lengths)
+ if max_frames % self.stride:
+ max_frames += self.stride - max_frames % self.stride
+ batch = torch.stack(
+ [
+ torch.nn.functional.pad(f, (0, 0, 0, max_frames - f.shape[0]), value=self.feature_padding_value)
+ for f in features
+ ]
+ )
+ mask = self._get_mask([(0, length) for length in frame_lengths], max_frames)
+
+ # Stride concatenation: (batch, frames, n_mels) -> (batch, frames // stride, n_mels * stride)
+ batch_size, num_frames, num_mel_bins = batch.shape
+ output["audio_features"] = batch.reshape(batch_size, num_frames // self.stride, num_mel_bins * self.stride)
+ output["audio_features_mask"] = (
+ mask.reshape(batch_size, num_frames // self.stride, self.stride).min(dim=-1).values
+ )
+ return output
+
+
+__all__ = ["Xcodec2AudioProcessor"]
diff --git a/src/transformers/models/xcodec2/feature_extraction_xcodec2.py b/src/transformers/models/xcodec2/feature_extraction_xcodec2.py
index 311f9be71a84..8cf1098ff0ba 100644
--- a/src/transformers/models/xcodec2/feature_extraction_xcodec2.py
+++ b/src/transformers/models/xcodec2/feature_extraction_xcodec2.py
@@ -1,235 +1,20 @@
-# Copyright 2026 The HuggingFace Inc. team. All rights reserved.
+# Copyright 2026 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import copy
-from typing import Any
-
-from ...audio_utils import AudioInput, make_list_of_audio
-from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
-from ...feature_extraction_utils import BatchFeature
-from ...utils import PaddingStrategy, TensorType, logging
-from ...utils.import_utils import is_torch_available, is_torchaudio_available
-
-
-if is_torch_available():
- import torch
- import torch.nn.functional as F
-
-if is_torchaudio_available():
- import torchaudio
-
-
-logger = logging.get_logger(__name__)
-
-
-class Xcodec2FeatureExtractor(SequenceFeatureExtractor):
- r"""
- Constructs a Xcodec2 feature extractor, which computes mel-filter bank features for the semantic encoder and padded
- audio for the acoustic encoder.
-
- This feature extractor inherits from [`SequenceFeatureExtractor`] which contains most of the main methods. Users
- should refer to this superclass for more information regarding those methods.
-
- Args:
- feature_size (`int`, *optional*, defaults to 80):
- The feature dimension of the extracted features.
- sampling_rate (`int`, *optional*, defaults to 16000):
- The sample rate at which the audio files should be digitalized expressed in hertz (Hz).
- padding_value (`float`, *optional*, defaults to 1.0):
- The value that is used to fill the padding vectors for the mel spectrogram.
- hop_length (`int`, *optional*, defaults to 320):
- Number of audio samples encoded per frame. Equivalent to product of downsampling ratios.
- Needed for acoustic encoder input padding.
- """
-
- model_input_names = ["input_features", "input_values", "padding_mask", "input_features_mask"]
-
- def __init__(
- self,
- feature_size=80,
- sampling_rate=16000,
- padding_value=1.0,
- hop_length=320,
- **kwargs,
- ):
- super().__init__(feature_size=feature_size, sampling_rate=sampling_rate, padding_value=padding_value, **kwargs)
-
- # Acoustic encoder feature extraction (similar to DAC). Defining sub feature extractor as workaround for
- # padding audio with hop_length multiple, and relying on the parent class for padding the spectrogram.
- self.hop_length = hop_length
- self.acoustic_encoder_padder = SequenceFeatureExtractor(
- feature_size=1,
- sampling_rate=sampling_rate,
- padding_value=0.0,
- )
- self.acoustic_encoder_padder.model_input_names = ["audio", "padding_mask"]
-
- # Semantic encoder feature extraction (similar to SeamlessM4T)
- self.stride = 2
- self.num_mel_bins = 80
- self.frame_length = 400
- self.frame_shift = 160
-
- def __call__(
- self,
- audio: AudioInput,
- padding: bool | str | PaddingStrategy = True,
- max_length: int | None = None,
- truncation: bool = False,
- return_tensors: str | TensorType | None = None,
- sampling_rate: int | None = None,
- device: str = "cpu",
- **kwargs,
- ) -> BatchFeature:
- """
- Args:
- audio (`np.ndarray`, `torch.Tensor`, `list[np.ndarray]`, `list[torch.Tensor]`):
- Numpy array or torch tensor with shape (num_channels, sequence_length). A list of such arrays or
- tensors can also be provided for a batch of inputs.
- padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `True`):
- Select a strategy to pad the returned sequences (according to the model's padding side and padding
- index) among:
-
- - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
- sequence if provided).
- - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
- acceptable input length for the model if that argument is not provided.
- - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
- lengths).
- max_length (`int`, *optional*):
- Maximum length of the returned list and optionally padding length (see above).
- truncation (`bool`):
- Activates truncation to cut input sequences longer than *max_length* to *max_length*.
- return_tensors (`str` or [`~utils.TensorType`], *optional*):
- If set, will return tensors instead of list of python integers. Acceptable values are:
-
- - `'tf'`: Return TensorFlow `tf.constant` objects.
- - `'pt'`: Return PyTorch `torch.Tensor` objects.
- - `'np'`: Return Numpy `np.ndarray` objects.
- sampling_rate (`int`, *optional*):
- The sample rate at which the `audio` input was sampled. It is strongly recommended to pass
- `sampling_rate` at the forward call to prevent silent errors.
- device (`str`, *optional*, defaults to `"cpu"`):
- Device for PyTorch tensors during mel-filter bank feature extraction.
- kwargs (*optional*):
- Remaining dictionary of keyword arguments that will be passed to the tokenizer or the feature
- extractor.
- """
- if not is_torch_available():
- raise ImportError("PyTorch is required for mel-filter bank feature extraction.")
-
- if sampling_rate is not None:
- if sampling_rate != self.sampling_rate:
- raise ValueError(
- f"The model corresponding to this feature extractor: {self} was trained using a sampling rate of"
- f" {self.sampling_rate}. Please make sure that the provided `audio` input was sampled with"
- f" {self.sampling_rate} and not {sampling_rate}."
- )
- else:
- logger.warning(
- f"It is strongly recommended to pass the `sampling_rate` argument to `{self.__class__.__name__}()`. "
- "Failing to do so can result in silent errors that might be hard to debug."
- )
-
- audio = make_list_of_audio(audio)
- for example in audio:
- if example.ndim > 2:
- raise ValueError(f"Expected input shape (channels, length) but got shape {example.shape}")
- batch_size = len(audio)
-
- # 1) Acoustic encoder padding
- audio = [F.pad(torch.as_tensor(a), (0, 1), value=0.0) for a in audio]
- padded_inputs = self.acoustic_encoder_padder.pad(
- BatchFeature({"audio": audio}),
- max_length=max_length,
- truncation=truncation,
- padding=padding,
- return_attention_mask=padding,
- pad_to_multiple_of=self.hop_length,
- return_tensors="pt",
- )
- padding_mask = padded_inputs.pop("attention_mask")
- padded_audio = padded_inputs["audio"][:, None, :]
+"""Backwards-compatibility shim: re-exports the legacy ``Xcodec2FeatureExtractor`` name as a
+deprecated alias of [`Xcodec2AudioProcessor`]. Importing or instantiating the alias emits a
+``FutureWarning``; the alias is removed in transformers v5.15 (see ADR 0002).
+"""
- # 2) Semantic encoder feature extraction (mel spectrogram) with normalization computed before padding
- # NOTE (ebezzam): looping over the batch to match the original implementation with `torchaudio.compliance.kaldi.fbank`. However it does not support batched inputs.
- # Original used `SeamlessM4TFeatureExtractor`, which also loops over individual audio, but was numpy-based.
- mel_features = []
- for i in range(batch_size):
- orig_len = int(padding_mask[i].sum().item()) if padding_mask is not None else padded_audio.shape[-1]
- per_sample_len = ((orig_len + self.hop_length - 1) // self.hop_length) * self.hop_length
- valid_len = min(per_sample_len, padded_audio.shape[-1])
- waveform = padded_audio[i, :, :valid_len]
- waveform = F.pad(waveform, (self.hop_length // 2, self.hop_length // 2), value=0.0)
- waveform = waveform.to(device)
- features = torchaudio.compliance.kaldi.fbank(
- waveform * (2**15),
- num_mel_bins=self.num_mel_bins,
- frame_length=self.frame_length / self.sampling_rate * 1000,
- frame_shift=self.frame_shift / self.sampling_rate * 1000,
- sample_frequency=self.sampling_rate,
- window_type="povey",
- preemphasis_coefficient=0.97,
- remove_dc_offset=True,
- use_log_fbank=True,
- use_energy=False,
- dither=0.0,
- snip_edges=True,
- low_freq=20,
- high_freq=self.sampling_rate // 2,
- )
- features = (features - features.mean(0)) / torch.sqrt(features.var(0, unbiased=True) + 1e-7)
- mel_features.append(features)
- encoded_inputs = BatchFeature({"input_features": mel_features})
- padded_mel = self.pad(
- encoded_inputs,
- padding=padding,
- max_length=max_length,
- truncation=truncation,
- pad_to_multiple_of=self.stride,
- return_attention_mask=padding,
- return_tensors="pt",
- )
- audio_spectrogram = padded_mel["input_features"]
- spectrogram_mask = padded_mel.get("attention_mask")
- trimmed_frames = audio_spectrogram.shape[1] - (audio_spectrogram.shape[1] % self.stride)
- audio_spectrogram = audio_spectrogram[:, :trimmed_frames, :].reshape(
- batch_size, trimmed_frames // self.stride, self.num_mel_bins * self.stride
- )
- if spectrogram_mask is not None:
- spectrogram_mask = (
- spectrogram_mask[:, :trimmed_frames]
- .reshape(batch_size, trimmed_frames // self.stride, self.stride)
- .min(dim=-1)
- .values
- )
+from ...audio_processing_base import make_legacy_audio_processor_alias
+from .audio_processing_xcodec2 import Xcodec2AudioProcessor
- return BatchFeature(
- {
- "input_values": padded_audio,
- "padding_mask": padding_mask,
- "input_features": audio_spectrogram,
- "input_features_mask": spectrogram_mask,
- },
- tensor_type=return_tensors,
- )
- def to_dict(self) -> dict[str, Any]:
- output = copy.deepcopy(self.__dict__)
- output["feature_extractor_type"] = self.__class__.__name__
- output.pop("acoustic_encoder_padder", None)
- return output
+Xcodec2FeatureExtractor = make_legacy_audio_processor_alias(Xcodec2AudioProcessor, "Xcodec2FeatureExtractor")
__all__ = ["Xcodec2FeatureExtractor"]
diff --git a/src/transformers/preprocessing_base.py b/src/transformers/preprocessing_base.py
new file mode 100644
index 000000000000..d96b407dd952
--- /dev/null
+++ b/src/transformers/preprocessing_base.py
@@ -0,0 +1,755 @@
+# Copyright 2020 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Base mixin for image processors and feature extractors, providing shared
+save/load/serialization logic.
+"""
+
+import copy
+import json
+import os
+from collections import UserDict
+from copy import deepcopy
+from typing import Any, TypeVar
+
+import numpy as np
+from huggingface_hub import create_repo, is_offline_mode
+from huggingface_hub.dataclasses import validate_typed_dict
+
+from .dynamic_module_utils import custom_object_save
+from .utils import (
+ PROCESSOR_NAME,
+ PushToHubMixin,
+ TensorType,
+ _is_tensor_or_array_like,
+ is_numpy_array,
+ is_torch_available,
+ is_torch_device,
+ is_torch_dtype,
+ logging,
+ requires_backends,
+ safe_load_json_file,
+)
+from .utils.hub import cached_file
+
+
+logger = logging.get_logger(__name__)
+
+PreprocessingMixinType = TypeVar("PreprocessingMixinType", bound="PreprocessingMixin")
+
+
+class BatchFeature(UserDict):
+ r"""
+ Holds the output of the [`~SequenceFeatureExtractor.pad`] and feature extractor specific `__call__` methods.
+
+ This class is derived from a python dictionary and can be used as a dictionary.
+
+ Args:
+ data (`dict`, *optional*):
+ Dictionary of lists/arrays/tensors returned by the __call__/pad methods ('input_values', 'attention_mask',
+ etc.).
+ tensor_type (`Union[None, str, TensorType]`, *optional*):
+ You can give a tensor_type here to convert the lists of integers in PyTorch/Numpy Tensors at
+ initialization.
+ skip_tensor_conversion (`list[str]` or `set[str]`, *optional*):
+ List or set of keys that should NOT be converted to tensors, even when `tensor_type` is specified.
+ """
+
+ def __init__(
+ self,
+ data: dict[str, Any] | None = None,
+ tensor_type: None | str | TensorType = None,
+ skip_tensor_conversion: list[str] | set[str] | None = None,
+ ):
+ super().__init__(data)
+ self.skip_tensor_conversion = skip_tensor_conversion
+ self.convert_to_tensors(tensor_type=tensor_type)
+
+ def __getitem__(self, item: str) -> Any:
+ """
+ If the key is a string, returns the value of the dict associated to `key` ('input_values', 'attention_mask',
+ etc.).
+ """
+ if isinstance(item, str):
+ return self.data[item]
+ else:
+ raise KeyError("Indexing with integers is not available when using Python based feature extractors")
+
+ def __getattr__(self, item: str):
+ try:
+ return self.data[item]
+ except KeyError:
+ raise AttributeError
+
+ def __getstate__(self):
+ return {"data": self.data}
+
+ def __setstate__(self, state):
+ if "data" in state:
+ self.data = state["data"]
+
+ def _get_is_as_tensor_fns(self, tensor_type: str | TensorType | None = None):
+ if tensor_type is None:
+ return None, None
+
+ # Convert to TensorType
+ if not isinstance(tensor_type, TensorType):
+ tensor_type = TensorType(tensor_type)
+
+ if tensor_type == TensorType.PYTORCH:
+ if not is_torch_available():
+ raise ImportError("Unable to convert output to PyTorch tensors format, PyTorch is not installed.")
+ import torch
+
+ def as_tensor(value):
+ if torch.is_tensor(value):
+ return value
+
+ # stack list of tensors if tensor_type is PyTorch (# torch.tensor() does not support list of tensors)
+ if isinstance(value, (list, tuple)) and len(value) > 0 and torch.is_tensor(value[0]):
+ return torch.stack(value)
+
+ # convert list of numpy arrays to numpy array (stack) if tensor_type is Numpy
+ if isinstance(value, (list, tuple)) and len(value) > 0:
+ if isinstance(value[0], np.ndarray):
+ value = np.array(value)
+ elif (
+ isinstance(value[0], (list, tuple))
+ and len(value[0]) > 0
+ and isinstance(value[0][0], np.ndarray)
+ ):
+ value = np.array(value)
+ if isinstance(value, np.ndarray):
+ return torch.from_numpy(value)
+ else:
+ return torch.tensor(value)
+
+ is_tensor = torch.is_tensor
+ else:
+
+ def as_tensor(value, dtype=None):
+ if isinstance(value, (list, tuple)) and isinstance(value[0], (list, tuple, np.ndarray)):
+ value_lens = [len(val) for val in value]
+ if len(set(value_lens)) > 1 and dtype is None:
+ # we have a ragged list so handle explicitly
+ value = as_tensor([np.asarray(val) for val in value], dtype=object)
+ return np.asarray(value, dtype=dtype)
+
+ is_tensor = is_numpy_array
+ return is_tensor, as_tensor
+
+ def convert_to_tensors(
+ self,
+ tensor_type: str | TensorType | None = None,
+ skip_tensor_conversion: list[str] | set[str] | None = None,
+ ):
+ """
+ Convert the inner content to tensors.
+
+ Args:
+ tensor_type (`str` or [`~utils.TensorType`], *optional*):
+ The type of tensors to use. If `str`, should be one of the values of the enum [`~utils.TensorType`]. If
+ `None`, no modification is done.
+ skip_tensor_conversion (`list[str]` or `set[str]`, *optional*):
+ List or set of keys that should NOT be converted to tensors, even when `tensor_type` is specified.
+
+ Note:
+ Values that don't have an array-like structure (e.g., strings, dicts, lists of strings) are
+ automatically skipped and won't be converted to tensors. Ragged arrays (lists of arrays with
+ different lengths) are still attempted, though they may raise errors during conversion.
+ """
+ if tensor_type is None:
+ return self
+
+ is_tensor, as_tensor = self._get_is_as_tensor_fns(tensor_type)
+ skip_tensor_conversion = (
+ skip_tensor_conversion if skip_tensor_conversion is not None else self.skip_tensor_conversion
+ )
+
+ # Do the tensor conversion in batch
+ for key, value in self.items():
+ # Skip keys explicitly marked for no conversion
+ if skip_tensor_conversion and key in skip_tensor_conversion:
+ continue
+
+ # Skip values that are not array-like
+ if not _is_tensor_or_array_like(value):
+ continue
+
+ try:
+ if not is_tensor(value):
+ tensor = as_tensor(value)
+ self[key] = tensor
+ except Exception as e:
+ if key == "overflowing_values":
+ raise ValueError(
+ f"Unable to create tensor for '{key}' with overflowing values of different lengths. "
+ f"Original error: {str(e)}"
+ ) from e
+ raise ValueError(
+ f"Unable to convert output '{key}' (type: {type(value).__name__}) to tensor: {str(e)}\n"
+ f"You can try:\n"
+ f" 1. Use padding=True to ensure all outputs have the same shape\n"
+ f" 2. Set return_tensors=None to return Python objects instead of tensors"
+ ) from e
+
+ return self
+
+ def to(self, *args, **kwargs) -> "BatchFeature":
+ """
+ Send all values to device by calling `v.to(*args, **kwargs)` (PyTorch only). This should support casting in
+ different `dtypes` and sending the `BatchFeature` to a different `device`.
+
+ Args:
+ args (`Tuple`):
+ Will be passed to the `to(...)` function of the tensors.
+ kwargs (`Dict`, *optional*):
+ Will be passed to the `to(...)` function of the tensors.
+ To enable asynchronous data transfer, set the `non_blocking` flag in `kwargs` (defaults to `False`).
+
+ Returns:
+ [`BatchFeature`]: The same instance after modification.
+ """
+ requires_backends(self, ["torch"])
+ import torch
+
+ device = kwargs.get("device")
+ non_blocking = kwargs.get("non_blocking", False)
+ # Check if the args are a device or a dtype
+ if device is None and len(args) > 0:
+ # device should be always the first argument
+ arg = args[0]
+ if is_torch_dtype(arg):
+ # The first argument is a dtype
+ pass
+ elif isinstance(arg, str) or is_torch_device(arg) or isinstance(arg, int):
+ device = arg
+ else:
+ # it's something else
+ raise ValueError(f"Attempting to cast a BatchFeature to type {str(arg)}. This is not supported.")
+
+ # We cast only floating point tensors to avoid issues with tokenizers casting `LongTensor` to `FloatTensor`
+ def maybe_to(v):
+ # check if v is a floating point tensor
+ if isinstance(v, torch.Tensor) and torch.is_floating_point(v):
+ # cast and send to device
+ return v.to(*args, **kwargs)
+ elif isinstance(v, torch.Tensor) and device is not None:
+ return v.to(device=device, non_blocking=non_blocking)
+ # recursively handle lists and tuples
+ elif isinstance(v, (list, tuple)):
+ return type(v)(maybe_to(item) for item in v)
+ else:
+ return v
+
+ self.data = {k: maybe_to(v) for k, v in self.items()}
+ return self
+
+
+class PreprocessingMixin(PushToHubMixin):
+ """
+ Base mixin providing saving/loading functionality shared by
+ ImageProcessingMixin, AudioProcessingMixin and FeatureExtractionMixin.
+
+ Subclasses must set the following class attributes:
+ _config_name: str — config file name (e.g. IMAGE_PROCESSOR_NAME)
+ _type_key: str — key added in to_dict() (e.g. "image_processor_type")
+ _nested_config_keys: list — keys to check in processor_config.json
+ _auto_class_default: str — default auto class for register_for_auto_class
+ _file_type_label: str — label for user-agent / error messages
+ Optional:
+ _excluded_dict_keys: set — keys to drop from to_dict() output
+ _extra_init_pops: list — extra keys to pop in __init__
+ _config_filename_kwarg: str — kwarg name that can override the config filename
+ _subfolder_default: str — default for the subfolder kwarg
+ """
+
+ _auto_class = None
+
+ # --- Must be overridden by subclasses ---
+ _config_name: str
+ _type_key: str
+ _nested_config_keys: list[str] = []
+ _auto_class_default: str
+ _file_type_label: str
+
+ # --- Optional overrides ---
+ _excluded_dict_keys: set[str] = set()
+ # Drop None-valued attrs whose class default is None from to_dict(). On by default for the
+ # modern processor classes; legacy FeatureExtractionMixin opts out to keep full serialization.
+ _filter_none_class_defaults: bool = True
+ _extra_init_pops: list[str] = []
+ _config_filename_kwarg: str | None = None
+ _subfolder_default: str | None = ""
+
+ def __init__(self, **kwargs):
+ """Set elements of `kwargs` as attributes."""
+ for key in self._extra_init_pops:
+ kwargs.pop(key, None)
+ # Pop "processor_class", should not be saved in config
+ kwargs.pop("processor_class", None)
+
+ if hasattr(self, "valid_kwargs") and hasattr(self.valid_kwargs, "__annotations__"):
+ self._init_kwargs_from_valid_kwargs(kwargs)
+
+ # Additional attributes without default values
+ for key, value in kwargs.items():
+ try:
+ setattr(self, key, value)
+ except AttributeError as err:
+ logger.error(f"Can't set {key} with value {value} for {self}")
+ raise err
+
+ def _init_kwargs_from_valid_kwargs(self, kwargs: dict):
+ """
+ Initialize instance attributes from `valid_kwargs` annotations.
+
+ For each key in `self.valid_kwargs.__annotations__`, pops it from `kwargs`
+ and sets it on the instance (or deep-copies the class default).
+ Also sets `self._valid_kwargs_names`.
+ """
+ for key in self.valid_kwargs.__annotations__:
+ kwarg = kwargs.pop(key, None)
+ if kwarg is not None:
+ setattr(self, key, kwarg)
+ else:
+ setattr(self, key, deepcopy(getattr(self, key, None)))
+ self._valid_kwargs_names = list(self.valid_kwargs.__annotations__.keys())
+
+ def _set_attributes(self, **kwargs):
+ """Standardize instance attributes for all valid kwargs (e.g. coerce dicts to their canonical form)."""
+ attributes = {key: getattr(self, key) for key in self._valid_kwargs_names}
+ attributes = self._standardize_kwargs(**attributes)
+ for key, value in attributes.items():
+ setattr(self, key, value)
+
+ def _standardize_kwargs(self, **kwargs) -> dict:
+ """
+ Hook: standardize kwargs to canonical format before validation (e.g. coerce dicts to
+ `SizeDict`/`SpectrogramConfig`). Overridden by modality base classes; default is a no-op.
+ """
+ return kwargs
+
+ def _validate_preprocess_kwargs(self, **kwargs):
+ """
+ Hook: validate the kwargs for the preprocess method. Overridden by modality base classes;
+ default is a no-op.
+ """
+
+ def preprocess(self, inputs, *args, **kwargs):
+ """
+ Common preprocess entrypoint: validate received kwargs against `valid_kwargs`, fill in
+ defaults from `self`, standardize and validate them, then dispatch to the modality-specific
+ `_preprocess_*_like_inputs` implementation via `_preprocess_like_inputs`.
+ """
+ # Perform type validation on received kwargs
+ validate_typed_dict(self.valid_kwargs, kwargs)
+
+ # Set default kwargs from self
+ for kwarg_name in self._valid_kwargs_names:
+ kwargs.setdefault(kwarg_name, getattr(self, kwarg_name, None))
+
+ # Update kwargs that need further processing before being validated
+ kwargs = self._standardize_kwargs(**kwargs)
+
+ # Validate kwargs
+ self._validate_preprocess_kwargs(**kwargs)
+
+ return self._preprocess_like_inputs(inputs, *args, **kwargs)
+
+ def _preprocess_like_inputs(self, inputs, *args, **kwargs):
+ """
+ Dispatch to the modality-specific `_preprocess_*_like_inputs` method
+ (e.g. `_preprocess_image_like_inputs`, `_preprocess_audio_like_inputs`).
+ Implemented by modality base classes.
+ """
+ raise NotImplementedError
+
+ def filter_out_unused_kwargs(self, kwargs: dict) -> dict:
+ """
+ Filter out the unused kwargs from the kwargs dictionary.
+ """
+ if self.unused_kwargs is None:
+ return kwargs
+
+ for kwarg_name in self.unused_kwargs:
+ if kwarg_name in kwargs:
+ logger.warning_once(f"This processor does not use the `{kwarg_name}` parameter. It will be ignored.")
+ kwargs.pop(kwarg_name)
+ return kwargs
+
+ @classmethod
+ def from_dict(cls, config_dict: dict[str, Any], **kwargs):
+ """
+ Instantiates a processor from a Python dictionary of parameters.
+
+ Args:
+ config_dict (`dict[str, Any]`):
+ Dictionary that will be used to instantiate the processor object.
+ kwargs (`dict[str, Any]`):
+ Additional parameters from which to initialize the processor object.
+
+ Returns:
+ A processor of type [`~PreprocessingMixin`].
+ """
+ config_dict = config_dict.copy()
+ return_unused_kwargs = kwargs.pop("return_unused_kwargs", False)
+
+ # Use valid_kwargs pattern when available (image/audio processors)
+ if hasattr(cls, "valid_kwargs") and hasattr(cls.valid_kwargs, "__annotations__"):
+ config_dict.update({k: v for k, v in kwargs.items() if k in cls.valid_kwargs.__annotations__})
+ processor = cls(**config_dict)
+
+ # Apply extra kwargs to instance (BC for remote code)
+ extra_keys = []
+ for key in reversed(list(kwargs.keys())):
+ if hasattr(processor, key) and key not in cls.valid_kwargs.__annotations__:
+ setattr(processor, key, kwargs.pop(key, None))
+ extra_keys.append(key)
+ if extra_keys:
+ logger.warning_once(
+ f"Processor {cls.__name__}: kwargs {extra_keys} were applied for backward compatibility. "
+ f"To avoid this warning, add them to valid_kwargs."
+ )
+ else:
+ processor = cls(**config_dict)
+
+ logger.info(f"Processor {processor}")
+ if return_unused_kwargs:
+ return processor, kwargs
+ else:
+ return processor
+
+ @classmethod
+ def from_pretrained(
+ cls: type[PreprocessingMixinType],
+ pretrained_model_name_or_path: str | os.PathLike,
+ cache_dir: str | os.PathLike | None = None,
+ force_download: bool = False,
+ local_files_only: bool = False,
+ token: str | bool | None = None,
+ revision: str = "main",
+ **kwargs,
+ ) -> PreprocessingMixinType:
+ r"""
+ Instantiate a processor from a pretrained model name or path.
+
+ Args:
+ pretrained_model_name_or_path (`str` or `os.PathLike`):
+ This can be either:
+
+ - a string, the *model id* of a pretrained processor hosted inside a model repo on
+ huggingface.co.
+ - a path to a *directory* containing a processor file saved using the
+ [`~PreprocessingMixin.save_pretrained`] method, e.g., `./my_model_directory/`.
+ - a path or url to a saved processor JSON *file*, e.g.,
+ `./my_model_directory/preprocessor_config.json`.
+ cache_dir (`str` or `os.PathLike`, *optional*):
+ Path to a directory in which a downloaded pretrained model processor should be cached if the
+ standard cache should not be used.
+ force_download (`bool`, *optional*, defaults to `False`):
+ Whether or not to force to (re-)download the processor files and override the cached versions if
+ they exist.
+ token (`str` or `bool`, *optional*):
+ The token to use as HTTP bearer authorization for remote files.
+ revision (`str`, *optional*, defaults to `"main"`):
+ The specific model version to use.
+ return_unused_kwargs (`bool`, *optional*, defaults to `False`):
+ If `False`, then this function returns just the final processor object. If `True`, then this
+ functions returns a `Tuple(processor, unused_kwargs)` where *unused_kwargs* is a dictionary
+ consisting of the key/value pairs whose keys are not processor attributes.
+ kwargs (`dict[str, Any]`, *optional*):
+ The values in kwargs of any keys which are processor attributes will be used to override the
+ loaded values.
+
+ Returns:
+ A processor of type [`~PreprocessingMixin`].
+ """
+ kwargs["cache_dir"] = cache_dir
+ kwargs["force_download"] = force_download
+ kwargs["local_files_only"] = local_files_only
+ kwargs["revision"] = revision
+
+ if token is not None:
+ kwargs["token"] = token
+
+ config_dict, kwargs = cls._get_config_dict(pretrained_model_name_or_path, **kwargs)
+
+ return cls.from_dict(config_dict, **kwargs)
+
+ def save_pretrained(self, save_directory: str | os.PathLike, push_to_hub: bool = False, **kwargs):
+ """
+ Save a processor object to the directory `save_directory`, so that it can be re-loaded using the
+ [`~PreprocessingMixin.from_pretrained`] class method.
+
+ Args:
+ save_directory (`str` or `os.PathLike`):
+ Directory where the processor JSON file will be saved (will be created if it does not exist).
+ push_to_hub (`bool`, *optional*, defaults to `False`):
+ Whether or not to push your model to the Hugging Face model hub after saving it.
+ kwargs (`dict[str, Any]`, *optional*):
+ Additional key word arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method.
+ """
+ if os.path.isfile(save_directory):
+ raise AssertionError(f"Provided path ({save_directory}) should be a directory, not a file")
+
+ os.makedirs(save_directory, exist_ok=True)
+
+ if push_to_hub:
+ commit_message = kwargs.pop("commit_message", None)
+ repo_id = kwargs.pop("repo_id", save_directory.split(os.path.sep)[-1])
+ repo_id = create_repo(repo_id, exist_ok=True, **kwargs).repo_id
+ files_timestamps = self._get_files_timestamps(save_directory)
+
+ # If we have a custom config, we copy the file defining it in the folder and set the attributes so it can be
+ # loaded from the Hub.
+ if self._auto_class is not None:
+ custom_object_save(self, save_directory, config=self)
+
+ # If we save using the predefined names, we can load using `from_pretrained`
+ output_file = os.path.join(save_directory, self._config_name)
+
+ self.to_json_file(output_file)
+ logger.info(f"{self._file_type_label} saved in {output_file}")
+
+ if push_to_hub:
+ self._upload_modified_files(
+ save_directory,
+ repo_id,
+ files_timestamps,
+ commit_message=commit_message,
+ token=kwargs.get("token"),
+ )
+
+ return [output_file]
+
+ @classmethod
+ def _get_config_dict(
+ cls, pretrained_model_name_or_path: str | os.PathLike, **kwargs
+ ) -> tuple[dict[str, Any], dict[str, Any]]:
+ """
+ From a `pretrained_model_name_or_path`, resolve to a dictionary of parameters, to be used for instantiating a
+ processor using `from_dict`.
+
+ Parameters:
+ pretrained_model_name_or_path (`str` or `os.PathLike`):
+ The identifier of the pre-trained checkpoint from which we want the dictionary of parameters.
+
+ Returns:
+ `tuple[Dict, Dict]`: The dictionary(ies) that will be used to instantiate the processor object.
+ """
+ cache_dir = kwargs.pop("cache_dir", None)
+ force_download = kwargs.pop("force_download", False)
+ proxies = kwargs.pop("proxies", None)
+ token = kwargs.pop("token", None)
+ local_files_only = kwargs.pop("local_files_only", False)
+ revision = kwargs.pop("revision", None)
+ subfolder = kwargs.pop("subfolder", cls._subfolder_default)
+
+ # Allow overriding the config filename via a kwarg (e.g. image_processor_filename)
+ if cls._config_filename_kwarg is not None:
+ config_filename = kwargs.pop(cls._config_filename_kwarg, cls._config_name)
+ else:
+ config_filename = cls._config_name
+
+ from_pipeline = kwargs.pop("_from_pipeline", None)
+ from_auto_class = kwargs.pop("_from_auto", False)
+
+ user_agent = {"file_type": cls._file_type_label, "from_auto_class": from_auto_class}
+ if from_pipeline is not None:
+ user_agent["using_pipeline"] = from_pipeline
+
+ if is_offline_mode() and not local_files_only:
+ logger.info("Offline mode: forcing local_files_only=True")
+ local_files_only = True
+
+ pretrained_model_name_or_path = str(pretrained_model_name_or_path)
+ is_local = os.path.isdir(pretrained_model_name_or_path)
+ if os.path.isdir(pretrained_model_name_or_path):
+ config_file = os.path.join(pretrained_model_name_or_path, config_filename)
+ if os.path.isfile(pretrained_model_name_or_path):
+ resolved_config_file = pretrained_model_name_or_path
+ resolved_processor_file = None
+ is_local = True
+ else:
+ config_file = config_filename
+ try:
+ resolved_processor_file = cached_file(
+ pretrained_model_name_or_path,
+ filename=PROCESSOR_NAME,
+ cache_dir=cache_dir,
+ force_download=force_download,
+ proxies=proxies,
+ local_files_only=local_files_only,
+ token=token,
+ user_agent=user_agent,
+ revision=revision,
+ subfolder=subfolder,
+ _raise_exceptions_for_missing_entries=False,
+ )
+ resolved_config_file = cached_file(
+ pretrained_model_name_or_path,
+ filename=config_file,
+ cache_dir=cache_dir,
+ force_download=force_download,
+ proxies=proxies,
+ local_files_only=local_files_only,
+ token=token,
+ user_agent=user_agent,
+ revision=revision,
+ subfolder=subfolder,
+ _raise_exceptions_for_missing_entries=False,
+ )
+ except OSError:
+ # Raise any environment error raise by `cached_file`. It will have a helpful error message adapted to
+ # the original exception.
+ raise
+ except Exception:
+ # For any other exception, we throw a generic error.
+ raise OSError(
+ f"Can't load {cls._file_type_label} for '{pretrained_model_name_or_path}'. If you were trying to load"
+ " it from 'https://huggingface.co/models', make sure you don't have a local directory with the"
+ f" same name. Otherwise, make sure '{pretrained_model_name_or_path}' is the correct path to a"
+ f" directory containing a {config_filename} file"
+ )
+
+ # Load config dict. Priority goes as (nested config if found -> standalone config)
+ # We are downloading both configs because almost all models have a `processor_config.json` but
+ # not all of these are nested. We need to check if it was saved recebtly as nested or if it is legacy style
+ config_dict = None
+ if resolved_processor_file is not None:
+ processor_dict = safe_load_json_file(resolved_processor_file)
+ for nested_key in cls._nested_config_keys:
+ if nested_key in processor_dict:
+ config_dict = processor_dict[nested_key]
+ break
+
+ if resolved_config_file is not None and config_dict is None:
+ config_dict = safe_load_json_file(resolved_config_file)
+
+ if config_dict is None:
+ raise OSError(
+ f"Can't load {cls._file_type_label} for '{pretrained_model_name_or_path}'. If you were trying to load"
+ " it from 'https://huggingface.co/models', make sure you don't have a local directory with the"
+ f" same name. Otherwise, make sure '{pretrained_model_name_or_path}' is the correct path to a"
+ f" directory containing a {config_filename} file"
+ )
+
+ if is_local:
+ logger.info(f"loading configuration file {resolved_config_file}")
+ else:
+ logger.info(
+ f"loading configuration file {config_file} from cache at {resolved_config_file}"
+ )
+
+ return config_dict, kwargs
+
+ def to_dict(self) -> dict[str, Any]:
+ """
+ Serializes this instance to a Python dictionary.
+
+ Returns:
+ `dict[str, Any]`: Dictionary of all the attributes that make up this instance.
+ """
+ output = copy.deepcopy(self.__dict__)
+ output[self._type_key] = self.__class__.__name__
+ output.pop("_valid_kwargs_names", None)
+ for key in self._excluded_dict_keys:
+ if key in output:
+ del output[key]
+ output = {key: self._serialize_value(key, value) for key, value in output.items()}
+ if self._filter_none_class_defaults:
+ output = {key: value for key, value in output.items() if self._keep_in_dict(key, value)}
+ return output
+
+ def _serialize_value(self, key, value):
+ """Hook: coerce a modality-specific attribute to a JSON-friendly form for `to_dict`
+ (e.g. `SizeDict`/`SpectrogramConfig` → plain dict). Default: identity."""
+ return value
+
+ def _keep_in_dict(self, key, value) -> bool:
+ """Keep non-None values; keep an explicit None only when the class default is itself
+ non-None (i.e. the user deliberately overrode a real default with None)."""
+ if value is not None:
+ return True
+ class_default = getattr(type(self), key, "NOT_FOUND")
+ return class_default != "NOT_FOUND" and class_default is not None
+
+ @classmethod
+ def from_json_file(cls, json_file: str | os.PathLike):
+ """
+ Instantiates a processor from the path to a JSON file of parameters.
+
+ Args:
+ json_file (`str` or `os.PathLike`):
+ Path to the JSON file containing the parameters.
+
+ Returns:
+ A processor of type [`~PreprocessingMixin`]: The processor object instantiated from that JSON file.
+ """
+ with open(json_file, encoding="utf-8") as reader:
+ text = reader.read()
+ config_dict = json.loads(text)
+ return cls(**config_dict)
+
+ def to_json_string(self) -> str:
+ """
+ Serializes this instance to a JSON string.
+
+ Returns:
+ `str`: String containing all the attributes that make up this instance in JSON format.
+ """
+ dictionary = self.to_dict()
+
+ for key, value in dictionary.items():
+ if isinstance(value, np.ndarray):
+ dictionary[key] = value.tolist()
+
+ return json.dumps(dictionary, indent=2, sort_keys=True) + "\n"
+
+ def to_json_file(self, json_file_path: str | os.PathLike):
+ """
+ Save this instance to a JSON file.
+
+ Args:
+ json_file_path (`str` or `os.PathLike`):
+ Path to the JSON file in which this instance's parameters will be saved.
+ """
+ with open(json_file_path, "w", encoding="utf-8") as writer:
+ writer.write(self.to_json_string())
+
+ def __repr__(self):
+ return f"{self.__class__.__name__} {self.to_json_string()}"
+
+ @classmethod
+ def register_for_auto_class(cls, auto_class=None):
+ """
+ Register this class with a given auto class.
+
+ Args:
+ auto_class (`str` or `type`, *optional*):
+ The auto class to register this new processor with. Defaults to the subclass's `_auto_class_default`.
+ """
+ if auto_class is None:
+ auto_class = cls._auto_class_default
+
+ if not isinstance(auto_class, str):
+ auto_class = auto_class.__name__
+
+ import transformers.models.auto as auto_module
+
+ if not hasattr(auto_module, auto_class):
+ raise ValueError(f"{auto_class} is not a valid auto class.")
+
+ cls._auto_class = auto_class
diff --git a/src/transformers/processing_utils.py b/src/transformers/processing_utils.py
index 17ffb0b10d30..f31f1b74ec37 100644
--- a/src/transformers/processing_utils.py
+++ b/src/transformers/processing_utils.py
@@ -34,7 +34,7 @@
from huggingface_hub.dataclasses import validate_typed_dict
from huggingface_hub.errors import EntryNotFoundError
-from .audio_utils import AudioInput, load_audio, make_list_of_audio
+from .audio_utils import AudioInput, SpectrogramConfig, load_audio, make_list_of_audio
from .dynamic_module_utils import custom_object_save
from .feature_extraction_utils import BatchFeature
from .image_utils import ChannelDimension, ImageInput, is_vision_available, make_flat_list_of_images
@@ -129,9 +129,9 @@ def keys(self):
"HiggsAudioV2TokenizerModel",
"DacModel",
), # TODO: @eustlb, to be replaced with PreTrainedAudioTokenizerBase
- "audio_processor": "FeatureExtractionMixin",
+ "audio_processor": ("FeatureExtractionMixin", "TorchAudioBackend", "NumpyAudioBackend"),
"tokenizer": ("PreTrainedTokenizerBase", "MistralCommonBackend"),
- "feature_extractor": "FeatureExtractionMixin",
+ "feature_extractor": ("FeatureExtractionMixin", "TorchAudioBackend", "NumpyAudioBackend"),
"image_processor": "ImageProcessingMixin",
"video_processor": "BaseVideoProcessor",
}
@@ -380,15 +380,29 @@ class VideosKwargs(TypedDict, total=False):
class AudioKwargs(TypedDict, total=False):
"""
- Keyword arguments for audio processing.
+ Keyword arguments for audio processing. For extended documentation, check the appropriate AudioProcessor
+ class methods and docstrings.
+
+ Note on `sampling_rate`: a model's native sampling rate is a processor *identity* attribute, set once at
+ init time as `sampling_rate` (e.g. `WhisperAudioProcessor.sampling_rate == 16000`). The per-call
+ `sampling_rate` keyword below reuses the same name as the *caller's assertion* of the rate at which the
+ provided arrays were actually sampled; it is checked against the processor's own `sampling_rate` and never
+ modifies it.
Attributes:
sampling_rate (`int`, *optional*):
- The sampling rate at which the `raw_speech` input was sampled.
- raw_speech (`np.ndarray`, `list[float]`, `list[np.ndarray]`, `list[list[float]]`):
- The sequence or batch of sequences to be padded. Each sequence can be a numpy array, a list of float
- values, a list of numpy arrays or a list of list of float values. Must be mono channel audio, not
- stereo, i.e. single float per timestep.
+ The sampling rate at which the input audio was sampled, asserted by the caller. Passing it lets the
+ processor verify it matches the model's native `sampling_rate` and avoid silent errors.
+ spectrogram_config (`dict` or [`~audio_utils.SpectrogramConfig`], *optional*):
+ Per-call override of the spectrogram extraction parameters (STFT, mel filterbank, log scaling).
+ A plain dict is coerced to [`~audio_utils.SpectrogramConfig`].
+ do_extract_spectrogram (`bool`, *optional*):
+ Whether to extract spectrogram features from the audio (otherwise padded raw waveforms are returned).
+ do_batch_spectrogram (`bool`, *optional*):
+ Whether to extract the spectrogram on the padded batch at once (`True`) or per waveform with
+ feature-level padding (`False`).
+ do_resample (`bool`, *optional*):
+ Whether to resample the input audio to the model's native `sampling_rate`.
padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*):
Select a strategy to pad the returned sequences (according to the model's padding side and padding
index) among:
@@ -405,24 +419,30 @@ class AudioKwargs(TypedDict, total=False):
pad_to_multiple_of (`int`, *optional*):
If set, will pad the sequence to a multiple of the provided value.
return_attention_mask (`bool`, *optional*):
- Whether or not [`~ASTFeatureExtractor.__call__`] should return `attention_mask`.
+ Whether the processor should return an attention/padding mask alongside the features.
return_tensors (`str` or [`~utils.TensorType`], *optional*):
If set, will return tensors of a particular framework. Acceptable values are:
- `'pt'`: Return PyTorch `torch.Tensor` objects.
- `'np'`: Return NumPy `np.ndarray` objects.
+ device (`str`, *optional*):
+ The device to use for processing (e.g. "cpu", "cuda"), only relevant for the torch backend.
load_audio_backend (`str`, *optional*):
Backend used by [`~audio_utils.load_audio`] to decode/resample audio referenced by URL/path
in `apply_chat_template`. One of `"auto"`, `"torchcodec"`, `"librosa"`, `"torchaudio"`.
"""
sampling_rate: Annotated[int | None, positive_int()]
- raw_speech: Union["np.ndarray", list[float], list["np.ndarray"], list[list[float]]] | None
+ spectrogram_config: dict | SpectrogramConfig | None
+ do_extract_spectrogram: bool | None
+ do_batch_spectrogram: bool | None
+ do_resample: bool | None
padding: Annotated[bool | str | PaddingStrategy | None, padding_validator()]
max_length: Annotated[int | None, positive_int()]
truncation: Annotated[bool | str | TruncationStrategy | None, truncation_validator()]
pad_to_multiple_of: Annotated[int | None, positive_int()]
return_attention_mask: bool | None
return_tensors: Annotated[str | TensorType | None, tensor_type_validator()]
+ device: Annotated[Union[str, "torch.device"] | None, device_validator()]
load_audio_backend: str | None
diff --git a/src/transformers/utils/auto_docstring.py b/src/transformers/utils/auto_docstring.py
index 3227c829423b..b4a9bee85143 100644
--- a/src/transformers/utils/auto_docstring.py
+++ b/src/transformers/utils/auto_docstring.py
@@ -2849,9 +2849,14 @@ def format_args_docstring(docstring: str, model_name: str) -> str:
placeholders_dict = get_placeholders_dict(placeholders, model_name)
# replace the placeholders in the docstring with the values from the placeholders_dict
for placeholder, value in placeholders_dict.items():
- if isinstance(value, dict) and placeholder == "image_processor_class":
- value = value.get("torchvision", value.get("pil", None))
- if placeholder is not None:
+ # Backend-keyed mapping values: image processors use {"torchvision": ..., "pil": ...};
+ # audio processors use {"torch": ..., "numpy": ...}. Resolve to the default backend's class.
+ if isinstance(value, dict):
+ if placeholder == "image_processor_class":
+ value = value.get("torchvision", value.get("pil"))
+ else:
+ value = value.get("torch", next(iter(value.values()), None))
+ if isinstance(value, str):
docstring = docstring.replace(f"{{{placeholder}}}", value)
return docstring
diff --git a/src/transformers/utils/deprecation.py b/src/transformers/utils/deprecation.py
index db0e67325d78..9b44e549df1b 100644
--- a/src/transformers/utils/deprecation.py
+++ b/src/transformers/utils/deprecation.py
@@ -33,6 +33,41 @@ class Action(ExplicitEnum):
RAISE = "raise"
+def deprecated_feature_extractor(audio_processor_class, old_class_name, version="5.5"):
+ """Create a deprecated FeatureExtractor alias for an AudioProcessor.
+
+ Uses dynamic class creation to reduce boilerplate across ~20 models.
+ """
+
+ def __init__(self, *args, **kwargs):
+ warnings.warn(
+ f"`{old_class_name}` is deprecated and will be removed in v{version}. "
+ f"Use `{audio_processor_class.__name__}` instead.",
+ FutureWarning,
+ stacklevel=2,
+ )
+ super(type(self), self).__init__(*args, **kwargs)
+
+ def __init_subclass__(cls, **kwargs):
+ warnings.warn(
+ f"`{old_class_name}` is deprecated and will be removed in v{version}. "
+ f"Use `{audio_processor_class.__name__}` instead.",
+ FutureWarning,
+ )
+ super(type(cls), cls).__init_subclass__(**kwargs)
+
+ return type(
+ old_class_name,
+ (audio_processor_class,),
+ {
+ "__init__": __init__,
+ "__init_subclass__": __init_subclass__,
+ "__module__": audio_processor_class.__module__,
+ "__doc__": f"Deprecated. Use {audio_processor_class.__name__} instead.",
+ },
+ )
+
+
def deprecate_kwarg(
old_name: str,
version: str,
diff --git a/tests/models/audio_spectrogram_transformer/test_audio_processing_audio_spectrogram_transformer.py b/tests/models/audio_spectrogram_transformer/test_audio_processing_audio_spectrogram_transformer.py
new file mode 100644
index 000000000000..0b41dbdb1aa7
--- /dev/null
+++ b/tests/models/audio_spectrogram_transformer/test_audio_processing_audio_spectrogram_transformer.py
@@ -0,0 +1,44 @@
+# Copyright 2026 HuggingFace Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+"""Tests for `AudioSpectrogramTransformerAudioProcessor` and its NumPy sibling."""
+
+from __future__ import annotations
+
+import unittest
+
+from transformers.models.auto.feature_extraction_auto import (
+ FEATURE_EXTRACTOR_MAPPING_NAMES,
+ feature_extractor_class_from_name,
+)
+from transformers.testing_utils import require_torch
+
+from ...test_audio_processing_common import AudioProcessingTestMixin
+
+
+class AudioSpectrogramTransformerAudioProcessingTester:
+ """Provides init kwargs and fixture parameters for the AST audio processor tests."""
+
+ sample_rate = 16000
+
+ def prepare_audio_processor_dict(self) -> dict:
+ return {}
+
+
+@require_torch
+class AudioSpectrogramTransformerAudioProcessingTest(AudioProcessingTestMixin, unittest.TestCase):
+ def setUp(self):
+ # AST is registered under `audio-spectrogram-transformer` (hyphenated) but the test
+ # directory uses underscores, so the mixin's auto-discovery cannot match.
+ self.audio_processor_tester = AudioSpectrogramTransformerAudioProcessingTester()
+ class_names_by_backend = FEATURE_EXTRACTOR_MAPPING_NAMES["audio-spectrogram-transformer"]
+ self.audio_processing_classes = {
+ backend: feature_extractor_class_from_name(class_name)
+ for backend, class_name in class_names_by_backend.items()
+ if class_name not in self.test_classes_to_skip
+ }
+ self.audio_processing_classes = {b: c for b, c in self.audio_processing_classes.items() if c is not None}
diff --git a/tests/models/audio_spectrogram_transformer/test_feature_extraction_audio_spectrogram_transformer.py b/tests/models/audio_spectrogram_transformer/test_feature_extraction_audio_spectrogram_transformer.py
deleted file mode 100644
index 66de4fd5b8b1..000000000000
--- a/tests/models/audio_spectrogram_transformer/test_feature_extraction_audio_spectrogram_transformer.py
+++ /dev/null
@@ -1,204 +0,0 @@
-# Copyright 2022 HuggingFace Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-import itertools
-import os
-import tempfile
-import unittest
-
-import numpy as np
-
-from transformers import ASTFeatureExtractor
-from transformers.testing_utils import check_json_file_has_correct_format, require_torch, require_torchaudio
-from transformers.utils.import_utils import is_torch_available
-
-from ...test_processing_common import floats_list
-from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin
-
-
-if is_torch_available():
- import torch
-
-
-class ASTFeatureExtractionTester:
- def __init__(
- self,
- parent,
- batch_size=7,
- min_seq_length=400,
- max_seq_length=2000,
- feature_size=1,
- padding_value=0.0,
- sampling_rate=16000,
- return_attention_mask=True,
- do_normalize=True,
- ):
- self.parent = parent
- self.batch_size = batch_size
- self.min_seq_length = min_seq_length
- self.max_seq_length = max_seq_length
- self.seq_length_diff = (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
- self.feature_size = feature_size
- self.padding_value = padding_value
- self.sampling_rate = sampling_rate
- self.return_attention_mask = return_attention_mask
- self.do_normalize = do_normalize
-
- def prepare_feat_extract_dict(self):
- return {
- "feature_size": self.feature_size,
- "padding_value": self.padding_value,
- "sampling_rate": self.sampling_rate,
- "return_attention_mask": self.return_attention_mask,
- "do_normalize": self.do_normalize,
- }
-
- def prepare_inputs_for_common(self, equal_length=False, numpify=False):
- def _flatten(list_of_lists):
- return list(itertools.chain(*list_of_lists))
-
- if equal_length:
- speech_inputs = floats_list((self.batch_size, self.max_seq_length))
- else:
- # make sure that inputs increase in size
- speech_inputs = [
- _flatten(floats_list((x, self.feature_size)))
- for x in range(self.min_seq_length, self.max_seq_length, self.seq_length_diff)
- ]
-
- if numpify:
- speech_inputs = [np.asarray(x) for x in speech_inputs]
-
- return speech_inputs
-
-
-@require_torch
-@require_torchaudio
-class ASTFeatureExtractionTest(SequenceFeatureExtractionTestMixin, unittest.TestCase):
- feature_extraction_class = ASTFeatureExtractor
-
- def setUp(self):
- self.feat_extract_tester = ASTFeatureExtractionTester(self)
-
- def test_call(self):
- # Tests that all call wrap to encode_plus and batch_encode_plus
- feat_extract = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- # create three inputs of length 800, 1000, and 1200
- speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)]
- np_speech_inputs = [np.asarray(speech_input) for speech_input in speech_inputs]
-
- # Test not batched input
- encoded_sequences_1 = feat_extract(speech_inputs[0], return_tensors="np").input_values
- encoded_sequences_2 = feat_extract(np_speech_inputs[0], return_tensors="np").input_values
- self.assertTrue(np.allclose(encoded_sequences_1, encoded_sequences_2, atol=1e-3))
-
- # Test batched
- encoded_sequences_1 = feat_extract(speech_inputs, padding=True, return_tensors="np").input_values
- encoded_sequences_2 = feat_extract(np_speech_inputs, padding=True, return_tensors="np").input_values
- for enc_seq_1, enc_seq_2 in zip(encoded_sequences_1, encoded_sequences_2):
- self.assertTrue(np.allclose(enc_seq_1, enc_seq_2, atol=1e-3))
-
- # Test 2-D numpy arrays are batched.
- speech_inputs = [floats_list((1, x))[0] for x in (800, 800, 800)]
- np_speech_inputs = np.asarray(speech_inputs)
- encoded_sequences_1 = feat_extract(speech_inputs, return_tensors="np").input_values
- encoded_sequences_2 = feat_extract(np_speech_inputs, return_tensors="np").input_values
- for enc_seq_1, enc_seq_2 in zip(encoded_sequences_1, encoded_sequences_2):
- self.assertTrue(np.allclose(enc_seq_1, enc_seq_2, atol=1e-3))
-
- @require_torch
- def test_double_precision_pad(self):
- import torch
-
- feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- np_speech_inputs = np.random.rand(100).astype(np.float64)
- py_speech_inputs = np_speech_inputs.tolist()
-
- for inputs in [py_speech_inputs, np_speech_inputs]:
- np_processed = feature_extractor.pad([{"input_values": inputs}], return_tensors="np")
- self.assertTrue(np_processed.input_values.dtype == np.float32)
- pt_processed = feature_extractor.pad([{"input_values": inputs}], return_tensors="pt")
- self.assertTrue(pt_processed.input_values.dtype == torch.float32)
-
- def _load_datasamples(self, num_samples):
- from datasets import load_dataset
-
- ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
- # automatic decoding with librispeech
- speech_samples = ds.sort("id")[:num_samples]["audio"]
-
- return [x["array"] for x in speech_samples]
-
- @require_torch
- def test_integration(self):
- # fmt: off
- EXPECTED_INPUT_VALUES = torch.tensor(
- [-0.9894, -1.2776, -0.9066, -1.2776, -0.9349, -1.2609, -1.0386, -1.2776,
- -1.1561, -1.2776, -1.2052, -1.2723, -1.2190, -1.2132, -1.2776, -1.1133,
- -1.1953, -1.1343, -1.1584, -1.2203, -1.1770, -1.2474, -1.2381, -1.1936,
- -0.9270, -0.8317, -0.8049, -0.7706, -0.7565, -0.7869]
- )
- # fmt: on
-
- input_speech = self._load_datasamples(1)
- feature_extractor = ASTFeatureExtractor()
- input_values = feature_extractor(input_speech, return_tensors="pt").input_values
- self.assertEqual(input_values.shape, (1, 1024, 128))
- torch.testing.assert_close(input_values[0, 0, :30], EXPECTED_INPUT_VALUES, rtol=1e-4, atol=1e-4)
-
- def test_feat_extract_from_and_save_pretrained(self):
- feat_extract_first = self.feature_extraction_class(**self.feat_extract_dict)
-
- with tempfile.TemporaryDirectory() as tmpdirname:
- saved_file = feat_extract_first.save_pretrained(tmpdirname)[0]
- check_json_file_has_correct_format(saved_file)
- feat_extract_second = self.feature_extraction_class.from_pretrained(tmpdirname)
-
- dict_first = feat_extract_first.to_dict()
- dict_second = feat_extract_second.to_dict()
- self.assertDictEqual(dict_first, dict_second)
-
- def test_feat_extract_to_json_file(self):
- feat_extract_first = self.feature_extraction_class(**self.feat_extract_dict)
-
- with tempfile.TemporaryDirectory() as tmpdirname:
- json_file_path = os.path.join(tmpdirname, "feat_extract.json")
- feat_extract_first.to_json_file(json_file_path)
- feat_extract_second = self.feature_extraction_class.from_json_file(json_file_path)
-
- dict_first = feat_extract_first.to_dict()
- dict_second = feat_extract_second.to_dict()
- self.assertEqual(dict_first, dict_second)
-
-
-# exact same tests than before, except that we simulate that torchaudio is not available
-@require_torch
-@unittest.mock.patch(
- "transformers.models.audio_spectrogram_transformer.feature_extraction_audio_spectrogram_transformer.is_speech_available",
- lambda: False,
-)
-class ASTFeatureExtractionWithoutTorchaudioTest(ASTFeatureExtractionTest):
- def test_using_audio_utils(self):
- # Tests that it uses audio_utils instead of torchaudio
- feat_extract = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
-
- self.assertTrue(hasattr(feat_extract, "window"))
- self.assertTrue(hasattr(feat_extract, "mel_filters"))
-
- from transformers.models.audio_spectrogram_transformer.feature_extraction_audio_spectrogram_transformer import (
- is_speech_available,
- )
-
- self.assertFalse(is_speech_available())
diff --git a/tests/models/clap/test_audio_processing_clap.py b/tests/models/clap/test_audio_processing_clap.py
new file mode 100644
index 000000000000..ef648e380f05
--- /dev/null
+++ b/tests/models/clap/test_audio_processing_clap.py
@@ -0,0 +1,37 @@
+# Copyright 2026 HuggingFace Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+"""Tests for `ClapAudioProcessor` and `ClapAudioProcessorNumpy`."""
+
+from __future__ import annotations
+
+import unittest
+
+from transformers.testing_utils import require_torch
+
+from ...test_audio_processing_common import AudioProcessingTestMixin
+
+
+class ClapAudioProcessingTester:
+ """Provides init kwargs and fixture parameters for the CLAP audio processor tests."""
+
+ sample_rate = 48000
+
+ def prepare_audio_processor_dict(self) -> dict:
+ return {}
+
+
+@require_torch
+class ClapAudioProcessingTest(AudioProcessingTestMixin, unittest.TestCase):
+ # CLAP's full-batch padded mel + 48 kHz STFT accumulates a slightly larger cross-backend
+ # divergence than the strict float32 noise floor — empirically up to ~8e-5 on batched inputs.
+ parity_atol = 1e-4
+ parity_rtol = 1e-4
+
+ def setUp(self):
+ self.audio_processor_tester = ClapAudioProcessingTester()
+ super().setUp()
diff --git a/tests/models/clap/test_feature_extraction_clap.py b/tests/models/clap/test_feature_extraction_clap.py
deleted file mode 100644
index 4cd73372dd87..000000000000
--- a/tests/models/clap/test_feature_extraction_clap.py
+++ /dev/null
@@ -1,529 +0,0 @@
-# Copyright 2023 HuggingFace Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-import itertools
-import unittest
-
-import numpy as np
-from datasets import load_dataset
-
-from transformers import ClapFeatureExtractor
-from transformers.testing_utils import require_torch, require_torchaudio
-from transformers.trainer_utils import set_seed
-from transformers.utils.import_utils import is_torch_available
-
-from ...test_processing_common import floats_list
-from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin
-
-
-if is_torch_available():
- import torch
-
-
-@require_torch
-@require_torchaudio
-# Copied from tests.models.whisper.test_feature_extraction_whisper.WhisperFeatureExtractionTester with Whisper->Clap
-class ClapFeatureExtractionTester:
- def __init__(
- self,
- parent,
- batch_size=7,
- min_seq_length=400,
- max_seq_length=2000,
- feature_size=10,
- hop_length=160,
- chunk_length=8,
- padding_value=0.0,
- sampling_rate=4_000,
- return_attention_mask=False,
- do_normalize=True,
- ):
- self.parent = parent
- self.batch_size = batch_size
- self.min_seq_length = min_seq_length
- self.max_seq_length = max_seq_length
- self.seq_length_diff = (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
- self.padding_value = padding_value
- self.sampling_rate = sampling_rate
- self.return_attention_mask = return_attention_mask
- self.do_normalize = do_normalize
- self.feature_size = feature_size
- self.chunk_length = chunk_length
- self.hop_length = hop_length
-
- def prepare_feat_extract_dict(self):
- return {
- "feature_size": self.feature_size,
- "hop_length": self.hop_length,
- "chunk_length": self.chunk_length,
- "padding_value": self.padding_value,
- "sampling_rate": self.sampling_rate,
- "return_attention_mask": self.return_attention_mask,
- "do_normalize": self.do_normalize,
- }
-
- def prepare_inputs_for_common(self, equal_length=False, numpify=False):
- def _flatten(list_of_lists):
- return list(itertools.chain(*list_of_lists))
-
- if equal_length:
- speech_inputs = [floats_list((self.max_seq_length, self.feature_size)) for _ in range(self.batch_size)]
- else:
- # make sure that inputs increase in size
- speech_inputs = [
- floats_list((x, self.feature_size))
- for x in range(self.min_seq_length, self.max_seq_length, self.seq_length_diff)
- ]
- if numpify:
- speech_inputs = [np.asarray(x) for x in speech_inputs]
- return speech_inputs
-
-
-@require_torch
-@require_torchaudio
-class ClapFeatureExtractionTest(SequenceFeatureExtractionTestMixin, unittest.TestCase):
- feature_extraction_class = ClapFeatureExtractor
-
- # Copied from tests.models.whisper.test_feature_extraction_whisper.WhisperFeatureExtractionTest.setUp with Whisper->Clap
- def setUp(self):
- self.feat_extract_tester = ClapFeatureExtractionTester(self)
-
- def test_call(self):
- # Tests that all call wrap to encode_plus and batch_encode_plus
- feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- # create three inputs of length 800, 1000, and 1200
- speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)]
- np_speech_inputs = [np.asarray(speech_input) for speech_input in speech_inputs]
-
- # Test feature size
- input_features = feature_extractor(np_speech_inputs, padding="max_length", return_tensors="np").input_features
- self.assertTrue(input_features.ndim == 4)
-
- # Test not batched input
- encoded_sequences_1 = feature_extractor(speech_inputs[0], return_tensors="np").input_features
- encoded_sequences_2 = feature_extractor(np_speech_inputs[0], return_tensors="np").input_features
- self.assertTrue(np.allclose(encoded_sequences_1, encoded_sequences_2, atol=1e-3))
-
- # Test batched
- encoded_sequences_1 = feature_extractor(speech_inputs, return_tensors="np").input_features
- encoded_sequences_2 = feature_extractor(np_speech_inputs, return_tensors="np").input_features
- for enc_seq_1, enc_seq_2 in zip(encoded_sequences_1, encoded_sequences_2):
- self.assertTrue(np.allclose(enc_seq_1, enc_seq_2, atol=1e-3))
-
- # Test 2-D numpy arrays are batched.
- speech_inputs = [floats_list((1, x))[0] for x in (800, 800, 800)]
- np_speech_inputs = np.asarray(speech_inputs)
- encoded_sequences_1 = feature_extractor(speech_inputs, return_tensors="np").input_features
- encoded_sequences_2 = feature_extractor(np_speech_inputs, return_tensors="np").input_features
- for enc_seq_1, enc_seq_2 in zip(encoded_sequences_1, encoded_sequences_2):
- self.assertTrue(np.allclose(enc_seq_1, enc_seq_2, atol=1e-3))
-
- # Copied from tests.models.whisper.test_feature_extraction_whisper.WhisperFeatureExtractionTest.test_double_precision_pad
- def test_double_precision_pad(self):
- import torch
-
- feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- np_speech_inputs = np.random.rand(100, 32).astype(np.float64)
- py_speech_inputs = np_speech_inputs.tolist()
-
- for inputs in [py_speech_inputs, np_speech_inputs]:
- np_processed = feature_extractor.pad([{"input_features": inputs}], return_tensors="np")
- self.assertTrue(np_processed.input_features.dtype == np.float32)
- pt_processed = feature_extractor.pad([{"input_features": inputs}], return_tensors="pt")
- self.assertTrue(pt_processed.input_features.dtype == torch.float32)
-
- # Copied from tests.models.whisper.test_feature_extraction_whisper.WhisperFeatureExtractionTest._load_datasamples
- def _load_datasamples(self, num_samples):
- ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
- # automatic decoding with librispeech
- speech_samples = ds.sort("id")[:num_samples]["audio"]
-
- return [x["array"] for x in speech_samples]
-
- def test_integration_fusion_short_input(self):
- # fmt: off
- EXPECTED_INPUT_FEATURES = torch.tensor(
- [
- [
- # "repeat"
- [
- -20.1049, -19.9764, -20.0731, -19.5055, -27.5018, -22.5761, -26.6071,
- -29.0091, -26.4659, -26.4236, -28.8808, -31.9190, -32.4848, -34.1186,
- -34.0340, -32.8803, -30.9895, -37.6238, -38.0347, -40.6263, -36.3496,
- -42.2533, -32.9132, -27.7068, -29.3704, -30.3208, -22.5972, -27.1494,
- -30.1975, -31.1005, -29.9372, -27.1917, -25.9806, -30.3489, -33.2380,
- -31.9062, -36.5498, -32.8721, -30.5629, -27.4674, -22.2232, -22.5653,
- -16.3868, -17.2713, -25.9738, -30.6256, -34.3766, -31.1292, -27.8950,
- -27.0588, -25.6206, -23.0712, -26.6050, -28.0112, -32.6847, -34.3396,
- -34.9738, -35.8463, -39.2324, -37.1188, -33.3705, -28.9230, -28.9112,
- -28.6578
- ],
- [
- -36.7233, -30.0587, -24.8431, -18.4611, -16.8149, -23.9319, -32.8580,
- -34.2264, -27.4332, -26.8027, -29.2721, -33.9033, -39.3403, -35.3232,
- -26.8076, -28.6460, -35.2780, -36.0738, -35.4996, -37.7631, -39.5056,
- -34.7112, -36.8741, -34.1066, -32.9474, -33.6604, -27.9937, -30.9594,
- -26.2928, -32.0485, -29.2151, -29.2917, -32.7308, -29.6542, -31.1454,
- -37.0088, -32.3388, -37.3086, -31.1024, -27.2889, -19.6788, -21.1488,
- -19.5144, -14.8889, -21.2006, -24.7488, -27.7940, -31.1058, -27.5068,
- -21.5737, -22.3780, -21.5151, -26.3086, -30.9223, -33.5043, -32.0307,
- -37.3806, -41.6188, -45.6650, -40.5131, -32.5023, -26.7385, -26.3709,
- -26.7761
- ]
- ],
- [
- # "repeatpad"
- [
- -25.7496, -24.9339, -24.1357, -23.1271, -23.7853, -26.1264, -29.1456,
- -33.2060, -37.8179, -42.4833, -41.9386, -41.2164, -42.3566, -44.2575,
- -40.0217, -36.6794, -36.6974, -38.7819, -42.0880, -45.5560, -39.9368,
- -36.3219, -35.5981, -36.6434, -35.1851, -33.0684, -30.0437, -30.2010,
- -34.3476, -42.1373, -38.8039, -37.3355, -40.4576, -41.0485, -40.6377,
- -38.2275, -42.7481, -34.6084, -34.7048, -29.5149, -26.3935, -26.8952,
- -34.1336, -26.2904, -28.2571, -32.5642, -36.7240, -35.5334, -38.2451,
- -34.8177, -28.9754, -25.1096, -27.9768, -32.3184, -37.0269, -40.5136,
- -40.8061, -36.4948, -40.3767, -38.9671, -38.3552, -34.1250, -30.9035,
- -31.6112
- ],
- [
- -100., -100., -100., -100., -100., -100., -100., -100., -100., -100.,
- -100., -100., -100., -100., -100., -100., -100., -100., -100., -100.,
- -100., -100., -100., -100., -100., -100., -100., -100., -100., -100.,
- -100., -100., -100., -100., -100., -100., -100., -100., -100., -100.,
- -100., -100., -100., -100., -100., -100., -100., -100., -100., -100.,
- -100., -100., -100., -100., -100., -100., -100., -100., -100., -100.,
- -100., -100., -100., -100.
- ]
- ],
- [
- # None, same as "repeatpad"
- [
- -25.7496, -24.9339, -24.1357, -23.1271, -23.7853, -26.1264, -29.1456,
- -33.2060, -37.8179, -42.4833, -41.9386, -41.2164, -42.3566, -44.2575,
- -40.0217, -36.6794, -36.6974, -38.7819, -42.0880, -45.5560, -39.9368,
- -36.3219, -35.5981, -36.6434, -35.1851, -33.0684, -30.0437, -30.2010,
- -34.3476, -42.1373, -38.8039, -37.3355, -40.4576, -41.0485, -40.6377,
- -38.2275, -42.7481, -34.6084, -34.7048, -29.5149, -26.3935, -26.8952,
- -34.1336, -26.2904, -28.2571, -32.5642, -36.7240, -35.5334, -38.2451,
- -34.8177, -28.9754, -25.1096, -27.9768, -32.3184, -37.0269, -40.5136,
- -40.8061, -36.4948, -40.3767, -38.9671, -38.3552, -34.1250, -30.9035,
- -31.6112
- ],
- [
- -100., -100., -100., -100., -100., -100., -100., -100., -100., -100.,
- -100., -100., -100., -100., -100., -100., -100., -100., -100., -100.,
- -100., -100., -100., -100., -100., -100., -100., -100., -100., -100.,
- -100., -100., -100., -100., -100., -100., -100., -100., -100., -100.,
- -100., -100., -100., -100., -100., -100., -100., -100., -100., -100.,
- -100., -100., -100., -100., -100., -100., -100., -100., -100., -100.,
- -100., -100., -100., -100.
- ]
- ],
- [
- # "pad"
- [
- -58.5260, -58.1155, -57.8623, -57.5059, -57.9178, -58.7171, -59.2343,
- -59.9833, -60.9764, -62.0722, -63.5723, -65.7111, -67.5153, -68.7088,
- -69.8325, -70.2987, -70.1548, -70.6233, -71.5702, -72.5159, -72.3821,
- -70.1817, -67.0315, -64.1387, -62.2202, -61.0717, -60.4951, -61.6005,
- -63.7358, -67.1400, -67.6185, -65.5635, -64.3593, -63.7138, -63.6209,
- -66.4950, -72.6284, -63.3961, -56.8334, -52.7319, -50.6310, -51.3728,
- -53.5619, -51.9190, -50.9708, -52.8684, -55.8073, -58.8227, -60.6991,
- -57.0547, -52.7611, -51.4388, -54.4892, -60.8950, -66.1024, -72.4352,
- -67.8538, -65.1463, -68.7588, -72.3080, -68.4864, -60.4688, -57.1516,
- -60.9460
- ],
- [
- -100., -100., -100., -100., -100., -100., -100., -100., -100., -100.,
- -100., -100., -100., -100., -100., -100., -100., -100., -100., -100.,
- -100., -100., -100., -100., -100., -100., -100., -100., -100., -100.,
- -100., -100., -100., -100., -100., -100., -100., -100., -100., -100.,
- -100., -100., -100., -100., -100., -100., -100., -100., -100., -100.,
- -100., -100., -100., -100., -100., -100., -100., -100., -100., -100.,
- -100., -100., -100., -100.
- ]
- ]
- ]
- )
- # fmt: on
- MEL_BIN = [[976, 977], [976, 977], [976, 977], [196, 197]]
- input_speech = self._load_datasamples(1)
- feature_extractor = ClapFeatureExtractor()
- for padding, EXPECTED_VALUES, idx_in_mel in zip(
- ["repeat", "repeatpad", None, "pad"], EXPECTED_INPUT_FEATURES, MEL_BIN
- ):
- input_features = feature_extractor(input_speech, return_tensors="pt", padding=padding).input_features
- self.assertEqual(input_features.shape, (1, 4, 1001, 64))
-
- torch.testing.assert_close(input_features[0, 0, idx_in_mel[0]], EXPECTED_VALUES[0], rtol=1e-4, atol=1e-4)
- torch.testing.assert_close(input_features[0, 0, idx_in_mel[1]], EXPECTED_VALUES[1], rtol=1e-4, atol=1e-4)
-
- self.assertTrue(torch.all(input_features[0, 0] == input_features[0, 1]))
- self.assertTrue(torch.all(input_features[0, 0] == input_features[0, 2]))
- self.assertTrue(torch.all(input_features[0, 0] == input_features[0, 3]))
-
- def test_integration_rand_trunc_short_input(self):
- # fmt: off
- EXPECTED_INPUT_FEATURES = torch.tensor(
- [
- [
- # "repeat"
- [
- -35.0483, -35.7865, -38.2884, -40.0220, -42.5349, -44.9489, -43.2228,
- -44.6499, -47.6253, -49.6983, -50.2127, -52.5483, -52.2223, -51.9157,
- -49.4082, -51.2024, -57.0476, -56.2803, -58.1618, -60.7474, -55.0389,
- -60.9514, -59.3080, -50.4419, -47.8172, -48.7570, -55.2552, -44.5036,
- -44.1148, -50.8218, -51.0968, -52.9408, -51.1037, -48.9789, -47.5897,
- -52.0915, -55.4216, -54.1529, -58.0149, -58.0866, -52.7798, -52.6154,
- -45.9144, -46.2008, -40.7603, -41.1703, -50.2250, -55.4112, -59.4818,
- -54.5795, -53.5552, -51.3668, -49.8358, -50.3186, -54.0452, -57.6030,
- -61.1589, -61.6415, -63.2756, -66.5890, -62.8543, -58.0665, -56.7203,
- -56.7632
- ],
- [
- -47.1320, -37.9961, -34.0076, -36.7109, -47.9057, -48.4924, -43.8371,
- -44.9728, -48.1689, -52.9141, -57.6077, -52.8520, -44.8502, -45.6764,
- -51.8389, -56.4284, -54.6972, -53.4889, -55.6077, -58.7149, -60.3760,
- -54.0136, -56.0730, -55.9870, -54.4017, -53.1094, -53.5640, -50.3064,
- -49.9520, -49.3239, -48.1668, -53.4852, -50.4561, -50.8688, -55.1970,
- -51.5538, -53.0260, -59.6933, -54.8183, -59.5895, -55.9589, -50.3761,
- -44.1282, -44.1463, -43.8540, -39.1168, -45.3893, -49.5542, -53.1505,
- -55.2870, -50.3921, -46.8511, -47.4444, -49.5633, -56.0034, -59.0815,
- -59.0018, -63.7589, -69.5745, -71.5789, -64.0498, -56.0558, -54.3475,
- -54.7004
- ]
- ],
- [
- # "repeatpad"
- [
- -40.3184, -39.7186, -39.8807, -41.6508, -45.3613, -50.4785, -57.0297,
- -60.4944, -59.1642, -58.9495, -60.4661, -62.5300, -58.4759, -55.2865,
- -54.8973, -56.0780, -57.5482, -59.6557, -64.3309, -65.0330, -59.4941,
- -56.8552, -55.0519, -55.9817, -56.9739, -55.2827, -54.5312, -51.4141,
- -50.4289, -51.9131, -57.5821, -63.9979, -59.9180, -58.9489, -62.3247,
- -62.6975, -63.7948, -60.5250, -64.6107, -58.7905, -57.0229, -54.3084,
- -49.8445, -50.4459, -57.0172, -50.6425, -52.5992, -57.4207, -61.6358,
- -60.6540, -63.1968, -57.4360, -52.3263, -51.7695, -57.1946, -62.9610,
- -66.7359, -67.0335, -63.7440, -68.1775, -66.3798, -62.8650, -59.8972,
- -59.3139
- ],
- [
- -100., -100., -100., -100., -100., -100., -100., -100., -100., -100.,
- -100., -100., -100., -100., -100., -100., -100., -100., -100., -100.,
- -100., -100., -100., -100., -100., -100., -100., -100., -100., -100.,
- -100., -100., -100., -100., -100., -100., -100., -100., -100., -100.,
- -100., -100., -100., -100., -100., -100., -100., -100., -100., -100.,
- -100., -100., -100., -100., -100., -100., -100., -100., -100., -100.,
- -100., -100., -100., -100.
- ]
- ],
- [
- # None, same as "repeatpad"
- [
- -40.3184, -39.7186, -39.8807, -41.6508, -45.3613, -50.4785, -57.0297,
- -60.4944, -59.1642, -58.9495, -60.4661, -62.5300, -58.4759, -55.2865,
- -54.8973, -56.0780, -57.5482, -59.6557, -64.3309, -65.0330, -59.4941,
- -56.8552, -55.0519, -55.9817, -56.9739, -55.2827, -54.5312, -51.4141,
- -50.4289, -51.9131, -57.5821, -63.9979, -59.9180, -58.9489, -62.3247,
- -62.6975, -63.7948, -60.5250, -64.6107, -58.7905, -57.0229, -54.3084,
- -49.8445, -50.4459, -57.0172, -50.6425, -52.5992, -57.4207, -61.6358,
- -60.6540, -63.1968, -57.4360, -52.3263, -51.7695, -57.1946, -62.9610,
- -66.7359, -67.0335, -63.7440, -68.1775, -66.3798, -62.8650, -59.8972,
- -59.3139
- ],
- [
- -100., -100., -100., -100., -100., -100., -100., -100., -100., -100.,
- -100., -100., -100., -100., -100., -100., -100., -100., -100., -100.,
- -100., -100., -100., -100., -100., -100., -100., -100., -100., -100.,
- -100., -100., -100., -100., -100., -100., -100., -100., -100., -100.,
- -100., -100., -100., -100., -100., -100., -100., -100., -100., -100.,
- -100., -100., -100., -100., -100., -100., -100., -100., -100., -100.,
- -100., -100., -100., -100.
- ]
- ],
- [
- # "pad"
- [
- -73.3190, -73.6349, -74.1451, -74.8539, -75.7476, -76.5438, -78.5540,
- -80.1339, -81.8911, -83.7560, -85.5387, -86.7466, -88.2072, -88.6090,
- -88.8243, -89.0784, -89.4364, -89.8179, -91.3146, -92.2833, -91.7221,
- -90.9440, -88.1315, -86.2425, -84.2281, -82.4893, -81.5993, -81.1328,
- -81.5759, -83.1068, -85.6525, -88.9520, -88.9187, -87.2703, -86.3052,
- -85.7188, -85.8802, -87.9996, -95.0464, -88.0133, -80.8561, -76.5597,
- -74.2816, -74.8109, -77.3615, -76.0719, -75.3426, -77.6428, -80.9663,
- -84.5275, -84.9907, -80.5205, -77.2851, -78.6259, -84.7740, -91.4535,
- -98.1894, -94.3872, -92.3735, -97.6807, -98.1501, -91.4344, -85.2842,
- -88.4338
- ],
- [
- -100., -100., -100., -100., -100., -100., -100., -100., -100., -100.,
- -100., -100., -100., -100., -100., -100., -100., -100., -100., -100.,
- -100., -100., -100., -100., -100., -100., -100., -100., -100., -100.,
- -100., -100., -100., -100., -100., -100., -100., -100., -100., -100.,
- -100., -100., -100., -100., -100., -100., -100., -100., -100., -100.,
- -100., -100., -100., -100., -100., -100., -100., -100., -100., -100.,
- -100., -100., -100., -100.
- ]
- ]
- ]
- )
- # fmt: on
- MEL_BIN = [[976, 977], [976, 977], [976, 977], [196, 197]]
- input_speech = self._load_datasamples(1)
- feature_extractor = ClapFeatureExtractor()
- for padding, EXPECTED_VALUES, idx_in_mel in zip(
- ["repeat", "repeatpad", None, "pad"], EXPECTED_INPUT_FEATURES, MEL_BIN
- ):
- input_features = feature_extractor(
- input_speech, return_tensors="pt", truncation="rand_trunc", padding=padding
- ).input_features
- self.assertEqual(input_features.shape, (1, 1, 1001, 64))
- torch.testing.assert_close(input_features[0, 0, idx_in_mel[0]], EXPECTED_VALUES[0], rtol=1e-4, atol=1e-4)
- torch.testing.assert_close(input_features[0, 0, idx_in_mel[1]], EXPECTED_VALUES[1], rtol=1e-4, atol=1e-4)
-
- def test_integration_fusion_long_input(self):
- # fmt: off
- EXPECTED_INPUT_FEATURES = torch.tensor(
- [
- [
- -11.1830, -10.1894, -8.6051, -4.8578, -1.3268, -8.4606, -14.5453,
- -9.2017, 0.5781, 16.2129, 14.8289, 3.6326, -3.8794, -6.5544,
- -2.4408, 1.9531, 6.0967, 1.7590, -7.6730, -6.1571, 2.0052,
- 16.6694, 20.6447, 21.2145, 13.4972, 15.9043, 16.8987, 4.1766,
- 11.9428, 21.2372, 12.3016, 4.8604, 6.7241, 1.8543, 4.9235,
- 5.3188, -0.9897, -1.2416, -6.5864, 2.9529, 2.9274, 6.4753,
- 10.2300, 11.2127, 3.4042, -1.0055, -6.0475, -6.7524, -3.9801,
- -1.4434, 0.4740, -0.1584, -4.5457, -8.5746, -8.8428, -13.1475,
- -9.6079, -8.5798, -4.1143, -3.7966, -7.1651, -6.1517, -8.0258,
- -12.1486
- ],
- [
- -10.2017, -7.9924, -5.9517, -3.9372, -1.9735, -4.3130, 16.1647,
- 25.0592, 23.5532, 14.4974, -7.0778, -10.2262, 6.4782, 20.3454,
- 19.4269, 1.7976, -16.5070, 4.9380, 12.3390, 6.9285, -13.6325,
- -8.5298, 1.0839, -5.9629, -8.4812, 3.1331, -2.0963, -16.6046,
- -14.0070, -17.5707, -13.2080, -17.2168, -17.7770, -12.1111, -18.6184,
- -17.1897, -13.9801, -12.0426, -23.5400, -25.6823, -23.5813, -18.7847,
- -20.5473, -25.6458, -19.7585, -27.6007, -28.9276, -24.8948, -25.4458,
- -22.2807, -19.6613, -19.2669, -15.7813, -19.6821, -24.3439, -22.2598,
- -28.2631, -30.1017, -32.7646, -33.6525, -27.5639, -22.0548, -27.8054,
- -29.6947
- ],
- [
- -9.2078, -7.2963, -6.2095, -7.9959, -2.9280, -11.1843, -6.1490,
- 5.0733, 19.2957, 21.4578, 14.6803, -3.3153, -6.3334, -2.3542,
- 6.9509, 15.2965, 14.6620, 5.2075, -0.0873, 1.1919, 18.1986,
- 20.8470, 10.8035, 2.2516, 7.6905, 7.7427, -1.2543, -5.0018,
- 0.9809, -2.1584, -5.4580, -5.4760, -11.8888, -9.0605, -8.4638,
- -9.9897, -0.0540, -5.1629, 0.0483, -4.1504, -4.8140, -7.8236,
- -9.0622, -10.1742, -8.9597, -11.5380, -16.5603, -17.1858, -17.5032,
- -20.9326, -23.9543, -25.2602, -25.3429, -27.4536, -26.8859, -22.7852,
- -25.8288, -24.8399, -23.8893, -24.2096, -26.5415, -23.7281, -25.6851,
- -22.3629
- ],
- [
- 1.3448, 2.9883, 4.0366, -0.8019, -10.4191, -10.0883, -4.3812,
- 0.8136, 2.1579, 0.0832, 1.0949, -0.9759, -5.5319, -4.6009,
- -6.5452, -14.9155, -20.1584, -9.3611, -2.4271, 1.4031, 4.9910,
- 8.6916, 8.6785, 10.1973, 9.9029, 5.3840, 7.5336, 5.2803,
- 2.8144, -0.3138, 2.2216, 5.7328, 7.5574, 7.7402, 1.0681,
- 3.1049, 7.0742, 6.5588, 7.3712, 5.7881, 8.6874, 8.7725,
- 2.8133, -4.5809, -6.1317, -5.1719, -5.0192, -9.0977, -10.9391,
- -6.0769, 1.6016, -0.8965, -7.2252, -7.8632, -11.4468, -11.7446,
- -10.7447, -7.0601, -2.7748, -4.1798, -2.8433, -3.1352, 0.8097,
- 6.4212
- ]
- ]
- )
- # fmt: on
- MEL_BIN = 963
- input_speech = torch.cat([torch.tensor(x) for x in self._load_datasamples(5)])
- feature_extractor = ClapFeatureExtractor()
- for padding, EXPECTED_VALUES, block_idx in zip(
- ["repeat", "repeatpad", None, "pad"], EXPECTED_INPUT_FEATURES, [1, 2, 0, 3]
- ):
- set_seed(987654321)
- input_features = feature_extractor(input_speech, return_tensors="pt", padding=padding).input_features
- self.assertEqual(input_features.shape, (1, 4, 1001, 64))
- torch.testing.assert_close(input_features[0, block_idx, MEL_BIN], EXPECTED_VALUES, rtol=1e-3, atol=1e-3)
-
- def test_integration_rand_trunc_long_input(self):
- # fmt: off
- EXPECTED_INPUT_FEATURES = torch.tensor(
- [
- [
- -35.4022, -32.7555, -31.2004, -32.7764, -42.5770, -41.6339, -43.1630,
- -44.5080, -44.3029, -48.9628, -39.5022, -39.2105, -43.1350, -43.2195,
- -48.4894, -52.2344, -57.6891, -52.2228, -45.5155, -44.2893, -43.4697,
- -46.6702, -43.7490, -40.4819, -42.7275, -46.3434, -46.8412, -41.2003,
- -43.1681, -46.2948, -46.1925, -47.8333, -45.6812, -44.9182, -41.7786,
- -43.3809, -44.3199, -42.8814, -45.4771, -46.7114, -46.9746, -42.7090,
- -41.6057, -38.3965, -40.1980, -41.0263, -34.1256, -28.3289, -29.0201,
- -30.4453, -29.5561, -30.1734, -25.9406, -19.0897, -15.8452, -20.1351,
- -23.6515, -23.1194, -17.1845, -19.4399, -23.6527, -22.8768, -20.7279,
- -22.7864
- ],
- [
- -35.7719, -27.2566, -23.6964, -27.5521, 0.2510, 7.4391, 1.3917,
- -13.3417, -28.1758, -17.0856, -5.7723, -0.8000, -7.8832, -15.5548,
- -30.5935, -24.7571, -13.7009, -10.3432, -21.2464, -24.8118, -19.4080,
- -14.9779, -11.7991, -18.4485, -20.1982, -17.3652, -20.6328, -28.2967,
- -25.7819, -21.8962, -28.5083, -29.5719, -30.2120, -35.7033, -31.8218,
- -34.0408, -37.7744, -33.9653, -31.3009, -30.9063, -28.6153, -32.2202,
- -28.5456, -28.8579, -32.5170, -37.9152, -43.0052, -46.4849, -44.0786,
- -39.1933, -33.2757, -31.6313, -42.6386, -52.3679, -53.5785, -55.6444,
- -47.0050, -47.6459, -56.6361, -60.6781, -61.5244, -55.8272, -60.4832,
- -58.1897
- ],
- [
- -38.2686, -36.6285, -32.5835, -35.1693, -37.7938, -37.4035, -35.3132,
- -35.6083, -36.3609, -40.9472, -36.7846, -36.1544, -38.9076, -39.3618,
- -35.4953, -34.2809, -39.9466, -39.7433, -34.8347, -37.5674, -41.5689,
- -38.9161, -34.3947, -30.2924, -30.4841, -34.5831, -28.9261, -24.8849,
- -31.2324, -27.1622, -27.2107, -25.9385, -30.1691, -30.9223, -23.9495,
- -25.6047, -26.7119, -28.5523, -27.7481, -32.8427, -35.4650, -31.0399,
- -31.2073, -30.5163, -22.9819, -20.8892, -19.2510, -24.7905, -28.9426,
- -28.1998, -26.7386, -25.0140, -27.9223, -32.9913, -33.1864, -34.9742,
- -38.5995, -39.6990, -29.3203, -22.4697, -25.6415, -33.5608, -33.0945,
- -27.1716
- ],
- [
- -33.2015, -28.7741, -21.9457, -23.4888, -32.1072, -8.6307, 3.2724,
- 5.9157, -0.9221, -30.1814, -31.0015, -27.4508, -27.0477, -9.5342,
- 0.3221, 0.6511, -7.1596, -25.9707, -32.8924, -32.2300, -13.8974,
- -0.4895, 0.9168, -10.7663, -27.1176, -35.0829, -11.6859, -4.8855,
- -11.8898, -26.6167, -5.6192, -3.8443, -19.7947, -14.4101, -8.6236,
- -21.2458, -21.0801, -17.9136, -24.4663, -18.6333, -24.8085, -15.5854,
- -15.4344, -11.5046, -22.3625, -27.3387, -32.4353, -30.9670, -31.3789,
- -35.4044, -34.4591, -25.2433, -28.0773, -33.8736, -33.0224, -33.3155,
- -38.5302, -39.2741, -36.6395, -34.7729, -32.4483, -42.4001, -49.2857,
- -39.1682
- ]
- ]
- )
- # fmt: on
- MEL_BIN = 963
- SEEDS = [987654321, 1234, 666, 5555]
- input_speech = torch.cat([torch.tensor(x) for x in self._load_datasamples(5)])
- feature_extractor = ClapFeatureExtractor()
- for padding, EXPECTED_VALUES, seed in zip(
- ["repeat", "repeatpad", None, "pad"], EXPECTED_INPUT_FEATURES, SEEDS
- ):
- set_seed(seed)
- input_features = feature_extractor(
- input_speech, return_tensors="pt", truncation="rand_trunc", padding=padding
- ).input_features
- self.assertEqual(input_features.shape, (1, 1, 1001, 64))
- torch.testing.assert_close(input_features[0, 0, MEL_BIN], EXPECTED_VALUES, rtol=1e-4, atol=1e-4)
diff --git a/tests/models/clap/test_processing_clap.py b/tests/models/clap/test_processing_clap.py
index dac375cb2a49..ae522cf2b220 100644
--- a/tests/models/clap/test_processing_clap.py
+++ b/tests/models/clap/test_processing_clap.py
@@ -20,7 +20,7 @@
from transformers.testing_utils import require_sentencepiece, require_torchaudio
from transformers.tokenization_utils_tokenizers import TokenizersBackend
-from .test_feature_extraction_clap import floats_list
+from ...test_processing_common import floats_list
@require_torchaudio
diff --git a/tests/models/clvp/test_audio_processing_clvp.py b/tests/models/clvp/test_audio_processing_clvp.py
new file mode 100644
index 000000000000..c433134e2b24
--- /dev/null
+++ b/tests/models/clvp/test_audio_processing_clvp.py
@@ -0,0 +1,37 @@
+# Copyright 2026 HuggingFace Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+"""Tests for `ClvpAudioProcessor` and `ClvpAudioProcessorNumpy`."""
+
+from __future__ import annotations
+
+import unittest
+
+from transformers.testing_utils import require_torch
+
+from ...test_audio_processing_common import AudioProcessingTestMixin
+
+
+class ClvpAudioProcessingTester:
+ """Provides init kwargs and fixture parameters for the CLVP audio processor tests."""
+
+ sample_rate = 22050
+
+ def prepare_audio_processor_dict(self) -> dict:
+ return {}
+
+
+@require_torch
+class ClvpAudioProcessingTest(AudioProcessingTestMixin, unittest.TestCase):
+ # CLVP's float64 log + per-mel-norm division before float32 cast accumulates a small
+ # cross-backend drift above the strict 1e-5 floor — empirically up to ~4e-5.
+ parity_atol = 1e-4
+ parity_rtol = 1e-4
+
+ def setUp(self):
+ self.audio_processor_tester = ClvpAudioProcessingTester()
+ super().setUp()
diff --git a/tests/models/clvp/test_feature_extraction_clvp.py b/tests/models/clvp/test_feature_extraction_clvp.py
deleted file mode 100644
index fc2e294ebf72..000000000000
--- a/tests/models/clvp/test_feature_extraction_clvp.py
+++ /dev/null
@@ -1,223 +0,0 @@
-# Copyright 2023 HuggingFace Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import itertools
-import os
-import tempfile
-import unittest
-
-import numpy as np
-from datasets import Audio, load_dataset
-
-from transformers import ClvpFeatureExtractor
-from transformers.testing_utils import (
- check_json_file_has_correct_format,
- cleanup,
- require_torch,
- slow,
- torch_device,
-)
-from transformers.utils.import_utils import is_torch_available
-
-from ...test_processing_common import floats_list
-from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin
-
-
-if is_torch_available():
- import torch
-
-
-@require_torch
-class ClvpFeatureExtractionTester:
- def __init__(
- self,
- parent,
- batch_size=7,
- min_seq_length=400,
- max_seq_length=2000,
- feature_size=10,
- hop_length=160,
- chunk_length=8,
- padding_value=0.0,
- sampling_rate=4_000,
- return_attention_mask=False,
- ):
- self.parent = parent
- self.batch_size = batch_size
- self.min_seq_length = min_seq_length
- self.max_seq_length = max_seq_length
- self.seq_length_diff = (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
- self.padding_value = padding_value
- self.sampling_rate = sampling_rate
- self.return_attention_mask = return_attention_mask
- self.feature_size = feature_size
- self.chunk_length = chunk_length
- self.hop_length = hop_length
-
- def prepare_feat_extract_dict(self):
- return {
- "feature_size": self.feature_size,
- "hop_length": self.hop_length,
- "chunk_length": self.chunk_length,
- "padding_value": self.padding_value,
- "sampling_rate": self.sampling_rate,
- "return_attention_mask": self.return_attention_mask,
- }
-
- # Copied from transformers.tests.models.whisper.test_feature_extraction_whisper.WhisperFeatureExtractionTester.prepare_inputs_for_common
- def prepare_inputs_for_common(self, equal_length=False, numpify=False):
- def _flatten(list_of_lists):
- return list(itertools.chain(*list_of_lists))
-
- if equal_length:
- speech_inputs = [floats_list((self.max_seq_length, self.feature_size)) for _ in range(self.batch_size)]
- else:
- # make sure that inputs increase in size
- speech_inputs = [
- floats_list((x, self.feature_size))
- for x in range(self.min_seq_length, self.max_seq_length, self.seq_length_diff)
- ]
- if numpify:
- speech_inputs = [np.asarray(x) for x in speech_inputs]
- return speech_inputs
-
-
-@require_torch
-class ClvpFeatureExtractionTest(SequenceFeatureExtractionTestMixin, unittest.TestCase):
- feature_extraction_class = ClvpFeatureExtractor
-
- def setUp(self):
- self.feat_extract_tester = ClvpFeatureExtractionTester(self)
-
- def tearDown(self):
- super().tearDown()
- # clean-up as much as possible GPU memory occupied by PyTorch
- cleanup(torch_device)
-
- # Copied from transformers.tests.models.whisper.test_feature_extraction_whisper.WhisperFeatureExtractionTest.test_feat_extract_from_and_save_pretrained
- def test_feat_extract_from_and_save_pretrained(self):
- feat_extract_first = self.feature_extraction_class(**self.feat_extract_dict)
-
- with tempfile.TemporaryDirectory() as tmpdirname:
- saved_file = feat_extract_first.save_pretrained(tmpdirname)[0]
- check_json_file_has_correct_format(saved_file)
- feat_extract_second = self.feature_extraction_class.from_pretrained(tmpdirname)
-
- dict_first = feat_extract_first.to_dict()
- dict_second = feat_extract_second.to_dict()
- mel_1 = feat_extract_first.mel_filters
- mel_2 = feat_extract_second.mel_filters
- self.assertTrue(np.allclose(mel_1, mel_2))
- self.assertEqual(dict_first, dict_second)
-
- # Copied from transformers.tests.models.whisper.test_feature_extraction_whisper.WhisperFeatureExtractionTest.test_feat_extract_to_json_file
- def test_feat_extract_to_json_file(self):
- feat_extract_first = self.feature_extraction_class(**self.feat_extract_dict)
-
- with tempfile.TemporaryDirectory() as tmpdirname:
- json_file_path = os.path.join(tmpdirname, "feat_extract.json")
- feat_extract_first.to_json_file(json_file_path)
- feat_extract_second = self.feature_extraction_class.from_json_file(json_file_path)
-
- dict_first = feat_extract_first.to_dict()
- dict_second = feat_extract_second.to_dict()
- mel_1 = feat_extract_first.mel_filters
- mel_2 = feat_extract_second.mel_filters
- self.assertTrue(np.allclose(mel_1, mel_2))
- self.assertEqual(dict_first, dict_second)
-
- def test_call(self):
- # Tests that all call wrap to encode_plus and batch_encode_plus
- feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- # create three inputs of length 800, 1000, and 1200
- speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)]
- np_speech_inputs = [np.asarray(speech_input) for speech_input in speech_inputs]
-
- # Test feature size
- input_features = feature_extractor(np_speech_inputs, padding="max_length", return_tensors="np").input_features
- self.assertTrue(input_features.ndim == 3)
- self.assertTrue(input_features.shape[-2] == feature_extractor.feature_size)
-
- # Test not batched input
- encoded_sequences_1 = feature_extractor(speech_inputs[0], return_tensors="np").input_features
- encoded_sequences_2 = feature_extractor(np_speech_inputs[0], return_tensors="np").input_features
- self.assertTrue(np.allclose(encoded_sequences_1, encoded_sequences_2, atol=1e-3))
-
- # Test batched
- encoded_sequences_1 = feature_extractor(speech_inputs, return_tensors="np").input_features
- encoded_sequences_2 = feature_extractor(np_speech_inputs, return_tensors="np").input_features
- for enc_seq_1, enc_seq_2 in zip(encoded_sequences_1, encoded_sequences_2):
- self.assertTrue(np.allclose(enc_seq_1, enc_seq_2, atol=1e-3))
-
- # Test 2-D numpy arrays are batched.
- speech_inputs = [floats_list((1, x))[0] for x in (800, 800, 800)]
- np_speech_inputs = np.asarray(speech_inputs)
- encoded_sequences_1 = feature_extractor(speech_inputs, return_tensors="np").input_features
- encoded_sequences_2 = feature_extractor(np_speech_inputs, return_tensors="np").input_features
- for enc_seq_1, enc_seq_2 in zip(encoded_sequences_1, encoded_sequences_2):
- self.assertTrue(np.allclose(enc_seq_1, enc_seq_2, atol=1e-3))
-
- # Test truncation required
- speech_inputs = [floats_list((1, x))[0] for x in range(200, (feature_extractor.n_samples + 500), 200)]
- np_speech_inputs = [np.asarray(speech_input) for speech_input in speech_inputs]
-
- speech_inputs_truncated = [x[: feature_extractor.n_samples] for x in speech_inputs]
- np_speech_inputs_truncated = [np.asarray(speech_input) for speech_input in speech_inputs_truncated]
-
- encoded_sequences_1 = feature_extractor(np_speech_inputs, return_tensors="np").input_features
- encoded_sequences_2 = feature_extractor(np_speech_inputs_truncated, return_tensors="np").input_features
- for enc_seq_1, enc_seq_2 in zip(encoded_sequences_1, encoded_sequences_2):
- self.assertTrue(np.allclose(enc_seq_1, enc_seq_2, atol=1e-3))
-
- # Copied from transformers.tests.models.whisper.test_feature_extraction_whisper.WhisperFeatureExtractionTest.test_double_precision_pad
- def test_double_precision_pad(self):
- import torch
-
- feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- np_speech_inputs = np.random.rand(100, 32).astype(np.float64)
- py_speech_inputs = np_speech_inputs.tolist()
-
- for inputs in [py_speech_inputs, np_speech_inputs]:
- np_processed = feature_extractor.pad([{"input_features": inputs}], return_tensors="np")
- self.assertTrue(np_processed.input_features.dtype == np.float32)
- pt_processed = feature_extractor.pad([{"input_features": inputs}], return_tensors="pt")
- self.assertTrue(pt_processed.input_features.dtype == torch.float32)
-
- def _load_datasamples(self, num_samples):
- ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
- ds = ds.cast_column("audio", Audio(sampling_rate=22050))
- # automatic decoding with librispeech
- speech_samples = ds.sort("id")[:num_samples]["audio"]
-
- return [x["array"] for x in speech_samples], [x["sampling_rate"] for x in speech_samples]
-
- @slow
- def test_integration(self):
- # fmt: off
- EXPECTED_INPUT_FEATURES = torch.tensor(
- [
- 0.9271, 1.1405, 1.4419, 1.2470, 1.2438, 1.1787, 1.0595, 1.0570, 1.1070,
- 1.2205, 1.2376, 1.2997, 1.1131, 1.0843, 1.0459, 1.1858, 1.2323, 1.3582,
- 1.3401, 1.3770, 1.4173, 1.3381, 1.2291, 1.0854, 1.2116, 1.1873, 1.2178,
- 1.2137, 1.3001, 1.4274
- ]
- )
- # fmt: on
-
- input_speech, sr = self._load_datasamples(1)
-
- feature_extractor = ClvpFeatureExtractor.from_pretrained("susnato/clvp_dev")
- input_features = feature_extractor(input_speech, sampling_rate=sr[0], return_tensors="pt").input_features
- self.assertEqual(input_features.shape, (1, 80, 517))
- torch.testing.assert_close(input_features[0, 0, :30], EXPECTED_INPUT_FEATURES, rtol=1e-4, atol=1e-4)
diff --git a/tests/models/clvp/test_processing_clvp.py b/tests/models/clvp/test_processing_clvp.py
index 34caf6df8a5a..cd186372e513 100644
--- a/tests/models/clvp/test_processing_clvp.py
+++ b/tests/models/clvp/test_processing_clvp.py
@@ -21,7 +21,7 @@
from transformers import ClvpFeatureExtractor, ClvpProcessor, ClvpTokenizer
from transformers.testing_utils import require_torch
-from .test_feature_extraction_clvp import floats_list
+from ...test_processing_common import floats_list
@require_torch
diff --git a/tests/models/cohere_asr/test_audio_processing_cohere_asr.py b/tests/models/cohere_asr/test_audio_processing_cohere_asr.py
new file mode 100644
index 000000000000..26b52eef2ced
--- /dev/null
+++ b/tests/models/cohere_asr/test_audio_processing_cohere_asr.py
@@ -0,0 +1,42 @@
+# Copyright 2026 HuggingFace Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+"""Tests for `CohereAsrAudioProcessor` and `CohereAsrAudioProcessorNumpy`."""
+
+from __future__ import annotations
+
+import unittest
+
+from transformers.testing_utils import require_torch
+
+from ...test_audio_processing_common import AudioProcessingTestMixin
+
+
+class CohereAsrAudioProcessingTester:
+ """Provides init kwargs and fixture parameters for the CohereAsr audio processor tests."""
+
+ sample_rate = 16000
+
+ def prepare_audio_processor_dict(self) -> dict:
+ # Disable dither for the parity test: the legacy FE seeds torch RNG by valid
+ # sample count, which we cannot reproduce bit-exactly with numpy's MT19937. Both
+ # backends implement deterministic dither in production; only this fixture turns
+ # it off.
+ return {"dither": 0.0}
+
+
+@require_torch
+class CohereAsrAudioProcessingTest(AudioProcessingTestMixin, unittest.TestCase):
+ # CohereAsr has a longer numerical chain (waveform-level preemphasis with masking, then
+ # log(x + 2^-24), then per-utterance mean/var on the padded batch). The float32 noise
+ # floor still holds for the unbatched test; the batched test relaxes slightly.
+ parity_atol = 1e-4
+ parity_rtol = 1e-4
+
+ def setUp(self):
+ self.audio_processor_tester = CohereAsrAudioProcessingTester()
+ super().setUp()
diff --git a/tests/models/dac/test_audio_processing_dac.py b/tests/models/dac/test_audio_processing_dac.py
new file mode 100644
index 000000000000..ccd29fcfbee9
--- /dev/null
+++ b/tests/models/dac/test_audio_processing_dac.py
@@ -0,0 +1,21 @@
+# Copyright 2026 HuggingFace Inc.
+# Licensed under the Apache License, Version 2.0
+import unittest
+
+from transformers.testing_utils import require_torch
+
+from ...test_audio_processing_common import AudioProcessingTestMixin
+
+
+class DacAudioProcessingTester:
+ sample_rate = 16000
+
+ def prepare_audio_processor_dict(self) -> dict:
+ return {}
+
+
+@require_torch
+class DacAudioProcessingTest(AudioProcessingTestMixin, unittest.TestCase):
+ def setUp(self):
+ self.audio_processor_tester = DacAudioProcessingTester()
+ super().setUp()
diff --git a/tests/models/dac/test_feature_extraction_dac.py b/tests/models/dac/test_feature_extraction_dac.py
deleted file mode 100644
index c1684edd704d..000000000000
--- a/tests/models/dac/test_feature_extraction_dac.py
+++ /dev/null
@@ -1,197 +0,0 @@
-# Copyright 2024 HuggingFace Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Tests for the dac feature extractor."""
-
-import itertools
-import unittest
-
-import numpy as np
-
-from transformers import DacFeatureExtractor
-from transformers.testing_utils import require_torch
-from transformers.utils.import_utils import is_torch_available
-
-from ...test_processing_common import floats_list
-from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin
-
-
-if is_torch_available():
- import torch
-
-
-@require_torch
-# Copied from transformers.tests.encodec.test_feature_extraction_encodec.EncodecFeatureExtractionTester with Encodec->Dac
-class DacFeatureExtractionTester:
- # Ignore copy
- def __init__(
- self,
- parent,
- batch_size=7,
- min_seq_length=400,
- max_seq_length=2000,
- feature_size=1,
- padding_value=0.0,
- sampling_rate=16000,
- hop_length=512,
- ):
- self.parent = parent
- self.batch_size = batch_size
- self.min_seq_length = min_seq_length
- self.max_seq_length = max_seq_length
- self.hop_length = hop_length
- self.seq_length_diff = (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
- self.feature_size = feature_size
- self.padding_value = padding_value
- self.sampling_rate = sampling_rate
-
- # Ignore copy
- def prepare_feat_extract_dict(self):
- return {
- "feature_size": self.feature_size,
- "padding_value": self.padding_value,
- "sampling_rate": self.sampling_rate,
- "hop_length": self.hop_length,
- }
-
- def prepare_inputs_for_common(self, equal_length=False, numpify=False):
- def _flatten(list_of_lists):
- return list(itertools.chain(*list_of_lists))
-
- if equal_length:
- audio_inputs = floats_list((self.batch_size, self.max_seq_length))
- else:
- # make sure that inputs increase in size
- audio_inputs = [
- _flatten(floats_list((x, self.feature_size)))
- for x in range(self.min_seq_length, self.max_seq_length, self.seq_length_diff)
- ]
-
- if numpify:
- audio_inputs = [np.asarray(x) for x in audio_inputs]
-
- return audio_inputs
-
-
-@require_torch
-# Copied from transformers.tests.encodec.test_feature_extraction_encodec.EnCodecFeatureExtractionTest with Encodec->Dac
-class DacFeatureExtractionTest(SequenceFeatureExtractionTestMixin, unittest.TestCase):
- feature_extraction_class = DacFeatureExtractor
-
- def setUp(self):
- self.feat_extract_tester = DacFeatureExtractionTester(self)
-
- def test_call(self):
- # Tests that all call wrap to encode_plus and batch_encode_plus
- feat_extract = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- # create three inputs of length 800, 1000, and 1200
- audio_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)]
- np_audio_inputs = [np.asarray(audio_input) for audio_input in audio_inputs]
-
- # Test not batched input
- encoded_sequences_1 = feat_extract(audio_inputs[0], return_tensors="np").input_values
- encoded_sequences_2 = feat_extract(np_audio_inputs[0], return_tensors="np").input_values
- self.assertTrue(np.allclose(encoded_sequences_1, encoded_sequences_2, atol=1e-3))
-
- # Test batched
- encoded_sequences_1 = feat_extract(audio_inputs, padding=True, return_tensors="np").input_values
- encoded_sequences_2 = feat_extract(np_audio_inputs, padding=True, return_tensors="np").input_values
- for enc_seq_1, enc_seq_2 in zip(encoded_sequences_1, encoded_sequences_2):
- self.assertTrue(np.allclose(enc_seq_1, enc_seq_2, atol=1e-3))
-
- def test_double_precision_pad(self):
- feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- np_audio_inputs = np.random.rand(100).astype(np.float64)
- py_audio_inputs = np_audio_inputs.tolist()
-
- for inputs in [py_audio_inputs, np_audio_inputs]:
- np_processed = feature_extractor.pad([{"input_values": inputs}], return_tensors="np")
- self.assertTrue(np_processed.input_values.dtype == np.float32)
- pt_processed = feature_extractor.pad([{"input_values": inputs}], return_tensors="pt")
- self.assertTrue(pt_processed.input_values.dtype == torch.float32)
-
- def _load_datasamples(self, num_samples):
- from datasets import load_dataset
-
- ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
- # automatic decoding with librispeech
- audio_samples = ds.sort("id")[:num_samples]["audio"]
-
- return [x["array"] for x in audio_samples]
-
- def test_integration(self):
- # fmt: off
- EXPECTED_INPUT_VALUES = torch.tensor(
- [ 2.3803711e-03, 2.0751953e-03, 1.9836426e-03, 2.1057129e-03,
- 1.6174316e-03, 3.0517578e-04, 9.1552734e-05, 3.3569336e-04,
- 9.7656250e-04, 1.8310547e-03, 2.0141602e-03, 2.1057129e-03,
- 1.7395020e-03, 4.5776367e-04, -3.9672852e-04, 4.5776367e-04,
- 1.0070801e-03, 9.1552734e-05, 4.8828125e-04, 1.1596680e-03,
- 7.3242188e-04, 9.4604492e-04, 1.8005371e-03, 1.8310547e-03,
- 8.8500977e-04, 4.2724609e-04, 4.8828125e-04, 7.3242188e-04,
- 1.0986328e-03, 2.1057129e-03]
- )
- # fmt: on
- input_audio = self._load_datasamples(1)
- feature_extractor = DacFeatureExtractor()
- input_values = feature_extractor(input_audio, return_tensors="pt")["input_values"]
- self.assertEqual(input_values.shape, (1, 1, 93696))
- torch.testing.assert_close(input_values[0, 0, :30], EXPECTED_INPUT_VALUES, rtol=1e-4, atol=1e-4)
- audio_input_end = torch.tensor(input_audio[0][-30:], dtype=torch.float32)
- torch.testing.assert_close(input_values[0, 0, -46:-16], audio_input_end, rtol=1e-4, atol=1e-4)
-
- # Ignore copy
- @unittest.skip("The DAC model doesn't support stereo logic")
- def test_integration_stereo(self):
- pass
-
- # Ignore copy
- def test_truncation_and_padding(self):
- input_audio = self._load_datasamples(2)
- # would be easier if the stride was like
- feature_extractor = DacFeatureExtractor()
-
- # pad and trunc raise an error ?
- with self.assertRaisesRegex(
- ValueError,
- "^Both padding and truncation were set. Make sure you only set one.$",
- ):
- truncated_outputs = feature_extractor(
- input_audio, padding="max_length", truncation=True, return_tensors="pt"
- ).input_values
-
- # force truncate to max_length
- truncated_outputs = feature_extractor(
- input_audio, truncation=True, max_length=48000, return_tensors="pt"
- ).input_values
- self.assertEqual(truncated_outputs.shape, (2, 1, 48128))
-
- # pad:
- padded_outputs = feature_extractor(input_audio, padding=True, return_tensors="pt").input_values
- self.assertEqual(padded_outputs.shape, (2, 1, 93696))
-
- # force pad to max length
- truncated_outputs = feature_extractor(
- input_audio, padding="max_length", max_length=100000, return_tensors="pt"
- ).input_values
- self.assertEqual(truncated_outputs.shape, (2, 1, 100352))
-
- # force no pad
- with self.assertRaisesRegex(
- ValueError,
- r"Unable to convert output[\s\S]*padding=True",
- ):
- truncated_outputs = feature_extractor(input_audio, padding=False, return_tensors="pt").input_values
-
- truncated_outputs = feature_extractor(input_audio[0], padding=False, return_tensors="pt").input_values
- self.assertEqual(truncated_outputs.shape, (1, 1, 93680))
diff --git a/tests/models/dia/test_audio_processing_dia.py b/tests/models/dia/test_audio_processing_dia.py
new file mode 100644
index 000000000000..c2a4d907360a
--- /dev/null
+++ b/tests/models/dia/test_audio_processing_dia.py
@@ -0,0 +1,21 @@
+# Copyright 2026 HuggingFace Inc.
+# Licensed under the Apache License, Version 2.0
+import unittest
+
+from transformers.testing_utils import require_torch
+
+from ...test_audio_processing_common import AudioProcessingTestMixin
+
+
+class DiaAudioProcessingTester:
+ sample_rate = 44100
+
+ def prepare_audio_processor_dict(self) -> dict:
+ return {}
+
+
+@require_torch
+class DiaAudioProcessingTest(AudioProcessingTestMixin, unittest.TestCase):
+ def setUp(self):
+ self.audio_processor_tester = DiaAudioProcessingTester()
+ super().setUp()
diff --git a/tests/models/dia/test_feature_extraction_dia.py b/tests/models/dia/test_feature_extraction_dia.py
deleted file mode 100644
index 82c17d82bad5..000000000000
--- a/tests/models/dia/test_feature_extraction_dia.py
+++ /dev/null
@@ -1,213 +0,0 @@
-# Copyright 2025 HuggingFace Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Tests for the Dia feature extractor."""
-
-import itertools
-import unittest
-
-import numpy as np
-
-from transformers import DiaFeatureExtractor
-from transformers.testing_utils import require_torch
-from transformers.utils.import_utils import is_torch_available
-
-from ...test_processing_common import floats_list
-from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin
-
-
-if is_torch_available():
- import torch
-
-
-@require_torch
-class DiaFeatureExtractionTester:
- # Copied from tests.models.dac.test_feature_extraction_dac.DacFeatureExtractionTester.__init__
- def __init__(
- self,
- parent,
- batch_size=7,
- min_seq_length=400,
- max_seq_length=2000,
- feature_size=1,
- padding_value=0.0,
- sampling_rate=16000,
- hop_length=512,
- ):
- self.parent = parent
- self.batch_size = batch_size
- self.min_seq_length = min_seq_length
- self.max_seq_length = max_seq_length
- self.hop_length = hop_length
- self.seq_length_diff = (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
- self.feature_size = feature_size
- self.padding_value = padding_value
- self.sampling_rate = sampling_rate
-
- # Copied from tests.models.dac.test_feature_extraction_dac.DacFeatureExtractionTester.prepare_feat_extract_dict
- def prepare_feat_extract_dict(self):
- return {
- "feature_size": self.feature_size,
- "padding_value": self.padding_value,
- "sampling_rate": self.sampling_rate,
- "hop_length": self.hop_length,
- }
-
- # Copied from tests.models.encodec.test_feature_extraction_encodec.EnCodecFeatureExtractionTester.prepare_inputs_for_common
- def prepare_inputs_for_common(self, equal_length=False, numpify=False):
- def _flatten(list_of_lists):
- return list(itertools.chain(*list_of_lists))
-
- if equal_length:
- audio_inputs = floats_list((self.batch_size, self.max_seq_length))
- else:
- # make sure that inputs increase in size
- audio_inputs = [
- _flatten(floats_list((x, self.feature_size)))
- for x in range(self.min_seq_length, self.max_seq_length, self.seq_length_diff)
- ]
-
- if numpify:
- audio_inputs = [np.asarray(x) for x in audio_inputs]
-
- return audio_inputs
-
-
-@require_torch
-class DiaFeatureExtractionTest(SequenceFeatureExtractionTestMixin, unittest.TestCase):
- feature_extraction_class = DiaFeatureExtractor
-
- def setUp(self):
- self.feat_extract_tester = DiaFeatureExtractionTester(self)
-
- # Copied from tests.models.dac.test_feature_extraction_dac.DacFeatureExtractionTest.test_call
- def test_call(self):
- # Tests that all call wrap to encode_plus and batch_encode_plus
- feat_extract = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- # create three inputs of length 800, 1000, and 1200
- audio_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)]
- np_audio_inputs = [np.asarray(audio_input) for audio_input in audio_inputs]
-
- # Test not batched input
- encoded_sequences_1 = feat_extract(audio_inputs[0], return_tensors="np").input_values
- encoded_sequences_2 = feat_extract(np_audio_inputs[0], return_tensors="np").input_values
- self.assertTrue(np.allclose(encoded_sequences_1, encoded_sequences_2, atol=1e-3))
-
- # Test batched
- encoded_sequences_1 = feat_extract(audio_inputs, padding=True, return_tensors="np").input_values
- encoded_sequences_2 = feat_extract(np_audio_inputs, padding=True, return_tensors="np").input_values
- for enc_seq_1, enc_seq_2 in zip(encoded_sequences_1, encoded_sequences_2):
- self.assertTrue(np.allclose(enc_seq_1, enc_seq_2, atol=1e-3))
-
- # Copied from tests.models.dac.test_feature_extraction_dac.DacFeatureExtractionTest.test_double_precision_pad
- def test_double_precision_pad(self):
- feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- np_audio_inputs = np.random.rand(100).astype(np.float64)
- py_audio_inputs = np_audio_inputs.tolist()
-
- for inputs in [py_audio_inputs, np_audio_inputs]:
- np_processed = feature_extractor.pad([{"input_values": inputs}], return_tensors="np")
- self.assertTrue(np_processed.input_values.dtype == np.float32)
- pt_processed = feature_extractor.pad([{"input_values": inputs}], return_tensors="pt")
- self.assertTrue(pt_processed.input_values.dtype == torch.float32)
-
- # Copied from tests.models.dac.test_feature_extraction_dac.DacFeatureExtractionTest._load_datasamples
- def _load_datasamples(self, num_samples):
- from datasets import load_dataset
-
- ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
- # automatic decoding with librispeech
- audio_samples = ds.sort("id")[:num_samples]["audio"]
-
- return [x["array"] for x in audio_samples]
-
- # Copied from tests.models.dac.test_feature_extraction_dac.DacFeatureExtractionTest.test_integration with Dac->Dia
- def test_integration(self):
- # fmt: off
- EXPECTED_INPUT_VALUES = torch.tensor(
- [ 2.3803711e-03, 2.0751953e-03, 1.9836426e-03, 2.1057129e-03,
- 1.6174316e-03, 3.0517578e-04, 9.1552734e-05, 3.3569336e-04,
- 9.7656250e-04, 1.8310547e-03, 2.0141602e-03, 2.1057129e-03,
- 1.7395020e-03, 4.5776367e-04, -3.9672852e-04, 4.5776367e-04,
- 1.0070801e-03, 9.1552734e-05, 4.8828125e-04, 1.1596680e-03,
- 7.3242188e-04, 9.4604492e-04, 1.8005371e-03, 1.8310547e-03,
- 8.8500977e-04, 4.2724609e-04, 4.8828125e-04, 7.3242188e-04,
- 1.0986328e-03, 2.1057129e-03]
- )
- # fmt: on
- input_audio = self._load_datasamples(1)
- feature_extractor = DiaFeatureExtractor()
- input_values = feature_extractor(input_audio, return_tensors="pt")["input_values"]
- self.assertEqual(input_values.shape, (1, 1, 93696))
- torch.testing.assert_close(input_values[0, 0, :30], EXPECTED_INPUT_VALUES, rtol=1e-4, atol=1e-4)
- audio_input_end = torch.tensor(input_audio[0][-30:], dtype=torch.float32)
- torch.testing.assert_close(input_values[0, 0, -46:-16], audio_input_end, rtol=1e-4, atol=1e-4)
-
- def test_integration_stereo(self):
- # fmt: off
- EXPECTED_INPUT_VALUES = torch.tensor(
- [2.3804e-03, 2.0752e-03, 1.9836e-03, 2.1057e-03, 1.6174e-03,
- 3.0518e-04, 9.1553e-05, 3.3569e-04, 9.7656e-04, 1.8311e-03,
- 2.0142e-03, 2.1057e-03, 1.7395e-03, 4.5776e-04, -3.9673e-04,
- 4.5776e-04, 1.0071e-03, 9.1553e-05, 4.8828e-04, 1.1597e-03,
- 7.3242e-04, 9.4604e-04, 1.8005e-03, 1.8311e-03, 8.8501e-04,
- 4.2725e-04, 4.8828e-04, 7.3242e-04, 1.0986e-03, 2.1057e-03]
- )
- # fmt: on
- input_audio = self._load_datasamples(1)
- input_audio = [np.tile(input_audio[0][None], reps=(2, 1))]
- feature_extractor = DiaFeatureExtractor(feature_size=2)
- input_values = feature_extractor(input_audio, return_tensors="pt").input_values
- self.assertEqual(input_values.shape, (1, 1, 93696))
- torch.testing.assert_close(input_values[0, 0, :30], EXPECTED_INPUT_VALUES, rtol=1e-4, atol=1e-4)
-
- # Copied from tests.models.dac.test_feature_extraction_dac.DacFeatureExtractionTest.test_truncation_and_padding with Dac->Dia
- def test_truncation_and_padding(self):
- input_audio = self._load_datasamples(2)
- # would be easier if the stride was like
- feature_extractor = DiaFeatureExtractor()
-
- # pad and trunc raise an error ?
- with self.assertRaisesRegex(
- ValueError,
- "^Both padding and truncation were set. Make sure you only set one.$",
- ):
- truncated_outputs = feature_extractor(
- input_audio, padding="max_length", truncation=True, return_tensors="pt"
- ).input_values
-
- # force truncate to max_length
- truncated_outputs = feature_extractor(
- input_audio, truncation=True, max_length=48000, return_tensors="pt"
- ).input_values
- self.assertEqual(truncated_outputs.shape, (2, 1, 48128))
-
- # pad:
- padded_outputs = feature_extractor(input_audio, padding=True, return_tensors="pt").input_values
- self.assertEqual(padded_outputs.shape, (2, 1, 93696))
-
- # force pad to max length
- truncated_outputs = feature_extractor(
- input_audio, padding="max_length", max_length=100000, return_tensors="pt"
- ).input_values
- self.assertEqual(truncated_outputs.shape, (2, 1, 100352))
-
- # force no pad
- with self.assertRaisesRegex(
- ValueError,
- r"Unable to convert output[\s\S]*padding=True",
- ):
- truncated_outputs = feature_extractor(input_audio, padding=False, return_tensors="pt").input_values
-
- truncated_outputs = feature_extractor(input_audio[0], padding=False, return_tensors="pt").input_values
- self.assertEqual(truncated_outputs.shape, (1, 1, 93680))
diff --git a/tests/models/encodec/test_audio_processing_encodec.py b/tests/models/encodec/test_audio_processing_encodec.py
new file mode 100644
index 000000000000..e9d2f3200916
--- /dev/null
+++ b/tests/models/encodec/test_audio_processing_encodec.py
@@ -0,0 +1,21 @@
+# Copyright 2026 HuggingFace Inc.
+# Licensed under the Apache License, Version 2.0
+import unittest
+
+from transformers.testing_utils import require_torch
+
+from ...test_audio_processing_common import AudioProcessingTestMixin
+
+
+class EncodecAudioProcessingTester:
+ sample_rate = 24000
+
+ def prepare_audio_processor_dict(self) -> dict:
+ return {}
+
+
+@require_torch
+class EncodecAudioProcessingTest(AudioProcessingTestMixin, unittest.TestCase):
+ def setUp(self):
+ self.audio_processor_tester = EncodecAudioProcessingTester()
+ super().setUp()
diff --git a/tests/models/encodec/test_feature_extraction_encodec.py b/tests/models/encodec/test_feature_extraction_encodec.py
deleted file mode 100644
index d3debb8bfd9d..000000000000
--- a/tests/models/encodec/test_feature_extraction_encodec.py
+++ /dev/null
@@ -1,234 +0,0 @@
-# Copyright 2021-2023 HuggingFace Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Tests for the EnCodec feature extractor."""
-
-import itertools
-import unittest
-
-import numpy as np
-
-from transformers import EncodecFeatureExtractor
-from transformers.testing_utils import require_torch
-from transformers.utils.import_utils import is_torch_available
-
-from ...test_processing_common import floats_list
-from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin
-
-
-if is_torch_available():
- import torch
-
-
-@require_torch
-class EnCodecFeatureExtractionTester:
- def __init__(
- self,
- parent,
- batch_size=7,
- min_seq_length=400,
- max_seq_length=2000,
- feature_size=1,
- padding_value=0.0,
- sampling_rate=24000,
- return_attention_mask=True,
- ):
- self.parent = parent
- self.batch_size = batch_size
- self.min_seq_length = min_seq_length
- self.max_seq_length = max_seq_length
- self.seq_length_diff = (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
- self.feature_size = feature_size
- self.padding_value = padding_value
- self.sampling_rate = sampling_rate
- self.return_attention_mask = return_attention_mask
-
- def prepare_feat_extract_dict(self):
- return {
- "feature_size": self.feature_size,
- "padding_value": self.padding_value,
- "sampling_rate": self.sampling_rate,
- "return_attention_mask": self.return_attention_mask,
- }
-
- def prepare_inputs_for_common(self, equal_length=False, numpify=False):
- def _flatten(list_of_lists):
- return list(itertools.chain(*list_of_lists))
-
- if equal_length:
- audio_inputs = floats_list((self.batch_size, self.max_seq_length))
- else:
- # make sure that inputs increase in size
- audio_inputs = [
- _flatten(floats_list((x, self.feature_size)))
- for x in range(self.min_seq_length, self.max_seq_length, self.seq_length_diff)
- ]
-
- if numpify:
- audio_inputs = [np.asarray(x) for x in audio_inputs]
-
- return audio_inputs
-
-
-@require_torch
-class EnCodecFeatureExtractionTest(SequenceFeatureExtractionTestMixin, unittest.TestCase):
- feature_extraction_class = EncodecFeatureExtractor
-
- def setUp(self):
- self.feat_extract_tester = EnCodecFeatureExtractionTester(self)
-
- def test_call(self):
- # Tests that all call wrap to encode_plus and batch_encode_plus
- feat_extract = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- # create three inputs of length 800, 1000, and 1200
- audio_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)]
- np_audio_inputs = [np.asarray(audio_input) for audio_input in audio_inputs]
-
- # Test not batched input
- encoded_sequences_1 = feat_extract(audio_inputs[0], return_tensors="np").input_values
- encoded_sequences_2 = feat_extract(np_audio_inputs[0], return_tensors="np").input_values
- self.assertTrue(np.allclose(encoded_sequences_1, encoded_sequences_2, atol=1e-3))
-
- # Test batched
- encoded_sequences_1 = feat_extract(audio_inputs, padding=True, return_tensors="np").input_values
- encoded_sequences_2 = feat_extract(np_audio_inputs, padding=True, return_tensors="np").input_values
- for enc_seq_1, enc_seq_2 in zip(encoded_sequences_1, encoded_sequences_2):
- self.assertTrue(np.allclose(enc_seq_1, enc_seq_2, atol=1e-3))
-
- def test_double_precision_pad(self):
- feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- np_audio_inputs = np.random.rand(100).astype(np.float64)
- py_audio_inputs = np_audio_inputs.tolist()
-
- for inputs in [py_audio_inputs, np_audio_inputs]:
- np_processed = feature_extractor.pad([{"input_values": inputs}], return_tensors="np")
- self.assertTrue(np_processed.input_values.dtype == np.float32)
- pt_processed = feature_extractor.pad([{"input_values": inputs}], return_tensors="pt")
- self.assertTrue(pt_processed.input_values.dtype == torch.float32)
-
- def _load_datasamples(self, num_samples):
- from datasets import load_dataset
-
- ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
- # automatic decoding with librispeech
- audio_samples = ds.sort("id")[:num_samples]["audio"]
-
- return [x["array"] for x in audio_samples]
-
- def test_integration(self):
- # fmt: off
- EXPECTED_INPUT_VALUES = torch.tensor(
- [2.3804e-03, 2.0752e-03, 1.9836e-03, 2.1057e-03, 1.6174e-03,
- 3.0518e-04, 9.1553e-05, 3.3569e-04, 9.7656e-04, 1.8311e-03,
- 2.0142e-03, 2.1057e-03, 1.7395e-03, 4.5776e-04, -3.9673e-04,
- 4.5776e-04, 1.0071e-03, 9.1553e-05, 4.8828e-04, 1.1597e-03,
- 7.3242e-04, 9.4604e-04, 1.8005e-03, 1.8311e-03, 8.8501e-04,
- 4.2725e-04, 4.8828e-04, 7.3242e-04, 1.0986e-03, 2.1057e-03]
- )
- # fmt: on
- input_audio = self._load_datasamples(1)
- feature_extractor = EncodecFeatureExtractor()
- input_values = feature_extractor(input_audio, return_tensors="pt").input_values
- self.assertEqual(input_values.shape, (1, 1, 93680))
- torch.testing.assert_close(input_values[0, 0, :30], EXPECTED_INPUT_VALUES, rtol=1e-6, atol=1e-6)
-
- def test_integration_stereo(self):
- # fmt: off
- EXPECTED_INPUT_VALUES = torch.tensor(
- [2.3804e-03, 2.0752e-03, 1.9836e-03, 2.1057e-03, 1.6174e-03,
- 3.0518e-04, 9.1553e-05, 3.3569e-04, 9.7656e-04, 1.8311e-03,
- 2.0142e-03, 2.1057e-03, 1.7395e-03, 4.5776e-04, -3.9673e-04,
- 4.5776e-04, 1.0071e-03, 9.1553e-05, 4.8828e-04, 1.1597e-03,
- 7.3242e-04, 9.4604e-04, 1.8005e-03, 1.8311e-03, 8.8501e-04,
- 4.2725e-04, 4.8828e-04, 7.3242e-04, 1.0986e-03, 2.1057e-03]
- )
- # fmt: on
- input_audio = self._load_datasamples(1)
- input_audio = [np.tile(input_audio[0][None], reps=(2, 1))]
- input_audio[0][1] *= 0.5
- feature_extractor = EncodecFeatureExtractor(feature_size=2)
- input_values = feature_extractor(input_audio, return_tensors="pt").input_values
- self.assertEqual(input_values.shape, (1, 2, 93680))
- torch.testing.assert_close(input_values[0, 0, :30], EXPECTED_INPUT_VALUES, rtol=1e-6, atol=1e-6)
- torch.testing.assert_close(input_values[0, 1, :30], EXPECTED_INPUT_VALUES * 0.5, rtol=1e-6, atol=1e-6)
-
- def test_truncation_and_padding(self):
- input_audio = self._load_datasamples(2)
- # would be easier if the stride was like
- feature_extractor = EncodecFeatureExtractor(feature_size=1, chunk_length_s=1, overlap=0.01)
-
- # pad and trunc raise an error ?
- with self.assertRaisesRegex(
- ValueError,
- "^Both padding and truncation were set. Make sure you only set one.$",
- ):
- truncated_outputs = feature_extractor(
- input_audio, padding="max_length", truncation=True, return_tensors="pt"
- ).input_values
-
- # truncate to chunk
- truncated_outputs = feature_extractor(input_audio, truncation=True, return_tensors="pt").input_values
- self.assertEqual(truncated_outputs.shape, (2, 1, 71520)) # 2 chunks
-
- # force truncate to max_length
- truncated_outputs = feature_extractor(
- input_audio, truncation=True, max_length=48000, return_tensors="pt"
- ).input_values
- self.assertEqual(truncated_outputs.shape, (2, 1, 48000))
-
- # pad to chunk
- padded_outputs = feature_extractor(input_audio, padding=True, return_tensors="pt").input_values
- self.assertEqual(padded_outputs.shape, (2, 1, 95280))
-
- # pad to chunk
- truncated_outputs = feature_extractor(input_audio, return_tensors="pt").input_values
- self.assertEqual(truncated_outputs.shape, (2, 1, 95280))
-
- # force pad to max length
- truncated_outputs = feature_extractor(
- input_audio, padding="max_length", max_length=100000, return_tensors="pt"
- ).input_values
- self.assertEqual(truncated_outputs.shape, (2, 1, 100000))
-
- # force no pad
- with self.assertRaisesRegex(
- ValueError,
- r"Unable to convert output[\s\S]*padding=True",
- ):
- truncated_outputs = feature_extractor(input_audio, padding=False, return_tensors="pt").input_values
-
- truncated_outputs = feature_extractor(input_audio[0], padding=False, return_tensors="pt").input_values
- self.assertEqual(truncated_outputs.shape, (1, 1, 93680))
-
- # no pad if no chunk_length_s
- feature_extractor.chunk_length_s = None
- with self.assertRaisesRegex(
- ValueError,
- r"Unable to convert output[\s\S]*padding=True",
- ):
- truncated_outputs = feature_extractor(input_audio, padding=False, return_tensors="pt").input_values
-
- truncated_outputs = feature_extractor(input_audio[0], padding=False, return_tensors="pt").input_values
- self.assertEqual(truncated_outputs.shape, (1, 1, 93680))
-
- # no pad if no overlap
- feature_extractor.chunk_length_s = 2
- feature_extractor.overlap = None
- with self.assertRaisesRegex(
- ValueError,
- r"Unable to convert output[\s\S]*padding=True",
- ):
- truncated_outputs = feature_extractor(input_audio, padding=False, return_tensors="pt").input_values
-
- truncated_outputs = feature_extractor(input_audio[0], padding=False, return_tensors="pt").input_values
- self.assertEqual(truncated_outputs.shape, (1, 1, 93680))
diff --git a/tests/models/gemma3n/test_audio_processing_gemma3n.py b/tests/models/gemma3n/test_audio_processing_gemma3n.py
new file mode 100644
index 000000000000..fd1a23d756ba
--- /dev/null
+++ b/tests/models/gemma3n/test_audio_processing_gemma3n.py
@@ -0,0 +1,25 @@
+import unittest
+
+from transformers.testing_utils import require_torch
+
+from ...test_audio_processing_common import AudioProcessingTestMixin
+
+
+class Gemma3nAudioProcessingTester:
+ sample_rate = 16000
+
+ def prepare_audio_processor_dict(self) -> dict:
+ return {}
+
+
+@require_torch
+class Gemma3nAudioProcessingTest(AudioProcessingTestMixin, unittest.TestCase):
+ # Gemma3n's unfold-based STFT with HTK preemphasis and float32-window-on-float64-frames
+ # multiplication produces a slightly larger cross-backend divergence than the strict
+ # float32 noise floor — empirically up to ~1.7e-4 on batched inputs.
+ parity_atol = 5e-4
+ parity_rtol = 1e-4
+
+ def setUp(self):
+ self.audio_processor_tester = Gemma3nAudioProcessingTester()
+ super().setUp()
diff --git a/tests/models/gemma3n/test_feature_extraction_gemma3n.py b/tests/models/gemma3n/test_feature_extraction_gemma3n.py
deleted file mode 100644
index 12512d0c081a..000000000000
--- a/tests/models/gemma3n/test_feature_extraction_gemma3n.py
+++ /dev/null
@@ -1,289 +0,0 @@
-# Copyright 2025 HuggingFace Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-import itertools
-import os
-import tempfile
-import unittest
-from collections.abc import Sequence
-
-import numpy as np
-from parameterized import parameterized
-
-from transformers.models.gemma3n import Gemma3nAudioFeatureExtractor
-from transformers.testing_utils import (
- check_json_file_has_correct_format,
- require_torch,
-)
-from transformers.utils.import_utils import is_torch_available
-
-from ...test_processing_common import floats_list
-from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin
-
-
-if is_torch_available():
- pass
-
-MAX_LENGTH_FOR_TESTING = 512
-
-
-class Gemma3nAudioFeatureExtractionTester:
- def __init__(
- self,
- parent,
- batch_size=7,
- min_seq_length=400,
- max_seq_length=2000,
- feature_size: int = 128,
- sampling_rate: int = 16_000,
- padding_value: float = 0.0,
- return_attention_mask: bool = False,
- # ignore hop_length / frame_length for now, as ms -> length conversion causes issues with serialization tests
- # frame_length_ms: float = 32.0,
- # hop_length: float = 10.0,
- min_frequency: float = 125.0,
- max_frequency: float = 7600.0,
- preemphasis: float = 0.97,
- preemphasis_htk_flavor: bool = True,
- fft_overdrive: bool = True,
- dither: float = 0.0,
- input_scale_factor: float = 1.0,
- mel_floor: float = 1e-5,
- per_bin_mean: Sequence[float] | None = None,
- per_bin_stddev: Sequence[float] | None = None,
- ):
- self.parent = parent
- self.batch_size = batch_size
- self.min_seq_length = min_seq_length
- self.max_seq_length = max_seq_length
- self.seq_length_diff = (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
- self.feature_size = feature_size
- self.sampling_rate = sampling_rate
- self.padding_value = padding_value
- self.return_attention_mask = return_attention_mask
- # ignore hop_length / frame_length for now, as ms -> length conversion causes issues with serialization tests
- # self.frame_length_ms = frame_length_ms
- # self.hop_length = hop_length
- self.min_frequency = min_frequency
- self.max_frequency = max_frequency
- self.preemphasis = preemphasis
- self.preemphasis_htk_flavor = preemphasis_htk_flavor
- self.fft_overdrive = fft_overdrive
- self.dither = dither
- self.input_scale_factor = input_scale_factor
- self.mel_floor = mel_floor
- self.per_bin_mean = per_bin_mean
- self.per_bin_stddev = per_bin_stddev
-
- def prepare_feat_extract_dict(self):
- return {
- "feature_size": self.feature_size,
- "sampling_rate": self.sampling_rate,
- "padding_value": self.padding_value,
- "return_attention_mask": self.return_attention_mask,
- "min_frequency": self.min_frequency,
- "max_frequency": self.max_frequency,
- "preemphasis": self.preemphasis,
- "preemphasis_htk_flavor": self.preemphasis_htk_flavor,
- "fft_overdrive": self.fft_overdrive,
- "dither": self.dither,
- "input_scale_factor": self.input_scale_factor,
- "mel_floor": self.mel_floor,
- "per_bin_mean": self.per_bin_mean,
- "per_bin_stddev": self.per_bin_stddev,
- }
-
- def prepare_inputs_for_common(self, equal_length=False, numpify=False):
- def _flatten(list_of_lists):
- return list(itertools.chain(*list_of_lists))
-
- if equal_length:
- speech_inputs = [floats_list((self.max_seq_length, self.feature_size)) for _ in range(self.batch_size)]
- else:
- # make sure that inputs increase in size
- speech_inputs = [
- floats_list((x, self.feature_size))
- for x in range(self.min_seq_length, self.max_seq_length, self.seq_length_diff)
- ]
- if numpify:
- speech_inputs = [np.asarray(x) for x in speech_inputs]
- return speech_inputs
-
-
-class Gemma3nAudioFeatureExtractionTest(SequenceFeatureExtractionTestMixin, unittest.TestCase):
- feature_extraction_class = Gemma3nAudioFeatureExtractor
-
- def setUp(self):
- self.feat_extract_tester = Gemma3nAudioFeatureExtractionTester(self)
-
- def test_feat_extract_from_and_save_pretrained(self):
- feat_extract_first = self.feature_extraction_class(**self.feat_extract_dict)
-
- with tempfile.TemporaryDirectory() as tmpdirname:
- saved_file = feat_extract_first.save_pretrained(tmpdirname)[0]
- check_json_file_has_correct_format(saved_file)
- feat_extract_second = self.feature_extraction_class.from_pretrained(tmpdirname)
-
- dict_first = feat_extract_first.to_dict()
- dict_second = feat_extract_second.to_dict()
- mel_1 = feat_extract_first.mel_filters
- mel_2 = feat_extract_second.mel_filters
- self.assertTrue(np.allclose(mel_1, mel_2))
- self.assertEqual(dict_first, dict_second)
-
- def test_feat_extract_to_json_file(self):
- feat_extract_first = self.feature_extraction_class(**self.feat_extract_dict)
-
- with tempfile.TemporaryDirectory() as tmpdirname:
- json_file_path = os.path.join(tmpdirname, "feat_extract.json")
- feat_extract_first.to_json_file(json_file_path)
- feat_extract_second = self.feature_extraction_class.from_json_file(json_file_path)
-
- dict_first = feat_extract_first.to_dict()
- dict_second = feat_extract_second.to_dict()
- mel_1 = feat_extract_first.mel_filters
- mel_2 = feat_extract_second.mel_filters
- self.assertTrue(np.allclose(mel_1, mel_2))
- self.assertEqual(dict_first, dict_second)
-
- def test_feat_extract_from_pretrained_kwargs(self):
- feat_extract_first = self.feature_extraction_class(**self.feat_extract_dict)
-
- with tempfile.TemporaryDirectory() as tmpdirname:
- saved_file = feat_extract_first.save_pretrained(tmpdirname)[0]
- check_json_file_has_correct_format(saved_file)
- feat_extract_second = self.feature_extraction_class.from_pretrained(
- tmpdirname, feature_size=2 * self.feat_extract_dict["feature_size"]
- )
-
- mel_1 = feat_extract_first.mel_filters
- mel_2 = feat_extract_second.mel_filters
- self.assertTrue(2 * mel_1.shape[1] == mel_2.shape[1])
-
- @parameterized.expand(
- [
- ([floats_list((1, x))[0] for x in range(800, 1400, 200)],),
- ([floats_list((1, x))[0] for x in (800, 800, 800)],),
- ([floats_list((1, x))[0] for x in range(200, (MAX_LENGTH_FOR_TESTING + 500), 200)], True),
- ]
- )
- def test_call(self, audio_inputs, test_truncation=False):
- feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- np_audio_inputs = [np.asarray(audio_input) for audio_input in audio_inputs]
-
- input_features = feature_extractor(np_audio_inputs, padding="max_length", return_tensors="np").input_features
- self.assertTrue(input_features.ndim == 3)
- # input_features.shape should be (batch, num_frames, n_mels) ~= (batch, num_frames, feature_size)
- # 480_000 is the max_length that inputs are padded to. we use that to calculate num_frames
- expected_num_frames = (480_000 - feature_extractor.frame_length) // (feature_extractor.hop_length) + 1
- self.assertTrue(
- input_features.shape[-2] == expected_num_frames,
- f"no match: {input_features.shape[-1]} vs {expected_num_frames}",
- )
- self.assertTrue(input_features.shape[-1] == feature_extractor.feature_size)
-
- encoded_sequences_1 = feature_extractor(audio_inputs, return_tensors="np").input_features
- encoded_sequences_2 = feature_extractor(np_audio_inputs, return_tensors="np").input_features
- for enc_seq_1, enc_seq_2 in zip(encoded_sequences_1, encoded_sequences_2):
- self.assertTrue(np.allclose(enc_seq_1, enc_seq_2, atol=1e-3))
-
- if test_truncation:
- audio_inputs_truncated = [x[:MAX_LENGTH_FOR_TESTING] for x in audio_inputs]
- np_audio_inputs_truncated = [np.asarray(audio_input) for audio_input in audio_inputs_truncated]
-
- encoded_sequences_1 = feature_extractor(
- audio_inputs_truncated, max_length=MAX_LENGTH_FOR_TESTING, return_tensors="np"
- ).input_features
- encoded_sequences_2 = feature_extractor(
- np_audio_inputs_truncated, max_length=MAX_LENGTH_FOR_TESTING, return_tensors="np"
- ).input_features
- for enc_seq_1, enc_seq_2 in zip(encoded_sequences_1, encoded_sequences_2):
- self.assertTrue(np.allclose(enc_seq_1, enc_seq_2, atol=1e-3))
-
- def test_call_unbatched(self):
- feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- np_audio = floats_list((1, 800))[0]
- input_features = feature_extractor(np_audio, return_tensors="np").input_features
- expected_input_features = feature_extractor([np_audio], return_tensors="np").input_features
- np.testing.assert_allclose(input_features, expected_input_features)
-
- def test_audio_features_attn_mask_consistent(self):
- # regression test for https://github.com/huggingface/transformers/issues/39911
- # Test input_features and input_features_mask have consistent shape
- np.random.seed(42)
- feature_extractor = self.feature_extraction_class(**self.feat_extract_dict)
- for i in [512, 640, 1024]:
- audio = np.random.randn(i)
- mm_data = {
- "raw_speech": [audio],
- "sampling_rate": 16000,
- }
- inputs = feature_extractor(**mm_data, return_tensors="np")
- out = inputs["input_features"]
- mask = inputs["input_features_mask"]
-
- assert out.ndim == 3
- assert mask.ndim == 2
- assert out.shape[:2] == mask.shape[:2]
-
- def test_dither(self):
- np.random.seed(42) # seed the dithering randn()
-
- # Tests that features with and without little dithering are similar, but not the same
- dict_no_dither = self.feat_extract_tester.prepare_feat_extract_dict()
- dict_no_dither["dither"] = 0.0
-
- dict_dither = self.feat_extract_tester.prepare_feat_extract_dict()
- dict_dither["dither"] = 0.00003 # approx. 1/32k
-
- feature_extractor_no_dither = self.feature_extraction_class(**dict_no_dither)
- feature_extractor_dither = self.feature_extraction_class(**dict_dither)
-
- # create three inputs of length 800, 1000, and 1200
- speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)]
- np_speech_inputs = [np.asarray(speech_input) for speech_input in speech_inputs]
-
- # compute features
- input_features_no_dither = feature_extractor_no_dither(
- np_speech_inputs, padding=True, return_tensors="np", sampling_rate=dict_no_dither["sampling_rate"]
- ).input_features
- input_features_dither = feature_extractor_dither(
- np_speech_inputs, padding=True, return_tensors="np", sampling_rate=dict_dither["sampling_rate"]
- ).input_features
-
- # test there is a difference between features (there's added noise to input signal)
- diff = input_features_dither - input_features_no_dither
-
- # features are not identical
- assert np.abs(diff).mean() > 1e-6
- # features are not too different
- # the heuristic value `7e-4` is obtained by running 50000 times (maximal value is around 3e-4).
- assert np.abs(diff).mean() < 7e-4
- # the heuristic value `8e-1` is obtained by running 50000 times (maximal value is around 5e-1).
- assert np.abs(diff).max() < 8e-1
-
- @require_torch
- def test_double_precision_pad(self):
- import torch
-
- feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- np_speech_inputs = np.random.rand(100, 32).astype(np.float64)
- py_speech_inputs = np_speech_inputs.tolist()
-
- for inputs in [py_speech_inputs, np_speech_inputs]:
- np_processed = feature_extractor.pad([{"input_features": inputs}], return_tensors="np")
- self.assertTrue(np_processed.input_features.dtype == np.float32)
- pt_processed = feature_extractor.pad([{"input_features": inputs}], return_tensors="pt")
- self.assertTrue(pt_processed.input_features.dtype == torch.float32)
diff --git a/tests/models/gemma3n/test_processing_gemma3n.py b/tests/models/gemma3n/test_processing_gemma3n.py
index 73119084a52d..3a6360eb22b3 100644
--- a/tests/models/gemma3n/test_processing_gemma3n.py
+++ b/tests/models/gemma3n/test_processing_gemma3n.py
@@ -22,8 +22,7 @@
require_vision,
)
-from ...test_processing_common import ProcessorTesterMixin
-from .test_feature_extraction_gemma3n import floats_list
+from ...test_processing_common import ProcessorTesterMixin, floats_list
# TODO: omni-modal processor can't run tests from `ProcessorTesterMixin`
diff --git a/tests/models/gemma4/test_audio_processing_gemma4.py b/tests/models/gemma4/test_audio_processing_gemma4.py
new file mode 100644
index 000000000000..826dde3ed726
--- /dev/null
+++ b/tests/models/gemma4/test_audio_processing_gemma4.py
@@ -0,0 +1,41 @@
+# Copyright 2026 HuggingFace Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+"""Tests for `Gemma4AudioProcessor` and `Gemma4AudioProcessorNumpy`."""
+
+from __future__ import annotations
+
+import unittest
+
+from transformers.testing_utils import require_torch
+
+from ...test_audio_processing_common import AudioProcessingTestMixin
+
+
+class Gemma4AudioProcessingTester:
+ """Provides init kwargs and fixture parameters for the Gemma4 audio processor tests."""
+
+ sample_rate = 16000
+
+ def prepare_audio_processor_dict(self) -> dict:
+ # Dither uses `np.random.randn` in the legacy FE — not seeded for cross-backend
+ # parity. Disable for the parity test (both backends still implement dither for
+ # production). HTK preemphasis is off by default, so the default flow is the
+ # interesting bit-exact path.
+ return {"dither": 0.0}
+
+
+@require_torch
+class Gemma4AudioProcessingTest(AudioProcessingTestMixin, unittest.TestCase):
+ # Like Gemma3n, the unfold-based STFT with float32-window-on-float64-frame multiplication
+ # produces a slightly larger cross-backend divergence than the strict float32 noise floor.
+ parity_atol = 5e-4
+ parity_rtol = 1e-4
+
+ def setUp(self):
+ self.audio_processor_tester = Gemma4AudioProcessingTester()
+ super().setUp()
diff --git a/tests/models/kyutai_speech_to_text/test_audio_processing_kyutai_speech_to_text.py b/tests/models/kyutai_speech_to_text/test_audio_processing_kyutai_speech_to_text.py
new file mode 100644
index 000000000000..1297ef4a75bc
--- /dev/null
+++ b/tests/models/kyutai_speech_to_text/test_audio_processing_kyutai_speech_to_text.py
@@ -0,0 +1,32 @@
+# Copyright 2026 HuggingFace Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+"""Tests for `KyutaiSpeechToTextAudioProcessor` and `KyutaiSpeechToTextAudioProcessorNumpy`."""
+
+from __future__ import annotations
+
+import unittest
+
+from transformers.testing_utils import require_torch
+
+from ...test_audio_processing_common import AudioProcessingTestMixin
+
+
+class KyutaiSpeechToTextAudioProcessingTester:
+ """Provides init kwargs and fixture parameters for the Kyutai STT audio processor tests."""
+
+ sample_rate = 24000
+
+ def prepare_audio_processor_dict(self) -> dict:
+ return {}
+
+
+@require_torch
+class KyutaiSpeechToTextAudioProcessingTest(AudioProcessingTestMixin, unittest.TestCase):
+ def setUp(self):
+ self.audio_processor_tester = KyutaiSpeechToTextAudioProcessingTester()
+ super().setUp()
diff --git a/tests/models/parakeet/test_audio_processing_parakeet.py b/tests/models/parakeet/test_audio_processing_parakeet.py
new file mode 100644
index 000000000000..d609e9b48c91
--- /dev/null
+++ b/tests/models/parakeet/test_audio_processing_parakeet.py
@@ -0,0 +1,53 @@
+# Copyright 2026 HuggingFace Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+"""Tests for `ParakeetAudioProcessor` and `ParakeetAudioProcessorNumpy`."""
+
+from __future__ import annotations
+
+import unittest
+
+from transformers.models.auto.feature_extraction_auto import (
+ FEATURE_EXTRACTOR_MAPPING_NAMES,
+ feature_extractor_class_from_name,
+)
+from transformers.testing_utils import require_torch
+
+from ...test_audio_processing_common import AudioProcessingTestMixin
+
+
+class ParakeetAudioProcessingTester:
+ """Provides init kwargs and fixture parameters for the Parakeet audio processor tests."""
+
+ sample_rate = 16000
+
+ def prepare_audio_processor_dict(self) -> dict:
+ return {}
+
+
+@require_torch
+class ParakeetAudioProcessingTest(AudioProcessingTestMixin, unittest.TestCase):
+ # Parakeet's `power=2.0` magnitudes, librosa-compatible mel filters, log(x + mel_floor)
+ # compression, and per-utterance mean/var normalization compound the underlying
+ # `np.fft.rfft` vs `torch.fft.rfft` float32 noise (a single STFT bin already drifts ~4e-6).
+ # Empirically the final audio_features drift up to ~6e-5 on batched inputs — within the
+ # float32 noise floor but above the strict default bar.
+ parity_atol = 1e-4
+ parity_rtol = 1e-4
+
+ def setUp(self):
+ # Parakeet is registered under `parakeet_ctc` / `parakeet_encoder` in the auto mapping
+ # rather than `parakeet` (the test directory name), so the mixin's auto-discovery
+ # cannot match. We override the lookup directly.
+ self.audio_processor_tester = ParakeetAudioProcessingTester()
+ class_names_by_backend = FEATURE_EXTRACTOR_MAPPING_NAMES["parakeet_ctc"]
+ self.audio_processing_classes = {
+ backend: feature_extractor_class_from_name(class_name)
+ for backend, class_name in class_names_by_backend.items()
+ if class_name not in self.test_classes_to_skip
+ }
+ self.audio_processing_classes = {b: c for b, c in self.audio_processing_classes.items() if c is not None}
diff --git a/tests/models/parakeet/test_feature_extraction_parakeet.py b/tests/models/parakeet/test_feature_extraction_parakeet.py
deleted file mode 100644
index 58c5b4f85bb2..000000000000
--- a/tests/models/parakeet/test_feature_extraction_parakeet.py
+++ /dev/null
@@ -1,181 +0,0 @@
-# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Testing suite for the Parakeet feature extraction."""
-
-import itertools
-import unittest
-
-import numpy as np
-
-from transformers import ParakeetFeatureExtractor
-from transformers.testing_utils import require_torch
-from transformers.utils import is_datasets_available, is_torch_available
-
-from ...test_processing_common import floats_list
-from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin
-
-
-if is_torch_available():
- import torch
-
-if is_datasets_available():
- from datasets import load_dataset
-
-
-class ParakeetFeatureExtractionTester:
- def __init__(
- self,
- parent,
- batch_size=7,
- min_seq_length=400,
- max_seq_length=2000,
- feature_size=80,
- hop_length=160,
- win_length=400,
- n_fft=512,
- sampling_rate=16000,
- padding_value=0.0,
- ):
- self.parent = parent
- self.batch_size = batch_size
- self.min_seq_length = min_seq_length
- self.max_seq_length = max_seq_length
- self.seq_length_diff = (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
- self.feature_size = feature_size
- self.hop_length = hop_length
- self.win_length = win_length
- self.n_fft = n_fft
- self.sampling_rate = sampling_rate
- self.padding_value = padding_value
-
- def prepare_feat_extract_dict(self):
- return {
- "feature_size": self.feature_size,
- "hop_length": self.hop_length,
- "win_length": self.win_length,
- "n_fft": self.n_fft,
- "sampling_rate": self.sampling_rate,
- "padding_value": self.padding_value,
- }
-
- # Copied from tests.models.whisper.test_feature_extraction_whisper.WhisperFeatureExtractionTester.prepare_inputs_for_common
- def prepare_inputs_for_common(self, equal_length=False, numpify=False):
- def _flatten(list_of_lists):
- return list(itertools.chain(*list_of_lists))
-
- if equal_length:
- speech_inputs = [floats_list((self.max_seq_length, self.feature_size)) for _ in range(self.batch_size)]
- else:
- # make sure that inputs increase in size
- speech_inputs = [
- floats_list((x, self.feature_size))
- for x in range(self.min_seq_length, self.max_seq_length, self.seq_length_diff)
- ]
- if numpify:
- speech_inputs = [np.asarray(x) for x in speech_inputs]
- return speech_inputs
-
-
-class ParakeetFeatureExtractionTest(SequenceFeatureExtractionTestMixin, unittest.TestCase):
- feature_extraction_class = ParakeetFeatureExtractor
-
- def setUp(self):
- self.feat_extract_tester = ParakeetFeatureExtractionTester(self)
-
- def _load_datasamples(self, num_samples):
- ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
- # automatic decoding with librispeech
- speech_samples = ds.sort("id")[:num_samples]["audio"]
-
- return [x["array"] for x in speech_samples]
-
- @require_torch
- def test_torch_integration(self):
- """
- reproducer: https://gist.github.com/eustlb/c4a0999e54466b7e8d8b040d8e0900df
- """
- # fmt: off
- EXPECTED_INPUT_FEATURES = torch.tensor(
- [
- 0.60935932, 1.18187428, 1.29877627, 1.36461377, 1.09311509, 1.39821815,
- 1.63753450, 1.37100816, 1.26510608, 1.70332706, 1.69067430, 1.28770995,
- 1.52999651, 1.77962756, 1.71420062, 1.21944094, 1.30884087, 1.44343364,
- 1.17694926, 1.42690814, 1.78877723, 1.68655288, 1.27155364, 1.66103351,
- 1.75820673, 1.41575801, 1.40622294, 1.70603478, 1.63117850, 1.13353217,
- ]
- )
- # fmt: on
-
- input_speech = self._load_datasamples(1)
- feature_extractor = ParakeetFeatureExtractor()
- inputs = feature_extractor(input_speech, return_tensors="pt")
-
- self.assertEqual(inputs.input_features.shape, (1, 586, 80))
- torch.testing.assert_close(inputs.input_features[0, 100, :30], EXPECTED_INPUT_FEATURES, atol=1e-4, rtol=1e-4)
-
- self.assertEqual(inputs.attention_mask.shape, (1, 586))
- # last frame should be masked
- self.assertEqual(inputs.attention_mask.sum(), 585)
-
- @require_torch
- def test_torch_integration_batch(self):
- """
- reproducer: https://gist.github.com/eustlb/c4a0999e54466b7e8d8b040d8e0900df
- """
- # fmt: off
- EXPECTED_INPUT_FEATURES = torch.tensor(
- [
- [ 0.60935932, 1.18187428, 1.29877627, 1.36461377, 1.09311533,
- 1.39821827, 1.63753450, 1.37100816, 1.26510608, 1.70332706,
- 1.69067478, 1.28770995, 1.52999651, 1.77962780, 1.71420062,
- 1.21944094, 1.30884087, 1.44343400, 1.17694926, 1.42690814,
- 1.78877664, 1.68655288, 1.27155364, 1.66103351, 1.75820673,
- 1.41575801, 1.40622294, 1.70603478, 1.63117862, 1.13353217],
- [ 0.58339858, 0.54317272, 0.46222782, 0.34154415, 0.17806509,
- 0.32182255, 0.28909618, 0.02141305, -0.09710173, -0.35818669,
- -0.48172510, -0.52942866, -0.58029658, -0.70519227, -0.67929971,
- -0.54698551, -0.28611183, -0.24780270, -0.31363955, -0.41913241,
- -0.32394424, -0.44897896, -0.68657434, -0.62047797, -0.46886450,
- -0.65987164, -1.02435589, -0.58527517, -0.56095684, -0.73582536],
- [-0.91937613, -0.97933632, -1.06843162, -1.02642107, -0.94232899,
- -0.83840621, -0.82306921, -0.45763230, -0.45182887, -0.75917768,
- -0.42541453, -0.28512970, -0.39637473, -0.66478080, -0.68004298,
- -0.49690303, -0.31799242, -0.12917191, 0.13149273, 0.10163058,
- -0.40041649, 0.05001565, 0.23906317, 0.28816083, 0.14308788,
- -0.29588422, -0.05428466, 0.14418560, 0.28865972, -0.12138986],
- [ 0.73217624, 0.84484011, 0.79323846, 0.66315967, 0.41556871,
- 0.88633078, 0.90718138, 0.91268104, 1.15920067, 1.26141894,
- 1.10222173, 0.92990804, 0.96352047, 0.88142169, 0.56635213,
- 0.71491158, 0.81301254, 0.67301887, 0.74780160, 0.64429688,
- 0.22885245, 0.47035533, 0.46498337, 0.17544533, 0.44458991,
- 0.79245001, 0.57207537, 0.85768145, 1.00491571, 0.93360955],
- [ 1.40496337, 1.32492661, 1.16519547, 0.98379827, 0.77614164,
- 0.95871657, 0.81910741, 1.23010278, 1.33011520, 1.16538525,
- 1.28319681, 1.45041633, 1.33421600, 0.91677380, 0.67107433,
- 0.52890682, 0.82009870, 1.15821445, 1.15343642, 1.10958862,
- 1.44962490, 1.44485891, 1.46043479, 1.90800595, 1.95863307,
- 1.63670933, 1.49021459, 1.18701911, 0.74906683, 0.84700620]
- ]
- )
- # fmt: on
-
- input_speech = self._load_datasamples(5)
- feature_extractor = ParakeetFeatureExtractor()
- inputs = feature_extractor(input_speech, return_tensors="pt")
-
- self.assertEqual(inputs.input_features.shape, (5, 2941, 80))
- torch.testing.assert_close(inputs.input_features[:, 100, :30], EXPECTED_INPUT_FEATURES, atol=1e-4, rtol=1e-4)
-
- self.assertEqual(inputs.attention_mask.shape, (5, 2941))
- self.assertTrue(inputs.attention_mask.sum(dim=-1).tolist(), [585, 481, 1248, 990, 2940])
diff --git a/tests/models/pe_audio/test_audio_processing_pe_audio.py b/tests/models/pe_audio/test_audio_processing_pe_audio.py
new file mode 100644
index 000000000000..9d846746aa2f
--- /dev/null
+++ b/tests/models/pe_audio/test_audio_processing_pe_audio.py
@@ -0,0 +1,19 @@
+import unittest
+
+from transformers.testing_utils import require_torch
+
+from ...test_audio_processing_common import AudioProcessingTestMixin
+
+
+class PeAudioAudioProcessingTester:
+ sample_rate = 16000
+
+ def prepare_audio_processor_dict(self) -> dict:
+ return {}
+
+
+@require_torch
+class PeAudioAudioProcessingTest(AudioProcessingTestMixin, unittest.TestCase):
+ def setUp(self):
+ self.audio_processor_tester = PeAudioAudioProcessingTester()
+ super().setUp()
diff --git a/tests/models/pop2piano/test_audio_processing_pop2piano.py b/tests/models/pop2piano/test_audio_processing_pop2piano.py
new file mode 100644
index 000000000000..fe119d651aeb
--- /dev/null
+++ b/tests/models/pop2piano/test_audio_processing_pop2piano.py
@@ -0,0 +1,39 @@
+# Copyright 2026 HuggingFace Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+"""Tests for `Pop2PianoAudioProcessor` and `Pop2PianoAudioProcessorNumpy`."""
+
+from __future__ import annotations
+
+import unittest
+
+from transformers.testing_utils import require_torch
+
+from ...test_audio_processing_common import AudioProcessingTestMixin
+
+
+class Pop2PianoAudioProcessingTester:
+ """Provides init kwargs and fixture parameters for the Pop2Piano audio processor tests."""
+
+ sample_rate = 22050
+
+ def prepare_audio_processor_dict(self) -> dict:
+ return {}
+
+
+@require_torch
+class Pop2PianoAudioProcessingTest(AudioProcessingTestMixin, unittest.TestCase):
+ # Pop2Piano's `n_fft=4096`, `power=2.0` magnitudes, large `n_mels=512` HTK mel filterbank,
+ # and `log10` compression amplify the `np.fft.rfft` vs `torch.fft.rfft` float32 drift.
+ # Empirically up to ~7e-4 on batched inputs — within the float32 noise floor for a
+ # spectrogram of this size but above the strict default bar.
+ parity_atol = 1e-3
+ parity_rtol = 1e-4
+
+ def setUp(self):
+ self.audio_processor_tester = Pop2PianoAudioProcessingTester()
+ super().setUp()
diff --git a/tests/models/pop2piano/test_feature_extraction_pop2piano.py b/tests/models/pop2piano/test_feature_extraction_pop2piano.py
deleted file mode 100644
index 560780662e5c..000000000000
--- a/tests/models/pop2piano/test_feature_extraction_pop2piano.py
+++ /dev/null
@@ -1,267 +0,0 @@
-# Copyright 2023 HuggingFace Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import tempfile
-import unittest
-
-import numpy as np
-from datasets import load_dataset
-
-from transformers.testing_utils import (
- check_json_file_has_correct_format,
- require_essentia,
- require_librosa,
- require_scipy,
- require_torch,
-)
-from transformers.utils.import_utils import (
- is_essentia_available,
- is_librosa_available,
- is_scipy_available,
- is_torch_available,
-)
-
-from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin
-
-
-requirements_available = (
- is_torch_available() and is_essentia_available() and is_scipy_available() and is_librosa_available()
-)
-
-if requirements_available:
- import torch
-
- from transformers import Pop2PianoFeatureExtractor
-
-
-class Pop2PianoFeatureExtractionTester:
- def __init__(
- self,
- parent,
- n_bars=2,
- sample_rate=22050,
- use_mel=True,
- padding_value=0,
- vocab_size_special=4,
- vocab_size_note=128,
- vocab_size_velocity=2,
- vocab_size_time=100,
- ):
- self.parent = parent
- self.n_bars = n_bars
- self.sample_rate = sample_rate
- self.use_mel = use_mel
- self.padding_value = padding_value
- self.vocab_size_special = vocab_size_special
- self.vocab_size_note = vocab_size_note
- self.vocab_size_velocity = vocab_size_velocity
- self.vocab_size_time = vocab_size_time
-
- def prepare_feat_extract_dict(self):
- return {
- "n_bars": self.n_bars,
- "sample_rate": self.sample_rate,
- "use_mel": self.use_mel,
- "padding_value": self.padding_value,
- "vocab_size_special": self.vocab_size_special,
- "vocab_size_note": self.vocab_size_note,
- "vocab_size_velocity": self.vocab_size_velocity,
- "vocab_size_time": self.vocab_size_time,
- }
-
-
-@require_torch
-@require_essentia
-@require_librosa
-@require_scipy
-class Pop2PianoFeatureExtractionTest(SequenceFeatureExtractionTestMixin, unittest.TestCase):
- feature_extraction_class = Pop2PianoFeatureExtractor if requirements_available else None
-
- def setUp(self):
- self.feat_extract_tester = Pop2PianoFeatureExtractionTester(self)
-
- def test_feat_extract_from_and_save_pretrained(self):
- feat_extract_first = self.feature_extraction_class(**self.feat_extract_dict)
-
- with tempfile.TemporaryDirectory() as tmpdirname:
- saved_file = feat_extract_first.save_pretrained(tmpdirname)[0]
- check_json_file_has_correct_format(saved_file)
- feat_extract_second = self.feature_extraction_class.from_pretrained(tmpdirname)
-
- dict_first = feat_extract_first.to_dict()
- dict_second = feat_extract_second.to_dict()
- mel_1 = feat_extract_first.use_mel
- mel_2 = feat_extract_second.use_mel
- self.assertTrue(np.allclose(mel_1, mel_2))
- self.assertEqual(dict_first, dict_second)
-
- def test_feat_extract_to_json_file(self):
- feat_extract_first = self.feature_extraction_class(**self.feat_extract_dict)
-
- with tempfile.TemporaryDirectory() as tmpdirname:
- json_file_path = os.path.join(tmpdirname, "feat_extract.json")
- feat_extract_first.to_json_file(json_file_path)
- feat_extract_second = self.feature_extraction_class.from_json_file(json_file_path)
-
- dict_first = feat_extract_first.to_dict()
- dict_second = feat_extract_second.to_dict()
- mel_1 = feat_extract_first.use_mel
- mel_2 = feat_extract_second.use_mel
- self.assertTrue(np.allclose(mel_1, mel_2))
- self.assertEqual(dict_first, dict_second)
-
- def test_call(self):
- feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- speech_input = np.zeros([1000000], dtype=np.float32)
-
- input_features = feature_extractor(speech_input, sampling_rate=16_000, return_tensors="np")
- self.assertTrue(input_features.input_features.ndim == 3)
- self.assertEqual(input_features.input_features.shape[-1], 512)
-
- self.assertTrue(input_features.beatsteps.ndim == 2)
- self.assertTrue(input_features.extrapolated_beatstep.ndim == 2)
-
- def test_integration(self):
- ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
- speech_samples = ds.sort("id").select([0])["audio"]
- input_speech = [x["array"] for x in speech_samples][0]
- sampling_rate = [x["sampling_rate"] for x in speech_samples][0]
- feature_extractor = Pop2PianoFeatureExtractor.from_pretrained("sweetcocoa/pop2piano")
- input_features = feature_extractor(
- input_speech, sampling_rate=sampling_rate, return_tensors="pt"
- ).input_features
-
- EXPECTED_INPUT_FEATURES = torch.tensor(
- [[-7.1493, -6.8701, -4.3214], [-5.9473, -5.7548, -3.8438], [-6.1324, -5.9018, -4.3778]]
- )
- torch.testing.assert_close(input_features[0, :3, :3], EXPECTED_INPUT_FEATURES, rtol=1e-4, atol=1e-4)
-
- def test_attention_mask(self):
- feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- speech_input1 = np.zeros([1_000_000], dtype=np.float32)
- speech_input2 = np.random.randint(low=0, high=10, size=500_000).astype(np.float32)
- input_features = feature_extractor(
- [speech_input1, speech_input2],
- sampling_rate=[44_100, 16_000],
- return_tensors="np",
- return_attention_mask=True,
- )
-
- self.assertTrue(hasattr(input_features, "attention_mask"))
-
- # check shapes
- self.assertTrue(input_features["attention_mask"].ndim == 2)
- self.assertEqual(input_features["attention_mask_beatsteps"].shape[0], 2)
- self.assertEqual(input_features["attention_mask_extrapolated_beatstep"].shape[0], 2)
-
- # check if they are any values except 0 and 1
- self.assertTrue(np.max(input_features["attention_mask"]) == 1)
- self.assertTrue(np.max(input_features["attention_mask_beatsteps"]) == 1)
- self.assertTrue(np.max(input_features["attention_mask_extrapolated_beatstep"]) == 1)
-
- self.assertTrue(np.min(input_features["attention_mask"]) == 0)
- self.assertTrue(np.min(input_features["attention_mask_beatsteps"]) == 0)
- self.assertTrue(np.min(input_features["attention_mask_extrapolated_beatstep"]) == 0)
-
- def test_batch_feature(self):
- feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- speech_input1 = np.zeros([1_000_000], dtype=np.float32)
- speech_input2 = np.ones([2_000_000], dtype=np.float32)
- speech_input3 = np.random.randint(low=0, high=10, size=500_000).astype(np.float32)
-
- input_features = feature_extractor(
- [speech_input1, speech_input2, speech_input3],
- sampling_rate=[44_100, 16_000, 48_000],
- return_attention_mask=True,
- )
-
- self.assertEqual(len(input_features["input_features"].shape), 3)
- # check shape
- self.assertEqual(input_features["beatsteps"].shape[0], 3)
- self.assertEqual(input_features["extrapolated_beatstep"].shape[0], 3)
-
- def test_batch_feature_np(self):
- feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- speech_input1 = np.zeros([1_000_000], dtype=np.float32)
- speech_input2 = np.ones([2_000_000], dtype=np.float32)
- speech_input3 = np.random.randint(low=0, high=10, size=500_000).astype(np.float32)
-
- input_features = feature_extractor(
- [speech_input1, speech_input2, speech_input3],
- sampling_rate=[44_100, 16_000, 48_000],
- return_tensors="np",
- return_attention_mask=True,
- )
-
- # check np array or not
- self.assertEqual(type(input_features["input_features"]), np.ndarray)
-
- # check shape
- self.assertEqual(len(input_features["input_features"].shape), 3)
-
- def test_batch_feature_pt(self):
- feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- speech_input1 = np.zeros([1_000_000], dtype=np.float32)
- speech_input2 = np.ones([2_000_000], dtype=np.float32)
- speech_input3 = np.random.randint(low=0, high=10, size=500_000).astype(np.float32)
-
- input_features = feature_extractor(
- [speech_input1, speech_input2, speech_input3],
- sampling_rate=[44_100, 16_000, 48_000],
- return_tensors="pt",
- return_attention_mask=True,
- )
-
- # check pt tensor or not
- self.assertEqual(type(input_features["input_features"]), torch.Tensor)
-
- # check shape
- self.assertEqual(len(input_features["input_features"].shape), 3)
-
- @unittest.skip(
- "Pop2PianoFeatureExtractor does not supports padding externally (while processing audios in batches padding is automatically applied to max_length)"
- )
- def test_padding_accepts_tensors_pt(self):
- pass
-
- @unittest.skip(
- "Pop2PianoFeatureExtractor does not supports padding externally (while processing audios in batches padding is automatically applied to max_length)"
- )
- def test_padding_accepts_tensors_tf(self):
- pass
-
- @unittest.skip(
- "Pop2PianoFeatureExtractor does not supports padding externally (while processing audios in batches padding is automatically applied to max_length)"
- )
- def test_padding_from_list(self):
- pass
-
- @unittest.skip(
- "Pop2PianoFeatureExtractor does not supports padding externally (while processing audios in batches padding is automatically applied to max_length)"
- )
- def test_padding_from_array(self):
- pass
-
- @unittest.skip(reason="Pop2PianoFeatureExtractor does not support truncation")
- def test_attention_mask_with_truncation(self):
- pass
-
- @unittest.skip(reason="Pop2PianoFeatureExtractor does not supports truncation")
- def test_truncation_from_array(self):
- pass
-
- @unittest.skip(reason="Pop2PianoFeatureExtractor does not supports truncation")
- def test_truncation_from_list(self):
- pass
diff --git a/tests/models/seamless_m4t/test_audio_processing_seamless_m4t.py b/tests/models/seamless_m4t/test_audio_processing_seamless_m4t.py
new file mode 100644
index 000000000000..d965481e049a
--- /dev/null
+++ b/tests/models/seamless_m4t/test_audio_processing_seamless_m4t.py
@@ -0,0 +1,39 @@
+# Copyright 2026 HuggingFace Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+"""Tests for `SeamlessM4tAudioProcessor` and `SeamlessM4tAudioProcessorNumpy`."""
+
+from __future__ import annotations
+
+import unittest
+
+from transformers.testing_utils import require_torch
+
+from ...test_audio_processing_common import AudioProcessingTestMixin
+
+
+class SeamlessM4tAudioProcessingTester:
+ """Provides init kwargs and fixture parameters for the SeamlessM4t audio processor tests."""
+
+ sample_rate = 16000
+
+ def prepare_audio_processor_dict(self) -> dict:
+ return {}
+
+
+@require_torch
+class SeamlessM4tAudioProcessingTest(AudioProcessingTestMixin, unittest.TestCase):
+ # SeamlessM4t's `power=2.0` magnitudes (computed in float64), kaldi-exact mel filters built
+ # in float32 (cast at the matmul site), and `ddof=1` per-utterance variance normalization
+ # amplify the underlying float32 STFT noise. Empirically the final audio_features drift up
+ # to ~1.2e-4 on batched inputs.
+ parity_atol = 5e-4
+ parity_rtol = 1e-4
+
+ def setUp(self):
+ self.audio_processor_tester = SeamlessM4tAudioProcessingTester()
+ super().setUp()
diff --git a/tests/models/seamless_m4t/test_feature_extraction_seamless_m4t.py b/tests/models/seamless_m4t/test_feature_extraction_seamless_m4t.py
deleted file mode 100644
index 23f9dedcb4ca..000000000000
--- a/tests/models/seamless_m4t/test_feature_extraction_seamless_m4t.py
+++ /dev/null
@@ -1,333 +0,0 @@
-# Copyright 2023 HuggingFace Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-import itertools
-import os
-import tempfile
-import unittest
-
-import numpy as np
-from datasets import load_dataset
-
-from transformers import SeamlessM4TFeatureExtractor, is_speech_available
-from transformers.testing_utils import check_json_file_has_correct_format, require_torch
-from transformers.utils.import_utils import is_torch_available
-
-from ...test_processing_common import floats_list
-from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin
-
-
-if is_torch_available():
- import torch
-
-
-@require_torch
-class SeamlessM4TFeatureExtractionTester:
- def __init__(
- self,
- parent,
- batch_size=7,
- min_seq_length=400,
- max_seq_length=2000,
- feature_size=10,
- padding_value=0.0,
- sampling_rate=4_000,
- return_attention_mask=True,
- do_normalize=True,
- stride=2,
- ):
- self.parent = parent
- self.batch_size = batch_size
- self.min_seq_length = min_seq_length
- self.max_seq_length = max_seq_length
- self.seq_length_diff = (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
- self.padding_value = padding_value
- self.sampling_rate = sampling_rate
- self.return_attention_mask = return_attention_mask
- self.do_normalize = do_normalize
- self.feature_size = feature_size
- self.stride = stride
- self.num_mel_bins = feature_size
-
- def prepare_feat_extract_dict(self):
- return {
- "feature_size": self.feature_size,
- "num_mel_bins": self.num_mel_bins,
- "padding_value": self.padding_value,
- "sampling_rate": self.sampling_rate,
- "stride": self.stride,
- "return_attention_mask": self.return_attention_mask,
- "do_normalize": self.do_normalize,
- }
-
- # Copied from tests.models.whisper.test_feature_extraction_whisper.WhisperFeatureExtractionTester.prepare_inputs_for_common
- def prepare_inputs_for_common(self, equal_length=False, numpify=False):
- def _flatten(list_of_lists):
- return list(itertools.chain(*list_of_lists))
-
- if equal_length:
- speech_inputs = [floats_list((self.max_seq_length, self.feature_size)) for _ in range(self.batch_size)]
- else:
- # make sure that inputs increase in size
- speech_inputs = [
- floats_list((x, self.feature_size))
- for x in range(self.min_seq_length, self.max_seq_length, self.seq_length_diff)
- ]
- if numpify:
- speech_inputs = [np.asarray(x) for x in speech_inputs]
- return speech_inputs
-
-
-@require_torch
-class SeamlessM4TFeatureExtractionTest(SequenceFeatureExtractionTestMixin, unittest.TestCase):
- feature_extraction_class = SeamlessM4TFeatureExtractor if is_speech_available() else None
-
- def setUp(self):
- self.feat_extract_tester = SeamlessM4TFeatureExtractionTester(self)
-
- def test_feat_extract_from_and_save_pretrained(self):
- feat_extract_first = self.feature_extraction_class(**self.feat_extract_dict)
-
- with tempfile.TemporaryDirectory() as tmpdirname:
- saved_file = feat_extract_first.save_pretrained(tmpdirname)[0]
- check_json_file_has_correct_format(saved_file)
- feat_extract_second = self.feature_extraction_class.from_pretrained(tmpdirname)
-
- dict_first = feat_extract_first.to_dict()
- dict_second = feat_extract_second.to_dict()
- self.assertDictEqual(dict_first, dict_second)
-
- def test_feat_extract_to_json_file(self):
- feat_extract_first = self.feature_extraction_class(**self.feat_extract_dict)
-
- with tempfile.TemporaryDirectory() as tmpdirname:
- json_file_path = os.path.join(tmpdirname, "feat_extract.json")
- feat_extract_first.to_json_file(json_file_path)
- feat_extract_second = self.feature_extraction_class.from_json_file(json_file_path)
-
- dict_first = feat_extract_first.to_dict()
- dict_second = feat_extract_second.to_dict()
- self.assertEqual(dict_first, dict_second)
-
- def test_call_numpy(self):
- # Tests that all call wrap to encode_plus and batch_encode_plus
- feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- # create three inputs of length 800, 1000, and 1200
- speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)]
- np_speech_inputs = [np.asarray(speech_input) for speech_input in speech_inputs]
-
- # Test feature size
- input_features = feature_extractor(np_speech_inputs, padding=True, return_tensors="np").input_features
- self.assertTrue(input_features.ndim == 3)
- self.assertTrue(input_features.shape[0] == 3)
- self.assertTrue(input_features.shape[-1] == feature_extractor.feature_size * feature_extractor.stride)
-
- # Test not batched input
- encoded_sequences_1 = feature_extractor(speech_inputs[0], return_tensors="np").input_features
- encoded_sequences_2 = feature_extractor(np_speech_inputs[0], return_tensors="np").input_features
- self.assertTrue(np.allclose(encoded_sequences_1, encoded_sequences_2, atol=1e-3))
-
- # Test batched
- encoded_sequences_1 = feature_extractor(speech_inputs, return_tensors="np").input_features
- encoded_sequences_2 = feature_extractor(np_speech_inputs, return_tensors="np").input_features
- for enc_seq_1, enc_seq_2 in zip(encoded_sequences_1, encoded_sequences_2):
- self.assertTrue(np.allclose(enc_seq_1, enc_seq_2, atol=1e-3))
-
- # Test 2-D numpy arrays are batched.
- speech_inputs = [floats_list((1, x))[0] for x in (800, 800, 800)]
- np_speech_inputs = np.asarray(speech_inputs)
- encoded_sequences_1 = feature_extractor(speech_inputs, return_tensors="np").input_features
- encoded_sequences_2 = feature_extractor(np_speech_inputs, return_tensors="np").input_features
- for enc_seq_1, enc_seq_2 in zip(encoded_sequences_1, encoded_sequences_2):
- self.assertTrue(np.allclose(enc_seq_1, enc_seq_2, atol=1e-3))
-
- def test_call_with_padded_input_not_multiple_of_stride(self):
- # same as test_call_numpy but with stride=6 and pad_to_multiple_of=8
- # the input sizes 800, 1400 and 200 are a multiple of pad_to_multiple_of but not a multiple of stride
- # therefore remainder = num_frames % self.stride will not be zero and must be subtracted from num_frames
- stride = 6
- pad_to_multiple_of = 8
-
- feature_extractor_args = self.feat_extract_tester.prepare_feat_extract_dict()
- feature_extractor_args["stride"] = stride
- feature_extractor = self.feature_extraction_class(**feature_extractor_args)
-
- speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)]
- np_speech_inputs = [np.asarray(speech_input) for speech_input in speech_inputs]
-
- # Test feature size and attention mask size
- output = feature_extractor(np_speech_inputs, pad_to_multiple_of=pad_to_multiple_of, return_tensors="np")
- input_features = output.input_features
- self.assertTrue(input_features.ndim == 3)
- self.assertTrue(input_features.shape[0] == 3)
- self.assertTrue(input_features.shape[-1] == feature_extractor.feature_size * feature_extractor.stride)
- # same as test_attention_mask
- attention_mask = output.attention_mask
- self.assertTrue(attention_mask.ndim == 2)
- self.assertTrue(attention_mask.shape[0] == 3)
- self.assertTrue(attention_mask.shape[-1] == input_features.shape[1])
-
- # Test not batched input
- encoded_sequences_1 = feature_extractor(
- speech_inputs[0], pad_to_multiple_of=pad_to_multiple_of, return_tensors="np"
- ).input_features
- encoded_sequences_2 = feature_extractor(
- np_speech_inputs[0], pad_to_multiple_of=pad_to_multiple_of, return_tensors="np"
- ).input_features
- self.assertTrue(np.allclose(encoded_sequences_1, encoded_sequences_2, atol=1e-3))
-
- # Test batched
- encoded_sequences_1 = feature_extractor(
- speech_inputs, pad_to_multiple_of=pad_to_multiple_of, return_tensors="np"
- ).input_features
- encoded_sequences_2 = feature_extractor(
- np_speech_inputs, pad_to_multiple_of=pad_to_multiple_of, return_tensors="np"
- ).input_features
- for enc_seq_1, enc_seq_2 in zip(encoded_sequences_1, encoded_sequences_2):
- self.assertTrue(np.allclose(enc_seq_1, enc_seq_2, atol=1e-3))
-
- # Test 2-D numpy arrays are batched.
- speech_inputs = [floats_list((1, x))[0] for x in (800, 800, 800)]
- np_speech_inputs = np.asarray(speech_inputs)
- encoded_sequences_1 = feature_extractor(
- speech_inputs, pad_to_multiple_of=pad_to_multiple_of, return_tensors="np"
- ).input_features
- encoded_sequences_2 = feature_extractor(
- np_speech_inputs, pad_to_multiple_of=pad_to_multiple_of, return_tensors="np"
- ).input_features
- for enc_seq_1, enc_seq_2 in zip(encoded_sequences_1, encoded_sequences_2):
- self.assertTrue(np.allclose(enc_seq_1, enc_seq_2, atol=1e-3))
-
- def test_call_without_attention_mask(self):
- feature_extractor_args = self.feat_extract_tester.prepare_feat_extract_dict()
- feature_extractor = self.feature_extraction_class(**feature_extractor_args)
-
- # create three inputs of length 800, 1000, and 1200
- speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)]
- np_speech_inputs = [np.asarray(speech_input) for speech_input in speech_inputs]
-
- # Test attention mask when passing no attention mask to forward call
- output = feature_extractor(np_speech_inputs, padding=True, return_tensors="np", return_attention_mask=False)
- self.assertTrue("attention_mask" not in output)
-
- # Test attention mask when no attention mask by default
- feature_extractor_args["return_attention_mask"] = False
- feature_extractor = self.feature_extraction_class(**feature_extractor_args)
- output = feature_extractor(np_speech_inputs, padding=True, return_tensors="np", return_attention_mask=False)
- self.assertTrue("attention_mask" not in output)
-
- def test_attention_mask(self):
- # test attention mask has the right output shape
- feature_extractor_args = self.feat_extract_tester.prepare_feat_extract_dict()
-
- feature_extractor = self.feature_extraction_class(**feature_extractor_args)
- # create three inputs of length 800, 1000, and 1200
- speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)]
- np_speech_inputs = [np.asarray(speech_input) for speech_input in speech_inputs]
-
- # Test attention mask when passing it to forward call
- output = feature_extractor(np_speech_inputs, padding=True, return_tensors="np")
- input_features = output.input_features
-
- attention_mask = output.attention_mask
- self.assertTrue(attention_mask.ndim == 2)
- self.assertTrue(attention_mask.shape[0] == 3)
- self.assertTrue(attention_mask.shape[-1] == input_features.shape[1])
-
- @require_torch
- def test_call_torch(self):
- import torch
-
- # Tests that all call wrap to encode_plus and batch_encode_plus
- feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- # create three inputs of length 800, 1000, and 1200
- speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)]
- pt_speech_inputs = [torch.tensor(speech_input) for speech_input in speech_inputs]
-
- # Test feature size
- input_features = feature_extractor(pt_speech_inputs, padding=True, return_tensors="pt").input_features
- self.assertTrue(input_features.ndim == 3)
- self.assertTrue(input_features.shape[0] == 3)
- self.assertTrue(input_features.shape[-1] == feature_extractor.feature_size * feature_extractor.stride)
-
- # Test not batched input
- encoded_sequences_1 = feature_extractor(speech_inputs[0], return_tensors="pt").input_features
- encoded_sequences_2 = feature_extractor(pt_speech_inputs[0], return_tensors="pt").input_features
- torch.testing.assert_close(encoded_sequences_1, encoded_sequences_2, rtol=1e-3, atol=1e-3)
-
- # Test batched
- encoded_sequences_1 = feature_extractor(speech_inputs, return_tensors="pt").input_features
- encoded_sequences_2 = feature_extractor(pt_speech_inputs, return_tensors="pt").input_features
- for enc_seq_1, enc_seq_2 in zip(encoded_sequences_1, encoded_sequences_2):
- torch.testing.assert_close(enc_seq_1, enc_seq_2, rtol=1e-3, atol=1e-3)
-
- # Test 2-D numpy arrays are batched.
- speech_inputs = [floats_list((1, x))[0] for x in (800, 800, 800)]
- pt_speech_inputs = torch.tensor(speech_inputs)
- encoded_sequences_1 = feature_extractor(speech_inputs, return_tensors="pt").input_features
- encoded_sequences_2 = feature_extractor(pt_speech_inputs, return_tensors="pt").input_features
- for enc_seq_1, enc_seq_2 in zip(encoded_sequences_1, encoded_sequences_2):
- torch.testing.assert_close(enc_seq_1, enc_seq_2, rtol=1e-3, atol=1e-3)
-
- @require_torch
- # Copied from tests.models.whisper.test_feature_extraction_whisper.WhisperFeatureExtractionTest.test_double_precision_pad
- def test_double_precision_pad(self):
- import torch
-
- feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- np_speech_inputs = np.random.rand(100, 32).astype(np.float64)
- py_speech_inputs = np_speech_inputs.tolist()
-
- for inputs in [py_speech_inputs, np_speech_inputs]:
- np_processed = feature_extractor.pad([{"input_features": inputs}], return_tensors="np")
- self.assertTrue(np_processed.input_features.dtype == np.float32)
- pt_processed = feature_extractor.pad([{"input_features": inputs}], return_tensors="pt")
- self.assertTrue(pt_processed.input_features.dtype == torch.float32)
-
- def _load_datasample(self, id):
- ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
- # automatic decoding with librispeech
- speech_sample = ds.sort("id")[id]["audio"]["array"]
-
- return torch.from_numpy(speech_sample).unsqueeze(0)
-
- def test_integration(self):
- # fmt: off
- EXPECTED_INPUT_FEATURES = torch.tensor(
- [
- -1.5621, -1.4236, -1.3335, -1.3991, -1.2881, -1.1133, -0.9710, -0.8895,
- -0.8280, -0.7376, -0.7194, -0.6896, -0.6849, -0.6788, -0.6545, -0.6610,
- -0.6566, -0.5738, -0.5252, -0.5533, -0.5887, -0.6116, -0.5971, -0.4956,
- -0.2881, -0.1512, 0.0299, 0.1762, 0.2728, 0.2236
- ]
- )
- # fmt: on
-
- input_speech = self._load_datasample(10)
- feature_extractor = SeamlessM4TFeatureExtractor()
- input_features = feature_extractor(input_speech, return_tensors="pt").input_features
-
- feature_extractor(input_speech, return_tensors="pt").input_features[0, 5, :30]
- self.assertEqual(input_features.shape, (1, 279, 160))
- torch.testing.assert_close(input_features[0, 5, :30], EXPECTED_INPUT_FEATURES, rtol=1e-4, atol=1e-4)
-
- def test_zero_mean_unit_variance_normalization_trunc_np_longest(self):
- feat_extract = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- audio = self._load_datasample(1)
- audio = ((audio - audio.min()) / (audio.max() - audio.min())) * 65535 # Rescale to [0, 65535] to show issue
- audio = feat_extract.zero_mean_unit_var_norm([audio], attention_mask=None)[0]
-
- self.assertTrue((audio.mean() < 1e-3).all())
- self.assertTrue(((audio.var() - 1).abs() < 1e-3).all())
diff --git a/tests/models/seamless_m4t/test_processing_seamless_m4t.py b/tests/models/seamless_m4t/test_processing_seamless_m4t.py
index ff3595c79f89..6c757b604233 100644
--- a/tests/models/seamless_m4t/test_processing_seamless_m4t.py
+++ b/tests/models/seamless_m4t/test_processing_seamless_m4t.py
@@ -23,7 +23,7 @@
)
from transformers.testing_utils import require_torch
-from .test_feature_extraction_seamless_m4t import floats_list
+from ...test_processing_common import floats_list
@require_torch
diff --git a/tests/models/speech_to_text/test_audio_processing_speech_to_text.py b/tests/models/speech_to_text/test_audio_processing_speech_to_text.py
new file mode 100644
index 000000000000..4531dd0b3511
--- /dev/null
+++ b/tests/models/speech_to_text/test_audio_processing_speech_to_text.py
@@ -0,0 +1,38 @@
+# Copyright 2026 HuggingFace Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+"""Tests for `SpeechToTextAudioProcessor` and `SpeechToTextAudioProcessorNumpy`."""
+
+from __future__ import annotations
+
+import unittest
+
+from transformers.testing_utils import require_torch
+
+from ...test_audio_processing_common import AudioProcessingTestMixin
+
+
+class SpeechToTextAudioProcessingTester:
+ """Provides init kwargs and fixture parameters for the SpeechToText audio processor tests."""
+
+ sample_rate = 16000
+
+ def prepare_audio_processor_dict(self) -> dict:
+ return {}
+
+
+@require_torch
+class SpeechToTextAudioProcessingTest(AudioProcessingTestMixin, unittest.TestCase):
+ # Per-waveform kaldi fbank + per-utterance CMVN (with numpy ddof=0 / torch unbiased=False
+ # matched) introduces small numerical drift above the strict 1e-5 floor — empirically
+ # up to ~3e-5 on batched inputs.
+ parity_atol = 1e-4
+ parity_rtol = 1e-4
+
+ def setUp(self):
+ self.audio_processor_tester = SpeechToTextAudioProcessingTester()
+ super().setUp()
diff --git a/tests/models/speech_to_text/test_feature_extraction_speech_to_text.py b/tests/models/speech_to_text/test_feature_extraction_speech_to_text.py
deleted file mode 100644
index 17bfa6d2f91b..000000000000
--- a/tests/models/speech_to_text/test_feature_extraction_speech_to_text.py
+++ /dev/null
@@ -1,342 +0,0 @@
-# Copyright 2021 HuggingFace Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-import itertools
-import os
-import tempfile
-import unittest
-
-import numpy as np
-
-from transformers import Speech2TextFeatureExtractor
-from transformers.testing_utils import (
- check_json_file_has_correct_format,
- require_torch,
- require_torchaudio,
-)
-
-from ...test_processing_common import floats_list
-from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin
-
-
-@require_torch
-@require_torchaudio
-class Speech2TextFeatureExtractionTester:
- def __init__(
- self,
- parent,
- batch_size=7,
- min_seq_length=400,
- max_seq_length=2000,
- feature_size=24,
- num_mel_bins=24,
- padding_value=0.0,
- sampling_rate=16_000,
- return_attention_mask=True,
- do_normalize=True,
- ):
- self.parent = parent
- self.batch_size = batch_size
- self.min_seq_length = min_seq_length
- self.max_seq_length = max_seq_length
- self.seq_length_diff = (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
- self.feature_size = feature_size
- self.num_mel_bins = num_mel_bins
- self.padding_value = padding_value
- self.sampling_rate = sampling_rate
- self.return_attention_mask = return_attention_mask
- self.do_normalize = do_normalize
-
- def prepare_feat_extract_dict(self):
- return {
- "feature_size": self.feature_size,
- "num_mel_bins": self.num_mel_bins,
- "padding_value": self.padding_value,
- "sampling_rate": self.sampling_rate,
- "return_attention_mask": self.return_attention_mask,
- "do_normalize": self.do_normalize,
- }
-
- def prepare_inputs_for_common(self, equal_length=False, numpify=False):
- def _flatten(list_of_lists):
- return list(itertools.chain(*list_of_lists))
-
- if equal_length:
- speech_inputs = [floats_list((self.max_seq_length, self.feature_size)) for _ in range(self.batch_size)]
- else:
- # make sure that inputs increase in size
- speech_inputs = [
- floats_list((x, self.feature_size))
- for x in range(self.min_seq_length, self.max_seq_length, self.seq_length_diff)
- ]
- if numpify:
- speech_inputs = [np.asarray(x) for x in speech_inputs]
- return speech_inputs
-
-
-@require_torch
-@require_torchaudio
-class Speech2TextFeatureExtractionTest(SequenceFeatureExtractionTestMixin, unittest.TestCase):
- feature_extraction_class = Speech2TextFeatureExtractor
-
- def setUp(self):
- self.feat_extract_tester = Speech2TextFeatureExtractionTester(self)
-
- def _check_zero_mean_unit_variance(self, input_vector):
- self.assertTrue(np.all(np.mean(input_vector, axis=0) < 1e-3))
- self.assertTrue(np.all(np.abs(np.var(input_vector, axis=0) - 1) < 1e-3))
-
- def test_call(self):
- # Tests that all call wrap to encode_plus and batch_encode_plus
- feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- # create three inputs of length 800, 1000, and 1200
- speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)]
- np_speech_inputs = [np.asarray(speech_input) for speech_input in speech_inputs]
-
- # Test feature size
- input_features = feature_extractor(np_speech_inputs, padding=True, return_tensors="np").input_features
- self.assertTrue(input_features.ndim == 3)
- self.assertTrue(input_features.shape[-1] == feature_extractor.feature_size)
-
- # Test not batched input
- encoded_sequences_1 = feature_extractor(speech_inputs[0], return_tensors="np").input_features
- encoded_sequences_2 = feature_extractor(np_speech_inputs[0], return_tensors="np").input_features
- self.assertTrue(np.allclose(encoded_sequences_1, encoded_sequences_2, atol=1e-3))
-
- # Test batched
- encoded_sequences_1 = feature_extractor(speech_inputs, return_tensors="np").input_features
- encoded_sequences_2 = feature_extractor(np_speech_inputs, return_tensors="np").input_features
- for enc_seq_1, enc_seq_2 in zip(encoded_sequences_1, encoded_sequences_2):
- self.assertTrue(np.allclose(enc_seq_1, enc_seq_2, atol=1e-3))
-
- # Test 2-D numpy arrays are batched.
- speech_inputs = [floats_list((1, x))[0] for x in (800, 800, 800)]
- np_speech_inputs = np.asarray(speech_inputs)
- encoded_sequences_1 = feature_extractor(speech_inputs, return_tensors="np").input_features
- encoded_sequences_2 = feature_extractor(np_speech_inputs, return_tensors="np").input_features
- for enc_seq_1, enc_seq_2 in zip(encoded_sequences_1, encoded_sequences_2):
- self.assertTrue(np.allclose(enc_seq_1, enc_seq_2, atol=1e-3))
-
- def test_dither(self):
- np.random.seed(42) # seed the dithering randn()
-
- # Tests that features with and without little dithering are similar, but not the same
- dict_no_dither = self.feat_extract_tester.prepare_feat_extract_dict()
- dict_no_dither["dither"] = 0.0
-
- dict_dither = self.feat_extract_tester.prepare_feat_extract_dict()
- dict_dither["dither"] = 1.0
-
- feature_extractor_no_dither = self.feature_extraction_class(**dict_no_dither)
- feature_extractor_dither = self.feature_extraction_class(**dict_dither)
-
- # create three inputs of length 800, 1000, and 1200
- speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)]
- np_speech_inputs = [np.asarray(speech_input) for speech_input in speech_inputs]
-
- # compute features
- input_features_no_dither = feature_extractor_no_dither(
- np_speech_inputs, padding=True, return_tensors="np", sampling_rate=dict_no_dither["sampling_rate"]
- ).input_features
- input_features_dither = feature_extractor_dither(
- np_speech_inputs, padding=True, return_tensors="np", sampling_rate=dict_dither["sampling_rate"]
- ).input_features
-
- # test there is a difference between features (there's added noise to input signal)
- diff = input_features_dither - input_features_no_dither
-
- # features are not identical
- self.assertTrue(np.abs(diff).mean() > 1e-5)
- # features are not too different
- self.assertTrue(np.abs(diff).mean() <= 1e-3)
- self.assertTrue(np.abs(diff).max() <= 5e-2)
-
- def test_cepstral_mean_and_variance_normalization(self):
- feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)]
-
- paddings = ["longest", "max_length", "do_not_pad"]
- max_lengths = [None, 16, None]
- for max_length, padding in zip(max_lengths, paddings):
- inputs = feature_extractor(
- speech_inputs, padding=padding, max_length=max_length, return_attention_mask=True
- )
- input_features = inputs.input_features
- attention_mask = inputs.attention_mask
- fbank_feat_lengths = [np.sum(x) for x in attention_mask]
-
- self._check_zero_mean_unit_variance(input_features[0][: fbank_feat_lengths[0]])
- self._check_zero_mean_unit_variance(input_features[1][: fbank_feat_lengths[1]])
- self._check_zero_mean_unit_variance(input_features[2][: fbank_feat_lengths[2]])
-
- def test_cepstral_mean_and_variance_normalization_np(self):
- feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)]
-
- paddings = ["longest", "max_length", "do_not_pad"]
- max_lengths = [None, 16, None]
- for max_length, padding in zip(max_lengths, paddings):
- inputs = feature_extractor(
- speech_inputs, max_length=max_length, padding=padding, return_tensors="np", return_attention_mask=True
- )
- input_features = inputs.input_features
- attention_mask = inputs.attention_mask
- fbank_feat_lengths = [np.sum(x) for x in attention_mask]
-
- self._check_zero_mean_unit_variance(input_features[0][: fbank_feat_lengths[0]])
- self.assertTrue(input_features[0][fbank_feat_lengths[0] :].sum() < 1e-6)
- self._check_zero_mean_unit_variance(input_features[1][: fbank_feat_lengths[1]])
- self.assertTrue(input_features[0][fbank_feat_lengths[1] :].sum() < 1e-6)
- self._check_zero_mean_unit_variance(input_features[2][: fbank_feat_lengths[2]])
-
- def test_cepstral_mean_and_variance_normalization_trunc_max_length(self):
- feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)]
- inputs = feature_extractor(
- speech_inputs,
- padding="max_length",
- max_length=4,
- truncation=True,
- return_tensors="np",
- return_attention_mask=True,
- )
- input_features = inputs.input_features
- attention_mask = inputs.attention_mask
- fbank_feat_lengths = np.sum(attention_mask == 1, axis=1)
-
- self._check_zero_mean_unit_variance(input_features[0, : fbank_feat_lengths[0]])
- self._check_zero_mean_unit_variance(input_features[1])
- self._check_zero_mean_unit_variance(input_features[2])
-
- def test_cepstral_mean_and_variance_normalization_trunc_longest(self):
- feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)]
- inputs = feature_extractor(
- speech_inputs,
- padding="longest",
- max_length=4,
- truncation=True,
- return_tensors="np",
- return_attention_mask=True,
- )
- input_features = inputs.input_features
- attention_mask = inputs.attention_mask
- fbank_feat_lengths = np.sum(attention_mask == 1, axis=1)
-
- self._check_zero_mean_unit_variance(input_features[0, : fbank_feat_lengths[0]])
- self._check_zero_mean_unit_variance(input_features[1, : fbank_feat_lengths[1]])
- self._check_zero_mean_unit_variance(input_features[2])
-
- # make sure that if max_length < longest -> then pad to max_length
- self.assertEqual(input_features.shape, (3, 4, 24))
-
- speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)]
- inputs = feature_extractor(
- speech_inputs,
- padding="longest",
- max_length=16,
- truncation=True,
- return_tensors="np",
- return_attention_mask=True,
- )
- input_features = inputs.input_features
- attention_mask = inputs.attention_mask
- fbank_feat_lengths = np.sum(attention_mask == 1, axis=1)
-
- self._check_zero_mean_unit_variance(input_features[0, : fbank_feat_lengths[0]])
- self._check_zero_mean_unit_variance(input_features[1, : fbank_feat_lengths[1]])
- self._check_zero_mean_unit_variance(input_features[2])
-
- # make sure that if max_length < longest -> then pad to max_length
- self.assertEqual(input_features.shape, (3, 6, 24))
-
- def test_double_precision_pad(self):
- import torch
-
- feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- np_speech_inputs = np.random.rand(100, 32).astype(np.float64)
- py_speech_inputs = np_speech_inputs.tolist()
-
- for inputs in [py_speech_inputs, np_speech_inputs]:
- np_processed = feature_extractor.pad([{"input_features": inputs}], return_tensors="np")
- self.assertTrue(np_processed.input_features.dtype == np.float32)
- pt_processed = feature_extractor.pad([{"input_features": inputs}], return_tensors="pt")
- self.assertTrue(pt_processed.input_features.dtype == torch.float32)
-
- def _load_datasamples(self, num_samples):
- from datasets import load_dataset
-
- ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
- # automatic decoding with librispeech
- speech_samples = ds.sort("id")[:num_samples]["audio"]
-
- return [x["array"] for x in speech_samples]
-
- def test_integration(self):
- # fmt: off
- expected = np.array([
- -1.5745, -1.7713, -1.7020, -1.6069, -1.2250, -1.1105, -0.9072, -0.8241,
- -1.2310, -0.8098, -0.3320, -0.4101, -0.7985, -0.4996, -0.8213, -0.9128,
- -1.0420, -1.1286, -1.0440, -0.7999, -0.8405, -1.2275, -1.5443, -1.4625,
- ])
- # fmt: on
-
- input_speech = self._load_datasamples(1)
- feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- input_features = feature_extractor(input_speech, return_tensors="pt").input_features
- self.assertEqual(input_features.shape, (1, 584, 24))
- self.assertTrue(np.allclose(input_features[0, 0, :30], expected, atol=1e-4))
-
- def test_feat_extract_from_and_save_pretrained(self):
- feat_extract_first = self.feature_extraction_class(**self.feat_extract_dict)
-
- with tempfile.TemporaryDirectory() as tmpdirname:
- saved_file = feat_extract_first.save_pretrained(tmpdirname)[0]
- check_json_file_has_correct_format(saved_file)
- feat_extract_second = self.feature_extraction_class.from_pretrained(tmpdirname)
-
- dict_first = feat_extract_first.to_dict()
- dict_second = feat_extract_second.to_dict()
- self.assertDictEqual(dict_first, dict_second)
-
- def test_feat_extract_to_json_file(self):
- feat_extract_first = self.feature_extraction_class(**self.feat_extract_dict)
-
- with tempfile.TemporaryDirectory() as tmpdirname:
- json_file_path = os.path.join(tmpdirname, "feat_extract.json")
- feat_extract_first.to_json_file(json_file_path)
- feat_extract_second = self.feature_extraction_class.from_json_file(json_file_path)
-
- dict_first = feat_extract_first.to_dict()
- dict_second = feat_extract_second.to_dict()
- self.assertEqual(dict_first, dict_second)
-
-
-# exact same tests than before, except that we simulate that torchaudio is not available
-@require_torch
-@unittest.mock.patch(
- "transformers.models.speech_to_text.feature_extraction_speech_to_text.is_speech_available", lambda: False
-)
-class Speech2TextFeatureExtractionWithoutTorchaudioTest(Speech2TextFeatureExtractionTest):
- def test_using_audio_utils(self):
- # Tests that it uses audio_utils instead of torchaudio
- feat_extract = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
-
- self.assertTrue(hasattr(feat_extract, "window"))
- self.assertTrue(hasattr(feat_extract, "mel_filters"))
-
- from transformers.models.speech_to_text.feature_extraction_speech_to_text import is_speech_available
-
- self.assertFalse(is_speech_available())
diff --git a/tests/models/speech_to_text/test_processing_speech_to_text.py b/tests/models/speech_to_text/test_processing_speech_to_text.py
index 0116445c91b1..c8dd5a9d05e7 100644
--- a/tests/models/speech_to_text/test_processing_speech_to_text.py
+++ b/tests/models/speech_to_text/test_processing_speech_to_text.py
@@ -22,7 +22,7 @@
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_torch, require_torchaudio
-from .test_feature_extraction_speech_to_text import floats_list
+from ...test_processing_common import floats_list
SAMPLE_SP = get_tests_dir("fixtures/test_sentencepiece.model")
diff --git a/tests/models/speecht5/test_audio_processing_speecht5.py b/tests/models/speecht5/test_audio_processing_speecht5.py
new file mode 100644
index 000000000000..6bf02f6e5a41
--- /dev/null
+++ b/tests/models/speecht5/test_audio_processing_speecht5.py
@@ -0,0 +1,32 @@
+# Copyright 2026 HuggingFace Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+"""Tests for `SpeechT5AudioProcessor` and `SpeechT5AudioProcessorNumpy`."""
+
+from __future__ import annotations
+
+import unittest
+
+from transformers.testing_utils import require_torch
+
+from ...test_audio_processing_common import AudioProcessingTestMixin
+
+
+class SpeechT5AudioProcessingTester:
+ """Provides init kwargs and fixture parameters for the SpeechT5 audio processor tests."""
+
+ sample_rate = 16000
+
+ def prepare_audio_processor_dict(self) -> dict:
+ return {}
+
+
+@require_torch
+class SpeechT5AudioProcessingTest(AudioProcessingTestMixin, unittest.TestCase):
+ def setUp(self):
+ self.audio_processor_tester = SpeechT5AudioProcessingTester()
+ super().setUp()
diff --git a/tests/models/speecht5/test_feature_extraction_speecht5.py b/tests/models/speecht5/test_feature_extraction_speecht5.py
deleted file mode 100644
index 4b7e65b770cc..000000000000
--- a/tests/models/speecht5/test_feature_extraction_speecht5.py
+++ /dev/null
@@ -1,402 +0,0 @@
-# Copyright 2021-2023 HuggingFace Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Tests for the SpeechT5 feature extractors."""
-
-import itertools
-import unittest
-
-import numpy as np
-
-from transformers import BatchFeature, SpeechT5FeatureExtractor
-from transformers.testing_utils import require_torch
-from transformers.utils.import_utils import is_torch_available
-
-from ...test_processing_common import floats_list
-from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin
-
-
-if is_torch_available():
- import torch
-
-
-@require_torch
-class SpeechT5FeatureExtractionTester:
- def __init__(
- self,
- parent,
- batch_size=7,
- min_seq_length=400,
- max_seq_length=2000,
- feature_size=1,
- padding_value=0.0,
- sampling_rate=16000,
- do_normalize=True,
- num_mel_bins=80,
- hop_length=16,
- win_length=64,
- win_function="hann_window",
- fmin=80,
- fmax=7600,
- mel_floor=1e-10,
- return_attention_mask=True,
- ):
- self.parent = parent
- self.batch_size = batch_size
- self.min_seq_length = min_seq_length
- self.max_seq_length = max_seq_length
- self.seq_length_diff = (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
- self.feature_size = feature_size
- self.padding_value = padding_value
- self.sampling_rate = sampling_rate
- self.do_normalize = do_normalize
- self.num_mel_bins = num_mel_bins
- self.hop_length = hop_length
- self.win_length = win_length
- self.win_function = win_function
- self.fmin = fmin
- self.fmax = fmax
- self.mel_floor = mel_floor
- self.return_attention_mask = return_attention_mask
-
- def prepare_feat_extract_dict(self):
- return {
- "feature_size": self.feature_size,
- "padding_value": self.padding_value,
- "sampling_rate": self.sampling_rate,
- "do_normalize": self.do_normalize,
- "num_mel_bins": self.num_mel_bins,
- "hop_length": self.hop_length,
- "win_length": self.win_length,
- "win_function": self.win_function,
- "fmin": self.fmin,
- "fmax": self.fmax,
- "mel_floor": self.mel_floor,
- "return_attention_mask": self.return_attention_mask,
- }
-
- def prepare_inputs_for_common(self, equal_length=False, numpify=False):
- def _flatten(list_of_lists):
- return list(itertools.chain(*list_of_lists))
-
- if equal_length:
- speech_inputs = floats_list((self.batch_size, self.max_seq_length))
- else:
- # make sure that inputs increase in size
- speech_inputs = [
- _flatten(floats_list((x, self.feature_size)))
- for x in range(self.min_seq_length, self.max_seq_length, self.seq_length_diff)
- ]
-
- if numpify:
- speech_inputs = [np.asarray(x) for x in speech_inputs]
-
- return speech_inputs
-
- def prepare_inputs_for_target(self, equal_length=False, numpify=False):
- if equal_length:
- speech_inputs = [floats_list((self.max_seq_length, self.num_mel_bins)) for _ in range(self.batch_size)]
- else:
- # make sure that inputs increase in size
- speech_inputs = [
- floats_list((x, self.num_mel_bins))
- for x in range(self.min_seq_length, self.max_seq_length, self.seq_length_diff)
- ]
-
- if numpify:
- speech_inputs = [np.asarray(x) for x in speech_inputs]
-
- return speech_inputs
-
-
-@require_torch
-class SpeechT5FeatureExtractionTest(SequenceFeatureExtractionTestMixin, unittest.TestCase):
- feature_extraction_class = SpeechT5FeatureExtractor
-
- def setUp(self):
- self.feat_extract_tester = SpeechT5FeatureExtractionTester(self)
-
- def _check_zero_mean_unit_variance(self, input_vector):
- self.assertTrue(np.all(np.mean(input_vector, axis=0) < 1e-3))
- self.assertTrue(np.all(np.abs(np.var(input_vector, axis=0) - 1) < 1e-3))
-
- def test_call(self):
- # Tests that all call wrap to encode_plus and batch_encode_plus
- feat_extract = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- # create three inputs of length 800, 1000, and 1200
- speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)]
- np_speech_inputs = [np.asarray(speech_input) for speech_input in speech_inputs]
-
- # Test not batched input
- encoded_sequences_1 = feat_extract(speech_inputs[0], return_tensors="np").input_values
- encoded_sequences_2 = feat_extract(np_speech_inputs[0], return_tensors="np").input_values
- self.assertTrue(np.allclose(encoded_sequences_1, encoded_sequences_2, atol=1e-3))
-
- # Test batched
- encoded_sequences_1 = feat_extract(speech_inputs, return_tensors="np").input_values
- encoded_sequences_2 = feat_extract(np_speech_inputs, return_tensors="np").input_values
- for enc_seq_1, enc_seq_2 in zip(encoded_sequences_1, encoded_sequences_2):
- self.assertTrue(np.allclose(enc_seq_1, enc_seq_2, atol=1e-3))
-
- def test_zero_mean_unit_variance_normalization_np(self):
- feat_extract = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)]
-
- paddings = ["longest", "max_length", "do_not_pad"]
- max_lengths = [None, 1600, None]
- for max_length, padding in zip(max_lengths, paddings):
- processed = feat_extract(speech_inputs, padding=padding, max_length=max_length, return_tensors="np")
- input_values = processed.input_values
-
- self._check_zero_mean_unit_variance(input_values[0][:800])
- self.assertTrue(input_values[0][800:].sum() < 1e-6)
- self._check_zero_mean_unit_variance(input_values[1][:1000])
- self.assertTrue(input_values[0][1000:].sum() < 1e-6)
- self._check_zero_mean_unit_variance(input_values[2][:1200])
-
- def test_zero_mean_unit_variance_normalization(self):
- feat_extract = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- lengths = range(800, 1400, 200)
- speech_inputs = [floats_list((1, x))[0] for x in lengths]
-
- paddings = ["longest", "max_length", "do_not_pad"]
- max_lengths = [None, 1600, None]
-
- for max_length, padding in zip(max_lengths, paddings):
- processed = feat_extract(speech_inputs, max_length=max_length, padding=padding)
- input_values = processed.input_values
-
- self._check_zero_mean_unit_variance(input_values[0][:800])
- self._check_zero_mean_unit_variance(input_values[1][:1000])
- self._check_zero_mean_unit_variance(input_values[2][:1200])
-
- def test_zero_mean_unit_variance_normalization_trunc_np_max_length(self):
- feat_extract = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)]
- processed = feat_extract(
- speech_inputs, truncation=True, max_length=1000, padding="max_length", return_tensors="np"
- )
- input_values = processed.input_values
-
- self._check_zero_mean_unit_variance(input_values[0, :800])
- self._check_zero_mean_unit_variance(input_values[1])
- self._check_zero_mean_unit_variance(input_values[2])
-
- def test_zero_mean_unit_variance_normalization_trunc_np_longest(self):
- feat_extract = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)]
- processed = feat_extract(
- speech_inputs, truncation=True, max_length=1000, padding="longest", return_tensors="np"
- )
- input_values = processed.input_values
-
- self._check_zero_mean_unit_variance(input_values[0, :800])
- self._check_zero_mean_unit_variance(input_values[1, :1000])
- self._check_zero_mean_unit_variance(input_values[2])
-
- # make sure that if max_length < longest -> then pad to max_length
- self.assertTrue(input_values.shape == (3, 1000))
-
- speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)]
- processed = feat_extract(
- speech_inputs, truncation=True, max_length=2000, padding="longest", return_tensors="np"
- )
- input_values = processed.input_values
-
- self._check_zero_mean_unit_variance(input_values[0, :800])
- self._check_zero_mean_unit_variance(input_values[1, :1000])
- self._check_zero_mean_unit_variance(input_values[2])
-
- # make sure that if max_length > longest -> then pad to longest
- self.assertTrue(input_values.shape == (3, 1200))
-
- def test_double_precision_pad(self):
- feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- np_speech_inputs = np.random.rand(100).astype(np.float64)
- py_speech_inputs = np_speech_inputs.tolist()
-
- for inputs in [py_speech_inputs, np_speech_inputs]:
- np_processed = feature_extractor.pad([{"input_values": inputs}], return_tensors="np")
- self.assertTrue(np_processed.input_values.dtype == np.float32)
- pt_processed = feature_extractor.pad([{"input_values": inputs}], return_tensors="pt")
- self.assertTrue(pt_processed.input_values.dtype == torch.float32)
-
- def test_call_target(self):
- # Tests that all call wrap to encode_plus and batch_encode_plus
- feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- # create three inputs of length 800, 1000, and 1200
- speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)]
- np_speech_inputs = [np.asarray(speech_input) for speech_input in speech_inputs]
-
- # Test feature size
- input_values = feature_extractor(audio_target=np_speech_inputs, padding=True, return_tensors="np").input_values
- self.assertTrue(input_values.ndim == 3)
- self.assertTrue(input_values.shape[-1] == feature_extractor.num_mel_bins)
-
- # Test not batched input
- encoded_sequences_1 = feature_extractor(speech_inputs[0], return_tensors="np").input_values
- encoded_sequences_2 = feature_extractor(np_speech_inputs[0], return_tensors="np").input_values
- self.assertTrue(np.allclose(encoded_sequences_1, encoded_sequences_2, atol=1e-3))
-
- # Test batched
- encoded_sequences_1 = feature_extractor(speech_inputs, return_tensors="np").input_values
- encoded_sequences_2 = feature_extractor(np_speech_inputs, return_tensors="np").input_values
- for enc_seq_1, enc_seq_2 in zip(encoded_sequences_1, encoded_sequences_2):
- self.assertTrue(np.allclose(enc_seq_1, enc_seq_2, atol=1e-3))
-
- # Test 2-D numpy arrays are batched.
- speech_inputs = [floats_list((1, x))[0] for x in (800, 800, 800)]
- np_speech_inputs = np.asarray(speech_inputs)
- encoded_sequences_1 = feature_extractor(speech_inputs, return_tensors="np").input_values
- encoded_sequences_2 = feature_extractor(np_speech_inputs, return_tensors="np").input_values
- for enc_seq_1, enc_seq_2 in zip(encoded_sequences_1, encoded_sequences_2):
- self.assertTrue(np.allclose(enc_seq_1, enc_seq_2, atol=1e-3))
-
- def test_batch_feature_target(self):
- speech_inputs = self.feat_extract_tester.prepare_inputs_for_target()
- feat_extract = self.feature_extraction_class(**self.feat_extract_dict)
- input_name = feat_extract.model_input_names[0]
-
- processed_features = BatchFeature({input_name: speech_inputs})
-
- self.assertTrue(all(len(x) == len(y) for x, y in zip(speech_inputs, processed_features[input_name])))
-
- speech_inputs = self.feat_extract_tester.prepare_inputs_for_target(equal_length=True)
- processed_features = BatchFeature({input_name: speech_inputs}, tensor_type="np")
-
- batch_features_input = processed_features[input_name]
-
- if len(batch_features_input.shape) < 3:
- batch_features_input = batch_features_input[:, :, None]
-
- self.assertTrue(
- batch_features_input.shape
- == (self.feat_extract_tester.batch_size, len(speech_inputs[0]), self.feat_extract_tester.num_mel_bins)
- )
-
- @require_torch
- def test_batch_feature_target_pt(self):
- speech_inputs = self.feat_extract_tester.prepare_inputs_for_target(equal_length=True)
- feat_extract = self.feature_extraction_class(**self.feat_extract_dict)
- input_name = feat_extract.model_input_names[0]
-
- processed_features = BatchFeature({input_name: speech_inputs}, tensor_type="pt")
-
- batch_features_input = processed_features[input_name]
-
- if len(batch_features_input.shape) < 3:
- batch_features_input = batch_features_input[:, :, None]
-
- self.assertTrue(
- batch_features_input.shape
- == (self.feat_extract_tester.batch_size, len(speech_inputs[0]), self.feat_extract_tester.num_mel_bins)
- )
-
- @require_torch
- def test_padding_accepts_tensors_target_pt(self):
- feat_extract = self.feature_extraction_class(**self.feat_extract_dict)
- speech_inputs = self.feat_extract_tester.prepare_inputs_for_target()
- input_name = feat_extract.model_input_names[0]
-
- processed_features = BatchFeature({input_name: speech_inputs})
-
- feat_extract.feature_size = feat_extract.num_mel_bins # hack!
-
- input_np = feat_extract.pad(processed_features, padding="longest", return_tensors="np")[input_name]
- input_pt = feat_extract.pad(processed_features, padding="longest", return_tensors="pt")[input_name]
-
- self.assertTrue(abs(input_np.astype(np.float32).sum() - input_pt.numpy().astype(np.float32).sum()) < 1e-2)
-
- def test_attention_mask_target(self):
- feat_dict = self.feat_extract_dict
- feat_dict["return_attention_mask"] = True
- feat_extract = self.feature_extraction_class(**feat_dict)
- speech_inputs = self.feat_extract_tester.prepare_inputs_for_target()
- input_lengths = [len(x) for x in speech_inputs]
- input_name = feat_extract.model_input_names[0]
-
- processed = BatchFeature({input_name: speech_inputs})
-
- feat_extract.feature_size = feat_extract.num_mel_bins # hack!
-
- processed = feat_extract.pad(processed, padding="longest", return_tensors="np")
- self.assertIn("attention_mask", processed)
- self.assertListEqual(list(processed.attention_mask.shape), list(processed[input_name].shape[:2]))
- self.assertListEqual(processed.attention_mask.sum(-1).tolist(), input_lengths)
-
- def test_attention_mask_with_truncation_target(self):
- feat_dict = self.feat_extract_dict
- feat_dict["return_attention_mask"] = True
- feat_extract = self.feature_extraction_class(**feat_dict)
- speech_inputs = self.feat_extract_tester.prepare_inputs_for_target()
- input_lengths = [len(x) for x in speech_inputs]
- input_name = feat_extract.model_input_names[0]
-
- processed = BatchFeature({input_name: speech_inputs})
- max_length = min(input_lengths)
-
- feat_extract.feature_size = feat_extract.num_mel_bins # hack!
-
- processed_pad = feat_extract.pad(
- processed, padding="max_length", max_length=max_length, truncation=True, return_tensors="np"
- )
- self.assertIn("attention_mask", processed_pad)
- self.assertListEqual(
- list(processed_pad.attention_mask.shape), [processed_pad[input_name].shape[0], max_length]
- )
- self.assertListEqual(
- processed_pad.attention_mask[:, :max_length].sum(-1).tolist(), [max_length for x in speech_inputs]
- )
-
- def _load_datasamples(self, num_samples):
- from datasets import load_dataset
-
- ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
- # automatic decoding with librispeech
- speech_samples = ds.sort("id")[:num_samples]["audio"]
-
- return [x["array"] for x in speech_samples]
-
- def test_integration(self):
- # fmt: off
- EXPECTED_INPUT_VALUES = torch.tensor(
- [2.3804e-03, 2.0752e-03, 1.9836e-03, 2.1057e-03, 1.6174e-03,
- 3.0518e-04, 9.1553e-05, 3.3569e-04, 9.7656e-04, 1.8311e-03,
- 2.0142e-03, 2.1057e-03, 1.7395e-03, 4.5776e-04, -3.9673e-04,
- 4.5776e-04, 1.0071e-03, 9.1553e-05, 4.8828e-04, 1.1597e-03,
- 7.3242e-04, 9.4604e-04, 1.8005e-03, 1.8311e-03, 8.8501e-04,
- 4.2725e-04, 4.8828e-04, 7.3242e-04, 1.0986e-03, 2.1057e-03]
- )
- # fmt: on
-
- input_speech = self._load_datasamples(1)
- feature_extractor = SpeechT5FeatureExtractor()
- input_values = feature_extractor(input_speech, return_tensors="pt").input_values
- self.assertEqual(input_values.shape, (1, 93680))
- torch.testing.assert_close(input_values[0, :30], EXPECTED_INPUT_VALUES, rtol=1e-6, atol=1e-6)
-
- def test_integration_target(self):
- # fmt: off
- EXPECTED_INPUT_VALUES = torch.tensor(
- [-2.6870, -3.0104, -3.1356, -3.5352, -3.0044, -3.0353, -3.4719, -3.6777,
- -3.1520, -2.9435, -2.6553, -2.8795, -2.9944, -2.5921, -3.0279, -3.0386,
- -3.0864, -3.1291, -3.2353, -2.7444, -2.6831, -2.7287, -3.1761, -3.1571,
- -3.2726, -3.0582, -3.1007, -3.4533, -3.4695, -3.0998]
- )
- # fmt: on
-
- input_speech = self._load_datasamples(1)
- feature_extractor = SpeechT5FeatureExtractor()
- input_values = feature_extractor(audio_target=input_speech, return_tensors="pt").input_values
- self.assertEqual(input_values.shape, (1, 366, 80))
- torch.testing.assert_close(input_values[0, 0, :30], EXPECTED_INPUT_VALUES, rtol=1e-4, atol=1e-4)
diff --git a/tests/models/speecht5/test_processing_speecht5.py b/tests/models/speecht5/test_processing_speecht5.py
index a6736132a390..6016a23207f9 100644
--- a/tests/models/speecht5/test_processing_speecht5.py
+++ b/tests/models/speecht5/test_processing_speecht5.py
@@ -25,7 +25,7 @@
if is_speech_available() and is_torch_available():
from transformers import SpeechT5FeatureExtractor, SpeechT5Processor
- from .test_feature_extraction_speecht5 import floats_list
+ from ...test_processing_common import floats_list
SAMPLE_VOCAB = get_tests_dir("fixtures/test_sentencepiece_bpe_char.model")
diff --git a/tests/models/univnet/test_audio_processing_univnet.py b/tests/models/univnet/test_audio_processing_univnet.py
new file mode 100644
index 000000000000..4997d7c9b517
--- /dev/null
+++ b/tests/models/univnet/test_audio_processing_univnet.py
@@ -0,0 +1,37 @@
+# Copyright 2026 HuggingFace Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+"""Tests for `UnivNetAudioProcessor` and `UnivNetAudioProcessorNumpy`."""
+
+from __future__ import annotations
+
+import unittest
+
+from transformers.testing_utils import require_torch
+
+from ...test_audio_processing_common import AudioProcessingTestMixin
+
+
+class UnivNetAudioProcessingTester:
+ """Provides init kwargs and fixture parameters for the UnivNet audio processor tests."""
+
+ sample_rate = 24000
+
+ def prepare_audio_processor_dict(self) -> dict:
+ return {}
+
+
+@require_torch
+class UnivNetAudioProcessingTest(AudioProcessingTestMixin, unittest.TestCase):
+ # UnivNet's reflect-padded STFT and float64 spectrogram path keep parity within the
+ # float32 noise floor but slightly above the strict 1e-5 bar — empirically up to ~3e-5.
+ parity_atol = 1e-4
+ parity_rtol = 1e-4
+
+ def setUp(self):
+ self.audio_processor_tester = UnivNetAudioProcessingTester()
+ super().setUp()
diff --git a/tests/models/univnet/test_feature_extraction_univnet.py b/tests/models/univnet/test_feature_extraction_univnet.py
deleted file mode 100644
index bd04f0b64f99..000000000000
--- a/tests/models/univnet/test_feature_extraction_univnet.py
+++ /dev/null
@@ -1,347 +0,0 @@
-# Copyright 2023 The HuggingFace Team. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import itertools
-import os
-import tempfile
-import unittest
-
-import numpy as np
-from datasets import Audio, load_dataset
-
-from transformers import UnivNetFeatureExtractor
-from transformers.testing_utils import check_json_file_has_correct_format, require_torch, slow
-from transformers.utils.import_utils import is_torch_available
-
-from ...test_processing_common import floats_list
-from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin
-
-
-if is_torch_available():
- import torch
-
-
-class UnivNetFeatureExtractionTester:
- def __init__(
- self,
- parent,
- batch_size=7,
- min_seq_length=400,
- max_seq_length=2000,
- feature_size=1,
- sampling_rate=24000,
- padding_value=0.0,
- do_normalize=True,
- num_mel_bins=100,
- hop_length=256,
- win_length=1024,
- win_function="hann_window",
- filter_length=1024,
- max_length_s=10,
- fmin=0.0,
- fmax=12000,
- mel_floor=1e-9,
- center=False,
- compression_factor=1.0,
- compression_clip_val=1e-5,
- normalize_min=-11.512925148010254,
- normalize_max=2.3143386840820312,
- model_in_channels=64,
- pad_end_length=10,
- ):
- self.parent = parent
- self.batch_size = batch_size
- self.min_seq_length = min_seq_length
- self.max_seq_length = max_seq_length
- self.seq_length_diff = (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
-
- self.feature_size = feature_size
- self.sampling_rate = sampling_rate
- self.padding_value = padding_value
- self.do_normalize = do_normalize
- self.num_mel_bins = num_mel_bins
- self.hop_length = hop_length
- self.win_length = win_length
- self.win_function = win_function
- self.filter_length = filter_length
- self.max_length_s = max_length_s
- self.fmin = fmin
- self.fmax = fmax
- self.mel_floor = mel_floor
- self.center = center
- self.compression_factor = compression_factor
- self.compression_clip_val = compression_clip_val
- self.normalize_min = normalize_min
- self.normalize_max = normalize_max
- self.model_in_channels = model_in_channels
- self.pad_end_length = pad_end_length
-
- def prepare_feat_extract_dict(self):
- return {
- "feature_size": self.feature_size,
- "sampling_rate": self.sampling_rate,
- "padding_value": self.padding_value,
- "do_normalize": self.do_normalize,
- "num_mel_bins": self.num_mel_bins,
- "hop_length": self.hop_length,
- "win_length": self.win_length,
- "win_function": self.win_function,
- "filter_length": self.filter_length,
- "max_length_s": self.max_length_s,
- "fmin": self.fmin,
- "fmax": self.fmax,
- "mel_floor": self.mel_floor,
- "center": self.center,
- "compression_factor": self.compression_factor,
- "compression_clip_val": self.compression_clip_val,
- "normalize_min": self.normalize_min,
- "normalize_max": self.normalize_max,
- "model_in_channels": self.model_in_channels,
- "pad_end_length": self.pad_end_length,
- }
-
- def prepare_inputs_for_common(self, equal_length=False, numpify=False):
- def _flatten(list_of_lists):
- return list(itertools.chain(*list_of_lists))
-
- if equal_length:
- speech_inputs = floats_list((self.batch_size, self.max_seq_length))
- else:
- # make sure that inputs increase in size
- speech_inputs = [
- _flatten(floats_list((x, self.feature_size)))
- for x in range(self.min_seq_length, self.max_seq_length, self.seq_length_diff)
- ]
-
- if numpify:
- speech_inputs = [np.asarray(x) for x in speech_inputs]
-
- return speech_inputs
-
-
-class UnivNetFeatureExtractionTest(SequenceFeatureExtractionTestMixin, unittest.TestCase):
- feature_extraction_class = UnivNetFeatureExtractor
-
- def setUp(self):
- self.feat_extract_tester = UnivNetFeatureExtractionTester(self)
-
- # Copied from tests.models.whisper.test_feature_extraction_whisper.WhisperFeatureExtractionTest.test_feat_extract_from_and_save_pretrained
- def test_feat_extract_from_and_save_pretrained(self):
- feat_extract_first = self.feature_extraction_class(**self.feat_extract_dict)
-
- with tempfile.TemporaryDirectory() as tmpdirname:
- saved_file = feat_extract_first.save_pretrained(tmpdirname)[0]
- check_json_file_has_correct_format(saved_file)
- feat_extract_second = self.feature_extraction_class.from_pretrained(tmpdirname)
-
- dict_first = feat_extract_first.to_dict()
- dict_second = feat_extract_second.to_dict()
- mel_1 = feat_extract_first.mel_filters
- mel_2 = feat_extract_second.mel_filters
- self.assertTrue(np.allclose(mel_1, mel_2))
- self.assertEqual(dict_first, dict_second)
-
- # Copied from tests.models.whisper.test_feature_extraction_whisper.WhisperFeatureExtractionTest.test_feat_extract_to_json_file
- def test_feat_extract_to_json_file(self):
- feat_extract_first = self.feature_extraction_class(**self.feat_extract_dict)
-
- with tempfile.TemporaryDirectory() as tmpdirname:
- json_file_path = os.path.join(tmpdirname, "feat_extract.json")
- feat_extract_first.to_json_file(json_file_path)
- feat_extract_second = self.feature_extraction_class.from_json_file(json_file_path)
-
- dict_first = feat_extract_first.to_dict()
- dict_second = feat_extract_second.to_dict()
- mel_1 = feat_extract_first.mel_filters
- mel_2 = feat_extract_second.mel_filters
- self.assertTrue(np.allclose(mel_1, mel_2))
- self.assertEqual(dict_first, dict_second)
-
- def test_call(self):
- # Tests that all call wrap to encode_plus and batch_encode_plus
- feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- # create three inputs of length 800, 1000, and 1200
- speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)]
- np_speech_inputs = [np.asarray(speech_input) for speech_input in speech_inputs]
-
- # Test feature size
- input_features = feature_extractor(
- np_speech_inputs, padding="max_length", max_length=1600, return_tensors="np"
- ).input_features
- self.assertTrue(input_features.ndim == 3)
- # Note: for some reason I get a weird padding error when feature_size > 1
- # self.assertTrue(input_features.shape[-2] == feature_extractor.feature_size)
- # Note: we use the shape convention (batch_size, seq_len, num_mel_bins)
- self.assertTrue(input_features.shape[-1] == feature_extractor.num_mel_bins)
-
- # Test not batched input
- encoded_sequences_1 = feature_extractor(speech_inputs[0], return_tensors="np").input_features
- encoded_sequences_2 = feature_extractor(np_speech_inputs[0], return_tensors="np").input_features
- self.assertTrue(np.allclose(encoded_sequences_1, encoded_sequences_2, atol=1e-3))
-
- # Test batched
- encoded_sequences_1 = feature_extractor(speech_inputs, return_tensors="np").input_features
- encoded_sequences_2 = feature_extractor(np_speech_inputs, return_tensors="np").input_features
- for enc_seq_1, enc_seq_2 in zip(encoded_sequences_1, encoded_sequences_2):
- self.assertTrue(np.allclose(enc_seq_1, enc_seq_2, atol=1e-3))
-
- # Test 2-D numpy arrays are batched.
- speech_inputs = [floats_list((1, x))[0] for x in (800, 800, 800)]
- np_speech_inputs = np.asarray(speech_inputs)
- encoded_sequences_1 = feature_extractor(speech_inputs, return_tensors="np").input_features
- encoded_sequences_2 = feature_extractor(np_speech_inputs, return_tensors="np").input_features
- for enc_seq_1, enc_seq_2 in zip(encoded_sequences_1, encoded_sequences_2):
- self.assertTrue(np.allclose(enc_seq_1, enc_seq_2, atol=1e-3))
-
- # Test truncation required
- speech_inputs = [
- floats_list((1, x))[0]
- for x in range((feature_extractor.num_max_samples - 100), (feature_extractor.num_max_samples + 500), 200)
- ]
- np_speech_inputs = [np.asarray(speech_input) for speech_input in speech_inputs]
-
- speech_inputs_truncated = [x[: feature_extractor.num_max_samples] for x in speech_inputs]
- np_speech_inputs_truncated = [np.asarray(speech_input) for speech_input in speech_inputs_truncated]
-
- encoded_sequences_1 = feature_extractor(np_speech_inputs, return_tensors="np").input_features
- encoded_sequences_2 = feature_extractor(np_speech_inputs_truncated, return_tensors="np").input_features
- for enc_seq_1, enc_seq_2 in zip(encoded_sequences_1, encoded_sequences_2):
- self.assertTrue(np.allclose(enc_seq_1, enc_seq_2, atol=1e-3))
-
- def test_batched_unbatched_consistency(self):
- feature_extractor = self.feature_extraction_class(**self.feat_extract_dict)
- speech_inputs = floats_list((1, 800))[0]
- np_speech_inputs = np.asarray(speech_inputs)
-
- # Test unbatched vs batched list
- encoded_sequences_1 = feature_extractor(speech_inputs, return_tensors="np").input_features
- encoded_sequences_2 = feature_extractor([speech_inputs], return_tensors="np").input_features
- for enc_seq_1, enc_seq_2 in zip(encoded_sequences_1, encoded_sequences_2):
- self.assertTrue(np.allclose(enc_seq_1, enc_seq_2, atol=1e-3))
-
- # Test np.ndarray vs list[np.ndarray]
- encoded_sequences_1 = feature_extractor(np_speech_inputs, return_tensors="np").input_features
- encoded_sequences_2 = feature_extractor([np_speech_inputs], return_tensors="np").input_features
- for enc_seq_1, enc_seq_2 in zip(encoded_sequences_1, encoded_sequences_2):
- self.assertTrue(np.allclose(enc_seq_1, enc_seq_2, atol=1e-3))
-
- # Test unbatched np.ndarray vs batched np.ndarray
- encoded_sequences_1 = feature_extractor(np_speech_inputs, return_tensors="np").input_features
- encoded_sequences_2 = feature_extractor(
- np.expand_dims(np_speech_inputs, axis=0), return_tensors="np"
- ).input_features
- for enc_seq_1, enc_seq_2 in zip(encoded_sequences_1, encoded_sequences_2):
- self.assertTrue(np.allclose(enc_seq_1, enc_seq_2, atol=1e-3))
-
- def test_generate_noise(self):
- feature_extractor = self.feature_extraction_class(**self.feat_extract_dict)
- speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)]
-
- features = feature_extractor(speech_inputs, return_noise=True)
- input_features = features.input_features
- noise_features = features.noise_sequence
-
- for spectrogram, noise in zip(input_features, noise_features):
- self.assertEqual(spectrogram.shape[0], noise.shape[0])
-
- def test_pad_end(self):
- feature_extractor = self.feature_extraction_class(**self.feat_extract_dict)
- speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)]
-
- input_features1 = feature_extractor(speech_inputs, padding=False, pad_end=False).input_features
- input_features2 = feature_extractor(speech_inputs, padding=False, pad_end=True).input_features
-
- for spectrogram1, spectrogram2 in zip(input_features1, input_features2):
- self.assertEqual(spectrogram1.shape[0] + self.feat_extract_tester.pad_end_length, spectrogram2.shape[0])
-
- def test_generate_noise_and_pad_end(self):
- feature_extractor = self.feature_extraction_class(**self.feat_extract_dict)
- speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)]
-
- features = feature_extractor(speech_inputs, padding=False, return_noise=True, pad_end=True)
- input_features = features.input_features
- noise_features = features.noise_sequence
-
- for spectrogram, noise in zip(input_features, noise_features):
- self.assertEqual(spectrogram.shape[0], noise.shape[0])
-
- @require_torch
- def test_batch_decode(self):
- import torch
-
- feature_extractor = self.feature_extraction_class(**self.feat_extract_dict)
- input_lengths = list(range(800, 1400, 200))
- pad_samples = feature_extractor.pad_end_length * feature_extractor.hop_length
- output_features = {
- "waveforms": torch.tensor(floats_list((3, max(input_lengths) + pad_samples))),
- "waveform_lengths": torch.tensor(input_lengths),
- }
- waveforms = feature_extractor.batch_decode(**output_features)
-
- for input_length, waveform in zip(input_lengths, waveforms):
- self.assertTrue(len(waveform.shape) == 1, msg="Individual output waveforms should be 1D")
- self.assertEqual(waveform.shape[0], input_length)
-
- @require_torch
- # Copied from tests.models.whisper.test_feature_extraction_whisper.WhisperFeatureExtractionTest.test_double_precision_pad
- def test_double_precision_pad(self):
- import torch
-
- feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- np_speech_inputs = np.random.rand(100, 32).astype(np.float64)
- py_speech_inputs = np_speech_inputs.tolist()
-
- for inputs in [py_speech_inputs, np_speech_inputs]:
- np_processed = feature_extractor.pad([{"input_features": inputs}], return_tensors="np")
- self.assertTrue(np_processed.input_features.dtype == np.float32)
- pt_processed = feature_extractor.pad([{"input_features": inputs}], return_tensors="pt")
- self.assertTrue(pt_processed.input_features.dtype == torch.float32)
-
- def _load_datasamples(self, num_samples):
- ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
- ds = ds.cast_column("audio", Audio(sampling_rate=self.feat_extract_tester.sampling_rate))
- # automatic decoding with librispeech
- speech_samples = ds.sort("id")[:num_samples]["audio"]
-
- return [x["array"] for x in speech_samples], [x["sampling_rate"] for x in speech_samples]
-
- @slow
- @require_torch
- def test_integration(self):
- # fmt: off
- EXPECTED_INPUT_FEATURES = torch.tensor(
- [
- -5.0229, -6.1358, -5.8346, -5.4447, -5.6707, -5.8577, -5.0464, -5.0058,
- -5.6015, -5.6410, -5.4325, -5.6116, -5.3700, -5.7956, -5.3196, -5.3274,
- -5.9655, -5.6057, -5.8382, -5.9602, -5.9005, -5.9123, -5.7669, -6.1441,
- -5.5168, -5.1405, -5.3927, -6.0032, -5.5784, -5.3728
- ],
- )
- # fmt: on
-
- input_speech, sr = self._load_datasamples(1)
-
- feature_extractor = UnivNetFeatureExtractor()
- input_features = feature_extractor(input_speech, sampling_rate=sr[0], return_tensors="pt").input_features
- self.assertEqual(input_features.shape, (1, 548, 100))
-
- input_features_mean = torch.mean(input_features)
- input_features_stddev = torch.std(input_features)
-
- EXPECTED_MEAN = torch.tensor(-6.18862009)
- EXPECTED_STDDEV = torch.tensor(2.80845642)
-
- torch.testing.assert_close(input_features_mean, EXPECTED_MEAN, rtol=5e-5, atol=5e-5)
- torch.testing.assert_close(input_features_stddev, EXPECTED_STDDEV)
- torch.testing.assert_close(input_features[0, :30, 0], EXPECTED_INPUT_FEATURES, rtol=1e-4, atol=1e-4)
diff --git a/tests/models/wav2vec2/test_audio_processing_wav2vec2.py b/tests/models/wav2vec2/test_audio_processing_wav2vec2.py
new file mode 100644
index 000000000000..5cfb2ce97856
--- /dev/null
+++ b/tests/models/wav2vec2/test_audio_processing_wav2vec2.py
@@ -0,0 +1,30 @@
+# Copyright 2026 HuggingFace Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+"""Tests for `Wav2Vec2AudioProcessor` and `Wav2Vec2AudioProcessorNumpy`."""
+
+from __future__ import annotations
+
+import unittest
+
+from transformers.testing_utils import require_torch
+
+from ...test_audio_processing_common import AudioProcessingTestMixin
+
+
+class Wav2Vec2AudioProcessingTester:
+ sample_rate = 16000
+
+ def prepare_audio_processor_dict(self) -> dict:
+ return {}
+
+
+@require_torch
+class Wav2Vec2AudioProcessingTest(AudioProcessingTestMixin, unittest.TestCase):
+ def setUp(self):
+ self.audio_processor_tester = Wav2Vec2AudioProcessingTester()
+ super().setUp()
diff --git a/tests/models/wav2vec2/test_feature_extraction_wav2vec2.py b/tests/models/wav2vec2/test_feature_extraction_wav2vec2.py
deleted file mode 100644
index 86a75464a04d..000000000000
--- a/tests/models/wav2vec2/test_feature_extraction_wav2vec2.py
+++ /dev/null
@@ -1,214 +0,0 @@
-# Copyright 2021 HuggingFace Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-import itertools
-import unittest
-
-import numpy as np
-
-from transformers import Wav2Vec2Config, Wav2Vec2FeatureExtractor
-from transformers.testing_utils import require_torch, slow
-
-from ...test_processing_common import floats_list
-from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin
-
-
-class Wav2Vec2FeatureExtractionTester:
- def __init__(
- self,
- parent,
- batch_size=7,
- min_seq_length=400,
- max_seq_length=2000,
- feature_size=1,
- padding_value=0.0,
- sampling_rate=16000,
- return_attention_mask=True,
- do_normalize=True,
- ):
- self.parent = parent
- self.batch_size = batch_size
- self.min_seq_length = min_seq_length
- self.max_seq_length = max_seq_length
- self.seq_length_diff = (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
- self.feature_size = feature_size
- self.padding_value = padding_value
- self.sampling_rate = sampling_rate
- self.return_attention_mask = return_attention_mask
- self.do_normalize = do_normalize
-
- def prepare_feat_extract_dict(self):
- return {
- "feature_size": self.feature_size,
- "padding_value": self.padding_value,
- "sampling_rate": self.sampling_rate,
- "return_attention_mask": self.return_attention_mask,
- "do_normalize": self.do_normalize,
- }
-
- def prepare_inputs_for_common(self, equal_length=False, numpify=False):
- def _flatten(list_of_lists):
- return list(itertools.chain(*list_of_lists))
-
- if equal_length:
- speech_inputs = floats_list((self.batch_size, self.max_seq_length))
- else:
- # make sure that inputs increase in size
- speech_inputs = [
- _flatten(floats_list((x, self.feature_size)))
- for x in range(self.min_seq_length, self.max_seq_length, self.seq_length_diff)
- ]
-
- if numpify:
- speech_inputs = [np.asarray(x) for x in speech_inputs]
-
- return speech_inputs
-
-
-class Wav2Vec2FeatureExtractionTest(SequenceFeatureExtractionTestMixin, unittest.TestCase):
- feature_extraction_class = Wav2Vec2FeatureExtractor
-
- def setUp(self):
- self.feat_extract_tester = Wav2Vec2FeatureExtractionTester(self)
-
- def _check_zero_mean_unit_variance(self, input_vector):
- self.assertTrue(np.all(np.mean(input_vector, axis=0) < 1e-3))
- self.assertTrue(np.all(np.abs(np.var(input_vector, axis=0) - 1) < 1e-3))
-
- def test_call(self):
- # Tests that all call wrap to encode_plus and batch_encode_plus
- feat_extract = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- # create three inputs of length 800, 1000, and 1200
- speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)]
- np_speech_inputs = [np.asarray(speech_input) for speech_input in speech_inputs]
-
- # Test not batched input
- encoded_sequences_1 = feat_extract(speech_inputs[0], return_tensors="np").input_values
- encoded_sequences_2 = feat_extract(np_speech_inputs[0], return_tensors="np").input_values
- self.assertTrue(np.allclose(encoded_sequences_1, encoded_sequences_2, atol=1e-3))
-
- # Test batched
- encoded_sequences_1 = feat_extract(speech_inputs, return_tensors="np").input_values
- encoded_sequences_2 = feat_extract(np_speech_inputs, return_tensors="np").input_values
- for enc_seq_1, enc_seq_2 in zip(encoded_sequences_1, encoded_sequences_2):
- self.assertTrue(np.allclose(enc_seq_1, enc_seq_2, atol=1e-3))
-
- # Test 2-D numpy arrays are batched.
- speech_inputs = [floats_list((1, x))[0] for x in (800, 800, 800)]
- np_speech_inputs = np.asarray(speech_inputs)
- encoded_sequences_1 = feat_extract(speech_inputs, return_tensors="np").input_values
- encoded_sequences_2 = feat_extract(np_speech_inputs, return_tensors="np").input_values
- for enc_seq_1, enc_seq_2 in zip(encoded_sequences_1, encoded_sequences_2):
- self.assertTrue(np.allclose(enc_seq_1, enc_seq_2, atol=1e-3))
-
- def test_zero_mean_unit_variance_normalization_np(self):
- feat_extract = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)]
-
- paddings = ["longest", "max_length", "do_not_pad"]
- max_lengths = [None, 1600, None]
- for max_length, padding in zip(max_lengths, paddings):
- processed = feat_extract(speech_inputs, padding=padding, max_length=max_length, return_tensors="np")
- input_values = processed.input_values
-
- self._check_zero_mean_unit_variance(input_values[0][:800])
- self.assertTrue(input_values[0][800:].sum() < 1e-6)
- self._check_zero_mean_unit_variance(input_values[1][:1000])
- self.assertTrue(input_values[0][1000:].sum() < 1e-6)
- self._check_zero_mean_unit_variance(input_values[2][:1200])
-
- def test_zero_mean_unit_variance_normalization(self):
- feat_extract = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- lengths = range(800, 1400, 200)
- speech_inputs = [floats_list((1, x))[0] for x in lengths]
-
- paddings = ["longest", "max_length", "do_not_pad"]
- max_lengths = [None, 1600, None]
-
- for max_length, padding in zip(max_lengths, paddings):
- processed = feat_extract(speech_inputs, max_length=max_length, padding=padding)
- input_values = processed.input_values
-
- self._check_zero_mean_unit_variance(input_values[0][:800])
- self._check_zero_mean_unit_variance(input_values[1][:1000])
- self._check_zero_mean_unit_variance(input_values[2][:1200])
-
- def test_zero_mean_unit_variance_normalization_trunc_np_max_length(self):
- feat_extract = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)]
- processed = feat_extract(
- speech_inputs, truncation=True, max_length=1000, padding="max_length", return_tensors="np"
- )
- input_values = processed.input_values
-
- self._check_zero_mean_unit_variance(input_values[0, :800])
- self._check_zero_mean_unit_variance(input_values[1])
- self._check_zero_mean_unit_variance(input_values[2])
-
- def test_zero_mean_unit_variance_normalization_trunc_np_longest(self):
- feat_extract = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)]
- processed = feat_extract(
- speech_inputs, truncation=True, max_length=1000, padding="longest", return_tensors="np"
- )
- input_values = processed.input_values
-
- self._check_zero_mean_unit_variance(input_values[0, :800])
- self._check_zero_mean_unit_variance(input_values[1, :1000])
- self._check_zero_mean_unit_variance(input_values[2])
-
- # make sure that if max_length < longest -> then pad to max_length
- self.assertTrue(input_values.shape == (3, 1000))
-
- speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)]
- processed = feat_extract(
- speech_inputs, truncation=True, max_length=2000, padding="longest", return_tensors="np"
- )
- input_values = processed.input_values
-
- self._check_zero_mean_unit_variance(input_values[0, :800])
- self._check_zero_mean_unit_variance(input_values[1, :1000])
- self._check_zero_mean_unit_variance(input_values[2])
-
- # make sure that if max_length > longest -> then pad to longest
- self.assertTrue(input_values.shape == (3, 1200))
-
- @require_torch
- def test_double_precision_pad(self):
- import torch
-
- feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- np_speech_inputs = np.random.rand(100).astype(np.float64)
- py_speech_inputs = np_speech_inputs.tolist()
-
- for inputs in [py_speech_inputs, np_speech_inputs]:
- np_processed = feature_extractor.pad([{"input_values": inputs}], return_tensors="np")
- self.assertTrue(np_processed.input_values.dtype == np.float32)
- pt_processed = feature_extractor.pad([{"input_values": inputs}], return_tensors="pt")
- self.assertTrue(pt_processed.input_values.dtype == torch.float32)
-
- @slow
- @require_torch
- def test_pretrained_checkpoints_are_set_correctly(self):
- # this test makes sure that models that are using
- # group norm don't have their feature extractor return the
- # attention_mask
- model_id = "facebook/wav2vec2-base-960h"
- config = Wav2Vec2Config.from_pretrained(model_id)
- feat_extract = Wav2Vec2FeatureExtractor.from_pretrained(model_id)
-
- # only "layer" feature extraction norm should make use of
- # attention_mask
- self.assertEqual(feat_extract.return_attention_mask, config.feat_extract_norm == "layer")
diff --git a/tests/models/wav2vec2/test_processing_wav2vec2.py b/tests/models/wav2vec2/test_processing_wav2vec2.py
index cb3a5cc8f872..889a0f721282 100644
--- a/tests/models/wav2vec2/test_processing_wav2vec2.py
+++ b/tests/models/wav2vec2/test_processing_wav2vec2.py
@@ -19,8 +19,7 @@
from transformers.models.wav2vec2 import Wav2Vec2Processor
from transformers.models.wav2vec2.tokenization_wav2vec2 import VOCAB_FILES_NAMES
-from ...test_processing_common import ProcessorTesterMixin
-from ..wav2vec2.test_feature_extraction_wav2vec2 import floats_list
+from ...test_processing_common import ProcessorTesterMixin, floats_list
class Wav2Vec2ProcessorTest(ProcessorTesterMixin, unittest.TestCase):
diff --git a/tests/models/wav2vec2_bert/test_processing_wav2vec2_bert.py b/tests/models/wav2vec2_bert/test_processing_wav2vec2_bert.py
index d188451da6d1..8dbad5c38cff 100644
--- a/tests/models/wav2vec2_bert/test_processing_wav2vec2_bert.py
+++ b/tests/models/wav2vec2_bert/test_processing_wav2vec2_bert.py
@@ -19,8 +19,7 @@
from transformers.models.wav2vec2.tokenization_wav2vec2 import VOCAB_FILES_NAMES
from transformers.models.wav2vec2_bert import Wav2Vec2BertProcessor
-from ...test_processing_common import ProcessorTesterMixin
-from ..wav2vec2.test_feature_extraction_wav2vec2 import floats_list
+from ...test_processing_common import ProcessorTesterMixin, floats_list
class Wav2Vec2BertProcessorTest(ProcessorTesterMixin, unittest.TestCase):
diff --git a/tests/models/wav2vec2_with_lm/test_processing_wav2vec2_with_lm.py b/tests/models/wav2vec2_with_lm/test_processing_wav2vec2_with_lm.py
index 2553bd3273b8..09a95c5e27ed 100644
--- a/tests/models/wav2vec2_with_lm/test_processing_wav2vec2_with_lm.py
+++ b/tests/models/wav2vec2_with_lm/test_processing_wav2vec2_with_lm.py
@@ -31,7 +31,7 @@
from transformers.testing_utils import require_pyctcdecode, require_torch, require_torchaudio, slow
from transformers.utils import is_pyctcdecode_available, is_torch_available
-from ..wav2vec2.test_feature_extraction_wav2vec2 import floats_list
+from ...test_processing_common import floats_list
if is_pyctcdecode_available():
diff --git a/tests/models/whisper/test_audio_processing_whisper.py b/tests/models/whisper/test_audio_processing_whisper.py
new file mode 100644
index 000000000000..82dd3c955c14
--- /dev/null
+++ b/tests/models/whisper/test_audio_processing_whisper.py
@@ -0,0 +1,32 @@
+# Copyright 2026 HuggingFace Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+"""Tests for `WhisperAudioProcessor` and `WhisperAudioProcessorNumpy`."""
+
+from __future__ import annotations
+
+import unittest
+
+from transformers.testing_utils import require_torch
+
+from ...test_audio_processing_common import AudioProcessingTestMixin
+
+
+class WhisperAudioProcessingTester:
+ """Provides init kwargs and fixture parameters for the Whisper audio processor tests."""
+
+ sample_rate = 16000
+
+ def prepare_audio_processor_dict(self) -> dict:
+ return {}
+
+
+@require_torch
+class WhisperAudioProcessingTest(AudioProcessingTestMixin, unittest.TestCase):
+ def setUp(self):
+ self.audio_processor_tester = WhisperAudioProcessingTester()
+ super().setUp()
diff --git a/tests/models/whisper/test_feature_extraction_whisper.py b/tests/models/whisper/test_feature_extraction_whisper.py
deleted file mode 100644
index 65f150cc1295..000000000000
--- a/tests/models/whisper/test_feature_extraction_whisper.py
+++ /dev/null
@@ -1,360 +0,0 @@
-# Copyright 2022 HuggingFace Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-import itertools
-import os
-import tempfile
-import unittest
-
-import numpy as np
-from datasets import load_dataset
-
-from transformers import WhisperFeatureExtractor
-from transformers.testing_utils import (
- check_json_file_has_correct_format,
- require_torch,
- require_torch_accelerator,
-)
-from transformers.utils.import_utils import is_torch_available
-
-from ...test_processing_common import floats_list
-from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin
-
-
-if is_torch_available():
- import torch
-
-
-class WhisperFeatureExtractionTester:
- def __init__(
- self,
- parent,
- batch_size=7,
- min_seq_length=400,
- max_seq_length=2000,
- feature_size=10,
- hop_length=160,
- chunk_length=8,
- padding_value=0.0,
- sampling_rate=4_000,
- return_attention_mask=False,
- do_normalize=True,
- ):
- self.parent = parent
- self.batch_size = batch_size
- self.min_seq_length = min_seq_length
- self.max_seq_length = max_seq_length
- self.seq_length_diff = (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
- self.padding_value = padding_value
- self.sampling_rate = sampling_rate
- self.return_attention_mask = return_attention_mask
- self.do_normalize = do_normalize
- self.feature_size = feature_size
- self.chunk_length = chunk_length
- self.hop_length = hop_length
-
- def prepare_feat_extract_dict(self):
- return {
- "feature_size": self.feature_size,
- "hop_length": self.hop_length,
- "chunk_length": self.chunk_length,
- "padding_value": self.padding_value,
- "sampling_rate": self.sampling_rate,
- "return_attention_mask": self.return_attention_mask,
- "do_normalize": self.do_normalize,
- }
-
- def prepare_inputs_for_common(self, equal_length=False, numpify=False):
- def _flatten(list_of_lists):
- return list(itertools.chain(*list_of_lists))
-
- if equal_length:
- speech_inputs = [floats_list((self.max_seq_length, self.feature_size)) for _ in range(self.batch_size)]
- else:
- # make sure that inputs increase in size
- speech_inputs = [
- floats_list((x, self.feature_size))
- for x in range(self.min_seq_length, self.max_seq_length, self.seq_length_diff)
- ]
- if numpify:
- speech_inputs = [np.asarray(x) for x in speech_inputs]
- return speech_inputs
-
-
-class WhisperFeatureExtractionTest(SequenceFeatureExtractionTestMixin, unittest.TestCase):
- feature_extraction_class = WhisperFeatureExtractor
-
- def setUp(self):
- self.feat_extract_tester = WhisperFeatureExtractionTester(self)
-
- def test_feat_extract_from_and_save_pretrained(self):
- feat_extract_first = self.feature_extraction_class(**self.feat_extract_dict)
-
- with tempfile.TemporaryDirectory() as tmpdirname:
- saved_file = feat_extract_first.save_pretrained(tmpdirname)[0]
- check_json_file_has_correct_format(saved_file)
- feat_extract_second = self.feature_extraction_class.from_pretrained(tmpdirname)
-
- dict_first = feat_extract_first.to_dict()
- dict_second = feat_extract_second.to_dict()
- mel_1 = feat_extract_first.mel_filters
- mel_2 = feat_extract_second.mel_filters
- self.assertTrue(np.allclose(mel_1, mel_2))
- self.assertEqual(dict_first, dict_second)
-
- def test_feat_extract_to_json_file(self):
- feat_extract_first = self.feature_extraction_class(**self.feat_extract_dict)
-
- with tempfile.TemporaryDirectory() as tmpdirname:
- json_file_path = os.path.join(tmpdirname, "feat_extract.json")
- feat_extract_first.to_json_file(json_file_path)
- feat_extract_second = self.feature_extraction_class.from_json_file(json_file_path)
-
- dict_first = feat_extract_first.to_dict()
- dict_second = feat_extract_second.to_dict()
- mel_1 = feat_extract_first.mel_filters
- mel_2 = feat_extract_second.mel_filters
- self.assertTrue(np.allclose(mel_1, mel_2))
- self.assertEqual(dict_first, dict_second)
-
- def test_feat_extract_from_pretrained_kwargs(self):
- feat_extract_first = self.feature_extraction_class(**self.feat_extract_dict)
-
- with tempfile.TemporaryDirectory() as tmpdirname:
- saved_file = feat_extract_first.save_pretrained(tmpdirname)[0]
- check_json_file_has_correct_format(saved_file)
- feat_extract_second = self.feature_extraction_class.from_pretrained(
- tmpdirname, feature_size=2 * self.feat_extract_dict["feature_size"]
- )
-
- mel_1 = feat_extract_first.mel_filters
- mel_2 = feat_extract_second.mel_filters
- self.assertTrue(2 * mel_1.shape[1] == mel_2.shape[1])
-
- def test_call(self):
- # Tests that all call wrap to encode_plus and batch_encode_plus
- feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- # create three inputs of length 800, 1000, and 1200
- speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)]
- np_speech_inputs = [np.asarray(speech_input) for speech_input in speech_inputs]
-
- # Test feature size
- input_features = feature_extractor(np_speech_inputs, padding="max_length", return_tensors="np").input_features
- self.assertTrue(input_features.ndim == 3)
- self.assertTrue(input_features.shape[-1] == feature_extractor.nb_max_frames)
- self.assertTrue(input_features.shape[-2] == feature_extractor.feature_size)
-
- # Test not batched input
- encoded_sequences_1 = feature_extractor(speech_inputs[0], return_tensors="np").input_features
- encoded_sequences_2 = feature_extractor(np_speech_inputs[0], return_tensors="np").input_features
- self.assertTrue(np.allclose(encoded_sequences_1, encoded_sequences_2, atol=1e-3))
-
- # Test batched
- encoded_sequences_1 = feature_extractor(speech_inputs, return_tensors="np").input_features
- encoded_sequences_2 = feature_extractor(np_speech_inputs, return_tensors="np").input_features
- for enc_seq_1, enc_seq_2 in zip(encoded_sequences_1, encoded_sequences_2):
- self.assertTrue(np.allclose(enc_seq_1, enc_seq_2, atol=1e-3))
-
- # Test 2-D numpy arrays are batched.
- speech_inputs = [floats_list((1, x))[0] for x in (800, 800, 800)]
- np_speech_inputs = np.asarray(speech_inputs)
- encoded_sequences_1 = feature_extractor(speech_inputs, return_tensors="np").input_features
- encoded_sequences_2 = feature_extractor(np_speech_inputs, return_tensors="np").input_features
- for enc_seq_1, enc_seq_2 in zip(encoded_sequences_1, encoded_sequences_2):
- self.assertTrue(np.allclose(enc_seq_1, enc_seq_2, atol=1e-3))
-
- # Test truncation required
- speech_inputs = [floats_list((1, x))[0] for x in range(200, (feature_extractor.n_samples + 500), 200)]
- np_speech_inputs = [np.asarray(speech_input) for speech_input in speech_inputs]
-
- speech_inputs_truncated = [x[: feature_extractor.n_samples] for x in speech_inputs]
- np_speech_inputs_truncated = [np.asarray(speech_input) for speech_input in speech_inputs_truncated]
-
- encoded_sequences_1 = feature_extractor(np_speech_inputs, return_tensors="np").input_features
- encoded_sequences_2 = feature_extractor(np_speech_inputs_truncated, return_tensors="np").input_features
- for enc_seq_1, enc_seq_2 in zip(encoded_sequences_1, encoded_sequences_2):
- self.assertTrue(np.allclose(enc_seq_1, enc_seq_2, atol=1e-3))
-
- def test_dither(self):
- np.random.seed(42) # seed the dithering randn()
-
- # Tests that features with and without little dithering are similar, but not the same
- dict_no_dither = self.feat_extract_tester.prepare_feat_extract_dict()
- dict_no_dither["dither"] = 0.0
-
- dict_dither = self.feat_extract_tester.prepare_feat_extract_dict()
- dict_dither["dither"] = 0.00003 # approx. 1/32k
-
- feature_extractor_no_dither = self.feature_extraction_class(**dict_no_dither)
- feature_extractor_dither = self.feature_extraction_class(**dict_dither)
-
- # create three inputs of length 800, 1000, and 1200
- speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)]
- np_speech_inputs = [np.asarray(speech_input) for speech_input in speech_inputs]
-
- # compute features
- input_features_no_dither = feature_extractor_no_dither(
- np_speech_inputs, padding=True, return_tensors="np", sampling_rate=dict_no_dither["sampling_rate"]
- ).input_features
- input_features_dither = feature_extractor_dither(
- np_speech_inputs, padding=True, return_tensors="np", sampling_rate=dict_dither["sampling_rate"]
- ).input_features
-
- # test there is a difference between features (there's added noise to input signal)
- diff = input_features_dither - input_features_no_dither
-
- # features are not identical
- self.assertTrue(np.abs(diff).mean() > 1e-6)
- # features are not too different
- self.assertTrue(np.abs(diff).mean() <= 1e-4)
- self.assertTrue(np.abs(diff).max() <= 5e-3)
-
- def test_feature_shape(self):
- feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- hop_length = feature_extractor.hop_length
- test_inputs = np.random.randn(16000)
-
- self.assertTrue(
- feature_extractor(
- [test_inputs[: hop_length * 5 + 1]],
- return_attention_mask=True,
- padding=False,
- return_tensors="np",
- ).attention_mask.shape[-1]
- == 5
- )
- self.assertTrue(
- feature_extractor(
- [test_inputs[: hop_length * 5]],
- return_attention_mask=True,
- padding=False,
- return_tensors="np",
- ).attention_mask.shape[-1]
- == 5
- )
- self.assertTrue(
- feature_extractor(
- [test_inputs[: hop_length * 5 - 1]],
- return_attention_mask=True,
- padding=False,
- return_tensors="np",
- ).attention_mask.shape[-1]
- == 4
- )
-
- @require_torch
- def test_double_precision_pad(self):
- import torch
-
- feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- np_speech_inputs = np.random.rand(100, 32).astype(np.float64)
- py_speech_inputs = np_speech_inputs.tolist()
-
- for inputs in [py_speech_inputs, np_speech_inputs]:
- np_processed = feature_extractor.pad([{"input_features": inputs}], return_tensors="np")
- self.assertTrue(np_processed.input_features.dtype == np.float32)
- pt_processed = feature_extractor.pad([{"input_features": inputs}], return_tensors="pt")
- self.assertTrue(pt_processed.input_features.dtype == torch.float32)
-
- def _load_datasamples(self, num_samples):
- ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
- # automatic decoding with librispeech
- speech_samples = ds.sort("id")[:num_samples]["audio"]
-
- return [x["array"] for x in speech_samples]
-
- @require_torch_accelerator
- @require_torch
- def test_torch_integration(self):
- # fmt: off
- EXPECTED_INPUT_FEATURES = torch.tensor(
- [
- 0.1193, -0.0946, -0.1098, -0.0196, 0.0225, -0.0690, -0.1736, 0.0951,
- 0.0971, -0.0817, -0.0702, 0.0162, 0.0260, 0.0017, -0.0192, -0.1678,
- 0.0709, -0.1867, -0.0655, -0.0274, -0.0234, -0.1884, -0.0516, -0.0554,
- -0.0274, -0.1425, -0.1423, 0.0837, 0.0377, -0.0854
- ]
- )
- # fmt: on
-
- input_speech = self._load_datasamples(1)
- feature_extractor = WhisperFeatureExtractor()
- input_features = feature_extractor(input_speech, return_tensors="pt").input_features
-
- self.assertEqual(input_features.shape, (1, 80, 3000))
- torch.testing.assert_close(input_features[0, 0, :30], EXPECTED_INPUT_FEATURES, rtol=1e-4, atol=1e-4)
-
- @unittest.mock.patch("transformers.models.whisper.feature_extraction_whisper.is_torch_available", lambda: False)
- def test_numpy_integration(self):
- # fmt: off
- EXPECTED_INPUT_FEATURES = np.array(
- [
- 0.1193, -0.0946, -0.1098, -0.0196, 0.0225, -0.0690, -0.1736, 0.0951,
- 0.0971, -0.0817, -0.0702, 0.0162, 0.0260, 0.0017, -0.0192, -0.1678,
- 0.0709, -0.1867, -0.0655, -0.0274, -0.0234, -0.1884, -0.0516, -0.0554,
- -0.0274, -0.1425, -0.1423, 0.0837, 0.0377, -0.0854
- ]
- )
- # fmt: on
-
- input_speech = self._load_datasamples(1)
- feature_extractor = WhisperFeatureExtractor()
- input_features = feature_extractor(input_speech, return_tensors="np").input_features
- self.assertEqual(input_features.shape, (1, 80, 3000))
- self.assertTrue(np.allclose(input_features[0, 0, :30], EXPECTED_INPUT_FEATURES, atol=1e-4))
-
- def test_zero_mean_unit_variance_normalization_trunc_np_longest(self):
- feat_extract = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
- audio = self._load_datasamples(1)[0]
- audio = ((audio - audio.min()) / (audio.max() - audio.min())) * 65535 # Rescale to [0, 65535] to show issue
- audio = feat_extract.zero_mean_unit_var_norm([audio], attention_mask=None)[0]
-
- self.assertTrue(np.all(np.mean(audio) < 1e-3))
- self.assertTrue(np.all(np.abs(np.var(audio) - 1) < 1e-3))
-
- @require_torch_accelerator
- @require_torch
- def test_torch_integration_batch(self):
- # fmt: off
- EXPECTED_INPUT_FEATURES = torch.tensor(
- [
- [
- 0.1193, -0.0946, -0.1098, -0.0196, 0.0225, -0.0690, -0.1736, 0.0951,
- 0.0971, -0.0817, -0.0702, 0.0162, 0.0260, 0.0017, -0.0192, -0.1678,
- 0.0709, -0.1867, -0.0655, -0.0274, -0.0234, -0.1884, -0.0516, -0.0554,
- -0.0274, -0.1425, -0.1423, 0.0837, 0.0377, -0.0854
- ],
- [
- -0.4696, -0.0751, 0.0276, -0.0312, -0.0540, -0.0383, 0.1295, 0.0568,
- -0.2071, -0.0548, 0.0389, -0.0316, -0.2346, -0.1068, -0.0322, 0.0475,
- -0.1709, -0.0041, 0.0872, 0.0537, 0.0075, -0.0392, 0.0371, 0.0189,
- -0.1522, -0.0270, 0.0744, 0.0738, -0.0245, -0.0667
- ],
- [
- -0.2337, -0.0060, -0.0063, -0.2353, -0.0431, 0.1102, -0.1492, -0.0292,
- 0.0787, -0.0608, 0.0143, 0.0582, 0.0072, 0.0101, -0.0444, -0.1701,
- -0.0064, -0.0027, -0.0826, -0.0730, -0.0099, -0.0762, -0.0170, 0.0446,
- -0.1153, 0.0960, -0.0361, 0.0652, 0.1207, 0.0277
- ]
- ]
- )
- # fmt: on
-
- with torch.device("cuda"):
- input_speech = self._load_datasamples(3)
- feature_extractor = WhisperFeatureExtractor()
- input_features = feature_extractor(input_speech, return_tensors="pt").input_features
- self.assertEqual(input_features.shape, (3, 80, 3000))
- torch.testing.assert_close(input_features[:, 0, :30], EXPECTED_INPUT_FEATURES, rtol=1e-4, atol=1e-4)
diff --git a/tests/models/whisper/test_processing_whisper.py b/tests/models/whisper/test_processing_whisper.py
index 23759ef3bd93..a9e05437815e 100644
--- a/tests/models/whisper/test_processing_whisper.py
+++ b/tests/models/whisper/test_processing_whisper.py
@@ -22,7 +22,7 @@
from transformers import WhisperTokenizer, WhisperTokenizerFast, is_speech_available
from transformers.testing_utils import require_sentencepiece, require_torch, require_torchaudio
-from .test_feature_extraction_whisper import floats_list
+from ...test_processing_common import floats_list
if is_speech_available():
diff --git a/tests/test_audio_processing_common.py b/tests/test_audio_processing_common.py
new file mode 100644
index 000000000000..7b44caec9bf9
--- /dev/null
+++ b/tests/test_audio_processing_common.py
@@ -0,0 +1,221 @@
+# Copyright 2026 HuggingFace Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+"""Common tests for `XxxAudioProcessor` classes.
+
+Mirrors `test_image_processing_common.ImageProcessingTestMixin`. Auto-discovers a model's
+sibling backend classes (`torch` and optionally `numpy`) from
+`FEATURE_EXTRACTOR_MAPPING_NAMES` keyed by model directory name. Per-model test files set
+``self.audio_processor_tester`` in their `setUp` and inherit from this mixin.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import pathlib
+import sys
+import tempfile
+
+import numpy as np
+
+from transformers.models.auto.feature_extraction_auto import (
+ FEATURE_EXTRACTOR_MAPPING_NAMES,
+ feature_extractor_class_from_name,
+)
+from transformers.testing_utils import check_json_file_has_correct_format, require_torch
+from transformers.utils import is_torch_available
+
+
+if is_torch_available():
+ import torch
+
+
+def prepare_audio_inputs(
+ batch_size: int = 3,
+ sample_rate: int = 16000,
+ min_length: float = 1.0,
+ max_length: float = 3.0,
+ equal_length: bool = False,
+ seed: int = 0,
+):
+ """Generate a batch of fake waveforms with varying lengths."""
+ rng = np.random.RandomState(seed)
+ if equal_length:
+ lengths = [int(max_length * sample_rate)] * batch_size
+ else:
+ lengths = [int(rng.uniform(min_length, max_length) * sample_rate) for _ in range(batch_size)]
+ return [rng.uniform(-1.0, 1.0, size=length).astype(np.float32) for length in lengths]
+
+
+class AudioProcessingTestMixin:
+ """Shared tests for every `XxxAudioProcessor` (and its sibling `XxxAudioProcessorNumpy`).
+
+ Subclasses must set ``self.audio_processor_tester`` in their `setUp`. The tester is
+ expected to expose:
+
+ - ``prepare_audio_processor_dict()`` → dict of init kwargs
+ - ``batch_size``, ``sample_rate`` (optional, used to generate fake inputs)
+ """
+
+ audio_processor_tester = None
+ test_classes_to_skip: set[str] = set()
+ # Per-model override of the cross-backend parity bar (ADR 0001). Default is the float32
+ # noise floor; models with longer numerical chains (custom STFT, unfold + preemphasis,
+ # float32/64 mixed ops) can relax to 1e-3 / 1e-4 if needed.
+ parity_atol: float = 1e-5
+ parity_rtol: float = 1e-5
+
+ def setUp(self):
+ # Infer the model_name from the test directory (e.g. "whisper" from tests/models/whisper/...).
+ test_file_path = pathlib.Path(sys.modules[self.__class__.__module__].__file__).resolve()
+ model_name = test_file_path.parent.name
+ try:
+ class_names_by_backend = FEATURE_EXTRACTOR_MAPPING_NAMES[model_name]
+ except KeyError as e:
+ raise ValueError(
+ f"No entry for model_name={model_name!r} in FEATURE_EXTRACTOR_MAPPING_NAMES. "
+ f"Override `setUp` in your test class to provide the backend mapping."
+ ) from e
+
+ self.audio_processing_classes = {
+ backend: feature_extractor_class_from_name(class_name)
+ for backend, class_name in class_names_by_backend.items()
+ if class_name not in self.test_classes_to_skip
+ }
+ self.audio_processing_classes = {b: c for b, c in self.audio_processing_classes.items() if c is not None}
+
+ # ── Cross-backend parity ──────────────────────────────────────────────
+
+ def _to_torch(self, x):
+ if isinstance(x, np.ndarray):
+ return torch.from_numpy(x)
+ if hasattr(x, "numpy"):
+ return x
+ return torch.as_tensor(x)
+
+ def _assert_outputs_bit_exact(self, output_a, output_b, *, atol=1e-5, rtol=1e-5):
+ """Per ADR 0001, sibling backends must agree within the float32 noise floor —
+ `torch.allclose(atol=1e-5, rtol=1e-5)`. The bar is intentionally not stricter
+ than `np.fft.rfft` vs `torch.fft.rfft` library divergence allows."""
+ keys_a = set(output_a.keys())
+ keys_b = set(output_b.keys())
+ self.assertEqual(keys_a, keys_b, f"Output keys differ: {keys_a} vs {keys_b}")
+ for key in keys_a:
+ a = self._to_torch(output_a[key])
+ b = self._to_torch(output_b[key])
+ self.assertEqual(a.shape, b.shape, f"Shape mismatch for {key!r}: {a.shape} vs {b.shape}")
+ # Integer masks must match exactly; only float outputs get tolerance.
+ if a.dtype in (torch.bool, torch.int32, torch.int64):
+ self.assertTrue(
+ torch.equal(a, b),
+ f"Mask/integer output mismatch for {key!r} (max abs diff: {(a.long() - b.long()).abs().max().item()})",
+ )
+ else:
+ self.assertTrue(
+ torch.allclose(a, b, atol=atol, rtol=rtol),
+ f"Numerical parity violated for output key {key!r}: "
+ f"max abs diff {(a - b).abs().max().item():.3e} exceeds atol={atol:.0e}, rtol={rtol:.0e}",
+ )
+
+ @require_torch
+ def test_backends_equivalence(self):
+ if len(self.audio_processing_classes) < 2:
+ self.skipTest("Only one backend registered; cross-backend parity test skipped.")
+ if self.audio_processor_tester is None:
+ self.skipTest("audio_processor_tester not set; cannot generate fixtures.")
+
+ init_dict = self.audio_processor_tester.prepare_audio_processor_dict()
+ waveform = prepare_audio_inputs(batch_size=1, seed=0)[0]
+
+ outputs = {}
+ for backend, cls in self.audio_processing_classes.items():
+ ap = cls(**init_dict)
+ outputs[backend] = ap(waveform, sampling_rate=ap.sampling_rate, return_tensors="pt")
+
+ reference_backend, reference_output = next(iter(outputs.items()))
+ for backend, output in outputs.items():
+ if backend == reference_backend:
+ continue
+ self._assert_outputs_bit_exact(reference_output, output, atol=self.parity_atol, rtol=self.parity_rtol)
+
+ @require_torch
+ def test_backends_equivalence_batched(self):
+ if len(self.audio_processing_classes) < 2:
+ self.skipTest("Only one backend registered; cross-backend parity test skipped.")
+ if self.audio_processor_tester is None:
+ self.skipTest("audio_processor_tester not set; cannot generate fixtures.")
+
+ init_dict = self.audio_processor_tester.prepare_audio_processor_dict()
+ waveforms = prepare_audio_inputs(batch_size=3, equal_length=False, seed=0)
+
+ outputs = {}
+ for backend, cls in self.audio_processing_classes.items():
+ ap = cls(**init_dict)
+ outputs[backend] = ap(waveforms, sampling_rate=ap.sampling_rate, return_tensors="pt")
+
+ reference_backend, reference_output = next(iter(outputs.items()))
+ for backend, output in outputs.items():
+ if backend == reference_backend:
+ continue
+ self._assert_outputs_bit_exact(reference_output, output, atol=self.parity_atol, rtol=self.parity_rtol)
+
+ # ── JSON round-trip ───────────────────────────────────────────────────
+
+ def test_audio_processor_to_json_string(self):
+ if self.audio_processor_tester is None:
+ self.skipTest("audio_processor_tester not set.")
+ init_dict = self.audio_processor_tester.prepare_audio_processor_dict()
+ for cls in self.audio_processing_classes.values():
+ ap = cls(**init_dict)
+ obj = json.loads(ap.to_json_string())
+ self.assertEqual(obj["audio_processor_type"], cls.__name__)
+
+ def test_audio_processor_to_json_file(self):
+ if self.audio_processor_tester is None:
+ self.skipTest("audio_processor_tester not set.")
+ init_dict = self.audio_processor_tester.prepare_audio_processor_dict()
+ for cls in self.audio_processing_classes.values():
+ ap_first = cls(**init_dict)
+ with tempfile.TemporaryDirectory() as tmpdirname:
+ json_file_path = os.path.join(tmpdirname, "audio_processor.json")
+ ap_first.to_json_file(json_file_path)
+ ap_second = cls.from_json_file(json_file_path)
+ self.assertEqual(ap_second.to_dict(), ap_first.to_dict())
+
+ def test_audio_processor_from_and_save_pretrained(self):
+ if self.audio_processor_tester is None:
+ self.skipTest("audio_processor_tester not set.")
+ init_dict = self.audio_processor_tester.prepare_audio_processor_dict()
+ for cls in self.audio_processing_classes.values():
+ ap_first = cls(**init_dict)
+ with tempfile.TemporaryDirectory() as tmpdirname:
+ saved_file = ap_first.save_pretrained(tmpdirname)[0]
+ check_json_file_has_correct_format(saved_file)
+ ap_second = cls.from_pretrained(tmpdirname)
+ self.assertEqual(ap_second.to_dict(), ap_first.to_dict())
+
+ def test_audio_processor_save_load_with_autoaudioprocessor(self):
+ if self.audio_processor_tester is None:
+ self.skipTest("audio_processor_tester not set.")
+ from transformers.models.auto.feature_extraction_auto import AutoAudioProcessor
+
+ init_dict = self.audio_processor_tester.prepare_audio_processor_dict()
+ for backend, cls in self.audio_processing_classes.items():
+ ap_first = cls(**init_dict)
+ with tempfile.TemporaryDirectory() as tmpdirname:
+ ap_first.save_pretrained(tmpdirname)
+ ap_second = AutoAudioProcessor.from_pretrained(tmpdirname, backend=backend)
+ self.assertEqual(type(ap_second), cls)
+ self.assertEqual(ap_second.to_dict(), ap_first.to_dict())
+
+ # ── Basic instantiation ───────────────────────────────────────────────
+
+ def test_init_without_params(self):
+ for cls in self.audio_processing_classes.values():
+ ap = cls()
+ self.assertIsNotNone(ap)