From 4cce31fc489231c17933597e05da1507f719a25c Mon Sep 17 00:00:00 2001 From: Eric B Date: Wed, 9 Jul 2025 17:50:52 +0200 Subject: [PATCH 1/8] Fix DAC (slow) integration tests. --- tests/models/dac/test_modeling_dac.py | 457 +++++++++++++++++--------- 1 file changed, 309 insertions(+), 148 deletions(-) diff --git a/tests/models/dac/test_modeling_dac.py b/tests/models/dac/test_modeling_dac.py index 8de3fb818b7b..e8634a3c62f5 100644 --- a/tests/models/dac/test_modeling_dac.py +++ b/tests/models/dac/test_modeling_dac.py @@ -382,34 +382,54 @@ def normalize(arr): def compute_rmse(arr1, arr2): - arr1_normalized = normalize(arr1) - arr2_normalized = normalize(arr2) + arr1_np = arr1.cpu().numpy().squeeze() + arr2_np = arr2.cpu().numpy().squeeze() + max_length = min(arr1.shape[-1], arr2.shape[-1]) + arr1_np = arr1_np[..., :max_length] + arr2_np = arr2_np[..., :max_length] + arr1_normalized = normalize(arr1_np) + arr2_normalized = normalize(arr2_np) return np.sqrt(((arr1_normalized - arr2_normalized) ** 2).mean()) +FIX_HOP_LENGTH = True @slow @require_torch class DacIntegrationTest(unittest.TestCase): def test_integration_16khz(self): expected_rmse = 0.004 - - expected_encoder_sums_dict = { - "loss": 24.8596, - "quantized_representation": -0.0745, - "audio_codes": 504.0948, - "projected_latents": 0.0682, + # Code for reproducing expected outputs: https://gist.github.com/ebezzam/bb315efa7a416db6336a6b2a2d424ffa#file-single-py + expected_encoder_means_dict = { + "loss": 24.8491, + "quantized_representation": -0.07544856518507004, + # "audio_codes": 505.13421630859375, + "projected_latents": 0.06593942642211914, } + expected_quantizer_codebook_mean = 504.3310546875 + expected_decoded_mean = -0.00018316633941140026 + expected_codec_error = 0.0038341842591762543 librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") model_name = "dac_16khz" + sample_rate = 16000 model_id = f"descript/{model_name}" model = DacModel.from_pretrained(model_id, force_download=True).to(torch_device).eval() - processor = AutoProcessor.from_pretrained(model_id) + processor = AutoProcessor.from_pretrained( + model_id, + hop_length=int(np.prod(model.config.downsampling_ratios)) if FIX_HOP_LENGTH else model.config.hop_length + ) librispeech_dummy = librispeech_dummy.cast_column("audio", Audio(sampling_rate=processor.sampling_rate)) audio_sample = librispeech_dummy[0]["audio"]["array"] + # Resample audio to 16kHz if necessary + if librispeech_dummy[0]["audio"]["sampling_rate"] != sample_rate: + import librosa + + audio_sample = librosa.resample( + audio_sample, orig_sr=librispeech_dummy[0]["audio"]["sampling_rate"], target_sr=sample_rate + ) inputs = processor( raw_audio=audio_sample, @@ -418,51 +438,84 @@ def test_integration_16khz(self): ).to(torch_device) with torch.no_grad(): + # compute HF encoder outputs encoder_outputs = model.encode(inputs["input_values"]) + hf_output_means_dict = { + "loss": encoder_outputs[0].item(), + "quantized_representation": encoder_outputs[1].mean().item(), + # "audio_codes": encoder_outputs[2].float().mean().item(), + "projected_latents": encoder_outputs[3].float().mean().item(), + } + hf_output_means = torch.tensor(list(hf_output_means_dict.values()), dtype=torch.float32) + + # make sure encoded outputs are similar + # TODO for all sampling rates, encoder error is relatively high compared to quantizer and decoder (but still minimal) + # they may be a bug in encoder weight mapping: + # https://github.com/ebezzam/transformers/blob/main/src/transformers/models/dac/convert_dac_checkpoint.py#L63 + # in any case, the error is small enough to not affect the codec performance + expected_encoder_means = torch.tensor(list(expected_encoder_means_dict.values()), dtype=torch.float32) + torch.testing.assert_close(hf_output_means, expected_encoder_means, rtol=1e-3, atol=1e-3) + + # check that quantizers behave similar (for same input) + encoded_hf = model.encoder(inputs["input_values"]) + hf_quantizer_codebook_mean = model.quantizer(encoded_hf)[1].float().mean().item() + torch.testing.assert_close( + hf_quantizer_codebook_mean, expected_quantizer_codebook_mean, rtol=1e-6, atol=1e-6 + ) - expected_encoder_sums = torch.tensor(list(expected_encoder_sums_dict.values()), dtype=torch.float32) - encoder_outputs_mean = torch.tensor([v.float().mean().cpu().item() for v in encoder_outputs.to_tuple()]) - - # make sure audio encoded codes are correct - torch.testing.assert_close(encoder_outputs_mean, expected_encoder_sums, rtol=1e-3, atol=1e-3) + # check that decoders behave similar (for same input) + hf_decoded_mean = model.decode(encoded_hf)["audio_values"].mean().item() + torch.testing.assert_close(hf_decoded_mean, expected_decoded_mean, rtol=1e-6, atol=1e-6) + # decode _, quantized_representation, _, _ = encoder_outputs.to_tuple() input_values_dec = model.decode(quantized_representation)[0] input_values_enc_dec = model(inputs["input_values"])[1] # make sure forward and decode gives same result - torch.testing.assert_close(input_values_dec, input_values_enc_dec, rtol=1e-3, atol=1e-3) - - arr = inputs["input_values"][0].cpu().numpy() - arr_enc_dec = input_values_enc_dec[0].cpu().numpy() - - max_length = min(arr_enc_dec.shape[-1], arr.shape[-1]) - - arr_cut = arr[0, :max_length].copy() - arr_enc_dec_cut = arr_enc_dec[:max_length].copy() + torch.testing.assert_close(input_values_dec, input_values_enc_dec, rtol=1e-6, atol=1e-6) # make sure audios are more or less equal - rmse = compute_rmse(arr_cut, arr_enc_dec_cut) + rmse = compute_rmse(input_values_enc_dec, inputs["input_values"]) self.assertTrue(rmse < expected_rmse) + # check that codec error is similar + torch.testing.assert_close(expected_codec_error, rmse, rtol=1e-6, atol=1e-6) + def test_integration_24khz(self): expected_rmse = 0.0039 - - expected_encoder_output_dict = { - "quantized_representation": torch.tensor([0.6257, 3.1245, 5.2514, 2.3160, 1.5774]), - "audio_codes": torch.tensor([919, 919, 234, 777, 234]), - "projected_latents": torch.tensor([-4.7841, -5.0063, -4.5595, -5.0372, -5.4280]), + # Code for reproducing expected outputs: https://gist.github.com/ebezzam/bb315efa7a416db6336a6b2a2d424ffa#file-single-py + expected_encoder_means_dict = { + "loss": 28.1121, + "quantized_representation": 0.016283338889479637, + # "audio_codes": 507.17724609375, + "projected_latents": -0.024361690506339073, } + expected_quantizer_codebook_mean = 506.8665466308594 + expected_decoded_mean = 0.0001686957839410752 + expected_codec_error = 0.002570481738075614 + librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") model_name = "dac_24khz" + sample_rate = 24000 model_id = f"descript/{model_name}" model = DacModel.from_pretrained(model_id, force_download=True).to(torch_device).eval() - processor = AutoProcessor.from_pretrained(model_id) + processor = AutoProcessor.from_pretrained( + model_id, + hop_length=int(np.prod(model.config.downsampling_ratios)) if FIX_HOP_LENGTH else model.config.hop_length + ) librispeech_dummy = librispeech_dummy.cast_column("audio", Audio(sampling_rate=processor.sampling_rate)) audio_sample = librispeech_dummy[0]["audio"]["array"] + # Resample audio to 24kHz if necessary + if librispeech_dummy[0]["audio"]["sampling_rate"] != sample_rate: + import librosa + + audio_sample = librosa.resample( + audio_sample, orig_sr=librispeech_dummy[0]["audio"]["sampling_rate"], target_sr=sample_rate + ) inputs = processor( raw_audio=audio_sample, @@ -471,72 +524,80 @@ def test_integration_24khz(self): ).to(torch_device) with torch.no_grad(): + # compute HF encoder outputs encoder_outputs = model.encode(inputs["input_values"]) + hf_output_means_dict = { + "loss": encoder_outputs[0].item(), + "quantized_representation": encoder_outputs[1].mean().item(), + # "audio_codes": encoder_outputs[2].float().mean().item(), + "projected_latents": encoder_outputs[3].float().mean().item(), + } + hf_output_means = torch.tensor(list(hf_output_means_dict.values()), dtype=torch.float32) - expected_quantized_representation = encoder_outputs["quantized_representation"][0, 0, :5].cpu() - expected_audio_codes = encoder_outputs["audio_codes"][0, 0, :5].cpu() - expected_projected_latents = encoder_outputs["projected_latents"][0, 0, :5].cpu() + # make sure encoded outputs are similar + expected_encoder_means = torch.tensor(list(expected_encoder_means_dict.values()), dtype=torch.float32) + torch.testing.assert_close(hf_output_means, expected_encoder_means, rtol=1e-2, atol=1e-2) - # make sure values are correct for audios slices - self.assertTrue( - torch.allclose( - expected_quantized_representation, - expected_encoder_output_dict["quantized_representation"], - atol=1e-3, - ) - ) - self.assertTrue( - torch.allclose(expected_audio_codes, expected_encoder_output_dict["audio_codes"], atol=1e-3) - ) - self.assertTrue( - torch.allclose( - expected_projected_latents, expected_encoder_output_dict["projected_latents"], atol=1e-3 - ) + # check that quantizers behave similar (for same input) + encoded_hf = model.encoder(inputs["input_values"]) + hf_quantizer_codebook_mean = model.quantizer(encoded_hf)[1].float().mean().item() + torch.testing.assert_close( + hf_quantizer_codebook_mean, expected_quantizer_codebook_mean, rtol=1e-6, atol=1e-6 ) + # check that decoders behave similar (for same input) + hf_decoded_mean = model.decode(encoded_hf)["audio_values"].mean().item() + torch.testing.assert_close(hf_decoded_mean, expected_decoded_mean, rtol=1e-6, atol=1e-6) + + # decode _, quantized_representation, _, _ = encoder_outputs.to_tuple() input_values_dec = model.decode(quantized_representation)[0] input_values_enc_dec = model(inputs["input_values"])[1] - input_values_from_codes = model.decode(audio_codes=encoder_outputs.audio_codes)[0] - - # make sure decode from audio codes and quantized values give more or less the same results - torch.testing.assert_close(input_values_from_codes, input_values_dec, rtol=1e-5, atol=1e-5) - # make sure forward and decode gives same result - torch.testing.assert_close(input_values_dec, input_values_enc_dec, rtol=1e-3, atol=1e-3) - - arr = inputs["input_values"][0].cpu().numpy() - arr_enc_dec = input_values_enc_dec[0].cpu().numpy() - - max_length = min(arr_enc_dec.shape[-1], arr.shape[-1]) - - arr_cut = arr[0, :max_length].copy() - arr_enc_dec_cut = arr_enc_dec[:max_length].copy() + torch.testing.assert_close(input_values_dec, input_values_enc_dec, rtol=1e-6, atol=1e-6) # make sure audios are more or less equal - rmse = compute_rmse(arr_cut, arr_enc_dec_cut) + rmse = compute_rmse(input_values_enc_dec, inputs["input_values"]) self.assertTrue(rmse < expected_rmse) + # check that codec error is similar + torch.testing.assert_close(expected_codec_error, rmse, rtol=1e-6, atol=1e-6) + def test_integration_44khz(self): expected_rmse = 0.002 - - expected_encoder_sums_dict = { - "loss": 34.3612, - "quantized_representation": 0.0078, - "audio_codes": 509.6812, - "projected_latents": -0.1054, + # Code for reproducing expected outputs: https://gist.github.com/ebezzam/bb315efa7a416db6336a6b2a2d424ffa#file-single-py + expected_encoder_means_dict = { + "loss": 23.7848, + "quantized_representation": 0.017807748168706894, + # "audio_codes": 513.7100219726562, + "projected_latents": 0.06925617158412933, } + expected_quantizer_codebook_mean = 514.03369140625 + expected_decoded_mean = -0.00010763177124317735 + expected_codec_error = 0.0007429996621794999 + librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") model_name = "dac_44khz" + sample_rate = 44100 model_id = f"descript/{model_name}" - model = DacModel.from_pretrained(model_id).to(torch_device).eval() - processor = AutoProcessor.from_pretrained(model_id) + model = DacModel.from_pretrained(model_id, force_download=True).to(torch_device).eval() + processor = AutoProcessor.from_pretrained( + model_id, + hop_length=int(np.prod(model.config.downsampling_ratios)) if FIX_HOP_LENGTH else model.config.hop_length + ) librispeech_dummy = librispeech_dummy.cast_column("audio", Audio(sampling_rate=processor.sampling_rate)) audio_sample = librispeech_dummy[0]["audio"]["array"] + # Resample audio to 24kHz if necessary + if librispeech_dummy[0]["audio"]["sampling_rate"] != sample_rate: + import librosa + + audio_sample = librosa.resample( + audio_sample, orig_sr=librispeech_dummy[0]["audio"]["sampling_rate"], target_sr=sample_rate + ) inputs = processor( raw_audio=audio_sample, @@ -545,54 +606,84 @@ def test_integration_44khz(self): ).to(torch_device) with torch.no_grad(): + # compute HF encoder outputs encoder_outputs = model.encode(inputs["input_values"]) + hf_output_means_dict = { + "loss": encoder_outputs[0].item(), + "quantized_representation": encoder_outputs[1].mean().item(), + # "audio_codes": encoder_outputs[2].float().mean().item(), + "projected_latents": encoder_outputs[3].float().mean().item(), + } + hf_output_means = torch.tensor(list(hf_output_means_dict.values()), dtype=torch.float32) - expected_encoder_sums = torch.tensor(list(expected_encoder_sums_dict.values()), dtype=torch.float32) - encoder_outputs_mean = torch.tensor([v.float().mean().cpu().item() for v in encoder_outputs.to_tuple()]) + # make sure encoded outputs are similar + expected_encoder_means = torch.tensor(list(expected_encoder_means_dict.values()), dtype=torch.float32) + torch.testing.assert_close(hf_output_means, expected_encoder_means, rtol=1e-3, atol=1e-3) + + # check that quantizers behave similar (for same input) + encoded_hf = model.encoder(inputs["input_values"]) + hf_quantizer_codebook_mean = model.quantizer(encoded_hf)[1].float().mean().item() + torch.testing.assert_close( + hf_quantizer_codebook_mean, expected_quantizer_codebook_mean, rtol=1e-6, atol=1e-6 + ) - # make sure audio encoded codes are correct - torch.testing.assert_close(encoder_outputs_mean, expected_encoder_sums, rtol=1e-3, atol=1e-3) + # check that decoders behave similar (for same input) + hf_decoded_mean = model.decode(encoded_hf)["audio_values"].mean().item() + torch.testing.assert_close(hf_decoded_mean, expected_decoded_mean, rtol=1e-6, atol=1e-6) + # decode _, quantized_representation, _, _ = encoder_outputs.to_tuple() input_values_dec = model.decode(quantized_representation)[0] input_values_enc_dec = model(inputs["input_values"])[1] # make sure forward and decode gives same result - torch.testing.assert_close(input_values_dec, input_values_enc_dec, rtol=1e-3, atol=1e-3) - - arr = inputs["input_values"][0].cpu().numpy() - arr_enc_dec = input_values_enc_dec[0].cpu().numpy() - - max_length = min(arr_enc_dec.shape[-1], arr.shape[-1]) - - arr_cut = arr[0, :max_length].copy() - arr_enc_dec_cut = arr_enc_dec[:max_length].copy() + torch.testing.assert_close(input_values_dec, input_values_enc_dec, rtol=1e-6, atol=1e-6) # make sure audios are more or less equal - rmse = compute_rmse(arr_cut, arr_enc_dec_cut) + rmse = compute_rmse(input_values_enc_dec, inputs["input_values"]) self.assertTrue(rmse < expected_rmse) + # check that codec error is similar + torch.testing.assert_close(expected_codec_error, rmse, rtol=1e-6, atol=1e-6) + def test_integration_batch_16khz(self): expected_rmse = 0.002 - - expected_encoder_sums_dict = { - "loss": 20.3913, - "quantized_representation": -0.0538, - "audio_codes": 487.8470, - "projected_latents": 0.0237, + # Code for reproducing expected outputs: https://gist.github.com/ebezzam/bb315efa7a416db6336a6b2a2d424ffa#file-batch-py + expected_encoder_means_dict = { + "loss": 20.370271682739258, + "quantized_representation": -0.05440079793334007, + "audio_codes": 488.02716064453125, + "projected_latents": 0.02350950613617897, } + expected_quantizer_codebook_mean = 488.4040222167969 + expected_decoded_mean = -7.977934001246467e-05 + expected_codec_error = 0.001973195234313607 librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") model_name = "dac_16khz" + sample_rate = 16000 model_id = f"descript/{model_name}" model = DacModel.from_pretrained(model_id).to(torch_device) - processor = AutoProcessor.from_pretrained(model_id) + processor = AutoProcessor.from_pretrained( + model_id, + hop_length=int(np.prod(model.config.downsampling_ratios)) if FIX_HOP_LENGTH else model.config.hop_length + ) librispeech_dummy = librispeech_dummy.cast_column("audio", Audio(sampling_rate=processor.sampling_rate)) audio_samples = [np.array([audio_sample["array"]])[0] for audio_sample in librispeech_dummy[-2:]["audio"]] + if sample_rate != librispeech_dummy[0]["audio"]["sampling_rate"]: + import librosa + + # resample audio if necessary + audio_samples = [ + librosa.resample( + audio_sample, orig_sr=librispeech_dummy[0]["audio"]["sampling_rate"], target_sr=sample_rate + ) + for audio_sample in audio_samples + ] inputs = processor( raw_audio=audio_samples, @@ -603,53 +694,82 @@ def test_integration_batch_16khz(self): with torch.no_grad(): encoder_outputs = model.encode(inputs["input_values"]) + hf_output_means_dict = { + "loss": encoder_outputs[0].mean().item(), + "quantized_representation": encoder_outputs[1].mean().item(), + "audio_codes": encoder_outputs[2].float().mean().item(), + "projected_latents": encoder_outputs[3].float().mean().item(), + } + hf_output_means = torch.tensor(list(hf_output_means_dict.values()), dtype=torch.float32) - expected_encoder_sums = torch.tensor(list(expected_encoder_sums_dict.values()), dtype=torch.float32) - encoder_outputs_mean = torch.tensor([v.float().mean().item() for v in encoder_outputs.to_tuple()]) + # make sure encoded outputs are similar + expected_encoder_means = torch.tensor(list(expected_encoder_means_dict.values()), dtype=torch.float32) + torch.testing.assert_close(hf_output_means, expected_encoder_means, rtol=1e-3, atol=1e-3) - # make sure audio encoded codes are correct - torch.testing.assert_close(encoder_outputs_mean, expected_encoder_sums, rtol=1e-3, atol=1e-3) + # check that quantizers behave similar (for same input) + encoded_hf = model.encoder(inputs["input_values"]) + hf_quantizer_codebook_mean = model.quantizer(encoded_hf)[1].float().mean().item() + torch.testing.assert_close( + hf_quantizer_codebook_mean, expected_quantizer_codebook_mean, rtol=1e-6, atol=1e-6 + ) + # check that decoders behave similar (for same input) + hf_decoded_mean = model.decode(encoded_hf)["audio_values"].mean().item() + torch.testing.assert_close(hf_decoded_mean, expected_decoded_mean, rtol=1e-6, atol=1e-6) + + # decode _, quantized_representation, _, _ = encoder_outputs.to_tuple() input_values_dec = model.decode(quantized_representation)[0] input_values_enc_dec = model(inputs["input_values"])[1] # make sure forward and decode gives same result - torch.testing.assert_close(input_values_dec, input_values_enc_dec, rtol=1e-3, atol=1e-3) - - arr = inputs["input_values"].cpu().numpy() - arr_enc_dec = input_values_enc_dec.cpu().numpy() - - max_length = min(arr_enc_dec.shape[-1], arr.shape[-1]) - - arr_cut = arr[:, 0, :max_length].copy() - arr_enc_dec_cut = arr_enc_dec[:, :max_length].copy() + torch.testing.assert_close(input_values_dec, input_values_enc_dec, rtol=1e-6, atol=1e-6) # make sure audios are more or less equal - rmse = compute_rmse(arr_cut, arr_enc_dec_cut) + rmse = compute_rmse(input_values_enc_dec, inputs["input_values"]) self.assertTrue(rmse < expected_rmse) + # check that codec error is similar + torch.testing.assert_close(expected_codec_error, rmse, rtol=1e-6, atol=1e-6) + def test_integration_batch_24khz(self): expected_rmse = 0.002 - - expected_encoder_sums_dict = { - "loss": 24.2309, - "quantized_representation": 0.0520, - "audio_codes": 510.2700, - "projected_latents": -0.0076, + # Code for reproducing expected outputs: https://gist.github.com/ebezzam/bb315efa7a416db6336a6b2a2d424ffa#file-batch-py + expected_encoder_means_dict = { + "loss": 24.505210876464844, + "quantized_representation": 0.03778776153922081, + "audio_codes": 509.5290222167969, + "projected_latents": -0.017138859257102013, } + expected_quantizer_codebook_mean = 509.381103515625 + expected_decoded_mean = 0.00010512518929317594 + expected_codec_error = 0.0012980918399989605 librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") model_name = "dac_24khz" + sample_rate = 24000 model_id = f"descript/{model_name}" model = DacModel.from_pretrained(model_id).to(torch_device) - processor = AutoProcessor.from_pretrained(model_id) + processor = AutoProcessor.from_pretrained( + model_id, + hop_length=int(np.prod(model.config.downsampling_ratios)) if FIX_HOP_LENGTH else model.config.hop_length + ) librispeech_dummy = librispeech_dummy.cast_column("audio", Audio(sampling_rate=processor.sampling_rate)) audio_samples = [np.array([audio_sample["array"]])[0] for audio_sample in librispeech_dummy[-2:]["audio"]] + if sample_rate != librispeech_dummy[0]["audio"]["sampling_rate"]: + import librosa + + # resample audio if necessary + audio_samples = [ + librosa.resample( + audio_sample, orig_sr=librispeech_dummy[0]["audio"]["sampling_rate"], target_sr=sample_rate + ) + for audio_sample in audio_samples + ] inputs = processor( raw_audio=audio_samples, @@ -660,53 +780,82 @@ def test_integration_batch_24khz(self): with torch.no_grad(): encoder_outputs = model.encode(inputs["input_values"]) + hf_output_means_dict = { + "loss": encoder_outputs[0].mean().item(), + "quantized_representation": encoder_outputs[1].mean().item(), + "audio_codes": encoder_outputs[2].float().mean().item(), + "projected_latents": encoder_outputs[3].float().mean().item(), + } + hf_output_means = torch.tensor(list(hf_output_means_dict.values()), dtype=torch.float32) + + # make sure encoded outputs are similar + expected_encoder_means = torch.tensor(list(expected_encoder_means_dict.values()), dtype=torch.float32) + torch.testing.assert_close(hf_output_means, expected_encoder_means, rtol=1e-3, atol=1e-3) - expected_encoder_sums = torch.tensor(list(expected_encoder_sums_dict.values()), dtype=torch.float32) - encoder_outputs_mean = torch.tensor([v.float().mean().cpu().item() for v in encoder_outputs.to_tuple()]) + # check that quantizers behave similar (for same input) + encoded_hf = model.encoder(inputs["input_values"]) + hf_quantizer_codebook_mean = model.quantizer(encoded_hf)[1].float().mean().item() + torch.testing.assert_close( + hf_quantizer_codebook_mean, expected_quantizer_codebook_mean, rtol=1e-6, atol=1e-6 + ) - # make sure audio encoded codes are correct - torch.testing.assert_close(encoder_outputs_mean, expected_encoder_sums, rtol=1e-3, atol=1e-3) + # check that decoders behave similar (for same input) + hf_decoded_mean = model.decode(encoded_hf)["audio_values"].mean().item() + torch.testing.assert_close(hf_decoded_mean, expected_decoded_mean, rtol=1e-6, atol=1e-6) + # decode _, quantized_representation, _, _ = encoder_outputs.to_tuple() input_values_dec = model.decode(quantized_representation)[0] input_values_enc_dec = model(inputs["input_values"])[1] # make sure forward and decode gives same result - torch.testing.assert_close(input_values_dec, input_values_enc_dec, rtol=1e-3, atol=1e-3) - - arr = inputs["input_values"].cpu().numpy() - arr_enc_dec = input_values_enc_dec.cpu().numpy() - - max_length = min(arr_enc_dec.shape[-1], arr.shape[-1]) - - arr_cut = arr[:, 0, :max_length].copy() - arr_enc_dec_cut = arr_enc_dec[:, :max_length].copy() + torch.testing.assert_close(input_values_dec, input_values_enc_dec, rtol=1e-6, atol=1e-6) # make sure audios are more or less equal - rmse = compute_rmse(arr_cut, arr_enc_dec_cut) + rmse = compute_rmse(input_values_enc_dec, inputs["input_values"]) self.assertTrue(rmse < expected_rmse) + # check that codec error is similar + torch.testing.assert_close(expected_codec_error, rmse, rtol=1e-6, atol=1e-6) + def test_integration_batch_44khz(self): expected_rmse = 0.001 - - expected_encoder_sums_dict = { - "loss": 25.9233, - "quantized_representation": 0.0013, - "audio_codes": 528.5620, - "projected_latents": -0.1194, + # Code for reproducing expected outputs: https://gist.github.com/ebezzam/bb315efa7a416db6336a6b2a2d424ffa#file-batch-py + expected_encoder_means_dict = { + "loss": 19.557754516601562, + "quantized_representation": 0.004012184217572212, + "audio_codes": 518.1870727539062, + "projected_latents": -0.0008539701229892671, } + expected_quantizer_codebook_mean = 518.0151977539062 + expected_decoded_mean = -2.039729770331178e-05 + expected_codec_error = 0.00037737112143076956 librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") model_name = "dac_44khz" + sample_rate = 44100 model_id = f"descript/{model_name}" model = DacModel.from_pretrained(model_id).to(torch_device) - processor = AutoProcessor.from_pretrained(model_id) + processor = AutoProcessor.from_pretrained( + model_id, + hop_length=int(np.prod(model.config.downsampling_ratios)) if FIX_HOP_LENGTH else model.config.hop_length + ) librispeech_dummy = librispeech_dummy.cast_column("audio", Audio(sampling_rate=processor.sampling_rate)) audio_samples = [np.array([audio_sample["array"]])[0] for audio_sample in librispeech_dummy[-2:]["audio"]] + if sample_rate != librispeech_dummy[0]["audio"]["sampling_rate"]: + import librosa + + # resample audio if necessary + audio_samples = [ + librosa.resample( + audio_sample, orig_sr=librispeech_dummy[0]["audio"]["sampling_rate"], target_sr=sample_rate + ) + for audio_sample in audio_samples + ] inputs = processor( raw_audio=audio_samples, @@ -717,28 +866,40 @@ def test_integration_batch_44khz(self): with torch.no_grad(): encoder_outputs = model.encode(inputs["input_values"]) + hf_output_means_dict = { + "loss": encoder_outputs[0].mean().item(), + "quantized_representation": encoder_outputs[1].mean().item(), + "audio_codes": encoder_outputs[2].float().mean().item(), + "projected_latents": encoder_outputs[3].float().mean().item(), + } + hf_output_means = torch.tensor(list(hf_output_means_dict.values()), dtype=torch.float32) - expected_encoder_sums = torch.tensor(list(expected_encoder_sums_dict.values()), dtype=torch.float32) - encoder_outputs_mean = torch.tensor([v.float().mean().cpu().item() for v in encoder_outputs.to_tuple()]) + # make sure encoded outputs are similar + expected_encoder_means = torch.tensor(list(expected_encoder_means_dict.values()), dtype=torch.float32) + torch.testing.assert_close(hf_output_means, expected_encoder_means, rtol=1e-3, atol=1e-3) + + # check that quantizers behave similar (for same input) + encoded_hf = model.encoder(inputs["input_values"]) + hf_quantizer_codebook_mean = model.quantizer(encoded_hf)[1].float().mean().item() + torch.testing.assert_close( + hf_quantizer_codebook_mean, expected_quantizer_codebook_mean, rtol=1e-6, atol=1e-6 + ) - # make sure audio encoded codes are correct - torch.testing.assert_close(encoder_outputs_mean, expected_encoder_sums, rtol=1e-3, atol=1e-3) + # check that decoders behave similar (for same input) + hf_decoded_mean = model.decode(encoded_hf)["audio_values"].mean().item() + torch.testing.assert_close(hf_decoded_mean, expected_decoded_mean, rtol=1e-6, atol=1e-6) + # decode _, quantized_representation, _, _ = encoder_outputs.to_tuple() input_values_dec = model.decode(quantized_representation)[0] input_values_enc_dec = model(inputs["input_values"])[1] # make sure forward and decode gives same result - torch.testing.assert_close(input_values_dec, input_values_enc_dec, rtol=1e-3, atol=1e-3) - - arr = inputs["input_values"].cpu().numpy() - arr_enc_dec = input_values_enc_dec.cpu().numpy() - - max_length = min(arr_enc_dec.shape[-1], arr.shape[-1]) - - arr_cut = arr[:, 0, :max_length].copy() - arr_enc_dec_cut = arr_enc_dec[:, :max_length].copy() + torch.testing.assert_close(input_values_dec, input_values_enc_dec, rtol=1e-6, atol=1e-6) # make sure audios are more or less equal - rmse = compute_rmse(arr_cut, arr_enc_dec_cut) + rmse = compute_rmse(input_values_enc_dec, inputs["input_values"]) self.assertTrue(rmse < expected_rmse) + + # check that codec error is similar + torch.testing.assert_close(expected_codec_error, rmse, rtol=1e-6, atol=1e-6) From 716baa6109b0eb62f7ebcc53bb5be68297772766 Mon Sep 17 00:00:00 2001 From: Eric B Date: Wed, 9 Jul 2025 17:51:43 +0200 Subject: [PATCH 2/8] Fix DAC conversion. --- src/transformers/models/dac/convert_dac_checkpoint.py | 2 ++ src/transformers/models/dac/modeling_dac.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/dac/convert_dac_checkpoint.py b/src/transformers/models/dac/convert_dac_checkpoint.py index 3608d3b4a9fe..df9863af7e6d 100644 --- a/src/transformers/models/dac/convert_dac_checkpoint.py +++ b/src/transformers/models/dac/convert_dac_checkpoint.py @@ -16,6 +16,7 @@ import fnmatch import re +import numpy as np import torch from transformers import ( @@ -207,6 +208,7 @@ def convert_checkpoint( config.upsampling_ratios = metadata["decoder_rates"] config.quantizer_dropout = float(metadata["quantizer_dropout"]) config.sampling_rate = sample_rate + config.hop_length = int(np.prod(config.downsampling_ratios)) model = DacModel(config) feature_extractor = DacFeatureExtractor() diff --git a/src/transformers/models/dac/modeling_dac.py b/src/transformers/models/dac/modeling_dac.py index 398d258bef08..01fa63a9a5a3 100644 --- a/src/transformers/models/dac/modeling_dac.py +++ b/src/transformers/models/dac/modeling_dac.py @@ -489,8 +489,8 @@ def _init_weights(self, module): def apply_weight_norm(self): weight_norm = nn.utils.weight_norm - if hasattr(nn.utils.parametrizations, "weight_norm"): - weight_norm = nn.utils.parametrizations.weight_norm + # if hasattr(nn.utils.parametrizations, "weight_norm"): + # weight_norm = nn.utils.parametrizations.weight_norm for layer in self.quantizer.quantizers: weight_norm(layer.in_proj) From 9e51f6faa19a94ce04d9a84ba9219be7ef75716a Mon Sep 17 00:00:00 2001 From: Eric B Date: Wed, 9 Jul 2025 19:57:08 +0200 Subject: [PATCH 3/8] Address comments --- tests/models/dac/test_modeling_dac.py | 108 ++++---------------------- 1 file changed, 15 insertions(+), 93 deletions(-) diff --git a/tests/models/dac/test_modeling_dac.py b/tests/models/dac/test_modeling_dac.py index e8634a3c62f5..3d0b6d914d56 100644 --- a/tests/models/dac/test_modeling_dac.py +++ b/tests/models/dac/test_modeling_dac.py @@ -391,14 +391,20 @@ def compute_rmse(arr1, arr2): arr2_normalized = normalize(arr2_np) return np.sqrt(((arr1_normalized - arr2_normalized) ** 2).mean()) -FIX_HOP_LENGTH = True @slow @require_torch class DacIntegrationTest(unittest.TestCase): + """ + Integration tests for DAC. + + Code for reproducing expected outputs can be found here: + - Single file: https://gist.github.com/ebezzam/bb315efa7a416db6336a6b2a2d424ffa#file-single-py + - Batched: https://gist.github.com/ebezzam/bb315efa7a416db6336a6b2a2d424ffa#file-batch-py + """ + def test_integration_16khz(self): expected_rmse = 0.004 - # Code for reproducing expected outputs: https://gist.github.com/ebezzam/bb315efa7a416db6336a6b2a2d424ffa#file-single-py expected_encoder_means_dict = { "loss": 24.8491, "quantized_representation": -0.07544856518507004, @@ -412,24 +418,13 @@ def test_integration_16khz(self): librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") model_name = "dac_16khz" - sample_rate = 16000 model_id = f"descript/{model_name}" model = DacModel.from_pretrained(model_id, force_download=True).to(torch_device).eval() - processor = AutoProcessor.from_pretrained( - model_id, - hop_length=int(np.prod(model.config.downsampling_ratios)) if FIX_HOP_LENGTH else model.config.hop_length - ) + processor = AutoProcessor.from_pretrained(model_id, hop_length=int(np.prod(model.config.downsampling_ratios))) librispeech_dummy = librispeech_dummy.cast_column("audio", Audio(sampling_rate=processor.sampling_rate)) audio_sample = librispeech_dummy[0]["audio"]["array"] - # Resample audio to 16kHz if necessary - if librispeech_dummy[0]["audio"]["sampling_rate"] != sample_rate: - import librosa - - audio_sample = librosa.resample( - audio_sample, orig_sr=librispeech_dummy[0]["audio"]["sampling_rate"], target_sr=sample_rate - ) inputs = processor( raw_audio=audio_sample, @@ -451,7 +446,7 @@ def test_integration_16khz(self): # make sure encoded outputs are similar # TODO for all sampling rates, encoder error is relatively high compared to quantizer and decoder (but still minimal) # they may be a bug in encoder weight mapping: - # https://github.com/ebezzam/transformers/blob/main/src/transformers/models/dac/convert_dac_checkpoint.py#L63 + # https://github.com/huggingface/transformers/blob/d61c0d087cedbfdbbee8c75b210d5837c35addb8/src/transformers/models/dac/convert_dac_checkpoint.py#L63 # in any case, the error is small enough to not affect the codec performance expected_encoder_means = torch.tensor(list(expected_encoder_means_dict.values()), dtype=torch.float32) torch.testing.assert_close(hf_output_means, expected_encoder_means, rtol=1e-3, atol=1e-3) @@ -484,7 +479,6 @@ def test_integration_16khz(self): def test_integration_24khz(self): expected_rmse = 0.0039 - # Code for reproducing expected outputs: https://gist.github.com/ebezzam/bb315efa7a416db6336a6b2a2d424ffa#file-single-py expected_encoder_means_dict = { "loss": 28.1121, "quantized_representation": 0.016283338889479637, @@ -498,24 +492,13 @@ def test_integration_24khz(self): librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") model_name = "dac_24khz" - sample_rate = 24000 model_id = f"descript/{model_name}" model = DacModel.from_pretrained(model_id, force_download=True).to(torch_device).eval() - processor = AutoProcessor.from_pretrained( - model_id, - hop_length=int(np.prod(model.config.downsampling_ratios)) if FIX_HOP_LENGTH else model.config.hop_length - ) + processor = AutoProcessor.from_pretrained(model_id, hop_length=int(np.prod(model.config.downsampling_ratios))) librispeech_dummy = librispeech_dummy.cast_column("audio", Audio(sampling_rate=processor.sampling_rate)) audio_sample = librispeech_dummy[0]["audio"]["array"] - # Resample audio to 24kHz if necessary - if librispeech_dummy[0]["audio"]["sampling_rate"] != sample_rate: - import librosa - - audio_sample = librosa.resample( - audio_sample, orig_sr=librispeech_dummy[0]["audio"]["sampling_rate"], target_sr=sample_rate - ) inputs = processor( raw_audio=audio_sample, @@ -566,7 +549,6 @@ def test_integration_24khz(self): def test_integration_44khz(self): expected_rmse = 0.002 - # Code for reproducing expected outputs: https://gist.github.com/ebezzam/bb315efa7a416db6336a6b2a2d424ffa#file-single-py expected_encoder_means_dict = { "loss": 23.7848, "quantized_representation": 0.017807748168706894, @@ -580,25 +562,13 @@ def test_integration_44khz(self): librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") model_name = "dac_44khz" - sample_rate = 44100 model_id = f"descript/{model_name}" model = DacModel.from_pretrained(model_id, force_download=True).to(torch_device).eval() - processor = AutoProcessor.from_pretrained( - model_id, - hop_length=int(np.prod(model.config.downsampling_ratios)) if FIX_HOP_LENGTH else model.config.hop_length - ) + processor = AutoProcessor.from_pretrained(model_id, hop_length=int(np.prod(model.config.downsampling_ratios))) librispeech_dummy = librispeech_dummy.cast_column("audio", Audio(sampling_rate=processor.sampling_rate)) audio_sample = librispeech_dummy[0]["audio"]["array"] - # Resample audio to 24kHz if necessary - if librispeech_dummy[0]["audio"]["sampling_rate"] != sample_rate: - import librosa - - audio_sample = librosa.resample( - audio_sample, orig_sr=librispeech_dummy[0]["audio"]["sampling_rate"], target_sr=sample_rate - ) - inputs = processor( raw_audio=audio_sample, sampling_rate=processor.sampling_rate, @@ -648,7 +618,6 @@ def test_integration_44khz(self): def test_integration_batch_16khz(self): expected_rmse = 0.002 - # Code for reproducing expected outputs: https://gist.github.com/ebezzam/bb315efa7a416db6336a6b2a2d424ffa#file-batch-py expected_encoder_means_dict = { "loss": 20.370271682739258, "quantized_representation": -0.05440079793334007, @@ -662,28 +631,13 @@ def test_integration_batch_16khz(self): librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") model_name = "dac_16khz" - sample_rate = 16000 model_id = f"descript/{model_name}" model = DacModel.from_pretrained(model_id).to(torch_device) - processor = AutoProcessor.from_pretrained( - model_id, - hop_length=int(np.prod(model.config.downsampling_ratios)) if FIX_HOP_LENGTH else model.config.hop_length - ) + processor = AutoProcessor.from_pretrained(model_id, hop_length=int(np.prod(model.config.downsampling_ratios))) librispeech_dummy = librispeech_dummy.cast_column("audio", Audio(sampling_rate=processor.sampling_rate)) - audio_samples = [np.array([audio_sample["array"]])[0] for audio_sample in librispeech_dummy[-2:]["audio"]] - if sample_rate != librispeech_dummy[0]["audio"]["sampling_rate"]: - import librosa - - # resample audio if necessary - audio_samples = [ - librosa.resample( - audio_sample, orig_sr=librispeech_dummy[0]["audio"]["sampling_rate"], target_sr=sample_rate - ) - for audio_sample in audio_samples - ] inputs = processor( raw_audio=audio_samples, @@ -734,7 +688,6 @@ def test_integration_batch_16khz(self): def test_integration_batch_24khz(self): expected_rmse = 0.002 - # Code for reproducing expected outputs: https://gist.github.com/ebezzam/bb315efa7a416db6336a6b2a2d424ffa#file-batch-py expected_encoder_means_dict = { "loss": 24.505210876464844, "quantized_representation": 0.03778776153922081, @@ -748,28 +701,13 @@ def test_integration_batch_24khz(self): librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") model_name = "dac_24khz" - sample_rate = 24000 model_id = f"descript/{model_name}" model = DacModel.from_pretrained(model_id).to(torch_device) - processor = AutoProcessor.from_pretrained( - model_id, - hop_length=int(np.prod(model.config.downsampling_ratios)) if FIX_HOP_LENGTH else model.config.hop_length - ) + processor = AutoProcessor.from_pretrained(model_id, hop_length=int(np.prod(model.config.downsampling_ratios))) librispeech_dummy = librispeech_dummy.cast_column("audio", Audio(sampling_rate=processor.sampling_rate)) - audio_samples = [np.array([audio_sample["array"]])[0] for audio_sample in librispeech_dummy[-2:]["audio"]] - if sample_rate != librispeech_dummy[0]["audio"]["sampling_rate"]: - import librosa - - # resample audio if necessary - audio_samples = [ - librosa.resample( - audio_sample, orig_sr=librispeech_dummy[0]["audio"]["sampling_rate"], target_sr=sample_rate - ) - for audio_sample in audio_samples - ] inputs = processor( raw_audio=audio_samples, @@ -820,7 +758,6 @@ def test_integration_batch_24khz(self): def test_integration_batch_44khz(self): expected_rmse = 0.001 - # Code for reproducing expected outputs: https://gist.github.com/ebezzam/bb315efa7a416db6336a6b2a2d424ffa#file-batch-py expected_encoder_means_dict = { "loss": 19.557754516601562, "quantized_representation": 0.004012184217572212, @@ -834,28 +771,13 @@ def test_integration_batch_44khz(self): librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") model_name = "dac_44khz" - sample_rate = 44100 model_id = f"descript/{model_name}" model = DacModel.from_pretrained(model_id).to(torch_device) - processor = AutoProcessor.from_pretrained( - model_id, - hop_length=int(np.prod(model.config.downsampling_ratios)) if FIX_HOP_LENGTH else model.config.hop_length - ) + processor = AutoProcessor.from_pretrained(model_id, hop_length=int(np.prod(model.config.downsampling_ratios))) librispeech_dummy = librispeech_dummy.cast_column("audio", Audio(sampling_rate=processor.sampling_rate)) - audio_samples = [np.array([audio_sample["array"]])[0] for audio_sample in librispeech_dummy[-2:]["audio"]] - if sample_rate != librispeech_dummy[0]["audio"]["sampling_rate"]: - import librosa - - # resample audio if necessary - audio_samples = [ - librosa.resample( - audio_sample, orig_sr=librispeech_dummy[0]["audio"]["sampling_rate"], target_sr=sample_rate - ) - for audio_sample in audio_samples - ] inputs = processor( raw_audio=audio_samples, From e5f02a2789eee311cda3997290028021f8ea36af Mon Sep 17 00:00:00 2001 From: Eric B Date: Thu, 10 Jul 2025 15:20:49 +0200 Subject: [PATCH 4/8] Sync with main, uncomment nn.utils.parametrizations.weight_norm. --- src/transformers/models/dac/modeling_dac.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/dac/modeling_dac.py b/src/transformers/models/dac/modeling_dac.py index 01fa63a9a5a3..398d258bef08 100644 --- a/src/transformers/models/dac/modeling_dac.py +++ b/src/transformers/models/dac/modeling_dac.py @@ -489,8 +489,8 @@ def _init_weights(self, module): def apply_weight_norm(self): weight_norm = nn.utils.weight_norm - # if hasattr(nn.utils.parametrizations, "weight_norm"): - # weight_norm = nn.utils.parametrizations.weight_norm + if hasattr(nn.utils.parametrizations, "weight_norm"): + weight_norm = nn.utils.parametrizations.weight_norm for layer in self.quantizer.quantizers: weight_norm(layer.in_proj) From 178c4d881e656ca91d86fdead605f10ebecb2ad8 Mon Sep 17 00:00:00 2001 From: Eric B Date: Fri, 11 Jul 2025 16:50:25 +0200 Subject: [PATCH 5/8] Update DAC integration tests with expected outputs. --- tests/models/dac/test_modeling_dac.py | 702 ++++++++++++++++---------- 1 file changed, 428 insertions(+), 274 deletions(-) diff --git a/tests/models/dac/test_modeling_dac.py b/tests/models/dac/test_modeling_dac.py index 3d0b6d914d56..7896dfa8541e 100644 --- a/tests/models/dac/test_modeling_dac.py +++ b/tests/models/dac/test_modeling_dac.py @@ -399,429 +399,583 @@ class DacIntegrationTest(unittest.TestCase): Integration tests for DAC. Code for reproducing expected outputs can be found here: - - Single file: https://gist.github.com/ebezzam/bb315efa7a416db6336a6b2a2d424ffa#file-single-py - - Batched: https://gist.github.com/ebezzam/bb315efa7a416db6336a6b2a2d424ffa#file-batch-py + - Single file: https://gist.github.com/ebezzam/bb315efa7a416db6336a6b2a2d424ffa#file-dac_integration_single-py + - Batched: https://gist.github.com/ebezzam/bb315efa7a416db6336a6b2a2d424ffa#file-dac_integration-py + + Moreover, here is a script to debug outputs and weights layer-by-layer: + https://gist.github.com/ebezzam/bb315efa7a416db6336a6b2a2d424ffa#file-dac_layer_by_layer_debugging-py """ def test_integration_16khz(self): - expected_rmse = 0.004 - expected_encoder_means_dict = { - "loss": 24.8491, - "quantized_representation": -0.07544856518507004, - # "audio_codes": 505.13421630859375, - "projected_latents": 0.06593942642211914, - } - expected_quantizer_codebook_mean = 504.3310546875 - expected_decoded_mean = -0.00018316633941140026 - expected_codec_error = 0.0038341842591762543 - - librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") - model_name = "dac_16khz" + # expected values + EXPECTED_PREPROC_SHAPE = torch.tensor([1, 1, 93760]) + EXPECTED_ENC_LOSS = 24.84908103942871 + EXPECTED_QUANT_CODES = torch.tensor( + [ + [ + [804, 25, 977, 52, 68, 867, 388, 653, 315, 706, 301, 305, 140, 25, 40], + [77, 955, 532, 601, 431, 375, 967, 56, 54, 261, 871, 552, 735, 341, 228], + [355, 908, 77, 927, 617, 443, 790, 149, 403, 707, 511, 226, 995, 883, 644], + [184, 162, 611, 54, 211, 890, 906, 253, 677, 1007, 302, 577, 378, 330, 778], + [763, 322, 6, 321, 116, 228, 911, 865, 1000, 234, 6, 901, 10, 174, 895], + [454, 1, 622, 622, 487, 668, 749, 833, 382, 900, 372, 959, 232, 418, 964], + [203, 43, 173, 307, 961, 593, 318, 1011, 386, 949, 343, 899, 536, 824, 38], + [82, 810, 692, 83, 131, 866, 483, 362, 519, 531, 853, 121, 1010, 512, 710], + [1003, 691, 530, 460, 827, 903, 81, 76, 629, 298, 168, 177, 368, 613, 762], + [571, 752, 544, 394, 198, 479, 952, 437, 222, 992, 934, 316, 741, 123, 538], + [686, 421, 393, 635, 246, 330, 908, 384, 962, 873, 92, 254, 912, 496, 83], + [721, 977, 148, 204, 993, 660, 176, 395, 901, 323, 342, 849, 474, 8, 513], + ] + ] + ).to(torch_device) + EXPECTED_QUANT_CODEBOOK_LOSS = 20.58063507080078 + EXPECTED_DEC_OUTPUTS = torch.tensor( + [[7.2661e-05, 5.9626e-04, 1.0609e-03, 1.4515e-03, 1.6704e-03, 1.0837e-03]] + ).to(torch_device) + EXPECTED_CODEC_ERROR = 0.0038341842591762543 + + # load model and processor model_id = f"descript/{model_name}" model = DacModel.from_pretrained(model_id, force_download=True).to(torch_device).eval() - processor = AutoProcessor.from_pretrained(model_id, hop_length=int(np.prod(model.config.downsampling_ratios))) + processor = AutoProcessor.from_pretrained(model_id) + # load audio sample + librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") librispeech_dummy = librispeech_dummy.cast_column("audio", Audio(sampling_rate=processor.sampling_rate)) audio_sample = librispeech_dummy[0]["audio"]["array"] + # check on processor audio shape inputs = processor( raw_audio=audio_sample, sampling_rate=processor.sampling_rate, return_tensors="pt", ).to(torch_device) + torch.equal(torch.tensor(inputs["input_values"].shape), EXPECTED_PREPROC_SHAPE) with torch.no_grad(): - # compute HF encoder outputs + # compare encoder loss encoder_outputs = model.encode(inputs["input_values"]) - hf_output_means_dict = { - "loss": encoder_outputs[0].item(), - "quantized_representation": encoder_outputs[1].mean().item(), - # "audio_codes": encoder_outputs[2].float().mean().item(), - "projected_latents": encoder_outputs[3].float().mean().item(), - } - hf_output_means = torch.tensor(list(hf_output_means_dict.values()), dtype=torch.float32) - - # make sure encoded outputs are similar - # TODO for all sampling rates, encoder error is relatively high compared to quantizer and decoder (but still minimal) - # they may be a bug in encoder weight mapping: - # https://github.com/huggingface/transformers/blob/d61c0d087cedbfdbbee8c75b210d5837c35addb8/src/transformers/models/dac/convert_dac_checkpoint.py#L63 - # in any case, the error is small enough to not affect the codec performance - expected_encoder_means = torch.tensor(list(expected_encoder_means_dict.values()), dtype=torch.float32) - torch.testing.assert_close(hf_output_means, expected_encoder_means, rtol=1e-3, atol=1e-3) - - # check that quantizers behave similar (for same input) - encoded_hf = model.encoder(inputs["input_values"]) - hf_quantizer_codebook_mean = model.quantizer(encoded_hf)[1].float().mean().item() + torch.testing.assert_close(EXPECTED_ENC_LOSS, encoder_outputs[0].squeeze().item(), rtol=1e-3, atol=1e-3) + + # compare quantizer outputs + quantizer_outputs = model.quantizer(encoder_outputs[1]) + torch.testing.assert_close( + EXPECTED_QUANT_CODES, quantizer_outputs[1][..., : EXPECTED_QUANT_CODES.shape[-1]], rtol=1e-6, atol=1e-6 + ) torch.testing.assert_close( - hf_quantizer_codebook_mean, expected_quantizer_codebook_mean, rtol=1e-6, atol=1e-6 + EXPECTED_QUANT_CODEBOOK_LOSS, quantizer_outputs[4].squeeze().item(), rtol=1e-6, atol=1e-6 ) - # check that decoders behave similar (for same input) - hf_decoded_mean = model.decode(encoded_hf)["audio_values"].mean().item() - torch.testing.assert_close(hf_decoded_mean, expected_decoded_mean, rtol=1e-6, atol=1e-6) + # compare decoder outputs + decoded_outputs = model.decode(encoder_outputs[1]) + torch.testing.assert_close( + EXPECTED_DEC_OUTPUTS, + decoded_outputs["audio_values"][..., : EXPECTED_DEC_OUTPUTS.shape[-1]], + rtol=1e-3, + atol=1e-3, + ) - # decode - _, quantized_representation, _, _ = encoder_outputs.to_tuple() - input_values_dec = model.decode(quantized_representation)[0] - input_values_enc_dec = model(inputs["input_values"])[1] + # compare codec error / lossiness + codec_err = compute_rmse(decoded_outputs["audio_values"], inputs["input_values"]) + torch.testing.assert_close(EXPECTED_CODEC_ERROR, codec_err, rtol=1e-6, atol=1e-6) # make sure forward and decode gives same result - torch.testing.assert_close(input_values_dec, input_values_enc_dec, rtol=1e-6, atol=1e-6) - - # make sure audios are more or less equal - rmse = compute_rmse(input_values_enc_dec, inputs["input_values"]) - self.assertTrue(rmse < expected_rmse) - - # check that codec error is similar - torch.testing.assert_close(expected_codec_error, rmse, rtol=1e-6, atol=1e-6) + enc_dec = model(inputs["input_values"])[1] + torch.testing.assert_close(decoded_outputs["audio_values"], enc_dec, rtol=1e-6, atol=1e-6) def test_integration_24khz(self): - expected_rmse = 0.0039 - expected_encoder_means_dict = { - "loss": 28.1121, - "quantized_representation": 0.016283338889479637, - # "audio_codes": 507.17724609375, - "projected_latents": -0.024361690506339073, - } - expected_quantizer_codebook_mean = 506.8665466308594 - expected_decoded_mean = 0.0001686957839410752 - expected_codec_error = 0.002570481738075614 - - librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") - model_name = "dac_24khz" + # expected values + EXPECTED_PREPROC_SHAPE = torch.tensor([1, 1, 140800]) + EXPECTED_ENC_LOSS = 28.112096786499023 + EXPECTED_QUANT_CODES = torch.tensor( + [ + [ + [160, 360, 826, 204, 239, 360, 90, 160, 851, 234, 252, 690, 360, 160, 665], + [189, 496, 717, 74, 847, 692, 496, 549, 847, 78, 669, 440, 9, 243, 117], + [497, 562, 161, 827, 408, 330, 562, 152, 80, 84, 320, 745, 1023, 544, 944], + [261, 140, 271, 843, 179, 239, 150, 211, 788, 343, 333, 760, 217, 243, 623], + [487, 846, 919, 947, 417, 787, 140, 186, 567, 129, 633, 328, 927, 932, 901], + [862, 953, 929, 184, 85, 433, 545, 672, 382, 666, 694, 382, 572, 38, 134], + [835, 260, 975, 144, 621, 800, 341, 1017, 28, 889, 521, 287, 805, 231, 474], + [470, 803, 475, 208, 574, 679, 382, 71, 413, 79, 571, 330, 408, 759, 79], + [452, 272, 257, 101, 76, 540, 378, 933, 83, 350, 334, 539, 808, 975, 860], + [450, 704, 839, 811, 705, 304, 895, 340, 979, 53, 573, 80, 241, 110, 571], + [801, 523, 138, 939, 729, 417, 588, 9, 501, 304, 820, 271, 497, 719, 141], + [579, 741, 42, 811, 561, 630, 528, 945, 1009, 637, 109, 702, 1005, 911, 748], + [96, 581, 853, 817, 256, 592, 23, 1014, 309, 3, 846, 780, 704, 481, 138], + [162, 193, 808, 498, 128, 949, 103, 928, 277, 599, 375, 718, 893, 388, 532], + [318, 498, 5, 696, 953, 1018, 442, 97, 573, 179, 850, 353, 548, 1002, 279], + [962, 911, 712, 684, 214, 240, 290, 467, 812, 588, 232, 588, 922, 101, 768], + [969, 785, 514, 168, 106, 423, 37, 683, 882, 657, 516, 819, 535, 50, 988], + [299, 914, 787, 584, 582, 449, 444, 366, 666, 721, 1022, 1015, 700, 752, 710], + [926, 669, 287, 618, 806, 309, 368, 502, 704, 573, 319, 562, 355, 994, 873], + [513, 75, 447, 290, 16, 370, 185, 43, 1015, 346, 450, 24, 490, 299, 231], + [616, 506, 867, 444, 648, 987, 6, 301, 556, 128, 898, 352, 657, 616, 798], + [382, 353, 420, 424, 107, 256, 163, 113, 832, 247, 415, 541, 893, 922, 918], + [135, 775, 363, 14, 603, 311, 346, 722, 746, 207, 695, 48, 821, 428, 53], + [626, 72, 220, 524, 256, 736, 86, 64, 618, 780, 607, 799, 734, 506, 868], + [310, 913, 13, 707, 177, 19, 856, 463, 400, 141, 959, 904, 910, 818, 734], + [948, 105, 835, 842, 802, 117, 340, 466, 774, 726, 389, 599, 558, 491, 420], + [916, 440, 167, 177, 842, 450, 744, 820, 906, 739, 702, 158, 745, 546, 636], + [135, 675, 544, 64, 955, 904, 1017, 862, 167, 564, 362, 1023, 774, 78, 914], + [216, 218, 494, 28, 605, 962, 212, 649, 249, 710, 83, 94, 437, 613, 54], + [611, 109, 743, 56, 493, 294, 364, 514, 980, 524, 474, 978, 35, 724, 767], + [719, 752, 343, 171, 776, 414, 217, 656, 717, 73, 955, 516, 582, 559, 241], + [821, 641, 740, 272, 468, 847, 699, 842, 20, 330, 216, 703, 581, 306, 137], + ] + ] + ).to(torch_device) + EXPECTED_QUANT_CODEBOOK_LOSS = 22.581758499145508 + EXPECTED_DEC_OUTPUTS = torch.tensor( + [[4.2660e-04, 4.0129e-04, 1.5403e-04, 5.0874e-05, 2.9436e-04, 1.0682e-03]] + ).to(torch_device) + EXPECTED_CODEC_ERROR = 0.002570481738075614 + + # load model and processor model_id = f"descript/{model_name}" model = DacModel.from_pretrained(model_id, force_download=True).to(torch_device).eval() - processor = AutoProcessor.from_pretrained(model_id, hop_length=int(np.prod(model.config.downsampling_ratios))) + processor = AutoProcessor.from_pretrained(model_id) + # load audio sample + librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") librispeech_dummy = librispeech_dummy.cast_column("audio", Audio(sampling_rate=processor.sampling_rate)) audio_sample = librispeech_dummy[0]["audio"]["array"] + # check on processor audio shape inputs = processor( raw_audio=audio_sample, sampling_rate=processor.sampling_rate, return_tensors="pt", ).to(torch_device) + torch.equal(torch.tensor(inputs["input_values"].shape), EXPECTED_PREPROC_SHAPE) with torch.no_grad(): - # compute HF encoder outputs + # compare encoder loss encoder_outputs = model.encode(inputs["input_values"]) - hf_output_means_dict = { - "loss": encoder_outputs[0].item(), - "quantized_representation": encoder_outputs[1].mean().item(), - # "audio_codes": encoder_outputs[2].float().mean().item(), - "projected_latents": encoder_outputs[3].float().mean().item(), - } - hf_output_means = torch.tensor(list(hf_output_means_dict.values()), dtype=torch.float32) - - # make sure encoded outputs are similar - expected_encoder_means = torch.tensor(list(expected_encoder_means_dict.values()), dtype=torch.float32) - torch.testing.assert_close(hf_output_means, expected_encoder_means, rtol=1e-2, atol=1e-2) + torch.testing.assert_close(EXPECTED_ENC_LOSS, encoder_outputs[0].squeeze().item(), rtol=1e-3, atol=1e-3) - # check that quantizers behave similar (for same input) - encoded_hf = model.encoder(inputs["input_values"]) - hf_quantizer_codebook_mean = model.quantizer(encoded_hf)[1].float().mean().item() + # compare quantizer outputs + quantizer_outputs = model.quantizer(encoder_outputs[1]) + torch.testing.assert_close( + EXPECTED_QUANT_CODES, quantizer_outputs[1][..., : EXPECTED_QUANT_CODES.shape[-1]], rtol=1e-6, atol=1e-6 + ) torch.testing.assert_close( - hf_quantizer_codebook_mean, expected_quantizer_codebook_mean, rtol=1e-6, atol=1e-6 + EXPECTED_QUANT_CODEBOOK_LOSS, quantizer_outputs[4].squeeze().item(), rtol=1e-6, atol=1e-6 ) - # check that decoders behave similar (for same input) - hf_decoded_mean = model.decode(encoded_hf)["audio_values"].mean().item() - torch.testing.assert_close(hf_decoded_mean, expected_decoded_mean, rtol=1e-6, atol=1e-6) + # compare decoder outputs + decoded_outputs = model.decode(encoder_outputs[1]) + torch.testing.assert_close( + EXPECTED_DEC_OUTPUTS, + decoded_outputs["audio_values"][..., : EXPECTED_DEC_OUTPUTS.shape[-1]], + rtol=1e-3, + atol=1e-3, + ) - # decode - _, quantized_representation, _, _ = encoder_outputs.to_tuple() - input_values_dec = model.decode(quantized_representation)[0] - input_values_enc_dec = model(inputs["input_values"])[1] + # compare codec error / lossiness + codec_err = compute_rmse(decoded_outputs["audio_values"], inputs["input_values"]) + torch.testing.assert_close(EXPECTED_CODEC_ERROR, codec_err, rtol=1e-6, atol=1e-6) # make sure forward and decode gives same result - torch.testing.assert_close(input_values_dec, input_values_enc_dec, rtol=1e-6, atol=1e-6) - - # make sure audios are more or less equal - rmse = compute_rmse(input_values_enc_dec, inputs["input_values"]) - self.assertTrue(rmse < expected_rmse) - - # check that codec error is similar - torch.testing.assert_close(expected_codec_error, rmse, rtol=1e-6, atol=1e-6) + enc_dec = model(inputs["input_values"])[1] + torch.testing.assert_close(decoded_outputs["audio_values"], enc_dec, rtol=1e-6, atol=1e-6) def test_integration_44khz(self): - expected_rmse = 0.002 - expected_encoder_means_dict = { - "loss": 23.7848, - "quantized_representation": 0.017807748168706894, - # "audio_codes": 513.7100219726562, - "projected_latents": 0.06925617158412933, - } - expected_quantizer_codebook_mean = 514.03369140625 - expected_decoded_mean = -0.00010763177124317735 - expected_codec_error = 0.0007429996621794999 - - librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") - model_name = "dac_44khz" + # expected values + EXPECTED_PREPROC_SHAPE = torch.tensor([1, 1, 258560]) + EXPECTED_ENC_LOSS = 23.78483772277832 + EXPECTED_QUANT_CODES = torch.tensor( + [ + [ + [332, 315, 105, 315, 616, 105, 494, 698, 315, 481, 330, 93, 105, 315, 105], + [670, 350, 249, 27, 232, 365, 311, 881, 186, 402, 311, 521, 527, 778, 254], + [569, 300, 361, 530, 1002, 419, 285, 501, 456, 471, 180, 615, 419, 491, 764], + [605, 436, 641, 291, 901, 556, 715, 780, 502, 410, 858, 125, 562, 174, 746], + [854, 706, 242, 294, 346, 88, 527, 961, 559, 664, 314, 963, 278, 90, 682], + [175, 152, 706, 884, 986, 457, 567, 176, 49, 535, 851, 417, 533, 349, 779], + [913, 710, 628, 162, 770, 254, 247, 6, 397, 264, 233, 704, 577, 111, 916], + [999, 693, 512, 884, 38, 223, 29, 744, 497, 123, 972, 120, 47, 301, 90], + [490, 163, 368, 507, 253, 283, 745, 65, 295, 935, 811, 587, 801, 255, 105], + ] + ] + ).to(torch_device) + EXPECTED_QUANT_CODEBOOK_LOSS = 16.2640438079834 + EXPECTED_DEC_OUTPUTS = torch.tensor([[0.0008, 0.0004, 0.0005, 0.0008, 0.0014, 0.0017]]).to(torch_device) + EXPECTED_CODEC_ERROR = 0.0007429996621794999 + + # load model and processor model_id = f"descript/{model_name}" model = DacModel.from_pretrained(model_id, force_download=True).to(torch_device).eval() - processor = AutoProcessor.from_pretrained(model_id, hop_length=int(np.prod(model.config.downsampling_ratios))) + processor = AutoProcessor.from_pretrained(model_id) + # load audio sample + librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") librispeech_dummy = librispeech_dummy.cast_column("audio", Audio(sampling_rate=processor.sampling_rate)) audio_sample = librispeech_dummy[0]["audio"]["array"] + + # check on processor audio shape inputs = processor( raw_audio=audio_sample, sampling_rate=processor.sampling_rate, return_tensors="pt", ).to(torch_device) + torch.equal(torch.tensor(inputs["input_values"].shape), EXPECTED_PREPROC_SHAPE) with torch.no_grad(): - # compute HF encoder outputs + # compare encoder loss encoder_outputs = model.encode(inputs["input_values"]) - hf_output_means_dict = { - "loss": encoder_outputs[0].item(), - "quantized_representation": encoder_outputs[1].mean().item(), - # "audio_codes": encoder_outputs[2].float().mean().item(), - "projected_latents": encoder_outputs[3].float().mean().item(), - } - hf_output_means = torch.tensor(list(hf_output_means_dict.values()), dtype=torch.float32) - - # make sure encoded outputs are similar - expected_encoder_means = torch.tensor(list(expected_encoder_means_dict.values()), dtype=torch.float32) - torch.testing.assert_close(hf_output_means, expected_encoder_means, rtol=1e-3, atol=1e-3) + torch.testing.assert_close(EXPECTED_ENC_LOSS, encoder_outputs[0].squeeze().item(), rtol=1e-3, atol=1e-3) - # check that quantizers behave similar (for same input) - encoded_hf = model.encoder(inputs["input_values"]) - hf_quantizer_codebook_mean = model.quantizer(encoded_hf)[1].float().mean().item() + # compare quantizer outputs + quantizer_outputs = model.quantizer(encoder_outputs[1]) torch.testing.assert_close( - hf_quantizer_codebook_mean, expected_quantizer_codebook_mean, rtol=1e-6, atol=1e-6 + EXPECTED_QUANT_CODES, quantizer_outputs[1][..., : EXPECTED_QUANT_CODES.shape[-1]], rtol=1e-6, atol=1e-6 + ) + torch.testing.assert_close( + EXPECTED_QUANT_CODEBOOK_LOSS, quantizer_outputs[4].squeeze().item(), rtol=1e-6, atol=1e-6 ) - # check that decoders behave similar (for same input) - hf_decoded_mean = model.decode(encoded_hf)["audio_values"].mean().item() - torch.testing.assert_close(hf_decoded_mean, expected_decoded_mean, rtol=1e-6, atol=1e-6) + # compare decoder outputs + decoded_outputs = model.decode(encoder_outputs[1]) + torch.testing.assert_close( + EXPECTED_DEC_OUTPUTS, + decoded_outputs["audio_values"][..., : EXPECTED_DEC_OUTPUTS.shape[-1]], + rtol=1e-3, + atol=1e-3, + ) - # decode - _, quantized_representation, _, _ = encoder_outputs.to_tuple() - input_values_dec = model.decode(quantized_representation)[0] - input_values_enc_dec = model(inputs["input_values"])[1] + # compare codec error / lossiness + codec_err = compute_rmse(decoded_outputs["audio_values"], inputs["input_values"]) + torch.testing.assert_close(EXPECTED_CODEC_ERROR, codec_err, rtol=1e-6, atol=1e-6) # make sure forward and decode gives same result - torch.testing.assert_close(input_values_dec, input_values_enc_dec, rtol=1e-6, atol=1e-6) - - # make sure audios are more or less equal - rmse = compute_rmse(input_values_enc_dec, inputs["input_values"]) - self.assertTrue(rmse < expected_rmse) - - # check that codec error is similar - torch.testing.assert_close(expected_codec_error, rmse, rtol=1e-6, atol=1e-6) + enc_dec = model(inputs["input_values"])[1] + torch.testing.assert_close(decoded_outputs["audio_values"], enc_dec, rtol=1e-6, atol=1e-6) def test_integration_batch_16khz(self): - expected_rmse = 0.002 - expected_encoder_means_dict = { - "loss": 20.370271682739258, - "quantized_representation": -0.05440079793334007, - "audio_codes": 488.02716064453125, - "projected_latents": 0.02350950613617897, - } - expected_quantizer_codebook_mean = 488.4040222167969 - expected_decoded_mean = -7.977934001246467e-05 - expected_codec_error = 0.001973195234313607 - - librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") - model_name = "dac_16khz" + # expected values + EXPECTED_PREPROC_SHAPE = torch.tensor([2, 1, 113920]) + EXPECTED_ENC_LOSS = 20.370271682739258 + EXPECTED_QUANT_CODES = torch.tensor( + [ + [ + [490, 664, 726, 166, 55, 379, 367, 664, 661, 726, 592, 301, 130, 198, 129], + [1020, 734, 23, 53, 134, 648, 549, 589, 790, 1000, 449, 271, 1021, 740, 36], + [701, 344, 955, 19, 927, 212, 212, 667, 212, 627, 453, 954, 777, 706, 496], + [526, 805, 444, 474, 870, 920, 394, 823, 814, 1021, 763, 677, 251, 485, 1021], + [721, 134, 280, 439, 287, 77, 175, 902, 973, 412, 739, 953, 130, 75, 543], + [675, 316, 285, 341, 783, 850, 131, 487, 701, 150, 749, 730, 900, 481, 498], + [377, 37, 237, 489, 55, 246, 427, 456, 755, 1011, 712, 631, 695, 576, 804], + [601, 557, 681, 52, 10, 299, 284, 216, 869, 276, 424, 364, 955, 41, 497], + [465, 553, 697, 59, 701, 195, 335, 225, 896, 804, 776, 928, 392, 192, 332], + [807, 306, 977, 801, 77, 172, 760, 747, 445, 38, 731, 31, 924, 724, 835], + [903, 561, 205, 421, 231, 873, 931, 361, 679, 854, 471, 884, 1011, 857, 248], + [490, 993, 122, 787, 178, 307, 141, 468, 652, 786, 879, 885, 226, 343, 501], + ], + [ + [140, 320, 210, 489, 444, 388, 210, 73, 821, 1004, 388, 686, 405, 563, 407], + [725, 449, 802, 85, 36, 532, 620, 28, 620, 418, 146, 532, 418, 453, 565], + [695, 725, 600, 371, 829, 237, 911, 927, 181, 707, 306, 337, 254, 577, 289], + [51, 648, 186, 129, 781, 570, 737, 563, 400, 839, 674, 689, 544, 767, 577], + [1007, 234, 145, 966, 734, 748, 68, 272, 473, 973, 414, 586, 618, 6, 909], + [410, 566, 507, 756, 943, 736, 269, 349, 549, 320, 303, 729, 507, 741, 76], + [172, 102, 548, 714, 225, 723, 149, 423, 307, 527, 844, 102, 747, 76, 586], + [656, 144, 407, 245, 140, 409, 48, 197, 126, 418, 112, 674, 582, 916, 223], + [776, 971, 291, 781, 833, 296, 817, 261, 937, 467, 352, 463, 530, 804, 683], + [1009, 284, 427, 907, 900, 630, 279, 285, 878, 315, 734, 751, 337, 699, 966], + [389, 748, 203, 585, 609, 474, 555, 64, 154, 443, 16, 139, 905, 172, 86], + [884, 34, 477, 1013, 335, 306, 724, 202, 356, 199, 728, 552, 755, 223, 371], + ], + ] + ).to(torch_device) + EXPECTED_QUANT_CODEBOOK_LOSS = 20.61562156677246 + EXPECTED_DEC_OUTPUTS = torch.tensor( + [ + [-1.9181e-04, 1.9380e-04, 3.1524e-04, 2.0670e-04, -2.8026e-05, -3.3014e-04], + [3.1081e-05, 4.7076e-04, -1.5066e-03, -1.7006e-05, -3.3131e-04, -1.1786e-03], + ] + ).to(torch_device) + EXPECTED_CODEC_ERROR = 0.001973195234313607 + + # load model and processor model_id = f"descript/{model_name}" model = DacModel.from_pretrained(model_id).to(torch_device) - processor = AutoProcessor.from_pretrained(model_id, hop_length=int(np.prod(model.config.downsampling_ratios))) + processor = AutoProcessor.from_pretrained(model_id) + # load audio samples + librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") librispeech_dummy = librispeech_dummy.cast_column("audio", Audio(sampling_rate=processor.sampling_rate)) audio_samples = [np.array([audio_sample["array"]])[0] for audio_sample in librispeech_dummy[-2:]["audio"]] + # check on processor audio shape inputs = processor( raw_audio=audio_samples, sampling_rate=processor.sampling_rate, truncation=False, return_tensors="pt", ).to(torch_device) + torch.equal(torch.tensor(inputs["input_values"].shape), EXPECTED_PREPROC_SHAPE) with torch.no_grad(): + # compare encoder loss encoder_outputs = model.encode(inputs["input_values"]) - hf_output_means_dict = { - "loss": encoder_outputs[0].mean().item(), - "quantized_representation": encoder_outputs[1].mean().item(), - "audio_codes": encoder_outputs[2].float().mean().item(), - "projected_latents": encoder_outputs[3].float().mean().item(), - } - hf_output_means = torch.tensor(list(hf_output_means_dict.values()), dtype=torch.float32) - - # make sure encoded outputs are similar - expected_encoder_means = torch.tensor(list(expected_encoder_means_dict.values()), dtype=torch.float32) - torch.testing.assert_close(hf_output_means, expected_encoder_means, rtol=1e-3, atol=1e-3) + torch.testing.assert_close(EXPECTED_ENC_LOSS, encoder_outputs[0].mean().item(), rtol=1e-3, atol=1e-3) - # check that quantizers behave similar (for same input) - encoded_hf = model.encoder(inputs["input_values"]) - hf_quantizer_codebook_mean = model.quantizer(encoded_hf)[1].float().mean().item() + # compare quantizer outputs + quantizer_outputs = model.quantizer(encoder_outputs[1]) torch.testing.assert_close( - hf_quantizer_codebook_mean, expected_quantizer_codebook_mean, rtol=1e-6, atol=1e-6 + EXPECTED_QUANT_CODES, quantizer_outputs[1][..., : EXPECTED_QUANT_CODES.shape[-1]], rtol=1e-6, atol=1e-6 + ) + torch.testing.assert_close( + EXPECTED_QUANT_CODEBOOK_LOSS, quantizer_outputs[4].mean().item(), rtol=1e-6, atol=1e-6 ) - # check that decoders behave similar (for same input) - hf_decoded_mean = model.decode(encoded_hf)["audio_values"].mean().item() - torch.testing.assert_close(hf_decoded_mean, expected_decoded_mean, rtol=1e-6, atol=1e-6) + # compare decoder outputs + decoded_outputs = model.decode(encoder_outputs[1]) + torch.testing.assert_close( + EXPECTED_DEC_OUTPUTS, + decoded_outputs["audio_values"][..., : EXPECTED_DEC_OUTPUTS.shape[-1]], + rtol=1e-3, + atol=1e-3, + ) - # decode - _, quantized_representation, _, _ = encoder_outputs.to_tuple() - input_values_dec = model.decode(quantized_representation)[0] - input_values_enc_dec = model(inputs["input_values"])[1] + # compare codec error / lossiness + codec_err = compute_rmse(decoded_outputs["audio_values"], inputs["input_values"]) + torch.testing.assert_close(EXPECTED_CODEC_ERROR, codec_err, rtol=1e-6, atol=1e-6) # make sure forward and decode gives same result - torch.testing.assert_close(input_values_dec, input_values_enc_dec, rtol=1e-6, atol=1e-6) - - # make sure audios are more or less equal - rmse = compute_rmse(input_values_enc_dec, inputs["input_values"]) - self.assertTrue(rmse < expected_rmse) - - # check that codec error is similar - torch.testing.assert_close(expected_codec_error, rmse, rtol=1e-6, atol=1e-6) + enc_dec = model(inputs["input_values"])[1] + torch.testing.assert_close(decoded_outputs["audio_values"], enc_dec, rtol=1e-6, atol=1e-6) def test_integration_batch_24khz(self): - expected_rmse = 0.002 - expected_encoder_means_dict = { - "loss": 24.505210876464844, - "quantized_representation": 0.03778776153922081, - "audio_codes": 509.5290222167969, - "projected_latents": -0.017138859257102013, - } - expected_quantizer_codebook_mean = 509.381103515625 - expected_decoded_mean = 0.00010512518929317594 - expected_codec_error = 0.0012980918399989605 - - librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") - model_name = "dac_24khz" + # expected values + EXPECTED_PREPROC_SHAPE = torch.tensor([2, 1, 170880]) + EXPECTED_ENC_LOSS = 24.505210876464844 + EXPECTED_QUANT_CODES = torch.tensor( + [ + [ + [234, 826, 826, 360, 204, 716, 766, 766, 360, 252, 919, 999, 360, 772, 668], + [117, 496, 229, 267, 9, 663, 1002, 629, 756, 372, 781, 496, 23, 780, 781], + [559, 712, 401, 423, 290, 27, 674, 340, 762, 410, 877, 558, 516, 5, 197], + [914, 8, 186, 766, 622, 547, 724, 101, 355, 634, 252, 517, 986, 348, 449], + [636, 148, 671, 232, 374, 24, 925, 118, 561, 760, 748, 964, 117, 126, 589], + [950, 825, 985, 600, 771, 949, 24, 629, 284, 398, 361, 893, 345, 840, 721], + [18, 263, 904, 778, 348, 839, 603, 447, 468, 117, 840, 631, 574, 898, 711], + [455, 359, 188, 148, 878, 246, 376, 509, 906, 759, 799, 991, 797, 833, 116], + [786, 275, 343, 492, 578, 952, 854, 833, 720, 730, 949, 72, 630, 305, 943], + [476, 696, 254, 283, 913, 407, 45, 408, 387, 904, 207, 206, 931, 621, 115], + [517, 73, 1019, 268, 238, 754, 188, 670, 923, 930, 110, 992, 870, 210, 953], + [311, 31, 371, 819, 949, 52, 650, 557, 573, 388, 222, 510, 908, 343, 559], + [405, 355, 520, 986, 179, 171, 49, 349, 706, 16, 439, 700, 704, 852, 759], + [854, 745, 982, 727, 466, 71, 530, 23, 125, 639, 254, 450, 397, 171, 766], + [863, 439, 415, 421, 463, 789, 551, 717, 641, 161, 882, 246, 576, 238, 464], + [331, 416, 322, 794, 416, 187, 689, 880, 29, 570, 283, 92, 310, 327, 748], + [149, 338, 105, 63, 848, 995, 824, 497, 792, 375, 745, 321, 914, 597, 101], + [588, 361, 77, 311, 483, 461, 889, 132, 724, 352, 187, 338, 72, 235, 761], + [434, 882, 522, 153, 462, 62, 725, 265, 597, 9, 161, 613, 576, 654, 1006], + [697, 927, 617, 1011, 561, 19, 181, 402, 830, 318, 248, 521, 645, 386, 111], + [787, 604, 809, 223, 21, 569, 817, 550, 253, 484, 718, 292, 358, 704, 556], + [821, 935, 743, 973, 982, 801, 799, 614, 988, 186, 337, 606, 166, 488, 116], + [789, 555, 32, 57, 671, 538, 712, 732, 524, 52, 869, 646, 91, 766, 516], + [481, 31, 464, 774, 756, 612, 619, 771, 372, 615, 697, 337, 28, 891, 706], + [293, 676, 468, 515, 777, 479, 625, 882, 725, 975, 491, 599, 594, 563, 235], + [170, 373, 462, 102, 335, 616, 880, 542, 989, 68, 154, 918, 716, 897, 33], + [228, 480, 610, 886, 733, 16, 924, 366, 490, 417, 790, 909, 88, 344, 351], + [243, 987, 683, 814, 104, 47, 173, 591, 376, 570, 181, 556, 955, 771, 464], + [1010, 62, 490, 536, 440, 174, 263, 849, 934, 544, 231, 908, 586, 558, 670], + [757, 604, 828, 519, 968, 862, 62, 182, 971, 627, 655, 518, 153, 666, 903], + [720, 192, 470, 262, 404, 920, 755, 138, 614, 245, 458, 182, 920, 398, 761], + [570, 527, 276, 994, 124, 174, 561, 150, 139, 988, 935, 327, 174, 1020, 383], + ], + [ + [851, 110, 668, 103, 826, 360, 919, 160, 826, 160, 204, 110, 360, 910, 160], + [325, 846, 245, 722, 664, 594, 1002, 130, 859, 261, 260, 496, 846, 146, 23], + [529, 465, 354, 408, 597, 710, 450, 460, 980, 1011, 577, 392, 631, 453, 861], + [344, 645, 255, 327, 101, 1017, 474, 296, 513, 903, 363, 823, 85, 83, 760], + [415, 208, 656, 878, 751, 798, 240, 326, 137, 393, 511, 253, 369, 110, 590], + [514, 639, 623, 632, 163, 77, 911, 168, 811, 314, 928, 365, 886, 571, 692], + [768, 700, 408, 359, 937, 540, 1018, 570, 401, 746, 541, 166, 813, 492, 659], + [141, 802, 880, 55, 557, 13, 440, 550, 250, 640, 92, 691, 671, 266, 707], + [539, 706, 445, 343, 984, 280, 667, 414, 525, 987, 272, 727, 247, 834, 383], + [668, 94, 376, 890, 975, 337, 178, 839, 449, 863, 980, 35, 929, 913, 661], + [489, 430, 874, 230, 318, 714, 732, 491, 460, 681, 897, 124, 653, 990, 203], + [352, 625, 110, 636, 618, 691, 976, 249, 165, 584, 92, 487, 940, 907, 83], + [168, 518, 471, 139, 693, 101, 761, 185, 415, 338, 330, 557, 1013, 530, 163], + [282, 355, 539, 464, 725, 808, 607, 691, 374, 502, 898, 960, 822, 680, 233], + [599, 15, 236, 918, 475, 45, 16, 631, 409, 662, 961, 868, 589, 820, 943], + [398, 238, 897, 395, 502, 972, 125, 219, 748, 1000, 310, 664, 371, 867, 163], + [415, 685, 758, 452, 615, 491, 298, 645, 180, 659, 137, 895, 158, 780, 803], + [14, 138, 789, 848, 203, 360, 66, 589, 842, 597, 296, 763, 157, 259, 176], + [432, 65, 342, 488, 399, 259, 869, 214, 490, 975, 349, 894, 691, 87, 850], + [20, 524, 1019, 333, 926, 632, 41, 1002, 75, 282, 319, 426, 513, 368, 241], + [252, 292, 705, 578, 937, 800, 861, 548, 732, 57, 914, 493, 415, 76, 626], + [1004, 799, 467, 438, 656, 397, 547, 882, 873, 675, 900, 360, 941, 25, 63], + [695, 7, 446, 799, 900, 821, 859, 760, 740, 398, 236, 936, 974, 305, 27], + [977, 58, 979, 294, 514, 525, 768, 381, 920, 147, 264, 675, 6, 318, 619], + [539, 315, 574, 938, 208, 454, 869, 220, 1007, 964, 906, 133, 247, 14, 357], + [555, 968, 337, 468, 767, 805, 991, 266, 620, 653, 882, 720, 592, 920, 1016], + [320, 824, 133, 631, 861, 176, 607, 5, 686, 187, 186, 982, 453, 479, 849], + [247, 191, 164, 884, 292, 289, 579, 996, 332, 480, 965, 856, 628, 522, 652], + [142, 388, 533, 548, 600, 1, 504, 663, 140, 246, 1, 80, 555, 739, 672], + [909, 361, 285, 925, 509, 358, 219, 725, 476, 626, 651, 511, 3, 456, 620], + [731, 421, 150, 573, 598, 936, 796, 57, 442, 821, 162, 359, 912, 139, 659], + [588, 398, 945, 404, 804, 494, 572, 124, 47, 809, 775, 266, 9, 596, 435], + ], + ] + ).to(torch_device) + EXPECTED_QUANT_CODEBOOK_LOSS = 23.9102783203125 + EXPECTED_DEC_OUTPUTS = torch.tensor( + [ + [2.9611e-04, 5.0039e-05, -5.4961e-04, -7.9769e-04, -6.9696e-04, -5.6013e-04], + [-4.3881e-04, 3.3771e-04, 1.0076e-03, 1.2748e-03, 1.4132e-03, 1.0326e-03], + ] + ).to(torch_device) + EXPECTED_CODEC_ERROR = 0.0012980918399989605 + + # load model and processor model_id = f"descript/{model_name}" model = DacModel.from_pretrained(model_id).to(torch_device) - processor = AutoProcessor.from_pretrained(model_id, hop_length=int(np.prod(model.config.downsampling_ratios))) + processor = AutoProcessor.from_pretrained(model_id) + # load audio samples + librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") librispeech_dummy = librispeech_dummy.cast_column("audio", Audio(sampling_rate=processor.sampling_rate)) audio_samples = [np.array([audio_sample["array"]])[0] for audio_sample in librispeech_dummy[-2:]["audio"]] + # check on processor audio shape inputs = processor( raw_audio=audio_samples, sampling_rate=processor.sampling_rate, truncation=False, return_tensors="pt", ).to(torch_device) + torch.equal(torch.tensor(inputs["input_values"].shape), EXPECTED_PREPROC_SHAPE) with torch.no_grad(): + # compare encoder loss encoder_outputs = model.encode(inputs["input_values"]) - hf_output_means_dict = { - "loss": encoder_outputs[0].mean().item(), - "quantized_representation": encoder_outputs[1].mean().item(), - "audio_codes": encoder_outputs[2].float().mean().item(), - "projected_latents": encoder_outputs[3].float().mean().item(), - } - hf_output_means = torch.tensor(list(hf_output_means_dict.values()), dtype=torch.float32) + torch.testing.assert_close(EXPECTED_ENC_LOSS, encoder_outputs[0].mean().item(), rtol=1e-3, atol=1e-3) - # make sure encoded outputs are similar - expected_encoder_means = torch.tensor(list(expected_encoder_means_dict.values()), dtype=torch.float32) - torch.testing.assert_close(hf_output_means, expected_encoder_means, rtol=1e-3, atol=1e-3) - - # check that quantizers behave similar (for same input) - encoded_hf = model.encoder(inputs["input_values"]) - hf_quantizer_codebook_mean = model.quantizer(encoded_hf)[1].float().mean().item() + # compare quantizer outputs + quantizer_outputs = model.quantizer(encoder_outputs[1]) + torch.testing.assert_close( + EXPECTED_QUANT_CODES, quantizer_outputs[1][..., : EXPECTED_QUANT_CODES.shape[-1]], rtol=1e-6, atol=1e-6 + ) torch.testing.assert_close( - hf_quantizer_codebook_mean, expected_quantizer_codebook_mean, rtol=1e-6, atol=1e-6 + EXPECTED_QUANT_CODEBOOK_LOSS, quantizer_outputs[4].mean().item(), rtol=1e-6, atol=1e-6 ) - # check that decoders behave similar (for same input) - hf_decoded_mean = model.decode(encoded_hf)["audio_values"].mean().item() - torch.testing.assert_close(hf_decoded_mean, expected_decoded_mean, rtol=1e-6, atol=1e-6) + # compare decoder outputs + decoded_outputs = model.decode(encoder_outputs[1]) + torch.testing.assert_close( + EXPECTED_DEC_OUTPUTS, + decoded_outputs["audio_values"][..., : EXPECTED_DEC_OUTPUTS.shape[-1]], + rtol=1e-3, + atol=1e-3, + ) - # decode - _, quantized_representation, _, _ = encoder_outputs.to_tuple() - input_values_dec = model.decode(quantized_representation)[0] - input_values_enc_dec = model(inputs["input_values"])[1] + # compare codec error / lossiness + codec_err = compute_rmse(decoded_outputs["audio_values"], inputs["input_values"]) + torch.testing.assert_close(EXPECTED_CODEC_ERROR, codec_err, rtol=1e-6, atol=1e-6) # make sure forward and decode gives same result - torch.testing.assert_close(input_values_dec, input_values_enc_dec, rtol=1e-6, atol=1e-6) - - # make sure audios are more or less equal - rmse = compute_rmse(input_values_enc_dec, inputs["input_values"]) - self.assertTrue(rmse < expected_rmse) - - # check that codec error is similar - torch.testing.assert_close(expected_codec_error, rmse, rtol=1e-6, atol=1e-6) + enc_dec = model(inputs["input_values"])[1] + torch.testing.assert_close(decoded_outputs["audio_values"], enc_dec, rtol=1e-6, atol=1e-6) def test_integration_batch_44khz(self): - expected_rmse = 0.001 - expected_encoder_means_dict = { - "loss": 19.557754516601562, - "quantized_representation": 0.004012184217572212, - "audio_codes": 518.1870727539062, - "projected_latents": -0.0008539701229892671, - } - expected_quantizer_codebook_mean = 518.0151977539062 - expected_decoded_mean = -2.039729770331178e-05 - expected_codec_error = 0.00037737112143076956 - - librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") - model_name = "dac_44khz" + # expected values + EXPECTED_PREPROC_SHAPE = torch.tensor([2, 1, 313856]) + EXPECTED_ENC_LOSS = 19.557754516601562 + EXPECTED_QUANT_CODES = torch.tensor( + [ + [ + [330, 315, 315, 619, 481, 315, 197, 315, 315, 105, 481, 481, 481, 481, 481], + [718, 1007, 309, 6, 906, 35, 402, 750, 396, 854, 962, 115, 609, 224, 329], + [417, 266, 150, 335, 300, 812, 325, 780, 1022, 605, 480, 342, 939, 150, 456], + [813, 811, 897, 334, 200, 852, 723, 497, 678, 922, 396, 333, 918, 548, 285], + [832, 315, 165, 106, 902, 326, 32, 572, 610, 170, 395, 223, 193, 807, 585], + [91, 941, 81, 684, 34, 340, 362, 946, 157, 640, 888, 215, 577, 483, 371], + [676, 859, 446, 664, 473, 815, 860, 640, 514, 385, 73, 201, 701, 78, 825], + [326, 426, 347, 970, 605, 997, 534, 111, 559, 538, 526, 208, 372, 709, 167], + [776, 315, 179, 232, 140, 456, 318, 155, 191, 674, 105, 992, 721, 406, 267], + ], + [ + [578, 592, 330, 330, 330, 330, 330, 801, 330, 330, 330, 698, 330, 330, 330], + [501, 204, 514, 215, 615, 580, 567, 684, 478, 905, 208, 32, 495, 84, 1000], + [141, 458, 489, 125, 691, 471, 522, 60, 978, 30, 125, 480, 424, 67, 1], + [908, 192, 865, 878, 137, 698, 965, 969, 565, 216, 535, 488, 441, 503, 181], + [850, 635, 993, 391, 500, 122, 365, 850, 905, 449, 586, 451, 840, 811, 797], + [307, 408, 497, 294, 24, 396, 417, 922, 161, 268, 100, 753, 778, 1014, 259], + [178, 918, 568, 28, 187, 375, 301, 889, 834, 406, 665, 7, 889, 909, 387], + [935, 566, 315, 13, 490, 37, 436, 801, 484, 62, 476, 551, 557, 232, 533], + [1017, 89, 585, 401, 13, 238, 744, 1017, 774, 872, 850, 468, 640, 833, 854], + ], + ] + ).to(torch_device) + EXPECTED_QUANT_CODEBOOK_LOSS = 16.177066802978516 + EXPECTED_DEC_OUTPUTS = torch.tensor( + [[-0.0004, -0.0001, 0.0001, 0.0003, 0.0004, 0.0005], [0.0001, 0.0005, 0.0001, -0.0006, -0.0012, -0.0011]] + ).to(torch_device) + EXPECTED_CODEC_ERROR = 0.00037737112143076956 + + # load model and processor model_id = f"descript/{model_name}" model = DacModel.from_pretrained(model_id).to(torch_device) - processor = AutoProcessor.from_pretrained(model_id, hop_length=int(np.prod(model.config.downsampling_ratios))) + processor = AutoProcessor.from_pretrained(model_id) + # load audio samples + librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") librispeech_dummy = librispeech_dummy.cast_column("audio", Audio(sampling_rate=processor.sampling_rate)) audio_samples = [np.array([audio_sample["array"]])[0] for audio_sample in librispeech_dummy[-2:]["audio"]] + # check on processor audio shape inputs = processor( raw_audio=audio_samples, sampling_rate=processor.sampling_rate, truncation=False, return_tensors="pt", ).to(torch_device) + torch.equal(torch.tensor(inputs["input_values"].shape), EXPECTED_PREPROC_SHAPE) with torch.no_grad(): + # compare encoder loss encoder_outputs = model.encode(inputs["input_values"]) - hf_output_means_dict = { - "loss": encoder_outputs[0].mean().item(), - "quantized_representation": encoder_outputs[1].mean().item(), - "audio_codes": encoder_outputs[2].float().mean().item(), - "projected_latents": encoder_outputs[3].float().mean().item(), - } - hf_output_means = torch.tensor(list(hf_output_means_dict.values()), dtype=torch.float32) - - # make sure encoded outputs are similar - expected_encoder_means = torch.tensor(list(expected_encoder_means_dict.values()), dtype=torch.float32) - torch.testing.assert_close(hf_output_means, expected_encoder_means, rtol=1e-3, atol=1e-3) + torch.testing.assert_close(EXPECTED_ENC_LOSS, encoder_outputs[0].mean().item(), rtol=1e-3, atol=1e-3) - # check that quantizers behave similar (for same input) - encoded_hf = model.encoder(inputs["input_values"]) - hf_quantizer_codebook_mean = model.quantizer(encoded_hf)[1].float().mean().item() + # compare quantizer outputs + quantizer_outputs = model.quantizer(encoder_outputs[1]) torch.testing.assert_close( - hf_quantizer_codebook_mean, expected_quantizer_codebook_mean, rtol=1e-6, atol=1e-6 + EXPECTED_QUANT_CODES, quantizer_outputs[1][..., : EXPECTED_QUANT_CODES.shape[-1]], rtol=1e-6, atol=1e-6 + ) + torch.testing.assert_close( + EXPECTED_QUANT_CODEBOOK_LOSS, quantizer_outputs[4].mean().item(), rtol=1e-6, atol=1e-6 ) - # check that decoders behave similar (for same input) - hf_decoded_mean = model.decode(encoded_hf)["audio_values"].mean().item() - torch.testing.assert_close(hf_decoded_mean, expected_decoded_mean, rtol=1e-6, atol=1e-6) + # compare decoder outputs + decoded_outputs = model.decode(encoder_outputs[1]) + torch.testing.assert_close( + EXPECTED_DEC_OUTPUTS, + decoded_outputs["audio_values"][..., : EXPECTED_DEC_OUTPUTS.shape[-1]], + rtol=1e-3, + atol=1e-3, + ) - # decode - _, quantized_representation, _, _ = encoder_outputs.to_tuple() - input_values_dec = model.decode(quantized_representation)[0] - input_values_enc_dec = model(inputs["input_values"])[1] + # compare codec error / lossiness + codec_err = compute_rmse(decoded_outputs["audio_values"], inputs["input_values"]) + torch.testing.assert_close(EXPECTED_CODEC_ERROR, codec_err, rtol=1e-6, atol=1e-6) # make sure forward and decode gives same result - torch.testing.assert_close(input_values_dec, input_values_enc_dec, rtol=1e-6, atol=1e-6) - - # make sure audios are more or less equal - rmse = compute_rmse(input_values_enc_dec, inputs["input_values"]) - self.assertTrue(rmse < expected_rmse) - - # check that codec error is similar - torch.testing.assert_close(expected_codec_error, rmse, rtol=1e-6, atol=1e-6) + enc_dec = model(inputs["input_values"])[1] + torch.testing.assert_close(decoded_outputs["audio_values"], enc_dec, rtol=1e-6, atol=1e-6) From da8243bd75f01b9f641f64d04cd05d253e3f9fd3 Mon Sep 17 00:00:00 2001 From: Eric B Date: Tue, 22 Jul 2025 15:43:53 +0200 Subject: [PATCH 6/8] Added info about encoder/decoder error and longer decoder outputs. --- tests/models/dac/test_modeling_dac.py | 131 +++++++++++++++++++++----- 1 file changed, 108 insertions(+), 23 deletions(-) diff --git a/tests/models/dac/test_modeling_dac.py b/tests/models/dac/test_modeling_dac.py index 7896dfa8541e..393e2fa5e94b 100644 --- a/tests/models/dac/test_modeling_dac.py +++ b/tests/models/dac/test_modeling_dac.py @@ -402,6 +402,11 @@ class DacIntegrationTest(unittest.TestCase): - Single file: https://gist.github.com/ebezzam/bb315efa7a416db6336a6b2a2d424ffa#file-dac_integration_single-py - Batched: https://gist.github.com/ebezzam/bb315efa7a416db6336a6b2a2d424ffa#file-dac_integration-py + See https://github.com/huggingface/transformers/pull/39313 for reason behind large tolerance between for encoder + and decoder outputs (1e-3). In summary, original model uses weight normalization, while Transformers does not. This + leads to accumulating error. However, this does not affect the quantizer codes, thanks to discretization being + robust to precision errors. Moreover, codec error is similar between Transformers and original. + Moreover, here is a script to debug outputs and weights layer-by-layer: https://gist.github.com/ebezzam/bb315efa7a416db6336a6b2a2d424ffa#file-dac_layer_by_layer_debugging-py """ @@ -430,10 +435,19 @@ def test_integration_16khz(self): ] ] ).to(torch_device) - EXPECTED_QUANT_CODEBOOK_LOSS = 20.58063507080078 - EXPECTED_DEC_OUTPUTS = torch.tensor( - [[7.2661e-05, 5.9626e-04, 1.0609e-03, 1.4515e-03, 1.6704e-03, 1.0837e-03]] - ).to(torch_device) + # fmt: off + EXPECTED_DEC_OUTPUTS = torch.tensor([[ 7.2661e-05, 5.9626e-04, 1.0609e-03, 1.4515e-03, 1.6704e-03, + 1.0837e-03, 4.6979e-04, -1.3811e-04, -2.7733e-04, 2.0613e-04, + 4.0715e-04, 8.4999e-04, 1.7112e-03, 2.7275e-03, 2.5560e-03, + 1.6202e-03, 1.4603e-03, 1.1447e-03, 7.4274e-04, 7.6758e-04, + 1.5931e-03, 2.5598e-03, 2.6844e-03, 2.9216e-03, 3.6430e-03, + 3.0532e-03, 2.1169e-03, 2.3657e-03, 2.0313e-03, 8.8282e-04, + -1.6314e-04, 2.0697e-05, 9.0119e-04, 1.5815e-03, 2.1719e-03, + 2.2010e-03, 1.4089e-03, -9.8639e-05, -7.1111e-04, -2.1185e-04, + 3.3837e-04, 5.2177e-04, 1.0538e-03, 2.2637e-03, 1.9972e-03, + 1.6396e-03, 1.6282e-03, 1.1689e-03, 2.7550e-04, -4.4859e-04]]).to(torch_device) + # fmt: on + EXPECTED_QUANT_CODEBOOK_LOSS = 20.5806350708007 EXPECTED_CODEC_ERROR = 0.0038341842591762543 # load model and processor @@ -529,10 +543,19 @@ def test_integration_24khz(self): ] ] ).to(torch_device) + # fmt: off + EXPECTED_DEC_OUTPUTS = torch.tensor([[ 4.2660e-04, 4.0129e-04, 1.5403e-04, 5.0874e-05, 2.9436e-04, + 1.0682e-03, 1.9777e-03, 1.9081e-03, 1.5145e-03, 1.2959e-03, + 1.1858e-03, 8.6308e-04, 7.6199e-05, -6.2039e-04, -2.8909e-04, + 7.2902e-04, 9.6803e-04, 3.5680e-04, -1.4637e-04, 7.8926e-05, + 7.9285e-04, 1.3313e-03, 1.1692e-03, 5.7410e-04, 7.0640e-04, + 1.5462e-03, 1.9182e-03, 1.3498e-03, 5.0153e-04, 1.5142e-04, + 2.1018e-04, 4.2771e-04, 7.4621e-04, 1.1082e-03, 1.5289e-03, + 1.9526e-03, 2.3434e-03, 2.6424e-03, 2.8369e-03, 2.7632e-03, + 2.3256e-03, 1.8973e-03, 1.8191e-03, 1.9133e-03, 1.7674e-03, + 1.0398e-03, 2.6915e-04, 1.3725e-04, 2.8598e-04, 2.5875e-04]]).to(torch_device) + # fmt: on EXPECTED_QUANT_CODEBOOK_LOSS = 22.581758499145508 - EXPECTED_DEC_OUTPUTS = torch.tensor( - [[4.2660e-04, 4.0129e-04, 1.5403e-04, 5.0874e-05, 2.9436e-04, 1.0682e-03]] - ).to(torch_device) EXPECTED_CODEC_ERROR = 0.002570481738075614 # load model and processor @@ -605,8 +628,19 @@ def test_integration_44khz(self): ] ] ).to(torch_device) + # fmt: off + EXPECTED_DEC_OUTPUTS = torch.tensor([[ 8.3748e-04, 3.7760e-04, 4.7135e-04, 8.2829e-04, 1.3677e-03, + 1.7487e-03, 1.8883e-03, 1.7437e-03, 1.4828e-03, 1.2284e-03, + 1.0894e-03, 1.0442e-03, 1.0558e-03, 1.0136e-03, 8.4781e-04, + 4.8677e-04, -2.0375e-05, -5.2144e-04, -8.6839e-04, -9.8977e-04, + -8.0130e-04, -3.6122e-04, 1.8086e-04, 6.4340e-04, 9.1103e-04, + 9.6243e-04, 8.6814e-04, 7.7186e-04, 7.5613e-04, 8.1264e-04, + 9.0747e-04, 9.5464e-04, 9.5436e-04, 8.7902e-04, 7.6080e-04, + 6.2870e-04, 5.5878e-04, 5.7444e-04, 6.6622e-04, 7.9741e-04, + 8.7610e-04, 8.4571e-04, 6.7909e-04, 4.2059e-04, 1.5131e-04, + -7.1465e-05, -1.8646e-04, -1.8300e-04, -1.2542e-04, -7.1933e-05]]).to(torch_device) + # fmt: on EXPECTED_QUANT_CODEBOOK_LOSS = 16.2640438079834 - EXPECTED_DEC_OUTPUTS = torch.tensor([[0.0008, 0.0004, 0.0005, 0.0008, 0.0014, 0.0017]]).to(torch_device) EXPECTED_CODEC_ERROR = 0.0007429996621794999 # load model and processor @@ -696,13 +730,29 @@ def test_integration_batch_16khz(self): ], ] ).to(torch_device) + # fmt: off + EXPECTED_DEC_OUTPUTS = torch.tensor([[-1.9181e-04, 1.9380e-04, 3.1524e-04, 2.0670e-04, -2.8026e-05, + -3.3014e-04, -4.6584e-04, -4.3935e-04, -2.8362e-04, 2.7245e-04, + 8.8112e-04, 1.1195e-03, 1.6224e-03, 1.9368e-03, 1.7803e-03, + 5.9601e-04, -4.4178e-04, -1.3736e-03, -1.9979e-03, -2.0477e-03, + -1.5583e-03, -4.1277e-04, 6.2742e-04, 1.2409e-03, 1.3380e-03, + 1.2884e-03, 6.0346e-04, 8.9812e-05, -6.1626e-04, -1.3760e-03, + -1.4970e-03, -9.8225e-04, -3.9102e-04, 5.3190e-04, 1.8696e-03, + 2.3731e-03, 2.1139e-03, 1.4220e-03, 7.3644e-04, -2.4944e-04, + -9.8294e-04, -1.3858e-03, -1.6684e-03, -1.0482e-03, -6.1834e-04, + -5.3312e-04, -2.1345e-04, 4.1917e-04, 7.7653e-04, 8.0206e-04], + [ 3.1081e-05, 4.7076e-04, -1.5066e-03, -1.7006e-05, -3.3131e-04, + -1.1786e-03, 8.2880e-04, -1.2492e-03, 4.6135e-04, -8.7780e-04, + -8.5493e-04, 3.2979e-04, 1.1218e-03, -1.8018e-03, 2.2795e-04, + 2.4981e-04, -3.1100e-03, 1.0356e-03, 1.1427e-03, 2.1378e-03, + -7.0038e-04, 1.6522e-03, -3.3599e-04, -2.3893e-03, -5.2286e-04, + 2.9462e-04, 1.2429e-03, -1.8078e-03, 3.3687e-03, 1.3336e-03, + -1.5815e-03, -1.5836e-04, -5.4054e-04, -7.2660e-04, -2.2980e-03, + -5.3254e-04, 1.4890e-03, -1.0853e-03, 1.0333e-03, 8.1283e-04, + -1.6996e-03, 6.0168e-05, -2.6916e-03, 3.7072e-04, -1.0729e-03, + 2.7891e-04, 3.3514e-03, -1.8029e-03, 5.5011e-04, -1.1905e-03]]).to(torch_device) + # fmt: on EXPECTED_QUANT_CODEBOOK_LOSS = 20.61562156677246 - EXPECTED_DEC_OUTPUTS = torch.tensor( - [ - [-1.9181e-04, 1.9380e-04, 3.1524e-04, 2.0670e-04, -2.8026e-05, -3.3014e-04], - [3.1081e-05, 4.7076e-04, -1.5066e-03, -1.7006e-05, -3.3131e-04, -1.1786e-03], - ] - ).to(torch_device) EXPECTED_CODEC_ERROR = 0.001973195234313607 # load model and processor @@ -833,13 +883,29 @@ def test_integration_batch_24khz(self): ], ] ).to(torch_device) + # fmt: off + EXPECTED_DEC_OUTPUTS = torch.tensor([[ 2.9611e-04, 5.0039e-05, -5.4961e-04, -7.9769e-04, -6.9696e-04, + -5.6013e-04, -4.7665e-04, -3.8039e-04, -6.8090e-05, 6.5704e-05, + 1.3205e-05, 1.3519e-04, 1.4002e-04, 4.3348e-05, 2.9029e-04, + 5.1533e-04, 1.4072e-04, -1.8430e-04, 6.3313e-05, 4.6729e-04, + 5.5076e-04, 5.6079e-04, 5.6557e-04, 3.2839e-04, 2.6326e-04, + 3.9028e-04, 3.1820e-04, 5.1251e-05, -7.0745e-05, -2.0471e-04, + -7.0736e-04, -1.2458e-03, -1.4124e-03, -1.3991e-03, -1.4890e-03, + -1.4013e-03, -1.0092e-03, -5.4982e-04, -3.5847e-05, 5.3150e-04, + 9.2390e-04, 1.0131e-03, 1.0362e-03, 1.0253e-03, 8.1528e-04, + 3.7854e-04, -1.3280e-05, -2.6982e-04, -4.8256e-04, -7.0810e-04], + [-4.3881e-04, 3.3771e-04, 1.0076e-03, 1.2748e-03, 1.4132e-03, + 1.0326e-03, 7.5779e-04, 5.3942e-04, -2.8545e-04, -2.0953e-03, + -2.2058e-03, 1.1152e-04, 5.6744e-04, -1.7912e-03, -1.4614e-03, + 1.8420e-03, 1.5202e-03, -1.0541e-03, 1.9058e-04, 1.3378e-03, + -2.0335e-03, -2.5633e-03, 2.4959e-03, 2.4356e-03, -3.1333e-03, + -2.8208e-03, 9.7969e-04, -1.0972e-03, -3.0217e-03, 4.1109e-04, + 2.3006e-04, -2.8686e-03, 1.2978e-03, 5.9192e-03, 7.3619e-04, + -3.9734e-03, -2.6965e-04, 1.3701e-03, -1.7230e-03, -9.4332e-04, + 4.2128e-04, -2.6123e-03, -1.8240e-03, 3.3554e-03, 1.7732e-03, + -3.2838e-03, -8.2577e-04, 3.1959e-03, 1.1458e-03, -2.4608e-04]]).to(torch_device) + # fmt: on EXPECTED_QUANT_CODEBOOK_LOSS = 23.9102783203125 - EXPECTED_DEC_OUTPUTS = torch.tensor( - [ - [2.9611e-04, 5.0039e-05, -5.4961e-04, -7.9769e-04, -6.9696e-04, -5.6013e-04], - [-4.3881e-04, 3.3771e-04, 1.0076e-03, 1.2748e-03, 1.4132e-03, 1.0326e-03], - ] - ).to(torch_device) EXPECTED_CODEC_ERROR = 0.0012980918399989605 # load model and processor @@ -924,10 +990,29 @@ def test_integration_batch_44khz(self): ], ] ).to(torch_device) + # fmt: off + EXPECTED_DEC_OUTPUTS = torch.tensor([[-3.7834e-04, -1.0849e-04, 1.1856e-04, 2.6852e-04, 3.7313e-04, + 5.0301e-04, 6.4261e-04, 8.0797e-04, 9.0969e-04, 9.9720e-04, + 1.0807e-03, 1.1217e-03, 1.1229e-03, 1.1208e-03, 1.0862e-03, + 9.5098e-04, 7.5477e-04, 5.2319e-04, 2.7449e-04, 2.4389e-05, + -1.9138e-04, -3.2046e-04, -4.0629e-04, -4.4804e-04, -5.0271e-04, + -5.8324e-04, -6.6573e-04, -6.9545e-04, -6.8046e-04, -6.1640e-04, + -5.3542e-04, -4.2302e-04, -3.0829e-04, -1.8475e-04, -3.9555e-05, + 9.0104e-05, 1.9291e-04, 2.7445e-04, 3.6738e-04, 4.7454e-04, + 6.0626e-04, 7.5514e-04, 8.5390e-04, 8.8749e-04, 8.5473e-04, + 7.5550e-04, 6.2329e-04, 4.9771e-04, 3.8809e-04, 3.0741e-04], + [ 1.1130e-04, 4.6536e-04, 1.0524e-04, -6.1460e-04, -1.1777e-03, + -1.0661e-03, -3.7962e-04, 5.3627e-04, 1.0481e-03, 8.7734e-04, + 1.3513e-04, -6.6297e-04, -9.5284e-04, -4.6333e-04, 5.5780e-04, + 1.4526e-03, 1.6264e-03, 1.0852e-03, 3.3766e-04, 1.0960e-04, + 7.7973e-04, 2.0579e-03, 3.0206e-03, 2.9674e-03, 1.8141e-03, + 3.1059e-04, -5.7140e-04, -3.4386e-04, 4.8406e-04, 8.6931e-04, + 2.1745e-05, -1.7647e-03, -3.2787e-03, -3.3368e-03, -1.7466e-03, + 4.3745e-04, 1.6595e-03, 1.1171e-03, -6.3018e-04, -2.0979e-03, + -2.1286e-03, -6.8752e-04, 1.1514e-03, 2.1590e-03, 1.9204e-03, + 1.0659e-03, 5.3295e-04, 6.6817e-04, 9.2716e-04, 5.3240e-04]]).to(torch_device) + # fmt: on EXPECTED_QUANT_CODEBOOK_LOSS = 16.177066802978516 - EXPECTED_DEC_OUTPUTS = torch.tensor( - [[-0.0004, -0.0001, 0.0001, 0.0003, 0.0004, 0.0005], [0.0001, 0.0005, 0.0001, -0.0006, -0.0012, -0.0011]] - ).to(torch_device) EXPECTED_CODEC_ERROR = 0.00037737112143076956 # load model and processor From 36a24cba350345e018fe45c05b945feabcde4019 Mon Sep 17 00:00:00 2001 From: Eric B Date: Wed, 23 Jul 2025 18:00:31 +0200 Subject: [PATCH 7/8] Parameterize tests. --- tests/models/dac/test_modeling_dac.py | 721 ++++++++++---------------- 1 file changed, 266 insertions(+), 455 deletions(-) diff --git a/tests/models/dac/test_modeling_dac.py b/tests/models/dac/test_modeling_dac.py index 393e2fa5e94b..b512d9c0c664 100644 --- a/tests/models/dac/test_modeling_dac.py +++ b/tests/models/dac/test_modeling_dac.py @@ -20,6 +20,7 @@ import numpy as np from datasets import Audio, load_dataset +from parameterized import parameterized from transformers import AutoProcessor, DacConfig, DacModel from transformers.testing_utils import is_torch_available, require_torch, slow, torch_device @@ -392,120 +393,54 @@ def compute_rmse(arr1, arr2): return np.sqrt(((arr1_normalized - arr2_normalized) ** 2).mean()) -@slow -@require_torch -class DacIntegrationTest(unittest.TestCase): - """ - Integration tests for DAC. - - Code for reproducing expected outputs can be found here: - - Single file: https://gist.github.com/ebezzam/bb315efa7a416db6336a6b2a2d424ffa#file-dac_integration_single-py - - Batched: https://gist.github.com/ebezzam/bb315efa7a416db6336a6b2a2d424ffa#file-dac_integration-py - - See https://github.com/huggingface/transformers/pull/39313 for reason behind large tolerance between for encoder - and decoder outputs (1e-3). In summary, original model uses weight normalization, while Transformers does not. This - leads to accumulating error. However, this does not affect the quantizer codes, thanks to discretization being - robust to precision errors. Moreover, codec error is similar between Transformers and original. - - Moreover, here is a script to debug outputs and weights layer-by-layer: - https://gist.github.com/ebezzam/bb315efa7a416db6336a6b2a2d424ffa#file-dac_layer_by_layer_debugging-py - """ - - def test_integration_16khz(self): - model_name = "dac_16khz" - - # expected values - EXPECTED_PREPROC_SHAPE = torch.tensor([1, 1, 93760]) - EXPECTED_ENC_LOSS = 24.84908103942871 - EXPECTED_QUANT_CODES = torch.tensor( +""" +Integration tests for DAC. + +Code for reproducing expected outputs can be found here: +- test_integration: https://gist.github.com/ebezzam/bb315efa7a416db6336a6b2a2d424ffa#file-dac_integration_single-py +- test_batch: https://gist.github.com/ebezzam/bb315efa7a416db6336a6b2a2d424ffa#file-dac_integration-py + +See https://github.com/huggingface/transformers/pull/39313 for reason behind large tolerance between for encoder +and decoder outputs (1e-3). In summary, original model uses weight normalization, while Transformers does not. This +leads to accumulating error. However, this does not affect the quantizer codes, thanks to discretization being +robust to precision errors. Moreover, codec error is similar between Transformers and original. + +Moreover, here is a script to debug outputs and weights layer-by-layer: +https://gist.github.com/ebezzam/bb315efa7a416db6336a6b2a2d424ffa#file-dac_layer_by_layer_debugging-py +""" + +# fmt: off +# -- test_integration +EXPECTED_PREPROC_SHAPE = { + "dac_16khz": torch.tensor([1, 1, 93760]), + "dac_24khz": torch.tensor([1, 1, 140800]), + "dac_44khz": torch.tensor([1, 1, 258560]), +} +EXPECTED_ENC_LOSS = { + "dac_16khz": 24.84908103942871, + "dac_24khz": 28.112096786499023, + "dac_44khz": 23.78483772277832, +} +EXPECTED_QUANT_CODES = { + "dac_16khz": torch.tensor( + [ [ - [ - [804, 25, 977, 52, 68, 867, 388, 653, 315, 706, 301, 305, 140, 25, 40], - [77, 955, 532, 601, 431, 375, 967, 56, 54, 261, 871, 552, 735, 341, 228], - [355, 908, 77, 927, 617, 443, 790, 149, 403, 707, 511, 226, 995, 883, 644], - [184, 162, 611, 54, 211, 890, 906, 253, 677, 1007, 302, 577, 378, 330, 778], - [763, 322, 6, 321, 116, 228, 911, 865, 1000, 234, 6, 901, 10, 174, 895], - [454, 1, 622, 622, 487, 668, 749, 833, 382, 900, 372, 959, 232, 418, 964], - [203, 43, 173, 307, 961, 593, 318, 1011, 386, 949, 343, 899, 536, 824, 38], - [82, 810, 692, 83, 131, 866, 483, 362, 519, 531, 853, 121, 1010, 512, 710], - [1003, 691, 530, 460, 827, 903, 81, 76, 629, 298, 168, 177, 368, 613, 762], - [571, 752, 544, 394, 198, 479, 952, 437, 222, 992, 934, 316, 741, 123, 538], - [686, 421, 393, 635, 246, 330, 908, 384, 962, 873, 92, 254, 912, 496, 83], - [721, 977, 148, 204, 993, 660, 176, 395, 901, 323, 342, 849, 474, 8, 513], - ] + [804, 25, 977, 52, 68, 867, 388, 653, 315, 706, 301, 305, 140, 25, 40], + [77, 955, 532, 601, 431, 375, 967, 56, 54, 261, 871, 552, 735, 341, 228], + [355, 908, 77, 927, 617, 443, 790, 149, 403, 707, 511, 226, 995, 883, 644], + [184, 162, 611, 54, 211, 890, 906, 253, 677, 1007, 302, 577, 378, 330, 778], + [763, 322, 6, 321, 116, 228, 911, 865, 1000, 234, 6, 901, 10, 174, 895], + [454, 1, 622, 622, 487, 668, 749, 833, 382, 900, 372, 959, 232, 418, 964], + [203, 43, 173, 307, 961, 593, 318, 1011, 386, 949, 343, 899, 536, 824, 38], + [82, 810, 692, 83, 131, 866, 483, 362, 519, 531, 853, 121, 1010, 512, 710], + [1003, 691, 530, 460, 827, 903, 81, 76, 629, 298, 168, 177, 368, 613, 762], + [571, 752, 544, 394, 198, 479, 952, 437, 222, 992, 934, 316, 741, 123, 538], + [686, 421, 393, 635, 246, 330, 908, 384, 962, 873, 92, 254, 912, 496, 83], + [721, 977, 148, 204, 993, 660, 176, 395, 901, 323, 342, 849, 474, 8, 513], ] - ).to(torch_device) - # fmt: off - EXPECTED_DEC_OUTPUTS = torch.tensor([[ 7.2661e-05, 5.9626e-04, 1.0609e-03, 1.4515e-03, 1.6704e-03, - 1.0837e-03, 4.6979e-04, -1.3811e-04, -2.7733e-04, 2.0613e-04, - 4.0715e-04, 8.4999e-04, 1.7112e-03, 2.7275e-03, 2.5560e-03, - 1.6202e-03, 1.4603e-03, 1.1447e-03, 7.4274e-04, 7.6758e-04, - 1.5931e-03, 2.5598e-03, 2.6844e-03, 2.9216e-03, 3.6430e-03, - 3.0532e-03, 2.1169e-03, 2.3657e-03, 2.0313e-03, 8.8282e-04, - -1.6314e-04, 2.0697e-05, 9.0119e-04, 1.5815e-03, 2.1719e-03, - 2.2010e-03, 1.4089e-03, -9.8639e-05, -7.1111e-04, -2.1185e-04, - 3.3837e-04, 5.2177e-04, 1.0538e-03, 2.2637e-03, 1.9972e-03, - 1.6396e-03, 1.6282e-03, 1.1689e-03, 2.7550e-04, -4.4859e-04]]).to(torch_device) - # fmt: on - EXPECTED_QUANT_CODEBOOK_LOSS = 20.5806350708007 - EXPECTED_CODEC_ERROR = 0.0038341842591762543 - - # load model and processor - model_id = f"descript/{model_name}" - model = DacModel.from_pretrained(model_id, force_download=True).to(torch_device).eval() - processor = AutoProcessor.from_pretrained(model_id) - - # load audio sample - librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") - librispeech_dummy = librispeech_dummy.cast_column("audio", Audio(sampling_rate=processor.sampling_rate)) - audio_sample = librispeech_dummy[0]["audio"]["array"] - - # check on processor audio shape - inputs = processor( - raw_audio=audio_sample, - sampling_rate=processor.sampling_rate, - return_tensors="pt", - ).to(torch_device) - torch.equal(torch.tensor(inputs["input_values"].shape), EXPECTED_PREPROC_SHAPE) - - with torch.no_grad(): - # compare encoder loss - encoder_outputs = model.encode(inputs["input_values"]) - torch.testing.assert_close(EXPECTED_ENC_LOSS, encoder_outputs[0].squeeze().item(), rtol=1e-3, atol=1e-3) - - # compare quantizer outputs - quantizer_outputs = model.quantizer(encoder_outputs[1]) - torch.testing.assert_close( - EXPECTED_QUANT_CODES, quantizer_outputs[1][..., : EXPECTED_QUANT_CODES.shape[-1]], rtol=1e-6, atol=1e-6 - ) - torch.testing.assert_close( - EXPECTED_QUANT_CODEBOOK_LOSS, quantizer_outputs[4].squeeze().item(), rtol=1e-6, atol=1e-6 - ) - - # compare decoder outputs - decoded_outputs = model.decode(encoder_outputs[1]) - torch.testing.assert_close( - EXPECTED_DEC_OUTPUTS, - decoded_outputs["audio_values"][..., : EXPECTED_DEC_OUTPUTS.shape[-1]], - rtol=1e-3, - atol=1e-3, - ) - - # compare codec error / lossiness - codec_err = compute_rmse(decoded_outputs["audio_values"], inputs["input_values"]) - torch.testing.assert_close(EXPECTED_CODEC_ERROR, codec_err, rtol=1e-6, atol=1e-6) - - # make sure forward and decode gives same result - enc_dec = model(inputs["input_values"])[1] - torch.testing.assert_close(decoded_outputs["audio_values"], enc_dec, rtol=1e-6, atol=1e-6) - - def test_integration_24khz(self): - model_name = "dac_24khz" - - # expected values - EXPECTED_PREPROC_SHAPE = torch.tensor([1, 1, 140800]) - EXPECTED_ENC_LOSS = 28.112096786499023 - EXPECTED_QUANT_CODES = torch.tensor( + ] + ).to(torch_device), + "dac_24khz": torch.tensor( [ [ [160, 360, 826, 204, 239, 360, 90, 160, 851, 234, 252, 690, 360, 160, 665], @@ -542,9 +477,38 @@ def test_integration_24khz(self): [821, 641, 740, 272, 468, 847, 699, 842, 20, 330, 216, 703, 581, 306, 137], ] ] - ).to(torch_device) - # fmt: off - EXPECTED_DEC_OUTPUTS = torch.tensor([[ 4.2660e-04, 4.0129e-04, 1.5403e-04, 5.0874e-05, 2.9436e-04, + ).to(torch_device), + "dac_44khz": torch.tensor([[[ 332, 315, 105, 315, 616, 105, 494, 698, 315, 481, 330, + 93, 105, 315, 105], + [ 670, 350, 249, 27, 232, 365, 311, 881, 186, 402, 311, + 521, 527, 778, 254], + [ 569, 300, 361, 530, 1002, 419, 285, 501, 456, 471, 180, + 615, 419, 491, 764], + [ 605, 436, 641, 291, 901, 556, 715, 780, 502, 410, 858, + 125, 562, 174, 746], + [ 854, 706, 242, 294, 346, 88, 527, 961, 559, 664, 314, + 963, 278, 90, 682], + [ 175, 152, 706, 884, 986, 457, 567, 176, 49, 535, 851, + 417, 533, 349, 779], + [ 913, 710, 628, 162, 770, 254, 247, 6, 397, 264, 233, + 704, 577, 111, 916], + [ 999, 693, 512, 884, 38, 223, 29, 744, 497, 123, 972, + 120, 47, 301, 90], + [ 490, 163, 368, 507, 253, 283, 745, 65, 295, 935, 811, + 587, 801, 255, 105]]]).to(torch_device), +} +EXPECTED_DEC_OUTPUTS = { + "dac_16khz": torch.tensor([[ 7.2661e-05, 5.9626e-04, 1.0609e-03, 1.4515e-03, 1.6704e-03, + 1.0837e-03, 4.6979e-04, -1.3811e-04, -2.7733e-04, 2.0613e-04, + 4.0715e-04, 8.4999e-04, 1.7112e-03, 2.7275e-03, 2.5560e-03, + 1.6202e-03, 1.4603e-03, 1.1447e-03, 7.4274e-04, 7.6758e-04, + 1.5931e-03, 2.5598e-03, 2.6844e-03, 2.9216e-03, 3.6430e-03, + 3.0532e-03, 2.1169e-03, 2.3657e-03, 2.0313e-03, 8.8282e-04, + -1.6314e-04, 2.0697e-05, 9.0119e-04, 1.5815e-03, 2.1719e-03, + 2.2010e-03, 1.4089e-03, -9.8639e-05, -7.1111e-04, -2.1185e-04, + 3.3837e-04, 5.2177e-04, 1.0538e-03, 2.2637e-03, 1.9972e-03, + 1.6396e-03, 1.6282e-03, 1.1689e-03, 2.7550e-04, -4.4859e-04]]).to(torch_device), + "dac_24khz": torch.tensor([[ 4.2660e-04, 4.0129e-04, 1.5403e-04, 5.0874e-05, 2.9436e-04, 1.0682e-03, 1.9777e-03, 1.9081e-03, 1.5145e-03, 1.2959e-03, 1.1858e-03, 8.6308e-04, 7.6199e-05, -6.2039e-04, -2.8909e-04, 7.2902e-04, 9.6803e-04, 3.5680e-04, -1.4637e-04, 7.8926e-05, @@ -553,265 +517,73 @@ def test_integration_24khz(self): 2.1018e-04, 4.2771e-04, 7.4621e-04, 1.1082e-03, 1.5289e-03, 1.9526e-03, 2.3434e-03, 2.6424e-03, 2.8369e-03, 2.7632e-03, 2.3256e-03, 1.8973e-03, 1.8191e-03, 1.9133e-03, 1.7674e-03, - 1.0398e-03, 2.6915e-04, 1.3725e-04, 2.8598e-04, 2.5875e-04]]).to(torch_device) - # fmt: on - EXPECTED_QUANT_CODEBOOK_LOSS = 22.581758499145508 - EXPECTED_CODEC_ERROR = 0.002570481738075614 - - # load model and processor - model_id = f"descript/{model_name}" - model = DacModel.from_pretrained(model_id, force_download=True).to(torch_device).eval() - processor = AutoProcessor.from_pretrained(model_id) - - # load audio sample - librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") - librispeech_dummy = librispeech_dummy.cast_column("audio", Audio(sampling_rate=processor.sampling_rate)) - audio_sample = librispeech_dummy[0]["audio"]["array"] - - # check on processor audio shape - inputs = processor( - raw_audio=audio_sample, - sampling_rate=processor.sampling_rate, - return_tensors="pt", - ).to(torch_device) - torch.equal(torch.tensor(inputs["input_values"].shape), EXPECTED_PREPROC_SHAPE) - - with torch.no_grad(): - # compare encoder loss - encoder_outputs = model.encode(inputs["input_values"]) - torch.testing.assert_close(EXPECTED_ENC_LOSS, encoder_outputs[0].squeeze().item(), rtol=1e-3, atol=1e-3) - - # compare quantizer outputs - quantizer_outputs = model.quantizer(encoder_outputs[1]) - torch.testing.assert_close( - EXPECTED_QUANT_CODES, quantizer_outputs[1][..., : EXPECTED_QUANT_CODES.shape[-1]], rtol=1e-6, atol=1e-6 - ) - torch.testing.assert_close( - EXPECTED_QUANT_CODEBOOK_LOSS, quantizer_outputs[4].squeeze().item(), rtol=1e-6, atol=1e-6 - ) - - # compare decoder outputs - decoded_outputs = model.decode(encoder_outputs[1]) - torch.testing.assert_close( - EXPECTED_DEC_OUTPUTS, - decoded_outputs["audio_values"][..., : EXPECTED_DEC_OUTPUTS.shape[-1]], - rtol=1e-3, - atol=1e-3, - ) - - # compare codec error / lossiness - codec_err = compute_rmse(decoded_outputs["audio_values"], inputs["input_values"]) - torch.testing.assert_close(EXPECTED_CODEC_ERROR, codec_err, rtol=1e-6, atol=1e-6) - - # make sure forward and decode gives same result - enc_dec = model(inputs["input_values"])[1] - torch.testing.assert_close(decoded_outputs["audio_values"], enc_dec, rtol=1e-6, atol=1e-6) - - def test_integration_44khz(self): - model_name = "dac_44khz" - - # expected values - EXPECTED_PREPROC_SHAPE = torch.tensor([1, 1, 258560]) - EXPECTED_ENC_LOSS = 23.78483772277832 - EXPECTED_QUANT_CODES = torch.tensor( + 1.0398e-03, 2.6915e-04, 1.3725e-04, 2.8598e-04, 2.5875e-04]]).to(torch_device), + "dac_44khz": torch.tensor([[ 8.3748e-04, 3.7760e-04, 4.7135e-04, 8.2829e-04, 1.3677e-03, + 1.7487e-03, 1.8883e-03, 1.7437e-03, 1.4828e-03, 1.2284e-03, + 1.0894e-03, 1.0442e-03, 1.0558e-03, 1.0136e-03, 8.4781e-04, + 4.8677e-04, -2.0375e-05, -5.2144e-04, -8.6839e-04, -9.8977e-04, + -8.0130e-04, -3.6122e-04, 1.8086e-04, 6.4340e-04, 9.1103e-04, + 9.6243e-04, 8.6814e-04, 7.7186e-04, 7.5613e-04, 8.1264e-04, + 9.0747e-04, 9.5464e-04, 9.5436e-04, 8.7902e-04, 7.6080e-04, + 6.2870e-04, 5.5878e-04, 5.7444e-04, 6.6622e-04, 7.9741e-04, + 8.7610e-04, 8.4571e-04, 6.7909e-04, 4.2059e-04, 1.5131e-04, + -7.1465e-05, -1.8646e-04, -1.8300e-04, -1.2542e-04, -7.1933e-05]]).to(torch_device), +} +EXPECTED_QUANT_CODEBOOK_LOSS = { + "dac_16khz": 20.5806350708007, + "dac_24khz": 22.581758499145508, + "dac_44khz": 16.2640438079834, +} +EXPECTED_CODEC_ERROR = { + "dac_16khz": 0.0038341842591762543, + "dac_24khz": 0.002570481738075614, + "dac_44khz": 0.0007429996621794999, +} +# -- test_batch +EXPECTED_PREPROC_SHAPE_BATCH = { + "dac_16khz": torch.tensor([2, 1, 113920]), + "dac_24khz": torch.tensor([2, 1, 170880]), + "dac_44khz": torch.tensor([2, 1, 313856]), +} +EXPECTED_ENC_LOSS_BATCH = { + "dac_16khz": 20.370271682739258, + "dac_24khz": 24.505210876464844, + "dac_44khz": 19.557754516601562, +} +EXPECTED_QUANT_CODES_BATCH = { + "dac_16khz": torch.tensor( + [ [ - [ - [332, 315, 105, 315, 616, 105, 494, 698, 315, 481, 330, 93, 105, 315, 105], - [670, 350, 249, 27, 232, 365, 311, 881, 186, 402, 311, 521, 527, 778, 254], - [569, 300, 361, 530, 1002, 419, 285, 501, 456, 471, 180, 615, 419, 491, 764], - [605, 436, 641, 291, 901, 556, 715, 780, 502, 410, 858, 125, 562, 174, 746], - [854, 706, 242, 294, 346, 88, 527, 961, 559, 664, 314, 963, 278, 90, 682], - [175, 152, 706, 884, 986, 457, 567, 176, 49, 535, 851, 417, 533, 349, 779], - [913, 710, 628, 162, 770, 254, 247, 6, 397, 264, 233, 704, 577, 111, 916], - [999, 693, 512, 884, 38, 223, 29, 744, 497, 123, 972, 120, 47, 301, 90], - [490, 163, 368, 507, 253, 283, 745, 65, 295, 935, 811, 587, 801, 255, 105], - ] - ] - ).to(torch_device) - # fmt: off - EXPECTED_DEC_OUTPUTS = torch.tensor([[ 8.3748e-04, 3.7760e-04, 4.7135e-04, 8.2829e-04, 1.3677e-03, - 1.7487e-03, 1.8883e-03, 1.7437e-03, 1.4828e-03, 1.2284e-03, - 1.0894e-03, 1.0442e-03, 1.0558e-03, 1.0136e-03, 8.4781e-04, - 4.8677e-04, -2.0375e-05, -5.2144e-04, -8.6839e-04, -9.8977e-04, - -8.0130e-04, -3.6122e-04, 1.8086e-04, 6.4340e-04, 9.1103e-04, - 9.6243e-04, 8.6814e-04, 7.7186e-04, 7.5613e-04, 8.1264e-04, - 9.0747e-04, 9.5464e-04, 9.5436e-04, 8.7902e-04, 7.6080e-04, - 6.2870e-04, 5.5878e-04, 5.7444e-04, 6.6622e-04, 7.9741e-04, - 8.7610e-04, 8.4571e-04, 6.7909e-04, 4.2059e-04, 1.5131e-04, - -7.1465e-05, -1.8646e-04, -1.8300e-04, -1.2542e-04, -7.1933e-05]]).to(torch_device) - # fmt: on - EXPECTED_QUANT_CODEBOOK_LOSS = 16.2640438079834 - EXPECTED_CODEC_ERROR = 0.0007429996621794999 - - # load model and processor - model_id = f"descript/{model_name}" - model = DacModel.from_pretrained(model_id, force_download=True).to(torch_device).eval() - processor = AutoProcessor.from_pretrained(model_id) - - # load audio sample - librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") - librispeech_dummy = librispeech_dummy.cast_column("audio", Audio(sampling_rate=processor.sampling_rate)) - audio_sample = librispeech_dummy[0]["audio"]["array"] - - # check on processor audio shape - inputs = processor( - raw_audio=audio_sample, - sampling_rate=processor.sampling_rate, - return_tensors="pt", - ).to(torch_device) - torch.equal(torch.tensor(inputs["input_values"].shape), EXPECTED_PREPROC_SHAPE) - - with torch.no_grad(): - # compare encoder loss - encoder_outputs = model.encode(inputs["input_values"]) - torch.testing.assert_close(EXPECTED_ENC_LOSS, encoder_outputs[0].squeeze().item(), rtol=1e-3, atol=1e-3) - - # compare quantizer outputs - quantizer_outputs = model.quantizer(encoder_outputs[1]) - torch.testing.assert_close( - EXPECTED_QUANT_CODES, quantizer_outputs[1][..., : EXPECTED_QUANT_CODES.shape[-1]], rtol=1e-6, atol=1e-6 - ) - torch.testing.assert_close( - EXPECTED_QUANT_CODEBOOK_LOSS, quantizer_outputs[4].squeeze().item(), rtol=1e-6, atol=1e-6 - ) - - # compare decoder outputs - decoded_outputs = model.decode(encoder_outputs[1]) - torch.testing.assert_close( - EXPECTED_DEC_OUTPUTS, - decoded_outputs["audio_values"][..., : EXPECTED_DEC_OUTPUTS.shape[-1]], - rtol=1e-3, - atol=1e-3, - ) - - # compare codec error / lossiness - codec_err = compute_rmse(decoded_outputs["audio_values"], inputs["input_values"]) - torch.testing.assert_close(EXPECTED_CODEC_ERROR, codec_err, rtol=1e-6, atol=1e-6) - - # make sure forward and decode gives same result - enc_dec = model(inputs["input_values"])[1] - torch.testing.assert_close(decoded_outputs["audio_values"], enc_dec, rtol=1e-6, atol=1e-6) - - def test_integration_batch_16khz(self): - model_name = "dac_16khz" - - # expected values - EXPECTED_PREPROC_SHAPE = torch.tensor([2, 1, 113920]) - EXPECTED_ENC_LOSS = 20.370271682739258 - EXPECTED_QUANT_CODES = torch.tensor( + [490, 664, 726, 166, 55, 379, 367, 664, 661, 726, 592, 301, 130, 198, 129], + [1020, 734, 23, 53, 134, 648, 549, 589, 790, 1000, 449, 271, 1021, 740, 36], + [701, 344, 955, 19, 927, 212, 212, 667, 212, 627, 453, 954, 777, 706, 496], + [526, 805, 444, 474, 870, 920, 394, 823, 814, 1021, 763, 677, 251, 485, 1021], + [721, 134, 280, 439, 287, 77, 175, 902, 973, 412, 739, 953, 130, 75, 543], + [675, 316, 285, 341, 783, 850, 131, 487, 701, 150, 749, 730, 900, 481, 498], + [377, 37, 237, 489, 55, 246, 427, 456, 755, 1011, 712, 631, 695, 576, 804], + [601, 557, 681, 52, 10, 299, 284, 216, 869, 276, 424, 364, 955, 41, 497], + [465, 553, 697, 59, 701, 195, 335, 225, 896, 804, 776, 928, 392, 192, 332], + [807, 306, 977, 801, 77, 172, 760, 747, 445, 38, 731, 31, 924, 724, 835], + [903, 561, 205, 421, 231, 873, 931, 361, 679, 854, 471, 884, 1011, 857, 248], + [490, 993, 122, 787, 178, 307, 141, 468, 652, 786, 879, 885, 226, 343, 501], + ], [ - [ - [490, 664, 726, 166, 55, 379, 367, 664, 661, 726, 592, 301, 130, 198, 129], - [1020, 734, 23, 53, 134, 648, 549, 589, 790, 1000, 449, 271, 1021, 740, 36], - [701, 344, 955, 19, 927, 212, 212, 667, 212, 627, 453, 954, 777, 706, 496], - [526, 805, 444, 474, 870, 920, 394, 823, 814, 1021, 763, 677, 251, 485, 1021], - [721, 134, 280, 439, 287, 77, 175, 902, 973, 412, 739, 953, 130, 75, 543], - [675, 316, 285, 341, 783, 850, 131, 487, 701, 150, 749, 730, 900, 481, 498], - [377, 37, 237, 489, 55, 246, 427, 456, 755, 1011, 712, 631, 695, 576, 804], - [601, 557, 681, 52, 10, 299, 284, 216, 869, 276, 424, 364, 955, 41, 497], - [465, 553, 697, 59, 701, 195, 335, 225, 896, 804, 776, 928, 392, 192, 332], - [807, 306, 977, 801, 77, 172, 760, 747, 445, 38, 731, 31, 924, 724, 835], - [903, 561, 205, 421, 231, 873, 931, 361, 679, 854, 471, 884, 1011, 857, 248], - [490, 993, 122, 787, 178, 307, 141, 468, 652, 786, 879, 885, 226, 343, 501], - ], - [ - [140, 320, 210, 489, 444, 388, 210, 73, 821, 1004, 388, 686, 405, 563, 407], - [725, 449, 802, 85, 36, 532, 620, 28, 620, 418, 146, 532, 418, 453, 565], - [695, 725, 600, 371, 829, 237, 911, 927, 181, 707, 306, 337, 254, 577, 289], - [51, 648, 186, 129, 781, 570, 737, 563, 400, 839, 674, 689, 544, 767, 577], - [1007, 234, 145, 966, 734, 748, 68, 272, 473, 973, 414, 586, 618, 6, 909], - [410, 566, 507, 756, 943, 736, 269, 349, 549, 320, 303, 729, 507, 741, 76], - [172, 102, 548, 714, 225, 723, 149, 423, 307, 527, 844, 102, 747, 76, 586], - [656, 144, 407, 245, 140, 409, 48, 197, 126, 418, 112, 674, 582, 916, 223], - [776, 971, 291, 781, 833, 296, 817, 261, 937, 467, 352, 463, 530, 804, 683], - [1009, 284, 427, 907, 900, 630, 279, 285, 878, 315, 734, 751, 337, 699, 966], - [389, 748, 203, 585, 609, 474, 555, 64, 154, 443, 16, 139, 905, 172, 86], - [884, 34, 477, 1013, 335, 306, 724, 202, 356, 199, 728, 552, 755, 223, 371], - ], - ] - ).to(torch_device) - # fmt: off - EXPECTED_DEC_OUTPUTS = torch.tensor([[-1.9181e-04, 1.9380e-04, 3.1524e-04, 2.0670e-04, -2.8026e-05, - -3.3014e-04, -4.6584e-04, -4.3935e-04, -2.8362e-04, 2.7245e-04, - 8.8112e-04, 1.1195e-03, 1.6224e-03, 1.9368e-03, 1.7803e-03, - 5.9601e-04, -4.4178e-04, -1.3736e-03, -1.9979e-03, -2.0477e-03, - -1.5583e-03, -4.1277e-04, 6.2742e-04, 1.2409e-03, 1.3380e-03, - 1.2884e-03, 6.0346e-04, 8.9812e-05, -6.1626e-04, -1.3760e-03, - -1.4970e-03, -9.8225e-04, -3.9102e-04, 5.3190e-04, 1.8696e-03, - 2.3731e-03, 2.1139e-03, 1.4220e-03, 7.3644e-04, -2.4944e-04, - -9.8294e-04, -1.3858e-03, -1.6684e-03, -1.0482e-03, -6.1834e-04, - -5.3312e-04, -2.1345e-04, 4.1917e-04, 7.7653e-04, 8.0206e-04], - [ 3.1081e-05, 4.7076e-04, -1.5066e-03, -1.7006e-05, -3.3131e-04, - -1.1786e-03, 8.2880e-04, -1.2492e-03, 4.6135e-04, -8.7780e-04, - -8.5493e-04, 3.2979e-04, 1.1218e-03, -1.8018e-03, 2.2795e-04, - 2.4981e-04, -3.1100e-03, 1.0356e-03, 1.1427e-03, 2.1378e-03, - -7.0038e-04, 1.6522e-03, -3.3599e-04, -2.3893e-03, -5.2286e-04, - 2.9462e-04, 1.2429e-03, -1.8078e-03, 3.3687e-03, 1.3336e-03, - -1.5815e-03, -1.5836e-04, -5.4054e-04, -7.2660e-04, -2.2980e-03, - -5.3254e-04, 1.4890e-03, -1.0853e-03, 1.0333e-03, 8.1283e-04, - -1.6996e-03, 6.0168e-05, -2.6916e-03, 3.7072e-04, -1.0729e-03, - 2.7891e-04, 3.3514e-03, -1.8029e-03, 5.5011e-04, -1.1905e-03]]).to(torch_device) - # fmt: on - EXPECTED_QUANT_CODEBOOK_LOSS = 20.61562156677246 - EXPECTED_CODEC_ERROR = 0.001973195234313607 - - # load model and processor - model_id = f"descript/{model_name}" - model = DacModel.from_pretrained(model_id).to(torch_device) - processor = AutoProcessor.from_pretrained(model_id) - - # load audio samples - librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") - librispeech_dummy = librispeech_dummy.cast_column("audio", Audio(sampling_rate=processor.sampling_rate)) - audio_samples = [np.array([audio_sample["array"]])[0] for audio_sample in librispeech_dummy[-2:]["audio"]] - - # check on processor audio shape - inputs = processor( - raw_audio=audio_samples, - sampling_rate=processor.sampling_rate, - truncation=False, - return_tensors="pt", - ).to(torch_device) - torch.equal(torch.tensor(inputs["input_values"].shape), EXPECTED_PREPROC_SHAPE) - - with torch.no_grad(): - # compare encoder loss - encoder_outputs = model.encode(inputs["input_values"]) - torch.testing.assert_close(EXPECTED_ENC_LOSS, encoder_outputs[0].mean().item(), rtol=1e-3, atol=1e-3) - - # compare quantizer outputs - quantizer_outputs = model.quantizer(encoder_outputs[1]) - torch.testing.assert_close( - EXPECTED_QUANT_CODES, quantizer_outputs[1][..., : EXPECTED_QUANT_CODES.shape[-1]], rtol=1e-6, atol=1e-6 - ) - torch.testing.assert_close( - EXPECTED_QUANT_CODEBOOK_LOSS, quantizer_outputs[4].mean().item(), rtol=1e-6, atol=1e-6 - ) - - # compare decoder outputs - decoded_outputs = model.decode(encoder_outputs[1]) - torch.testing.assert_close( - EXPECTED_DEC_OUTPUTS, - decoded_outputs["audio_values"][..., : EXPECTED_DEC_OUTPUTS.shape[-1]], - rtol=1e-3, - atol=1e-3, - ) - - # compare codec error / lossiness - codec_err = compute_rmse(decoded_outputs["audio_values"], inputs["input_values"]) - torch.testing.assert_close(EXPECTED_CODEC_ERROR, codec_err, rtol=1e-6, atol=1e-6) - - # make sure forward and decode gives same result - enc_dec = model(inputs["input_values"])[1] - torch.testing.assert_close(decoded_outputs["audio_values"], enc_dec, rtol=1e-6, atol=1e-6) - - def test_integration_batch_24khz(self): - model_name = "dac_24khz" - - # expected values - EXPECTED_PREPROC_SHAPE = torch.tensor([2, 1, 170880]) - EXPECTED_ENC_LOSS = 24.505210876464844 - EXPECTED_QUANT_CODES = torch.tensor( + [140, 320, 210, 489, 444, 388, 210, 73, 821, 1004, 388, 686, 405, 563, 407], + [725, 449, 802, 85, 36, 532, 620, 28, 620, 418, 146, 532, 418, 453, 565], + [695, 725, 600, 371, 829, 237, 911, 927, 181, 707, 306, 337, 254, 577, 289], + [51, 648, 186, 129, 781, 570, 737, 563, 400, 839, 674, 689, 544, 767, 577], + [1007, 234, 145, 966, 734, 748, 68, 272, 473, 973, 414, 586, 618, 6, 909], + [410, 566, 507, 756, 943, 736, 269, 349, 549, 320, 303, 729, 507, 741, 76], + [172, 102, 548, 714, 225, 723, 149, 423, 307, 527, 844, 102, 747, 76, 586], + [656, 144, 407, 245, 140, 409, 48, 197, 126, 418, 112, 674, 582, 916, 223], + [776, 971, 291, 781, 833, 296, 817, 261, 937, 467, 352, 463, 530, 804, 683], + [1009, 284, 427, 907, 900, 630, 279, 285, 878, 315, 734, 751, 337, 699, 966], + [389, 748, 203, 585, 609, 474, 555, 64, 154, 443, 16, 139, 905, 172, 86], + [884, 34, 477, 1013, 335, 306, 724, 202, 356, 199, 728, 552, 755, 223, 371], + ], + ] + ).to(torch_device), + "dac_24khz": torch.tensor( [ [ [234, 826, 826, 360, 204, 716, 766, 766, 360, 252, 919, 999, 360, 772, 668], @@ -882,9 +654,56 @@ def test_integration_batch_24khz(self): [588, 398, 945, 404, 804, 494, 572, 124, 47, 809, 775, 266, 9, 596, 435], ], ] - ).to(torch_device) - # fmt: off - EXPECTED_DEC_OUTPUTS = torch.tensor([[ 2.9611e-04, 5.0039e-05, -5.4961e-04, -7.9769e-04, -6.9696e-04, + ).to(torch_device), + "dac_44khz": torch.tensor( + [ + [ + [330, 315, 315, 619, 481, 315, 197, 315, 315, 105, 481, 481, 481, 481, 481], + [718, 1007, 309, 6, 906, 35, 402, 750, 396, 854, 962, 115, 609, 224, 329], + [417, 266, 150, 335, 300, 812, 325, 780, 1022, 605, 480, 342, 939, 150, 456], + [813, 811, 897, 334, 200, 852, 723, 497, 678, 922, 396, 333, 918, 548, 285], + [832, 315, 165, 106, 902, 326, 32, 572, 610, 170, 395, 223, 193, 807, 585], + [91, 941, 81, 684, 34, 340, 362, 946, 157, 640, 888, 215, 577, 483, 371], + [676, 859, 446, 664, 473, 815, 860, 640, 514, 385, 73, 201, 701, 78, 825], + [326, 426, 347, 970, 605, 997, 534, 111, 559, 538, 526, 208, 372, 709, 167], + [776, 315, 179, 232, 140, 456, 318, 155, 191, 674, 105, 992, 721, 406, 267], + ], + [ + [578, 592, 330, 330, 330, 330, 330, 801, 330, 330, 330, 698, 330, 330, 330], + [501, 204, 514, 215, 615, 580, 567, 684, 478, 905, 208, 32, 495, 84, 1000], + [141, 458, 489, 125, 691, 471, 522, 60, 978, 30, 125, 480, 424, 67, 1], + [908, 192, 865, 878, 137, 698, 965, 969, 565, 216, 535, 488, 441, 503, 181], + [850, 635, 993, 391, 500, 122, 365, 850, 905, 449, 586, 451, 840, 811, 797], + [307, 408, 497, 294, 24, 396, 417, 922, 161, 268, 100, 753, 778, 1014, 259], + [178, 918, 568, 28, 187, 375, 301, 889, 834, 406, 665, 7, 889, 909, 387], + [935, 566, 315, 13, 490, 37, 436, 801, 484, 62, 476, 551, 557, 232, 533], + [1017, 89, 585, 401, 13, 238, 744, 1017, 774, 872, 850, 468, 640, 833, 854], + ], + ] + ).to(torch_device), +} +EXPECTED_DEC_OUTPUTS_BATCH = { + "dac_16khz": torch.tensor([[-1.9181e-04, 1.9380e-04, 3.1524e-04, 2.0670e-04, -2.8026e-05, + -3.3014e-04, -4.6584e-04, -4.3935e-04, -2.8362e-04, 2.7245e-04, + 8.8112e-04, 1.1195e-03, 1.6224e-03, 1.9368e-03, 1.7803e-03, + 5.9601e-04, -4.4178e-04, -1.3736e-03, -1.9979e-03, -2.0477e-03, + -1.5583e-03, -4.1277e-04, 6.2742e-04, 1.2409e-03, 1.3380e-03, + 1.2884e-03, 6.0346e-04, 8.9812e-05, -6.1626e-04, -1.3760e-03, + -1.4970e-03, -9.8225e-04, -3.9102e-04, 5.3190e-04, 1.8696e-03, + 2.3731e-03, 2.1139e-03, 1.4220e-03, 7.3644e-04, -2.4944e-04, + -9.8294e-04, -1.3858e-03, -1.6684e-03, -1.0482e-03, -6.1834e-04, + -5.3312e-04, -2.1345e-04, 4.1917e-04, 7.7653e-04, 8.0206e-04], + [ 3.1081e-05, 4.7076e-04, -1.5066e-03, -1.7006e-05, -3.3131e-04, + -1.1786e-03, 8.2880e-04, -1.2492e-03, 4.6135e-04, -8.7780e-04, + -8.5493e-04, 3.2979e-04, 1.1218e-03, -1.8018e-03, 2.2795e-04, + 2.4981e-04, -3.1100e-03, 1.0356e-03, 1.1427e-03, 2.1378e-03, + -7.0038e-04, 1.6522e-03, -3.3599e-04, -2.3893e-03, -5.2286e-04, + 2.9462e-04, 1.2429e-03, -1.8078e-03, 3.3687e-03, 1.3336e-03, + -1.5815e-03, -1.5836e-04, -5.4054e-04, -7.2660e-04, -2.2980e-03, + -5.3254e-04, 1.4890e-03, -1.0853e-03, 1.0333e-03, 8.1283e-04, + -1.6996e-03, 6.0168e-05, -2.6916e-03, 3.7072e-04, -1.0729e-03, + 2.7891e-04, 3.3514e-03, -1.8029e-03, 5.5011e-04, -1.1905e-03]]).to(torch_device), + "dac_24khz": torch.tensor([[ 2.9611e-04, 5.0039e-05, -5.4961e-04, -7.9769e-04, -6.9696e-04, -5.6013e-04, -4.7665e-04, -3.8039e-04, -6.8090e-05, 6.5704e-05, 1.3205e-05, 1.3519e-04, 1.4002e-04, 4.3348e-05, 2.9029e-04, 5.1533e-04, 1.4072e-04, -1.8430e-04, 6.3313e-05, 4.6729e-04, @@ -903,118 +722,102 @@ def test_integration_batch_24khz(self): 2.3006e-04, -2.8686e-03, 1.2978e-03, 5.9192e-03, 7.3619e-04, -3.9734e-03, -2.6965e-04, 1.3701e-03, -1.7230e-03, -9.4332e-04, 4.2128e-04, -2.6123e-03, -1.8240e-03, 3.3554e-03, 1.7732e-03, - -3.2838e-03, -8.2577e-04, 3.1959e-03, 1.1458e-03, -2.4608e-04]]).to(torch_device) - # fmt: on - EXPECTED_QUANT_CODEBOOK_LOSS = 23.9102783203125 - EXPECTED_CODEC_ERROR = 0.0012980918399989605 + -3.2838e-03, -8.2577e-04, 3.1959e-03, 1.1458e-03, -2.4608e-04]]).to(torch_device), + "dac_44khz": torch.tensor([[-3.7834e-04, -1.0849e-04, 1.1856e-04, 2.6852e-04, 3.7313e-04, + 5.0301e-04, 6.4261e-04, 8.0797e-04, 9.0969e-04, 9.9720e-04, + 1.0807e-03, 1.1217e-03, 1.1229e-03, 1.1208e-03, 1.0862e-03, + 9.5098e-04, 7.5477e-04, 5.2319e-04, 2.7449e-04, 2.4389e-05, + -1.9138e-04, -3.2046e-04, -4.0629e-04, -4.4804e-04, -5.0271e-04, + -5.8324e-04, -6.6573e-04, -6.9545e-04, -6.8046e-04, -6.1640e-04, + -5.3542e-04, -4.2302e-04, -3.0829e-04, -1.8475e-04, -3.9555e-05, + 9.0104e-05, 1.9291e-04, 2.7445e-04, 3.6738e-04, 4.7454e-04, + 6.0626e-04, 7.5514e-04, 8.5390e-04, 8.8749e-04, 8.5473e-04, + 7.5550e-04, 6.2329e-04, 4.9771e-04, 3.8809e-04, 3.0741e-04], + [ 1.1130e-04, 4.6536e-04, 1.0524e-04, -6.1460e-04, -1.1777e-03, + -1.0661e-03, -3.7962e-04, 5.3627e-04, 1.0481e-03, 8.7734e-04, + 1.3513e-04, -6.6297e-04, -9.5284e-04, -4.6333e-04, 5.5780e-04, + 1.4526e-03, 1.6264e-03, 1.0852e-03, 3.3766e-04, 1.0960e-04, + 7.7973e-04, 2.0579e-03, 3.0206e-03, 2.9674e-03, 1.8141e-03, + 3.1059e-04, -5.7140e-04, -3.4386e-04, 4.8406e-04, 8.6931e-04, + 2.1745e-05, -1.7647e-03, -3.2787e-03, -3.3368e-03, -1.7466e-03, + 4.3745e-04, 1.6595e-03, 1.1171e-03, -6.3018e-04, -2.0979e-03, + -2.1286e-03, -6.8752e-04, 1.1514e-03, 2.1590e-03, 1.9204e-03, + 1.0659e-03, 5.3295e-04, 6.6817e-04, 9.2716e-04, 5.3240e-04]]).to(torch_device), +} +EXPECTED_QUANT_CODEBOOK_LOSS_BATCH = { + "dac_16khz": 20.61562156677246, + "dac_24khz": 23.9102783203125, + "dac_44khz": 16.177066802978516, +} +EXPECTED_CODEC_ERROR_BATCH = { + "dac_16khz": 0.001973195234313607, + "dac_24khz": 0.0012980918399989605, + "dac_44khz": 0.00037737112143076956, +} +# fmt: on + +@slow +@require_torch +class DacIntegrationTest(unittest.TestCase): + @parameterized.expand([(model_name,) for model_name in EXPECTED_PREPROC_SHAPE.keys()]) + def test_integration(self, model_name): # load model and processor model_id = f"descript/{model_name}" - model = DacModel.from_pretrained(model_id).to(torch_device) + model = DacModel.from_pretrained(model_id, force_download=True).to(torch_device).eval() processor = AutoProcessor.from_pretrained(model_id) - # load audio samples + # load audio sample librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") librispeech_dummy = librispeech_dummy.cast_column("audio", Audio(sampling_rate=processor.sampling_rate)) - audio_samples = [np.array([audio_sample["array"]])[0] for audio_sample in librispeech_dummy[-2:]["audio"]] + audio_sample = librispeech_dummy[0]["audio"]["array"] # check on processor audio shape inputs = processor( - raw_audio=audio_samples, + raw_audio=audio_sample, sampling_rate=processor.sampling_rate, - truncation=False, return_tensors="pt", ).to(torch_device) - torch.equal(torch.tensor(inputs["input_values"].shape), EXPECTED_PREPROC_SHAPE) + torch.equal(torch.tensor(inputs["input_values"].shape), EXPECTED_PREPROC_SHAPE[model_name]) with torch.no_grad(): # compare encoder loss encoder_outputs = model.encode(inputs["input_values"]) - torch.testing.assert_close(EXPECTED_ENC_LOSS, encoder_outputs[0].mean().item(), rtol=1e-3, atol=1e-3) + torch.testing.assert_close( + EXPECTED_ENC_LOSS[model_name], encoder_outputs[0].squeeze().item(), rtol=1e-3, atol=1e-3 + ) # compare quantizer outputs quantizer_outputs = model.quantizer(encoder_outputs[1]) torch.testing.assert_close( - EXPECTED_QUANT_CODES, quantizer_outputs[1][..., : EXPECTED_QUANT_CODES.shape[-1]], rtol=1e-6, atol=1e-6 + EXPECTED_QUANT_CODES[model_name], + quantizer_outputs[1][..., : EXPECTED_QUANT_CODES[model_name].shape[-1]], + rtol=1e-6, + atol=1e-6, ) torch.testing.assert_close( - EXPECTED_QUANT_CODEBOOK_LOSS, quantizer_outputs[4].mean().item(), rtol=1e-6, atol=1e-6 + EXPECTED_QUANT_CODEBOOK_LOSS[model_name], quantizer_outputs[4].squeeze().item(), rtol=1e-6, atol=1e-6 ) # compare decoder outputs decoded_outputs = model.decode(encoder_outputs[1]) torch.testing.assert_close( - EXPECTED_DEC_OUTPUTS, - decoded_outputs["audio_values"][..., : EXPECTED_DEC_OUTPUTS.shape[-1]], + EXPECTED_DEC_OUTPUTS[model_name], + decoded_outputs["audio_values"][..., : EXPECTED_DEC_OUTPUTS[model_name].shape[-1]], rtol=1e-3, atol=1e-3, ) # compare codec error / lossiness codec_err = compute_rmse(decoded_outputs["audio_values"], inputs["input_values"]) - torch.testing.assert_close(EXPECTED_CODEC_ERROR, codec_err, rtol=1e-6, atol=1e-6) + torch.testing.assert_close(EXPECTED_CODEC_ERROR[model_name], codec_err, rtol=1e-6, atol=1e-6) # make sure forward and decode gives same result enc_dec = model(inputs["input_values"])[1] torch.testing.assert_close(decoded_outputs["audio_values"], enc_dec, rtol=1e-6, atol=1e-6) - def test_integration_batch_44khz(self): - model_name = "dac_44khz" - - # expected values - EXPECTED_PREPROC_SHAPE = torch.tensor([2, 1, 313856]) - EXPECTED_ENC_LOSS = 19.557754516601562 - EXPECTED_QUANT_CODES = torch.tensor( - [ - [ - [330, 315, 315, 619, 481, 315, 197, 315, 315, 105, 481, 481, 481, 481, 481], - [718, 1007, 309, 6, 906, 35, 402, 750, 396, 854, 962, 115, 609, 224, 329], - [417, 266, 150, 335, 300, 812, 325, 780, 1022, 605, 480, 342, 939, 150, 456], - [813, 811, 897, 334, 200, 852, 723, 497, 678, 922, 396, 333, 918, 548, 285], - [832, 315, 165, 106, 902, 326, 32, 572, 610, 170, 395, 223, 193, 807, 585], - [91, 941, 81, 684, 34, 340, 362, 946, 157, 640, 888, 215, 577, 483, 371], - [676, 859, 446, 664, 473, 815, 860, 640, 514, 385, 73, 201, 701, 78, 825], - [326, 426, 347, 970, 605, 997, 534, 111, 559, 538, 526, 208, 372, 709, 167], - [776, 315, 179, 232, 140, 456, 318, 155, 191, 674, 105, 992, 721, 406, 267], - ], - [ - [578, 592, 330, 330, 330, 330, 330, 801, 330, 330, 330, 698, 330, 330, 330], - [501, 204, 514, 215, 615, 580, 567, 684, 478, 905, 208, 32, 495, 84, 1000], - [141, 458, 489, 125, 691, 471, 522, 60, 978, 30, 125, 480, 424, 67, 1], - [908, 192, 865, 878, 137, 698, 965, 969, 565, 216, 535, 488, 441, 503, 181], - [850, 635, 993, 391, 500, 122, 365, 850, 905, 449, 586, 451, 840, 811, 797], - [307, 408, 497, 294, 24, 396, 417, 922, 161, 268, 100, 753, 778, 1014, 259], - [178, 918, 568, 28, 187, 375, 301, 889, 834, 406, 665, 7, 889, 909, 387], - [935, 566, 315, 13, 490, 37, 436, 801, 484, 62, 476, 551, 557, 232, 533], - [1017, 89, 585, 401, 13, 238, 744, 1017, 774, 872, 850, 468, 640, 833, 854], - ], - ] - ).to(torch_device) - # fmt: off - EXPECTED_DEC_OUTPUTS = torch.tensor([[-3.7834e-04, -1.0849e-04, 1.1856e-04, 2.6852e-04, 3.7313e-04, - 5.0301e-04, 6.4261e-04, 8.0797e-04, 9.0969e-04, 9.9720e-04, - 1.0807e-03, 1.1217e-03, 1.1229e-03, 1.1208e-03, 1.0862e-03, - 9.5098e-04, 7.5477e-04, 5.2319e-04, 2.7449e-04, 2.4389e-05, - -1.9138e-04, -3.2046e-04, -4.0629e-04, -4.4804e-04, -5.0271e-04, - -5.8324e-04, -6.6573e-04, -6.9545e-04, -6.8046e-04, -6.1640e-04, - -5.3542e-04, -4.2302e-04, -3.0829e-04, -1.8475e-04, -3.9555e-05, - 9.0104e-05, 1.9291e-04, 2.7445e-04, 3.6738e-04, 4.7454e-04, - 6.0626e-04, 7.5514e-04, 8.5390e-04, 8.8749e-04, 8.5473e-04, - 7.5550e-04, 6.2329e-04, 4.9771e-04, 3.8809e-04, 3.0741e-04], - [ 1.1130e-04, 4.6536e-04, 1.0524e-04, -6.1460e-04, -1.1777e-03, - -1.0661e-03, -3.7962e-04, 5.3627e-04, 1.0481e-03, 8.7734e-04, - 1.3513e-04, -6.6297e-04, -9.5284e-04, -4.6333e-04, 5.5780e-04, - 1.4526e-03, 1.6264e-03, 1.0852e-03, 3.3766e-04, 1.0960e-04, - 7.7973e-04, 2.0579e-03, 3.0206e-03, 2.9674e-03, 1.8141e-03, - 3.1059e-04, -5.7140e-04, -3.4386e-04, 4.8406e-04, 8.6931e-04, - 2.1745e-05, -1.7647e-03, -3.2787e-03, -3.3368e-03, -1.7466e-03, - 4.3745e-04, 1.6595e-03, 1.1171e-03, -6.3018e-04, -2.0979e-03, - -2.1286e-03, -6.8752e-04, 1.1514e-03, 2.1590e-03, 1.9204e-03, - 1.0659e-03, 5.3295e-04, 6.6817e-04, 9.2716e-04, 5.3240e-04]]).to(torch_device) - # fmt: on - EXPECTED_QUANT_CODEBOOK_LOSS = 16.177066802978516 - EXPECTED_CODEC_ERROR = 0.00037737112143076956 - + @parameterized.expand([(model_name,) for model_name in EXPECTED_PREPROC_SHAPE_BATCH.keys()]) + def test_integration_batch(self, model_name): # load model and processor model_id = f"descript/{model_name}" model = DacModel.from_pretrained(model_id).to(torch_device) @@ -1032,34 +835,42 @@ def test_integration_batch_44khz(self): truncation=False, return_tensors="pt", ).to(torch_device) - torch.equal(torch.tensor(inputs["input_values"].shape), EXPECTED_PREPROC_SHAPE) + torch.equal(torch.tensor(inputs["input_values"].shape), EXPECTED_PREPROC_SHAPE_BATCH[model_name]) with torch.no_grad(): # compare encoder loss encoder_outputs = model.encode(inputs["input_values"]) - torch.testing.assert_close(EXPECTED_ENC_LOSS, encoder_outputs[0].mean().item(), rtol=1e-3, atol=1e-3) + torch.testing.assert_close( + EXPECTED_ENC_LOSS_BATCH[model_name], encoder_outputs[0].mean().item(), rtol=1e-3, atol=1e-3 + ) # compare quantizer outputs quantizer_outputs = model.quantizer(encoder_outputs[1]) torch.testing.assert_close( - EXPECTED_QUANT_CODES, quantizer_outputs[1][..., : EXPECTED_QUANT_CODES.shape[-1]], rtol=1e-6, atol=1e-6 + EXPECTED_QUANT_CODES_BATCH[model_name], + quantizer_outputs[1][..., : EXPECTED_QUANT_CODES_BATCH[model_name].shape[-1]], + rtol=1e-6, + atol=1e-6, ) torch.testing.assert_close( - EXPECTED_QUANT_CODEBOOK_LOSS, quantizer_outputs[4].mean().item(), rtol=1e-6, atol=1e-6 + EXPECTED_QUANT_CODEBOOK_LOSS_BATCH[model_name], + quantizer_outputs[4].mean().item(), + rtol=1e-6, + atol=1e-6, ) # compare decoder outputs decoded_outputs = model.decode(encoder_outputs[1]) torch.testing.assert_close( - EXPECTED_DEC_OUTPUTS, - decoded_outputs["audio_values"][..., : EXPECTED_DEC_OUTPUTS.shape[-1]], + EXPECTED_DEC_OUTPUTS_BATCH[model_name], + decoded_outputs["audio_values"][..., : EXPECTED_DEC_OUTPUTS_BATCH[model_name].shape[-1]], rtol=1e-3, atol=1e-3, ) # compare codec error / lossiness codec_err = compute_rmse(decoded_outputs["audio_values"], inputs["input_values"]) - torch.testing.assert_close(EXPECTED_CODEC_ERROR, codec_err, rtol=1e-6, atol=1e-6) + torch.testing.assert_close(EXPECTED_CODEC_ERROR_BATCH[model_name], codec_err, rtol=1e-6, atol=1e-6) # make sure forward and decode gives same result enc_dec = model(inputs["input_values"])[1] From 7d27ea10fc4f668a4683af358fea48ddf8a88d3e Mon Sep 17 00:00:00 2001 From: Eric B Date: Wed, 23 Jul 2025 16:37:07 +0000 Subject: [PATCH 8/8] Set expected values to GitHub runners. --- tests/models/dac/test_modeling_dac.py | 741 +++++++++++++++----------- 1 file changed, 426 insertions(+), 315 deletions(-) diff --git a/tests/models/dac/test_modeling_dac.py b/tests/models/dac/test_modeling_dac.py index b512d9c0c664..93f61f418626 100644 --- a/tests/models/dac/test_modeling_dac.py +++ b/tests/models/dac/test_modeling_dac.py @@ -397,8 +397,8 @@ def compute_rmse(arr1, arr2): Integration tests for DAC. Code for reproducing expected outputs can be found here: -- test_integration: https://gist.github.com/ebezzam/bb315efa7a416db6336a6b2a2d424ffa#file-dac_integration_single-py -- test_batch: https://gist.github.com/ebezzam/bb315efa7a416db6336a6b2a2d424ffa#file-dac_integration-py +- test_integration: https://gist.github.com/ebezzam/bb315efa7a416db6336a6b2a2d424ffa#file-test_dac-py +- test_batch: https://gist.github.com/ebezzam/bb315efa7a416db6336a6b2a2d424ffa#file-test_dac_batch-py See https://github.com/huggingface/transformers/pull/39313 for reason behind large tolerance between for encoder and decoder outputs (1e-3). In summary, original model uses weight normalization, while Transformers does not. This @@ -417,127 +417,156 @@ def compute_rmse(arr1, arr2): "dac_44khz": torch.tensor([1, 1, 258560]), } EXPECTED_ENC_LOSS = { - "dac_16khz": 24.84908103942871, - "dac_24khz": 28.112096786499023, - "dac_44khz": 23.78483772277832, + "dac_16khz": 24.889205932617188, + "dac_24khz": 27.661380767822266, + "dac_44khz": 23.87179183959961, } EXPECTED_QUANT_CODES = { - "dac_16khz": torch.tensor( - [ - [ - [804, 25, 977, 52, 68, 867, 388, 653, 315, 706, 301, 305, 140, 25, 40], - [77, 955, 532, 601, 431, 375, 967, 56, 54, 261, 871, 552, 735, 341, 228], - [355, 908, 77, 927, 617, 443, 790, 149, 403, 707, 511, 226, 995, 883, 644], - [184, 162, 611, 54, 211, 890, 906, 253, 677, 1007, 302, 577, 378, 330, 778], - [763, 322, 6, 321, 116, 228, 911, 865, 1000, 234, 6, 901, 10, 174, 895], - [454, 1, 622, 622, 487, 668, 749, 833, 382, 900, 372, 959, 232, 418, 964], - [203, 43, 173, 307, 961, 593, 318, 1011, 386, 949, 343, 899, 536, 824, 38], - [82, 810, 692, 83, 131, 866, 483, 362, 519, 531, 853, 121, 1010, 512, 710], - [1003, 691, 530, 460, 827, 903, 81, 76, 629, 298, 168, 177, 368, 613, 762], - [571, 752, 544, 394, 198, 479, 952, 437, 222, 992, 934, 316, 741, 123, 538], - [686, 421, 393, 635, 246, 330, 908, 384, 962, 873, 92, 254, 912, 496, 83], - [721, 977, 148, 204, 993, 660, 176, 395, 901, 323, 342, 849, 474, 8, 513], - ] - ] - ).to(torch_device), - "dac_24khz": torch.tensor( - [ - [ - [160, 360, 826, 204, 239, 360, 90, 160, 851, 234, 252, 690, 360, 160, 665], - [189, 496, 717, 74, 847, 692, 496, 549, 847, 78, 669, 440, 9, 243, 117], - [497, 562, 161, 827, 408, 330, 562, 152, 80, 84, 320, 745, 1023, 544, 944], - [261, 140, 271, 843, 179, 239, 150, 211, 788, 343, 333, 760, 217, 243, 623], - [487, 846, 919, 947, 417, 787, 140, 186, 567, 129, 633, 328, 927, 932, 901], - [862, 953, 929, 184, 85, 433, 545, 672, 382, 666, 694, 382, 572, 38, 134], - [835, 260, 975, 144, 621, 800, 341, 1017, 28, 889, 521, 287, 805, 231, 474], - [470, 803, 475, 208, 574, 679, 382, 71, 413, 79, 571, 330, 408, 759, 79], - [452, 272, 257, 101, 76, 540, 378, 933, 83, 350, 334, 539, 808, 975, 860], - [450, 704, 839, 811, 705, 304, 895, 340, 979, 53, 573, 80, 241, 110, 571], - [801, 523, 138, 939, 729, 417, 588, 9, 501, 304, 820, 271, 497, 719, 141], - [579, 741, 42, 811, 561, 630, 528, 945, 1009, 637, 109, 702, 1005, 911, 748], - [96, 581, 853, 817, 256, 592, 23, 1014, 309, 3, 846, 780, 704, 481, 138], - [162, 193, 808, 498, 128, 949, 103, 928, 277, 599, 375, 718, 893, 388, 532], - [318, 498, 5, 696, 953, 1018, 442, 97, 573, 179, 850, 353, 548, 1002, 279], - [962, 911, 712, 684, 214, 240, 290, 467, 812, 588, 232, 588, 922, 101, 768], - [969, 785, 514, 168, 106, 423, 37, 683, 882, 657, 516, 819, 535, 50, 988], - [299, 914, 787, 584, 582, 449, 444, 366, 666, 721, 1022, 1015, 700, 752, 710], - [926, 669, 287, 618, 806, 309, 368, 502, 704, 573, 319, 562, 355, 994, 873], - [513, 75, 447, 290, 16, 370, 185, 43, 1015, 346, 450, 24, 490, 299, 231], - [616, 506, 867, 444, 648, 987, 6, 301, 556, 128, 898, 352, 657, 616, 798], - [382, 353, 420, 424, 107, 256, 163, 113, 832, 247, 415, 541, 893, 922, 918], - [135, 775, 363, 14, 603, 311, 346, 722, 746, 207, 695, 48, 821, 428, 53], - [626, 72, 220, 524, 256, 736, 86, 64, 618, 780, 607, 799, 734, 506, 868], - [310, 913, 13, 707, 177, 19, 856, 463, 400, 141, 959, 904, 910, 818, 734], - [948, 105, 835, 842, 802, 117, 340, 466, 774, 726, 389, 599, 558, 491, 420], - [916, 440, 167, 177, 842, 450, 744, 820, 906, 739, 702, 158, 745, 546, 636], - [135, 675, 544, 64, 955, 904, 1017, 862, 167, 564, 362, 1023, 774, 78, 914], - [216, 218, 494, 28, 605, 962, 212, 649, 249, 710, 83, 94, 437, 613, 54], - [611, 109, 743, 56, 493, 294, 364, 514, 980, 524, 474, 978, 35, 724, 767], - [719, 752, 343, 171, 776, 414, 217, 656, 717, 73, 955, 516, 582, 559, 241], - [821, 641, 740, 272, 468, 847, 699, 842, 20, 330, 216, 703, 581, 306, 137], - ] - ] - ).to(torch_device), - "dac_44khz": torch.tensor([[[ 332, 315, 105, 315, 616, 105, 494, 698, 315, 481, 330, - 93, 105, 315, 105], - [ 670, 350, 249, 27, 232, 365, 311, 881, 186, 402, 311, - 521, 527, 778, 254], - [ 569, 300, 361, 530, 1002, 419, 285, 501, 456, 471, 180, - 615, 419, 491, 764], - [ 605, 436, 641, 291, 901, 556, 715, 780, 502, 410, 858, - 125, 562, 174, 746], - [ 854, 706, 242, 294, 346, 88, 527, 961, 559, 664, 314, - 963, 278, 90, 682], - [ 175, 152, 706, 884, 986, 457, 567, 176, 49, 535, 851, - 417, 533, 349, 779], - [ 913, 710, 628, 162, 770, 254, 247, 6, 397, 264, 233, - 704, 577, 111, 916], - [ 999, 693, 512, 884, 38, 223, 29, 744, 497, 123, 972, - 120, 47, 301, 90], - [ 490, 163, 368, 507, 253, 283, 745, 65, 295, 935, 811, - 587, 801, 255, 105]]]).to(torch_device), + "dac_16khz": torch.tensor([[[ 804, 25, 536, 52, 68, 867, 388, 653, 484, 706, 301, + 305, 752, 25, 40], + [ 77, 955, 134, 601, 431, 375, 967, 56, 684, 261, 871, + 552, 232, 341, 228], + [ 355, 701, 172, 927, 617, 765, 790, 149, 117, 707, 511, + 226, 254, 883, 644], + [ 184, 85, 828, 54, 211, 1007, 906, 253, 406, 1007, 302, + 577, 644, 330, 601], + [ 763, 865, 586, 321, 116, 357, 911, 865, 234, 234, 6, + 630, 6, 174, 895], + [ 454, 241, 67, 622, 487, 426, 749, 833, 639, 900, 372, + 481, 622, 418, 964], + [ 203, 609, 730, 307, 961, 609, 318, 1011, 747, 949, 343, + 548, 657, 824, 21], + [ 82, 92, 692, 83, 131, 866, 483, 362, 596, 531, 853, + 121, 404, 512, 373], + [1003, 260, 431, 460, 827, 927, 81, 76, 444, 298, 168, + 673, 466, 613, 383], + [ 571, 203, 594, 394, 198, 560, 952, 437, 343, 992, 934, + 316, 497, 123, 305], + [ 686, 715, 393, 635, 246, 716, 908, 384, 98, 873, 92, + 878, 592, 496, 104], + [ 721, 502, 606, 204, 993, 428, 176, 395, 617, 323, 342, + 530, 226, 8, 600]]]).to(torch_device), + "dac_24khz": torch.tensor([[[ 252, 851, 919, 204, 239, 360, 90, 103, 851, 876, 160, + 160, 103, 234, 665], + [ 908, 658, 479, 556, 847, 265, 496, 32, 847, 773, 623, + 375, 9, 497, 117], + [ 385, 278, 221, 778, 408, 330, 562, 215, 80, 84, 320, + 728, 931, 470, 944], + [ 383, 134, 271, 494, 179, 304, 150, 804, 788, 780, 356, + 416, 297, 903, 623], + [ 487, 263, 414, 947, 608, 810, 140, 74, 372, 129, 417, + 592, 671, 479, 901], + [ 692, 953, 508, 359, 85, 396, 545, 375, 382, 382, 511, + 382, 383, 643, 134], + [ 652, 213, 210, 385, 326, 899, 341, 925, 908, 68, 216, + 21, 568, 1008, 635], + [ 938, 848, 570, 515, 574, 693, 382, 71, 42, 742, 603, + 109, 193, 629, 79], + [ 847, 101, 874, 894, 384, 832, 378, 658, 1, 487, 976, + 993, 932, 886, 860], + [ 220, 344, 307, 69, 705, 974, 895, 438, 8, 806, 573, + 690, 543, 709, 303], + [ 394, 594, 144, 10, 832, 4, 588, 659, 501, 218, 351, + 861, 915, 148, 141], + [ 447, 763, 930, 894, 196, 668, 528, 862, 70, 598, 136, + 119, 395, 474, 1000], + [ 677, 178, 637, 874, 471, 113, 23, 534, 333, 6, 821, + 777, 635, 932, 475], + [ 932, 345, 436, 335, 555, 355, 103, 436, 277, 816, 400, + 356, 73, 23, 450], + [ 592, 402, 177, 31, 693, 459, 442, 193, 615, 940, 927, + 917, 676, 327, 658], + [ 192, 458, 540, 808, 626, 340, 290, 700, 190, 345, 381, + 137, 280, 611, 794], + [ 834, 5, 522, 685, 146, 754, 37, 580, 78, 2, 1008, + 808, 281, 375, 366], + [ 892, 790, 948, 662, 355, 437, 444, 790, 450, 850, 316, + 529, 385, 480, 178], + [ 36, 696, 125, 753, 143, 562, 368, 824, 491, 507, 892, + 880, 355, 152, 253], + [ 934, 829, 457, 261, 668, 1014, 185, 464, 78, 332, 374, + 869, 530, 67, 884], + [ 567, 914, 334, 38, 313, 744, 6, 210, 489, 867, 200, + 799, 540, 318, 706], + [ 178, 882, 776, 992, 651, 800, 163, 470, 687, 906, 508, + 260, 36, 783, 64], + [ 169, 66, 179, 711, 598, 938, 346, 251, 773, 108, 873, + 813, 479, 425, 669], + [ 981, 692, 143, 589, 224, 282, 86, 712, 689, 907, 586, + 595, 444, 265, 198], + [ 856, 540, 556, 302, 883, 96, 856, 560, 529, 91, 707, + 286, 142, 553, 252], + [ 103, 868, 879, 779, 882, 34, 340, 603, 186, 808, 397, + 673, 919, 989, 626], + [ 933, 215, 775, 747, 842, 836, 744, 272, 604, 202, 288, + 164, 242, 542, 207], + [ 969, 373, 999, 524, 927, 879, 1017, 14, 526, 385, 478, + 690, 347, 589, 10], + [ 716, 503, 781, 119, 176, 316, 212, 836, 850, 26, 685, + 973, 606, 796, 593], + [ 164, 418, 929, 523, 571, 917, 364, 964, 480, 1021, 0, + 994, 876, 887, 379], + [ 416, 957, 819, 478, 640, 479, 217, 842, 926, 771, 129, + 537, 899, 680, 547], + [ 623, 596, 332, 517, 947, 376, 699, 918, 1012, 995, 858, + 516, 56, 43, 268]]]).to(torch_device), + "dac_44khz": torch.tensor([[[ 698, 315, 105, 315, 330, 105, 105, 698, 315, 481, 330, + 93, 629, 315, 105], + [ 30, 232, 249, 881, 962, 365, 56, 881, 186, 402, 311, + 521, 558, 778, 254], + [1022, 22, 361, 491, 233, 419, 909, 456, 456, 471, 420, + 569, 455, 491, 16], + [ 599, 143, 641, 352, 40, 556, 860, 780, 138, 137, 304, + 563, 863, 174, 370], + [ 485, 350, 242, 555, 174, 581, 666, 744, 559, 810, 127, + 558, 453, 90, 124], + [ 851, 423, 706, 178, 36, 564, 650, 539, 733, 720, 18, + 265, 619, 545, 581], + [ 755, 891, 628, 674, 724, 764, 420, 51, 566, 315, 178, + 881, 461, 111, 675], + [ 52, 995, 512, 139, 538, 666, 1017, 868, 619, 0, 449, + 1005, 982, 106, 139], + [ 357, 180, 368, 892, 856, 567, 960, 148, 36, 708, 945, + 285, 531, 331, 440]]]).to(torch_device), } EXPECTED_DEC_OUTPUTS = { - "dac_16khz": torch.tensor([[ 7.2661e-05, 5.9626e-04, 1.0609e-03, 1.4515e-03, 1.6704e-03, - 1.0837e-03, 4.6979e-04, -1.3811e-04, -2.7733e-04, 2.0613e-04, - 4.0715e-04, 8.4999e-04, 1.7112e-03, 2.7275e-03, 2.5560e-03, - 1.6202e-03, 1.4603e-03, 1.1447e-03, 7.4274e-04, 7.6758e-04, - 1.5931e-03, 2.5598e-03, 2.6844e-03, 2.9216e-03, 3.6430e-03, - 3.0532e-03, 2.1169e-03, 2.3657e-03, 2.0313e-03, 8.8282e-04, - -1.6314e-04, 2.0697e-05, 9.0119e-04, 1.5815e-03, 2.1719e-03, - 2.2010e-03, 1.4089e-03, -9.8639e-05, -7.1111e-04, -2.1185e-04, - 3.3837e-04, 5.2177e-04, 1.0538e-03, 2.2637e-03, 1.9972e-03, - 1.6396e-03, 1.6282e-03, 1.1689e-03, 2.7550e-04, -4.4859e-04]]).to(torch_device), - "dac_24khz": torch.tensor([[ 4.2660e-04, 4.0129e-04, 1.5403e-04, 5.0874e-05, 2.9436e-04, - 1.0682e-03, 1.9777e-03, 1.9081e-03, 1.5145e-03, 1.2959e-03, - 1.1858e-03, 8.6308e-04, 7.6199e-05, -6.2039e-04, -2.8909e-04, - 7.2902e-04, 9.6803e-04, 3.5680e-04, -1.4637e-04, 7.8926e-05, - 7.9285e-04, 1.3313e-03, 1.1692e-03, 5.7410e-04, 7.0640e-04, - 1.5462e-03, 1.9182e-03, 1.3498e-03, 5.0153e-04, 1.5142e-04, - 2.1018e-04, 4.2771e-04, 7.4621e-04, 1.1082e-03, 1.5289e-03, - 1.9526e-03, 2.3434e-03, 2.6424e-03, 2.8369e-03, 2.7632e-03, - 2.3256e-03, 1.8973e-03, 1.8191e-03, 1.9133e-03, 1.7674e-03, - 1.0398e-03, 2.6915e-04, 1.3725e-04, 2.8598e-04, 2.5875e-04]]).to(torch_device), - "dac_44khz": torch.tensor([[ 8.3748e-04, 3.7760e-04, 4.7135e-04, 8.2829e-04, 1.3677e-03, - 1.7487e-03, 1.8883e-03, 1.7437e-03, 1.4828e-03, 1.2284e-03, - 1.0894e-03, 1.0442e-03, 1.0558e-03, 1.0136e-03, 8.4781e-04, - 4.8677e-04, -2.0375e-05, -5.2144e-04, -8.6839e-04, -9.8977e-04, - -8.0130e-04, -3.6122e-04, 1.8086e-04, 6.4340e-04, 9.1103e-04, - 9.6243e-04, 8.6814e-04, 7.7186e-04, 7.5613e-04, 8.1264e-04, - 9.0747e-04, 9.5464e-04, 9.5436e-04, 8.7902e-04, 7.6080e-04, - 6.2870e-04, 5.5878e-04, 5.7444e-04, 6.6622e-04, 7.9741e-04, - 8.7610e-04, 8.4571e-04, 6.7909e-04, 4.2059e-04, 1.5131e-04, - -7.1465e-05, -1.8646e-04, -1.8300e-04, -1.2542e-04, -7.1933e-05]]).to(torch_device), + "dac_16khz": torch.tensor([[ 0.0002, 0.0007, 0.0012, 0.0015, 0.0017, 0.0011, 0.0004, -0.0002, + -0.0003, 0.0002, 0.0006, 0.0012, 0.0020, 0.0029, 0.0026, 0.0015, + 0.0015, 0.0014, 0.0010, 0.0011, 0.0019, 0.0026, 0.0028, 0.0032, + 0.0040, 0.0031, 0.0022, 0.0025, 0.0020, 0.0010, 0.0001, 0.0001, + 0.0007, 0.0016, 0.0024, 0.0024, 0.0017, 0.0002, -0.0006, -0.0002, + 0.0003, 0.0006, 0.0011, 0.0023, 0.0020, 0.0016, 0.0015, 0.0012, + 0.0005, -0.0003]]).to(torch_device), + "dac_24khz": torch.tensor([[ 1.8275e-04, 1.8167e-04, -3.1626e-05, -6.4468e-05, 2.1254e-04, + 8.4161e-04, 1.5839e-03, 1.6693e-03, 1.5439e-03, 1.3923e-03, + 1.1167e-03, 6.2019e-04, -1.2014e-04, -5.7301e-04, -1.7829e-04, + 6.0980e-04, 6.7130e-04, 1.6166e-04, -6.9366e-06, 3.1507e-04, + 6.3976e-04, 7.1702e-04, 6.3391e-04, 5.7553e-04, 1.1151e-03, + 1.9032e-03, 1.9737e-03, 1.2812e-03, 5.6187e-04, 3.9073e-04, + 3.8875e-04, 3.0256e-04, 3.8140e-04, 7.6331e-04, 1.3098e-03, + 1.7796e-03, 2.1707e-03, 2.5330e-03, 2.9214e-03, 3.0557e-03, + 2.7402e-03, 2.2303e-03, 1.8196e-03, 1.6796e-03, 1.6199e-03, + 1.0460e-03, 3.5502e-04, 2.8095e-04, 3.8291e-04, 2.2683e-04]]).to(torch_device), + "dac_44khz": torch.tensor([[ 1.3282e-03, 1.4784e-03, 1.6923e-03, 1.8359e-03, 1.8795e-03, + 1.9519e-03, 1.9145e-03, 1.7839e-03, 1.5222e-03, 1.2423e-03, + 9.9689e-04, 8.4000e-04, 7.6656e-04, 7.7500e-04, 7.7684e-04, + 6.9986e-04, 5.3156e-04, 3.2828e-04, 1.7750e-04, 1.6440e-04, + 2.9904e-04, 5.4582e-04, 8.2008e-04, 1.0400e-03, 1.1518e-03, + 1.1718e-03, 1.1220e-03, 1.0717e-03, 1.0772e-03, 1.1534e-03, + 1.3257e-03, 1.5572e-03, 1.7794e-03, 1.9112e-03, 1.9242e-03, + 1.7837e-03, 1.5347e-03, 1.2386e-03, 9.3313e-04, 6.4671e-04, + 3.5892e-04, 8.4733e-05, -1.6930e-04, -3.9932e-04, -5.8345e-04, + -6.9382e-04, -7.0792e-04, -5.6856e-04, -2.6751e-04, 1.5914e-04]]).to(torch_device), } EXPECTED_QUANT_CODEBOOK_LOSS = { - "dac_16khz": 20.5806350708007, - "dac_24khz": 22.581758499145508, - "dac_44khz": 16.2640438079834, + "dac_16khz": 20.62909698486328, + "dac_24khz": 22.47393798828125, + "dac_44khz": 16.229290008544922, } EXPECTED_CODEC_ERROR = { - "dac_16khz": 0.0038341842591762543, - "dac_24khz": 0.002570481738075614, - "dac_44khz": 0.0007429996621794999, + "dac_16khz": 0.003831653157249093, + "dac_24khz": 0.0025609051808714867, + "dac_44khz": 0.0007433777209371328, } # -- test_batch EXPECTED_PREPROC_SHAPE_BATCH = { @@ -546,213 +575,295 @@ def compute_rmse(arr1, arr2): "dac_44khz": torch.tensor([2, 1, 313856]), } EXPECTED_ENC_LOSS_BATCH = { - "dac_16khz": 20.370271682739258, - "dac_24khz": 24.505210876464844, - "dac_44khz": 19.557754516601562, + "dac_16khz": 20.3460636138916, + "dac_24khz": 23.54486846923828, + "dac_44khz": 19.58145523071289, } EXPECTED_QUANT_CODES_BATCH = { - "dac_16khz": torch.tensor( - [ - [ - [490, 664, 726, 166, 55, 379, 367, 664, 661, 726, 592, 301, 130, 198, 129], - [1020, 734, 23, 53, 134, 648, 549, 589, 790, 1000, 449, 271, 1021, 740, 36], - [701, 344, 955, 19, 927, 212, 212, 667, 212, 627, 453, 954, 777, 706, 496], - [526, 805, 444, 474, 870, 920, 394, 823, 814, 1021, 763, 677, 251, 485, 1021], - [721, 134, 280, 439, 287, 77, 175, 902, 973, 412, 739, 953, 130, 75, 543], - [675, 316, 285, 341, 783, 850, 131, 487, 701, 150, 749, 730, 900, 481, 498], - [377, 37, 237, 489, 55, 246, 427, 456, 755, 1011, 712, 631, 695, 576, 804], - [601, 557, 681, 52, 10, 299, 284, 216, 869, 276, 424, 364, 955, 41, 497], - [465, 553, 697, 59, 701, 195, 335, 225, 896, 804, 776, 928, 392, 192, 332], - [807, 306, 977, 801, 77, 172, 760, 747, 445, 38, 731, 31, 924, 724, 835], - [903, 561, 205, 421, 231, 873, 931, 361, 679, 854, 471, 884, 1011, 857, 248], - [490, 993, 122, 787, 178, 307, 141, 468, 652, 786, 879, 885, 226, 343, 501], - ], - [ - [140, 320, 210, 489, 444, 388, 210, 73, 821, 1004, 388, 686, 405, 563, 407], - [725, 449, 802, 85, 36, 532, 620, 28, 620, 418, 146, 532, 418, 453, 565], - [695, 725, 600, 371, 829, 237, 911, 927, 181, 707, 306, 337, 254, 577, 289], - [51, 648, 186, 129, 781, 570, 737, 563, 400, 839, 674, 689, 544, 767, 577], - [1007, 234, 145, 966, 734, 748, 68, 272, 473, 973, 414, 586, 618, 6, 909], - [410, 566, 507, 756, 943, 736, 269, 349, 549, 320, 303, 729, 507, 741, 76], - [172, 102, 548, 714, 225, 723, 149, 423, 307, 527, 844, 102, 747, 76, 586], - [656, 144, 407, 245, 140, 409, 48, 197, 126, 418, 112, 674, 582, 916, 223], - [776, 971, 291, 781, 833, 296, 817, 261, 937, 467, 352, 463, 530, 804, 683], - [1009, 284, 427, 907, 900, 630, 279, 285, 878, 315, 734, 751, 337, 699, 966], - [389, 748, 203, 585, 609, 474, 555, 64, 154, 443, 16, 139, 905, 172, 86], - [884, 34, 477, 1013, 335, 306, 724, 202, 356, 199, 728, 552, 755, 223, 371], - ], - ] - ).to(torch_device), - "dac_24khz": torch.tensor( - [ - [ - [234, 826, 826, 360, 204, 716, 766, 766, 360, 252, 919, 999, 360, 772, 668], - [117, 496, 229, 267, 9, 663, 1002, 629, 756, 372, 781, 496, 23, 780, 781], - [559, 712, 401, 423, 290, 27, 674, 340, 762, 410, 877, 558, 516, 5, 197], - [914, 8, 186, 766, 622, 547, 724, 101, 355, 634, 252, 517, 986, 348, 449], - [636, 148, 671, 232, 374, 24, 925, 118, 561, 760, 748, 964, 117, 126, 589], - [950, 825, 985, 600, 771, 949, 24, 629, 284, 398, 361, 893, 345, 840, 721], - [18, 263, 904, 778, 348, 839, 603, 447, 468, 117, 840, 631, 574, 898, 711], - [455, 359, 188, 148, 878, 246, 376, 509, 906, 759, 799, 991, 797, 833, 116], - [786, 275, 343, 492, 578, 952, 854, 833, 720, 730, 949, 72, 630, 305, 943], - [476, 696, 254, 283, 913, 407, 45, 408, 387, 904, 207, 206, 931, 621, 115], - [517, 73, 1019, 268, 238, 754, 188, 670, 923, 930, 110, 992, 870, 210, 953], - [311, 31, 371, 819, 949, 52, 650, 557, 573, 388, 222, 510, 908, 343, 559], - [405, 355, 520, 986, 179, 171, 49, 349, 706, 16, 439, 700, 704, 852, 759], - [854, 745, 982, 727, 466, 71, 530, 23, 125, 639, 254, 450, 397, 171, 766], - [863, 439, 415, 421, 463, 789, 551, 717, 641, 161, 882, 246, 576, 238, 464], - [331, 416, 322, 794, 416, 187, 689, 880, 29, 570, 283, 92, 310, 327, 748], - [149, 338, 105, 63, 848, 995, 824, 497, 792, 375, 745, 321, 914, 597, 101], - [588, 361, 77, 311, 483, 461, 889, 132, 724, 352, 187, 338, 72, 235, 761], - [434, 882, 522, 153, 462, 62, 725, 265, 597, 9, 161, 613, 576, 654, 1006], - [697, 927, 617, 1011, 561, 19, 181, 402, 830, 318, 248, 521, 645, 386, 111], - [787, 604, 809, 223, 21, 569, 817, 550, 253, 484, 718, 292, 358, 704, 556], - [821, 935, 743, 973, 982, 801, 799, 614, 988, 186, 337, 606, 166, 488, 116], - [789, 555, 32, 57, 671, 538, 712, 732, 524, 52, 869, 646, 91, 766, 516], - [481, 31, 464, 774, 756, 612, 619, 771, 372, 615, 697, 337, 28, 891, 706], - [293, 676, 468, 515, 777, 479, 625, 882, 725, 975, 491, 599, 594, 563, 235], - [170, 373, 462, 102, 335, 616, 880, 542, 989, 68, 154, 918, 716, 897, 33], - [228, 480, 610, 886, 733, 16, 924, 366, 490, 417, 790, 909, 88, 344, 351], - [243, 987, 683, 814, 104, 47, 173, 591, 376, 570, 181, 556, 955, 771, 464], - [1010, 62, 490, 536, 440, 174, 263, 849, 934, 544, 231, 908, 586, 558, 670], - [757, 604, 828, 519, 968, 862, 62, 182, 971, 627, 655, 518, 153, 666, 903], - [720, 192, 470, 262, 404, 920, 755, 138, 614, 245, 458, 182, 920, 398, 761], - [570, 527, 276, 994, 124, 174, 561, 150, 139, 988, 935, 327, 174, 1020, 383], - ], - [ - [851, 110, 668, 103, 826, 360, 919, 160, 826, 160, 204, 110, 360, 910, 160], - [325, 846, 245, 722, 664, 594, 1002, 130, 859, 261, 260, 496, 846, 146, 23], - [529, 465, 354, 408, 597, 710, 450, 460, 980, 1011, 577, 392, 631, 453, 861], - [344, 645, 255, 327, 101, 1017, 474, 296, 513, 903, 363, 823, 85, 83, 760], - [415, 208, 656, 878, 751, 798, 240, 326, 137, 393, 511, 253, 369, 110, 590], - [514, 639, 623, 632, 163, 77, 911, 168, 811, 314, 928, 365, 886, 571, 692], - [768, 700, 408, 359, 937, 540, 1018, 570, 401, 746, 541, 166, 813, 492, 659], - [141, 802, 880, 55, 557, 13, 440, 550, 250, 640, 92, 691, 671, 266, 707], - [539, 706, 445, 343, 984, 280, 667, 414, 525, 987, 272, 727, 247, 834, 383], - [668, 94, 376, 890, 975, 337, 178, 839, 449, 863, 980, 35, 929, 913, 661], - [489, 430, 874, 230, 318, 714, 732, 491, 460, 681, 897, 124, 653, 990, 203], - [352, 625, 110, 636, 618, 691, 976, 249, 165, 584, 92, 487, 940, 907, 83], - [168, 518, 471, 139, 693, 101, 761, 185, 415, 338, 330, 557, 1013, 530, 163], - [282, 355, 539, 464, 725, 808, 607, 691, 374, 502, 898, 960, 822, 680, 233], - [599, 15, 236, 918, 475, 45, 16, 631, 409, 662, 961, 868, 589, 820, 943], - [398, 238, 897, 395, 502, 972, 125, 219, 748, 1000, 310, 664, 371, 867, 163], - [415, 685, 758, 452, 615, 491, 298, 645, 180, 659, 137, 895, 158, 780, 803], - [14, 138, 789, 848, 203, 360, 66, 589, 842, 597, 296, 763, 157, 259, 176], - [432, 65, 342, 488, 399, 259, 869, 214, 490, 975, 349, 894, 691, 87, 850], - [20, 524, 1019, 333, 926, 632, 41, 1002, 75, 282, 319, 426, 513, 368, 241], - [252, 292, 705, 578, 937, 800, 861, 548, 732, 57, 914, 493, 415, 76, 626], - [1004, 799, 467, 438, 656, 397, 547, 882, 873, 675, 900, 360, 941, 25, 63], - [695, 7, 446, 799, 900, 821, 859, 760, 740, 398, 236, 936, 974, 305, 27], - [977, 58, 979, 294, 514, 525, 768, 381, 920, 147, 264, 675, 6, 318, 619], - [539, 315, 574, 938, 208, 454, 869, 220, 1007, 964, 906, 133, 247, 14, 357], - [555, 968, 337, 468, 767, 805, 991, 266, 620, 653, 882, 720, 592, 920, 1016], - [320, 824, 133, 631, 861, 176, 607, 5, 686, 187, 186, 982, 453, 479, 849], - [247, 191, 164, 884, 292, 289, 579, 996, 332, 480, 965, 856, 628, 522, 652], - [142, 388, 533, 548, 600, 1, 504, 663, 140, 246, 1, 80, 555, 739, 672], - [909, 361, 285, 925, 509, 358, 219, 725, 476, 626, 651, 511, 3, 456, 620], - [731, 421, 150, 573, 598, 936, 796, 57, 442, 821, 162, 359, 912, 139, 659], - [588, 398, 945, 404, 804, 494, 572, 124, 47, 809, 775, 266, 9, 596, 435], - ], - ] - ).to(torch_device), - "dac_44khz": torch.tensor( - [ - [ - [330, 315, 315, 619, 481, 315, 197, 315, 315, 105, 481, 481, 481, 481, 481], - [718, 1007, 309, 6, 906, 35, 402, 750, 396, 854, 962, 115, 609, 224, 329], - [417, 266, 150, 335, 300, 812, 325, 780, 1022, 605, 480, 342, 939, 150, 456], - [813, 811, 897, 334, 200, 852, 723, 497, 678, 922, 396, 333, 918, 548, 285], - [832, 315, 165, 106, 902, 326, 32, 572, 610, 170, 395, 223, 193, 807, 585], - [91, 941, 81, 684, 34, 340, 362, 946, 157, 640, 888, 215, 577, 483, 371], - [676, 859, 446, 664, 473, 815, 860, 640, 514, 385, 73, 201, 701, 78, 825], - [326, 426, 347, 970, 605, 997, 534, 111, 559, 538, 526, 208, 372, 709, 167], - [776, 315, 179, 232, 140, 456, 318, 155, 191, 674, 105, 992, 721, 406, 267], - ], - [ - [578, 592, 330, 330, 330, 330, 330, 801, 330, 330, 330, 698, 330, 330, 330], - [501, 204, 514, 215, 615, 580, 567, 684, 478, 905, 208, 32, 495, 84, 1000], - [141, 458, 489, 125, 691, 471, 522, 60, 978, 30, 125, 480, 424, 67, 1], - [908, 192, 865, 878, 137, 698, 965, 969, 565, 216, 535, 488, 441, 503, 181], - [850, 635, 993, 391, 500, 122, 365, 850, 905, 449, 586, 451, 840, 811, 797], - [307, 408, 497, 294, 24, 396, 417, 922, 161, 268, 100, 753, 778, 1014, 259], - [178, 918, 568, 28, 187, 375, 301, 889, 834, 406, 665, 7, 889, 909, 387], - [935, 566, 315, 13, 490, 37, 436, 801, 484, 62, 476, 551, 557, 232, 533], - [1017, 89, 585, 401, 13, 238, 744, 1017, 774, 872, 850, 468, 640, 833, 854], - ], - ] - ).to(torch_device), + "dac_16khz": torch.tensor([[[ 490, 664, 726, 166, 55, 379, 367, 664, 661, 726, 592, + 301, 130, 198, 129], + [1020, 734, 23, 53, 134, 648, 549, 589, 790, 1000, 420, + 271, 1021, 740, 36], + [ 701, 344, 955, 19, 927, 212, 212, 667, 212, 627, 837, + 954, 777, 706, 496], + [ 526, 805, 444, 474, 870, 920, 394, 823, 814, 1021, 319, + 677, 251, 485, 1021], + [ 721, 134, 280, 439, 287, 77, 175, 902, 973, 412, 548, + 953, 130, 75, 543], + [ 675, 316, 285, 341, 783, 850, 131, 487, 701, 150, 674, + 730, 900, 481, 498], + [ 377, 37, 237, 489, 55, 246, 427, 456, 755, 1011, 171, + 631, 695, 576, 804], + [ 601, 557, 681, 52, 10, 299, 284, 216, 869, 276, 907, + 364, 955, 41, 497], + [ 465, 553, 697, 59, 701, 195, 335, 225, 896, 804, 240, + 928, 392, 192, 332], + [ 807, 306, 977, 801, 77, 172, 760, 747, 445, 38, 395, + 31, 924, 724, 835], + [ 903, 561, 205, 421, 231, 873, 931, 361, 679, 854, 248, + 884, 1011, 857, 248], + [ 490, 993, 122, 787, 178, 307, 141, 468, 652, 786, 959, + 885, 226, 343, 501]], + [[ 140, 320, 140, 489, 444, 320, 210, 73, 821, 1004, 388, + 686, 405, 563, 517], + [ 725, 449, 715, 85, 761, 532, 620, 28, 620, 418, 146, + 532, 418, 453, 565], + [ 695, 725, 994, 371, 829, 1008, 911, 927, 181, 707, 306, + 337, 254, 577, 857], + [ 51, 648, 474, 129, 781, 968, 737, 718, 400, 839, 674, + 689, 544, 767, 540], + [1007, 234, 865, 966, 734, 748, 68, 454, 473, 973, 414, + 586, 618, 6, 612], + [ 410, 566, 692, 756, 307, 1008, 269, 743, 549, 320, 303, + 729, 507, 741, 362], + [ 172, 102, 959, 714, 292, 173, 149, 308, 307, 527, 844, + 102, 747, 76, 295], + [ 656, 144, 994, 245, 686, 925, 48, 356, 126, 418, 112, + 674, 582, 916, 296], + [ 776, 971, 967, 781, 174, 688, 817, 278, 937, 467, 352, + 463, 530, 804, 619], + [1009, 284, 966, 907, 397, 875, 279, 643, 878, 315, 734, + 751, 337, 699, 382], + [ 389, 748, 50, 585, 69, 565, 555, 931, 154, 443, 16, + 139, 905, 172, 361], + [ 884, 34, 945, 1013, 212, 493, 724, 775, 356, 199, 728, + 552, 755, 223, 378]]]).to(torch_device), + "dac_24khz": torch.tensor([[[ 234, 322, 826, 360, 204, 208, 766, 826, 458, 322, 919, + 999, 360, 772, 204], + [ 780, 201, 229, 497, 9, 663, 1002, 243, 556, 300, 781, + 496, 77, 780, 781], + [ 714, 342, 401, 553, 728, 196, 181, 109, 949, 528, 39, + 558, 180, 5, 197], + [ 112, 408, 186, 933, 543, 829, 724, 1001, 425, 39, 163, + 517, 986, 348, 653], + [1001, 207, 671, 551, 742, 231, 870, 577, 353, 1016, 259, + 282, 247, 126, 63], + [ 924, 59, 799, 739, 771, 568, 280, 673, 639, 1002, 35, + 143, 270, 749, 571], + [ 310, 982, 904, 666, 819, 67, 161, 373, 945, 871, 597, + 466, 388, 898, 584], + [ 69, 357, 188, 969, 213, 162, 376, 35, 638, 657, 731, + 991, 625, 833, 801], + [ 333, 885, 343, 621, 752, 319, 292, 389, 947, 776, 78, + 585, 193, 834, 622], + [ 958, 144, 680, 819, 303, 832, 56, 683, 366, 996, 609, + 784, 305, 621, 36], + [ 561, 766, 69, 768, 219, 126, 945, 798, 568, 554, 115, + 245, 31, 384, 167], + [ 727, 684, 371, 447, 50, 309, 407, 121, 839, 1019, 816, + 423, 604, 489, 738], + [ 598, 490, 578, 353, 517, 283, 927, 432, 464, 608, 927, + 32, 240, 852, 326], + [ 337, 226, 450, 862, 549, 799, 887, 925, 392, 841, 539, + 633, 351, 7, 386], + [ 668, 497, 586, 937, 516, 898, 768, 1014, 420, 173, 116, + 602, 786, 940, 56], + [ 575, 927, 322, 885, 367, 175, 691, 337, 21, 796, 317, + 826, 109, 604, 54], + [ 50, 854, 118, 231, 567, 332, 827, 422, 339, 958, 529, + 63, 992, 597, 428], + [ 480, 619, 605, 598, 912, 1012, 365, 926, 538, 915, 22, + 675, 460, 667, 255], + [ 578, 373, 355, 92, 920, 454, 979, 536, 645, 442, 783, + 956, 693, 457, 842], + [1019, 0, 998, 958, 159, 159, 332, 94, 886, 1, 455, + 981, 418, 758, 358], + [ 698, 843, 1008, 626, 776, 342, 53, 518, 636, 997, 22, + 36, 997, 12, 374], + [ 904, 408, 802, 456, 645, 899, 15, 447, 857, 265, 185, + 983, 1018, 282, 607], + [ 459, 467, 461, 358, 389, 792, 385, 678, 50, 888, 63, + 3, 792, 588, 972], + [ 877, 180, 212, 656, 60, 73, 261, 644, 755, 496, 137, + 948, 879, 361, 863], + [ 172, 588, 948, 452, 297, 1009, 49, 426, 853, 843, 249, + 957, 1008, 730, 860], + [ 677, 125, 519, 975, 686, 404, 321, 310, 38, 138, 424, + 457, 98, 736, 1004], + [ 784, 262, 289, 299, 1022, 170, 865, 869, 951, 839, 100, + 301, 828, 62, 511], + [ 726, 693, 235, 208, 668, 777, 284, 61, 376, 203, 784, + 101, 344, 587, 736], + [ 851, 83, 484, 951, 839, 180, 801, 525, 890, 373, 206, + 467, 524, 572, 614], + [ 48, 297, 674, 895, 740, 179, 782, 242, 721, 815, 85, + 74, 179, 650, 554], + [ 336, 166, 203, 1021, 89, 991, 410, 518, 1019, 742, 235, + 810, 782, 623, 176], + [ 110, 999, 360, 260, 278, 582, 921, 470, 242, 667, 21, + 463, 335, 566, 897]], + [[ 851, 160, 851, 877, 665, 110, 581, 936, 826, 910, 110, + 110, 160, 103, 160], + [ 325, 342, 722, 260, 549, 617, 508, 0, 221, 631, 846, + 446, 457, 124, 23], + [ 529, 921, 767, 408, 628, 980, 80, 460, 255, 209, 768, + 255, 773, 759, 861], + [ 344, 600, 255, 271, 402, 228, 805, 662, 497, 94, 852, + 337, 812, 140, 760], + [ 415, 423, 322, 337, 599, 703, 520, 332, 377, 539, 511, + 511, 124, 110, 638], + [ 514, 501, 660, 1014, 678, 77, 563, 793, 608, 464, 405, + 24, 630, 176, 692], + [ 768, 497, 276, 353, 968, 214, 527, 447, 680, 746, 281, + 972, 681, 708, 907], + [ 461, 802, 81, 411, 271, 186, 530, 670, 952, 1001, 828, + 270, 568, 74, 606], + [ 539, 178, 451, 343, 235, 336, 346, 272, 992, 958, 924, + 91, 606, 408, 104], + [ 668, 629, 817, 872, 526, 369, 889, 265, 297, 140, 229, + 240, 360, 811, 189], + [ 973, 419, 164, 855, 767, 168, 378, 968, 698, 10, 610, + 297, 236, 976, 668], + [ 162, 291, 66, 67, 749, 433, 428, 573, 421, 467, 202, + 838, 125, 452, 873], + [ 5, 949, 393, 322, 563, 679, 306, 467, 779, 326, 624, + 27, 447, 142, 965], + [ 981, 105, 116, 51, 674, 584, 351, 322, 81, 320, 476, + 527, 668, 212, 944], + [ 813, 156, 1013, 675, 964, 788, 137, 475, 436, 109, 400, + 899, 599, 820, 746], + [ 398, 21, 63, 720, 304, 1017, 1009, 889, 475, 619, 684, + 571, 430, 642, 69], + [ 405, 140, 531, 526, 657, 991, 624, 1014, 818, 256, 300, + 1013, 255, 567, 0], + [ 153, 469, 23, 553, 210, 812, 327, 527, 251, 406, 38, + 893, 974, 777, 58], + [ 324, 399, 4, 563, 703, 499, 256, 136, 112, 164, 979, + 524, 975, 596, 520], + [ 792, 511, 224, 225, 229, 424, 436, 124, 27, 267, 806, + 8, 657, 914, 808], + [ 595, 491, 993, 961, 722, 756, 937, 723, 195, 991, 436, + 392, 464, 837, 604], + [ 918, 647, 931, 658, 594, 677, 106, 194, 466, 92, 728, + 575, 302, 864, 930], + [ 672, 685, 997, 36, 344, 956, 260, 781, 108, 348, 755, + 142, 65, 754, 284], + [ 327, 987, 859, 525, 115, 551, 384, 202, 10, 669, 84, + 481, 193, 392, 246], + [ 206, 432, 1018, 954, 534, 350, 902, 30, 428, 701, 913, + 408, 456, 135, 726], + [ 483, 953, 684, 843, 478, 406, 931, 189, 426, 596, 459, + 34, 306, 140, 22], + [ 508, 990, 988, 862, 265, 437, 277, 876, 874, 301, 759, + 759, 989, 85, 292], + [ 586, 487, 860, 525, 90, 436, 15, 475, 625, 714, 697, + 180, 453, 279, 524], + [ 639, 844, 513, 487, 853, 185, 690, 664, 688, 842, 439, + 1002, 468, 745, 298], + [ 551, 764, 383, 422, 768, 760, 244, 332, 722, 567, 352, + 654, 579, 1019, 787], + [ 207, 365, 766, 423, 792, 470, 582, 978, 692, 408, 573, + 19, 314, 471, 587], + [ 776, 854, 529, 113, 927, 187, 362, 791, 131, 570, 559, + 61, 763, 83, 1015]]]).to(torch_device), + "dac_44khz": torch.tensor([[[ 330, 315, 315, 619, 481, 315, 197, 315, 315, 105, 481, + 315, 481, 481, 481], + [ 718, 1007, 929, 6, 906, 944, 402, 750, 675, 854, 336, + 426, 609, 356, 329], + [ 417, 266, 697, 456, 300, 941, 325, 923, 1022, 605, 991, + 7, 939, 329, 456], + [ 813, 811, 271, 148, 184, 838, 723, 497, 330, 922, 12, + 333, 918, 963, 285], + [ 832, 307, 635, 794, 334, 114, 32, 505, 344, 170, 161, + 907, 193, 180, 585], + [ 91, 941, 912, 1001, 507, 486, 362, 1006, 228, 640, 760, + 215, 577, 633, 371], + [ 676, 27, 903, 472, 473, 219, 860, 477, 969, 385, 533, + 911, 701, 241, 825], + [ 326, 399, 116, 443, 605, 373, 534, 199, 748, 538, 516, + 983, 372, 565, 167], + [ 776, 843, 185, 326, 723, 756, 318, 34, 818, 674, 728, + 554, 721, 369, 267]], + [[ 578, 698, 330, 330, 330, 578, 330, 801, 330, 330, 330, + 330, 330, 330, 330], + [ 171, 503, 725, 215, 814, 861, 139, 684, 880, 905, 937, + 418, 359, 190, 823], + [ 141, 482, 780, 489, 845, 499, 59, 480, 296, 30, 631, + 540, 399, 23, 385], + [ 402, 837, 216, 116, 535, 456, 1006, 969, 994, 125, 1011, + 285, 851, 832, 197], + [ 46, 950, 728, 645, 850, 839, 527, 850, 81, 205, 590, + 166, 22, 148, 402], + [ 98, 758, 474, 941, 217, 667, 681, 109, 719, 824, 162, + 160, 329, 627, 716], + [ 999, 228, 752, 639, 404, 333, 993, 177, 888, 158, 644, + 221, 1011, 302, 79], + [ 669, 535, 164, 665, 809, 798, 448, 800, 123, 936, 639, + 361, 353, 402, 160], + [ 345, 355, 940, 261, 71, 946, 750, 120, 565, 164, 813, + 976, 946, 50, 516]]]).to(torch_device), } EXPECTED_DEC_OUTPUTS_BATCH = { - "dac_16khz": torch.tensor([[-1.9181e-04, 1.9380e-04, 3.1524e-04, 2.0670e-04, -2.8026e-05, - -3.3014e-04, -4.6584e-04, -4.3935e-04, -2.8362e-04, 2.7245e-04, - 8.8112e-04, 1.1195e-03, 1.6224e-03, 1.9368e-03, 1.7803e-03, - 5.9601e-04, -4.4178e-04, -1.3736e-03, -1.9979e-03, -2.0477e-03, - -1.5583e-03, -4.1277e-04, 6.2742e-04, 1.2409e-03, 1.3380e-03, - 1.2884e-03, 6.0346e-04, 8.9812e-05, -6.1626e-04, -1.3760e-03, - -1.4970e-03, -9.8225e-04, -3.9102e-04, 5.3190e-04, 1.8696e-03, - 2.3731e-03, 2.1139e-03, 1.4220e-03, 7.3644e-04, -2.4944e-04, - -9.8294e-04, -1.3858e-03, -1.6684e-03, -1.0482e-03, -6.1834e-04, - -5.3312e-04, -2.1345e-04, 4.1917e-04, 7.7653e-04, 8.0206e-04], - [ 3.1081e-05, 4.7076e-04, -1.5066e-03, -1.7006e-05, -3.3131e-04, - -1.1786e-03, 8.2880e-04, -1.2492e-03, 4.6135e-04, -8.7780e-04, - -8.5493e-04, 3.2979e-04, 1.1218e-03, -1.8018e-03, 2.2795e-04, - 2.4981e-04, -3.1100e-03, 1.0356e-03, 1.1427e-03, 2.1378e-03, - -7.0038e-04, 1.6522e-03, -3.3599e-04, -2.3893e-03, -5.2286e-04, - 2.9462e-04, 1.2429e-03, -1.8078e-03, 3.3687e-03, 1.3336e-03, - -1.5815e-03, -1.5836e-04, -5.4054e-04, -7.2660e-04, -2.2980e-03, - -5.3254e-04, 1.4890e-03, -1.0853e-03, 1.0333e-03, 8.1283e-04, - -1.6996e-03, 6.0168e-05, -2.6916e-03, 3.7072e-04, -1.0729e-03, - 2.7891e-04, 3.3514e-03, -1.8029e-03, 5.5011e-04, -1.1905e-03]]).to(torch_device), - "dac_24khz": torch.tensor([[ 2.9611e-04, 5.0039e-05, -5.4961e-04, -7.9769e-04, -6.9696e-04, - -5.6013e-04, -4.7665e-04, -3.8039e-04, -6.8090e-05, 6.5704e-05, - 1.3205e-05, 1.3519e-04, 1.4002e-04, 4.3348e-05, 2.9029e-04, - 5.1533e-04, 1.4072e-04, -1.8430e-04, 6.3313e-05, 4.6729e-04, - 5.5076e-04, 5.6079e-04, 5.6557e-04, 3.2839e-04, 2.6326e-04, - 3.9028e-04, 3.1820e-04, 5.1251e-05, -7.0745e-05, -2.0471e-04, - -7.0736e-04, -1.2458e-03, -1.4124e-03, -1.3991e-03, -1.4890e-03, - -1.4013e-03, -1.0092e-03, -5.4982e-04, -3.5847e-05, 5.3150e-04, - 9.2390e-04, 1.0131e-03, 1.0362e-03, 1.0253e-03, 8.1528e-04, - 3.7854e-04, -1.3280e-05, -2.6982e-04, -4.8256e-04, -7.0810e-04], - [-4.3881e-04, 3.3771e-04, 1.0076e-03, 1.2748e-03, 1.4132e-03, - 1.0326e-03, 7.5779e-04, 5.3942e-04, -2.8545e-04, -2.0953e-03, - -2.2058e-03, 1.1152e-04, 5.6744e-04, -1.7912e-03, -1.4614e-03, - 1.8420e-03, 1.5202e-03, -1.0541e-03, 1.9058e-04, 1.3378e-03, - -2.0335e-03, -2.5633e-03, 2.4959e-03, 2.4356e-03, -3.1333e-03, - -2.8208e-03, 9.7969e-04, -1.0972e-03, -3.0217e-03, 4.1109e-04, - 2.3006e-04, -2.8686e-03, 1.2978e-03, 5.9192e-03, 7.3619e-04, - -3.9734e-03, -2.6965e-04, 1.3701e-03, -1.7230e-03, -9.4332e-04, - 4.2128e-04, -2.6123e-03, -1.8240e-03, 3.3554e-03, 1.7732e-03, - -3.2838e-03, -8.2577e-04, 3.1959e-03, 1.1458e-03, -2.4608e-04]]).to(torch_device), - "dac_44khz": torch.tensor([[-3.7834e-04, -1.0849e-04, 1.1856e-04, 2.6852e-04, 3.7313e-04, - 5.0301e-04, 6.4261e-04, 8.0797e-04, 9.0969e-04, 9.9720e-04, - 1.0807e-03, 1.1217e-03, 1.1229e-03, 1.1208e-03, 1.0862e-03, - 9.5098e-04, 7.5477e-04, 5.2319e-04, 2.7449e-04, 2.4389e-05, - -1.9138e-04, -3.2046e-04, -4.0629e-04, -4.4804e-04, -5.0271e-04, - -5.8324e-04, -6.6573e-04, -6.9545e-04, -6.8046e-04, -6.1640e-04, - -5.3542e-04, -4.2302e-04, -3.0829e-04, -1.8475e-04, -3.9555e-05, - 9.0104e-05, 1.9291e-04, 2.7445e-04, 3.6738e-04, 4.7454e-04, - 6.0626e-04, 7.5514e-04, 8.5390e-04, 8.8749e-04, 8.5473e-04, - 7.5550e-04, 6.2329e-04, 4.9771e-04, 3.8809e-04, 3.0741e-04], - [ 1.1130e-04, 4.6536e-04, 1.0524e-04, -6.1460e-04, -1.1777e-03, - -1.0661e-03, -3.7962e-04, 5.3627e-04, 1.0481e-03, 8.7734e-04, - 1.3513e-04, -6.6297e-04, -9.5284e-04, -4.6333e-04, 5.5780e-04, - 1.4526e-03, 1.6264e-03, 1.0852e-03, 3.3766e-04, 1.0960e-04, - 7.7973e-04, 2.0579e-03, 3.0206e-03, 2.9674e-03, 1.8141e-03, - 3.1059e-04, -5.7140e-04, -3.4386e-04, 4.8406e-04, 8.6931e-04, - 2.1745e-05, -1.7647e-03, -3.2787e-03, -3.3368e-03, -1.7466e-03, - 4.3745e-04, 1.6595e-03, 1.1171e-03, -6.3018e-04, -2.0979e-03, - -2.1286e-03, -6.8752e-04, 1.1514e-03, 2.1590e-03, 1.9204e-03, - 1.0659e-03, 5.3295e-04, 6.6817e-04, 9.2716e-04, 5.3240e-04]]).to(torch_device), + "dac_16khz": torch.tensor([[-1.9537e-04, 1.9159e-04, 3.1591e-04, 2.0804e-04, -3.1973e-05, + -3.3672e-04, -4.6511e-04, -4.3928e-04, -2.8604e-04, 2.7375e-04, + 8.8118e-04, 1.1193e-03, 1.6241e-03, 1.9374e-03, 1.7826e-03, + 5.9879e-04, -4.4053e-04, -1.3708e-03, -1.9989e-03, -2.0518e-03, + -1.5591e-03, -4.0491e-04, 6.3700e-04, 1.2456e-03, 1.3381e-03, + 1.2848e-03, 6.0356e-04, 9.4392e-05, -6.1609e-04, -1.3806e-03, + -1.4977e-03, -9.7825e-04, -3.8692e-04, 5.3131e-04, 1.8666e-03, + 2.3713e-03, 2.1134e-03, 1.4220e-03, 7.3615e-04, -2.5369e-04, + -9.8636e-04, -1.3868e-03, -1.6701e-03, -1.0521e-03, -6.2109e-04, + -5.3288e-04, -2.1532e-04, 4.1671e-04, 7.7438e-04, 8.0039e-04], + [ 6.5413e-05, 3.6614e-04, -1.4457e-03, -2.3634e-04, -3.6627e-04, + -1.3334e-03, 1.0519e-03, -1.4445e-03, 2.1915e-04, -3.3080e-04, + -1.3308e-03, 4.8407e-04, 8.6294e-04, -1.7639e-03, 4.2044e-05, + 2.0936e-04, -2.9692e-03, 8.7512e-04, 1.3507e-03, 2.0057e-03, + -5.5121e-04, 1.3708e-03, -3.1085e-05, -2.6315e-03, -6.7661e-04, + 6.2430e-04, 8.3580e-04, -1.5940e-03, 3.3061e-03, 1.3702e-03, + -1.7913e-03, -4.0576e-05, -5.5106e-04, -9.3050e-04, -2.3780e-03, + -5.3527e-04, 1.5840e-03, -1.4020e-03, 1.2090e-03, 6.0580e-04, + -1.8049e-03, 3.5135e-05, -3.0823e-03, 5.0042e-04, -1.1099e-03, + 1.1512e-04, 3.3324e-03, -1.7616e-03, 5.2421e-04, -1.3589e-03]]).to(torch_device), + "dac_24khz": torch.tensor([[ 2.5545e-04, 8.9353e-05, -4.1158e-04, -6.1750e-04, -5.9480e-04, + -5.6071e-04, -5.2090e-04, -4.2821e-04, -1.4335e-04, -6.9339e-05, + -9.0480e-05, 6.5549e-05, 7.5300e-05, 1.9337e-07, 2.0931e-04, + 4.1511e-04, 1.1008e-04, -1.6662e-04, 4.9021e-05, 4.0946e-04, + 4.3870e-04, 3.9847e-04, 4.1346e-04, 2.3158e-04, 2.4527e-04, + 4.4284e-04, 3.8170e-04, 1.2579e-04, -4.0307e-05, -2.8757e-04, + -8.5801e-04, -1.4023e-03, -1.5856e-03, -1.5326e-03, -1.5314e-03, + -1.4345e-03, -1.0435e-03, -5.2566e-04, 2.8071e-05, 5.4406e-04, + 8.9030e-04, 1.0047e-03, 1.0342e-03, 9.4115e-04, 6.8876e-04, + 3.2003e-04, -7.9418e-05, -4.0320e-04, -5.7941e-04, -7.3025e-04], + [-4.7845e-04, 3.8872e-04, 4.0155e-04, 3.6504e-04, 1.5022e-03, + 1.2856e-03, -1.8015e-04, -7.2616e-05, 6.3906e-04, -1.1491e-03, + -2.7369e-03, -1.5336e-03, -8.2313e-04, -1.6791e-03, -9.4759e-06, + 2.3807e-03, -2.2854e-04, -2.9693e-03, 2.9812e-04, 2.7258e-03, + -3.8019e-04, -2.2031e-03, -3.6195e-04, -6.6059e-04, -2.0270e-03, + -9.9469e-05, 5.4256e-04, -3.3896e-03, -3.9328e-03, 5.6228e-04, + 1.1226e-03, -1.0931e-03, 1.0939e-03, 2.9646e-03, -4.1916e-04, + -1.8292e-03, 1.0766e-03, 2.3094e-04, -3.4554e-03, -2.0085e-03, + 5.9608e-04, -1.3147e-03, -1.3603e-03, 1.8352e-03, 4.6342e-04, + -2.6805e-03, -1.3435e-05, 2.8397e-03, 1.0937e-04, -1.7540e-03]]).to(torch_device), + "dac_44khz": torch.tensor([[-4.8139e-04, -2.2367e-04, 3.1570e-06, 1.6349e-04, 2.6632e-04, + 3.9803e-04, 5.3275e-04, 7.0730e-04, 8.0937e-04, 9.2120e-04, + 1.0271e-03, 1.0728e-03, 1.0603e-03, 1.0328e-03, 9.8452e-04, + 8.4670e-04, 6.5249e-04, 4.2936e-04, 1.9743e-04, -4.4033e-06, + -1.5679e-04, -2.3475e-04, -2.6826e-04, -2.6645e-04, -2.9844e-04, + -3.6448e-04, -4.6388e-04, -5.5712e-04, -6.4478e-04, -7.0090e-04, + -7.1978e-04, -6.8389e-04, -6.1487e-04, -4.9192e-04, -3.1528e-04, + -1.3920e-04, 1.6591e-05, 1.4938e-04, 2.6723e-04, 4.0855e-04, + 6.0641e-04, 8.1632e-04, 9.6742e-04, 1.0481e-03, 1.0581e-03, + 1.0213e-03, 9.3807e-04, 8.1994e-04, 6.9299e-04, 5.8774e-04], + [ 7.2770e-04, 8.2807e-04, 3.7124e-04, -4.1002e-04, -8.7899e-04, + -6.0642e-04, 2.0435e-04, 1.0668e-03, 1.3318e-03, 7.8307e-04, + -3.2117e-04, -1.3448e-03, -1.6520e-03, -1.0778e-03, 2.4146e-05, + 9.8221e-04, 1.2399e-03, 7.6147e-04, -2.2230e-05, -4.7415e-04, + -1.4114e-04, 8.9560e-04, 1.9897e-03, 2.4969e-03, 2.0585e-03, + 1.0263e-03, 1.5015e-04, 9.2623e-05, 7.8239e-04, 1.3270e-03, + 7.3531e-04, -1.1100e-03, -3.1865e-03, -3.9610e-03, -2.6410e-03, + -6.5765e-06, 1.9960e-03, 1.7654e-03, -5.9006e-04, -3.2932e-03, + -4.2902e-03, -2.8423e-03, -6.7126e-05, 2.0438e-03, 2.2075e-03, + 8.8849e-04, -3.6330e-04, -3.9405e-04, 6.1344e-04, 1.4316e-03]]).to(torch_device), } EXPECTED_QUANT_CODEBOOK_LOSS_BATCH = { - "dac_16khz": 20.61562156677246, - "dac_24khz": 23.9102783203125, - "dac_44khz": 16.177066802978516, + "dac_16khz": 20.685312271118164, + "dac_24khz": 23.66303253173828, + "dac_44khz": 16.129348754882812, } EXPECTED_CODEC_ERROR_BATCH = { - "dac_16khz": 0.001973195234313607, - "dac_24khz": 0.0012980918399989605, - "dac_44khz": 0.00037737112143076956, + "dac_16khz": 0.0019726448226720095, + "dac_24khz": 0.0013017073506489396, + "dac_44khz": 0.0003825263702310622, } # fmt: on @@ -810,7 +921,7 @@ def test_integration(self, model_name): # compare codec error / lossiness codec_err = compute_rmse(decoded_outputs["audio_values"], inputs["input_values"]) - torch.testing.assert_close(EXPECTED_CODEC_ERROR[model_name], codec_err, rtol=1e-6, atol=1e-6) + torch.testing.assert_close(EXPECTED_CODEC_ERROR[model_name], codec_err, rtol=1e-5, atol=1e-5) # make sure forward and decode gives same result enc_dec = model(inputs["input_values"])[1]