Merge pull request #40 from goatchurchprime/fixothersampling

Fixothersampling

Author: goatchurchprime

README.md

@@ -155,6 +155,7 @@ scons apply_patches  # optional
 scons build_opus     # build opus using cmake
 scons build_rnnoise  # build opus using cmake
 scons                # build this library
+cp addons/twovoip/libs/*.so example/addons/twovoip/libs/
 ```
 
 To compile for another platform like web, the commands are
@@ -183,7 +184,7 @@ as your `GodotEngine.exe` so that it finds and links it.
 For the addon to work correctly, `twovoip_lipsync` and `twovoip` cannot be used in the same project.
 
 
-### Nixos automated
+### Nixos automated (not working)
 
 The build system is defined by the flake.nix file
 

example/default_bus_layout.tres

@@ -1,11 +1,7 @@
-[gd_resource type="AudioBusLayout" load_steps=3 format=3 uid="uid://orx4iw038t0c"]
+[gd_resource type="AudioBusLayout" load_steps=2 format=3 uid="uid://orx4iw038t0c"]
 
-[sub_resource type="AudioEffectPitchShift" id="AudioEffectPitchShift_jq6ag"]
+[sub_resource type="AudioEffectPitchShift" id="AudioEffectPitchShift_urnej"]
 resource_name = "PitchShift"
-fft_size = 1
-
-[sub_resource type="AudioEffectOpusChunked" id="AudioEffectOpusChunked_v1e16"]
-resource_name = "OpusChunked"
 
 [resource]
 bus/1/name = &"SpeechBus"
@@ -14,13 +10,11 @@ bus/1/mute = false
 bus/1/bypass_fx = false
 bus/1/volume_db = 0.0
 bus/1/send = &"Master"
-bus/1/effect/0/effect = SubResource("AudioEffectPitchShift_jq6ag")
+bus/1/effect/0/effect = SubResource("AudioEffectPitchShift_urnej")
 bus/1/effect/0/enabled = true
 bus/2/name = &"MicrophoneBus"
 bus/2/solo = false
 bus/2/mute = true
 bus/2/bypass_fx = false
 bus/2/volume_db = 0.0
 bus/2/send = &"Master"
-bus/2/effect/0/effect = SubResource("AudioEffectOpusChunked_v1e16")
-bus/2/effect/0/enabled = true

example/radiomqtt/member.gd

@@ -82,8 +82,10 @@ func processheaderpacket(h):
 		#$AudioStreamPlayer.play()
 		setupaudioshader()
 
-	if opusframesize != 0 and audiostreamopuschunked == null:
+	if opusframesize != 0 and audiostreamopuschunked == null and not h["noopuscompression"]:
 		print("Compressed opus stream received that we cannot decompress")
+	if audiostreamopuschunked != null:
+		audiostreamopuschunked.resetdecoder()
 	audioserveroutputlatency = AudioServer.get_output_latency()
 	print("audioserveroutputlatency ", audioserveroutputlatency)
 
@@ -130,8 +132,8 @@ func _process(delta):
 		var chunkv1 = 0.0
 		while audiostreamopuschunked.chunk_space_available():
 			if resampledpacketsbuffer != null and len(resampledpacketsbuffer) != 0:
-				#audiostreamopuschunked.push_audio_chunk(resampledpacketsbuffer.pop_front())
-				var audiochunk = audiostreamopuschunked.resample_chunk(resampledpacketsbuffer.pop_front())
+				var resampledaudiochunk = resampledpacketsbuffer.pop_front()
+				var audiochunk = audiostreamopuschunked.resample_chunk(resampledaudiochunk)
 				audiostreamopuschunked.push_audio_chunk(audiochunk)
 			elif len(audiopacketsbuffer) != 0:
 				audiostreamopuschunked.push_audio_chunk(audiopacketsbuffer.pop_front())
@@ -161,7 +163,7 @@ func _process(delta):
 		elif bufferlengthtime < audiobufferregulationtime:
 			$AudioStreamPlayer.pitch_scale = 1.0
 		else:
-			var w = inverse_lerp(audiobufferregulationtime, audioserveroutputlatency + audiobuffersize/audiosamplerate, bufferlengthtime)
+			var w = inverse_lerp(audiobufferregulationtime, audioserveroutputlatency + audiobuffersize*1.0/audiosamplerate, bufferlengthtime)
 			$AudioStreamPlayer.pitch_scale = lerp(1.0, audiobufferregulationpitch, w)
 #show some view of the speedup rate on here
 

example/radiomqtt/radiomqtt.gd

@@ -22,8 +22,9 @@ var opusframesize : int = 960
 var opuscomplexity : int = 5
 var opusoptimizeforvoice : bool = true
 
-var prefixbytes = PackedByteArray([0,0,0,0,1])
+var prefixbytes = PackedByteArray([23])
 var mqttpacketencodebase64 : bool = false
+var noopuscompression = false
 
 var recordedsamples = [ ]
 var recordedopuspackets = [ ]
@@ -87,12 +88,13 @@ func _ready():
 		if ClassDB.can_instantiate("AudioEffectOpusChunked"):
 			audioopuschunkedeffect = ClassDB.instantiate("AudioEffectOpusChunked")
 
-	for effect_idx in range(AudioServer.get_bus_effect_count(speechbusidx)):
-		var laudioeffectonspeechbus : AudioEffect = AudioServer.get_bus_effect(speechbusidx, effect_idx)
-		if laudioeffectonspeechbus.is_class("AudioEffectPitchShift"):
-			audioeffectpitchshift = laudioeffectonspeechbus
-			audioeffectpitchshiftidx = effect_idx
-			break
+	if speechbusidx != -1:
+		for effect_idx in range(AudioServer.get_bus_effect_count(speechbusidx)):
+			var laudioeffectonspeechbus : AudioEffect = AudioServer.get_bus_effect(speechbusidx, effect_idx)
+			if laudioeffectonspeechbus.is_class("AudioEffectPitchShift"):
+				audioeffectpitchshift = laudioeffectonspeechbus
+				audioeffectpitchshiftidx = effect_idx
+				break
 
 	updatesamplerates()
 	for i in range(1, len(visemes)):
@@ -104,7 +106,7 @@ func _ready():
 
 	SelfMember.audiobufferregulationtime = 3600.0
 
-func resamplerecordedsamples(orgsamples, newsamplesize):
+func rechunkrecordedchunks(orgsamples, newsamplesize):
 	assert (newsamplesize > 0)
 	var res = [ ]
 	var currentsample = PackedVector2Array()
@@ -133,12 +135,13 @@ func updatesamplerates():
 
 	print("aaa audiosamplesize ", audiosamplesize, "  audiosamplerate ", audiosamplerate)
 
-	var noopuscompression = false
+	noopuscompression = false
 	if opussamplerate == audioresamplerate:
 		$VBoxFrameLength/HBoxOpusExtra/Compressed.disabled = false
 		if not $VBoxFrameLength/HBoxOpusExtra/Compressed.button_pressed:
 			noopuscompression = true
 	else:
+		$VBoxFrameLength/HBoxOpusExtra/Compressed.button_pressed = false
 		$VBoxFrameLength/HBoxOpusExtra/Compressed.disabled = true
 		noopuscompression = true
 	opusbitrate = int($VBoxFrameLength/HBoxOpusBitRate/BitRate.value)
@@ -163,9 +166,10 @@ func updatesamplerates():
 	recordedheader = { "opusframesize":audioresamplesize, 
 					   "opussamplerate":audioresamplerate, 
 					   "prefixbyteslength":len(prefixbytes), 
+					   "noopuscompression":noopuscompression,
 					   "mqttpacketencoding":"base64" if mqttpacketencodebase64 else "binary" }
 	if len(recordedsamples) != 0 and len(recordedsamples[0]) != audiosamplesize:
-		recordedsamples = resamplerecordedsamples(recordedsamples, audiosamplesize)
+		recordedsamples = rechunkrecordedchunks(recordedsamples, audiosamplesize)
 	recordedopuspacketsMemSize = 0
 	recordedopuspackets = null
 	recordedresampledpackets = null
@@ -176,7 +180,7 @@ func updatesamplerates():
 			recordedopuspackets.append(opuspacket)
 			recordedopuspacketsMemSize += opuspacket.size() 
 		$VBoxPlayback/HBoxPlaycount/GridContainer/FrameCount.text = str(len(recordedopuspackets))
-	else:
+	elif audioopuschunkedeffect != null:
 		recordedresampledpackets = [ ]
 		var denoise = not $HBoxBigButtons/VBoxPTT/Denoise.disabled and $HBoxBigButtons/VBoxPTT/Denoise.button_pressed
 		for s in recordedsamples:
@@ -187,6 +191,9 @@ func updatesamplerates():
 		$VBoxPlayback/HBoxPlaycount/GridContainer/FrameCount.text = "1"
 		if len(recordedresampledpackets):
 			recordedopuspacketsMemSize = len(recordedresampledpackets)*len(recordedresampledpackets[0])*4
+	else:
+		recordedresampledpackets = null
+
 
 	$VBoxPlayback/HBoxPlaycount/GridContainer/Totalbytes.text = str(recordedopuspacketsMemSize)
 	var tm = len(recordedsamples)*frametimems*0.001
@@ -229,8 +236,16 @@ var talkingstarttime = 0
 func starttalking():
 	currentlytalking = true
 	recordedsamples = [ ]
-	recordedopuspackets = [ ]
-	recordedresampledpackets = null
+	if not noopuscompression:
+		recordedopuspackets = [ ]
+		recordedresampledpackets = null
+	else:
+		recordedopuspackets = null
+		if audioopuschunkedeffect != null:
+			recordedresampledpackets = [ ]
+		else:
+			recordedresampledpackets = null
+
 	$VBoxPlayback/HBoxPlaycount/GridContainer/FrameCount.text = str(0)
 	$VBoxPlayback/HBoxPlaycount/GridContainer/TimeSecs.text = str(0)
 	recordedopuspacketsMemSize = 0
@@ -248,8 +263,8 @@ func starttalking():
 			leadtimems -= frametimems
 			Dundroppedchunks += 1
 		print("Undropped ", Dundroppedchunks, " chunks")
-		if opusframesize != 0:
-			audioopuschunkedeffect.flush_opus_encoder(false)
+		if opusframesize != 0 and $VBoxFrameLength/HBoxOpusExtra/Compressed.button_pressed:
+			audioopuschunkedeffect.resetencoder()
 
 func _on_mic_working_toggled(toggled_on):
 	print("_on_mic_working_toggled ", $AudioStreamMicrophone.playing, " to ", toggled_on)
@@ -321,7 +336,9 @@ func _process(_delta):
 			if currentlytalking:
 				if len(recordedsamples) < maxrecordedsamples:
 					recordedsamples.append(audiosamples)
-				if opusframesize != 0:
+				if noopuscompression:
+					recordedresampledpackets.append(audioopuschunkedeffect.read_chunk(true))
+				elif opusframesize != 0:
 					var opuspacket = audioopuschunkedeffect.read_opus_packet(prefixbytes)
 					$MQTTnetwork.transportaudiopacket(opuspacket, mqttpacketencodebase64)
 					if len(recordedopuspackets) < maxrecordedsamples:
@@ -392,6 +409,11 @@ func _on_play_pressed():
 	elif recordedresampledpackets != null:
 		SelfMember.processheaderpacket(h)
 		SelfMember.resampledpacketsbuffer = recordedresampledpackets.duplicate()
+		var resampledaudiochunk_blank = PackedVector2Array()
+		resampledaudiochunk_blank.resize(h["opusframesize"])
+		for i in range(5):
+			SelfMember.audiostreamopuschunked.resample_chunk(resampledaudiochunk_blank)
+
 	elif recordedsamples and SelfMember.audiostreamgeneratorplayback != null:
 		SelfMember.audiosamplesize = audiosamplesize
 		SelfMember.audiopacketsbuffer = recordedsamples.duplicate()

example/radiomqtt/radiomqtt.tscn

@@ -356,6 +356,7 @@ text = "Resampled:"
 [node name="ResampleRate" type="SpinBox" parent="VBoxFrameLength/HBoxAudioFrame"]
 custom_minimum_size = Vector2(95, 0)
 layout_mode = 2
+tooltip_text = "Needs to match the Opus sample rate to be compressed"
 min_value = 20.0
 max_value = 96000.0
 value = 48000.0

src/audio_effect_opus_chunked.cpp

@@ -75,7 +75,7 @@ void AudioEffectOpusChunked::_bind_methods() {
     ClassDB::bind_method(D_METHOD("drop_chunk"), &AudioEffectOpusChunked::drop_chunk);
     ClassDB::bind_method(D_METHOD("undrop_chunk"), &AudioEffectOpusChunked::undrop_chunk);
     ClassDB::bind_method(D_METHOD("read_opus_packet", "prefixbytes"), &AudioEffectOpusChunked::read_opus_packet);
-    ClassDB::bind_method(D_METHOD("flush_opus_encoder", "denoise"), &AudioEffectOpusChunked::flush_opus_encoder);
+    ClassDB::bind_method(D_METHOD("resetencoder"), &AudioEffectOpusChunked::resetencoder);
     ClassDB::bind_method(D_METHOD("chunk_to_opus_packet", "prefixbytes", "audiosamples", "denoise"), &AudioEffectOpusChunked::chunk_to_opus_packet);
     ClassDB::bind_method(D_METHOD("chunk_resample", "audiosamples", "denoise", "backresample"), &AudioEffectOpusChunked::chunk_resample);
 }
@@ -92,7 +92,7 @@ AudioEffectOpusChunked::AudioEffectOpusChunked() {
 
 AudioEffectOpusChunked::~AudioEffectOpusChunked() 
 {
-    resetencoder(17);
+    deleteencoder();
 };
 
 Ref<AudioEffectInstance> AudioEffectOpusChunked::_instantiate() {
@@ -102,8 +102,22 @@ Ref<AudioEffectInstance> AudioEffectOpusChunked::_instantiate() {
     return ins;
 }
 
-void AudioEffectOpusChunked::resetencoder(int Dreason) {
-    godot::UtilityFunctions::prints("resetting AudioEffectOpusChunked", Dreason);
+void AudioEffectOpusChunked::resetencoder() {
+    if ((opusframesize == 0) || (chunknumber == -1)) 
+        return;
+    if (speexresampler != NULL)
+        speex_resampler_reset_mem(speexresampler);
+    if (speexbackresampler != NULL) 
+        speex_resampler_reset_mem(speexbackresampler);
+    if (st != NULL) 
+        rnnoise_init(st, NULL);        
+    if (opusencoder != NULL) 
+        opus_encoder_ctl(opusencoder, OPUS_RESET_STATE);
+    lastopuschunk = chunknumber - 1;
+}
+
+
+void AudioEffectOpusChunked::deleteencoder() {
     if (speexresampler != NULL) {
         speex_resampler_destroy(speexresampler);
         speexresampler = NULL;
@@ -133,7 +147,7 @@ void AudioEffectOpusChunked::resetencoder(int Dreason) {
 }
 
 void AudioEffectOpusChunked::createencoder() {
-    resetencoder(4);  // In case called from GDScript
+    deleteencoder();  
     audiosamplebuffer.resize(audiosamplesize*ringbufferchunks); 
     chunknumber = 0;
     bufferend = 0;
@@ -191,22 +205,25 @@ void AudioEffectOpusChunked::createencoder() {
     }
     int opuserror2 = opus_encoder_ctl(opusencoder, OPUS_SET_BITRATE(opusbitrate));
     if (opuserror2 != 0) {
-        godot::UtilityFunctions::printerr("opus_encoder_ctl bitrate error error ", opuserror2);
+        godot::UtilityFunctions::printerr("opus_encoder_ctl bitrate error ", opuserror2);
         chunknumber = -2;
         return;
     }
     int opuserror3 = opus_encoder_ctl(opusencoder, OPUS_SET_COMPLEXITY(complexity));
     if (opuserror3 != 0) {
-        godot::UtilityFunctions::printerr("opus_encoder_ctl complexity error error ", opuserror3);
+        godot::UtilityFunctions::printerr("opus_encoder_ctl complexity error ", opuserror3);
         chunknumber = -2;
         return;
     }
     int opuserror4 = opus_encoder_ctl(opusencoder, OPUS_SET_SIGNAL(signal_type));
     if (opuserror4 != 0) {
-        godot::UtilityFunctions::printerr("opus_encoder_ctl signal_type error error ", opuserror4);
+        godot::UtilityFunctions::printerr("opus_encoder_ctl signal_type error ", opuserror4);
         chunknumber = -2;
         return;
     }
+    // we don't set DTX because it's for letting opus decide internally when it is quiet https://github.com/xiph/opus/issues/381
+    //int opuserror5 = opus_encoder_ctl(opusencoder, OPUS_SET_DTX(1));
+
     opusbytebuffer.resize(sizeof(float)*channels*opusframesize + MAXPREFIXBYTES);
     lastopuschunk = -1;
 }
@@ -343,28 +360,6 @@ PackedByteArray AudioEffectOpusChunked::read_opus_packet(const PackedByteArray&
     return opus_frame_to_opus_packet(prefixbytes, paudioresamples);
 }
 
-void AudioEffectOpusChunked::flush_opus_encoder(bool denoise) {
-    if ((opusframesize == 0) || (chunknumber == -1)) 
-        return;
-    // this just sends 5 empty chunks into the encoder.  doesn't necessarily work
-    float* paudioresamples = (float*)singleresamplebuffer.ptrw();
-    for (int j = 0; j < opusframesize*2; j++)
-        paudioresamples[j] = 0.0F;
-    for (int i = 0; i < 5; i++)
-        opus_frame_to_opus_packet(PackedByteArray(), paudioresamples);
-
-    if ((st != NULL) && denoise) {
-        // we could use rnnoise_init (if it doesn't involve a heavy reload of the model)
-        float* rin = (float*)rnnoise_in.ptr();
-        for (int j = 0; j < rnnoiseframesize; j++)
-            rin[j] = 0.0F;
-        float* rout = (float*)rnnoise_out.ptr();
-        for (int i = 0; i < 5; i++)
-            rnnoise_process_frame(st, rout, rin);
-    }
-    
-    lastopuschunk = chunknumber - 1;
-}
 
 int AudioEffectOpusChunked::chunk_to_lipsync(bool resampled) {
 #ifdef OVR_LIP_SYNC

src/audio_effect_opus_chunked.h

@@ -43,6 +43,7 @@
 #include "opus.h"
 #include "speex_resampler/speex_resampler.h"
 
+
 #ifdef OVR_LIP_SYNC
     #include "OVRLipSync.h"
 #else
@@ -92,15 +93,6 @@ typedef enum {
 
 // chunk_to_opus_packet() is for encoding a series of chunks not in the ring buffer.
 
-// TODO
-// fix any crashes  
-// plot the float value of the noise detector in the screen as a threshold
-// plot the resampled denoised view in the same texture too. (aligned)
-// hack the main scons module so it builds on the actions
-// make a stub so it can run without the rnnoise library if necessary
-// get help with this compiling
-
-// finish folding and delivering GP leaflets
 
 
 class AudioEffectOpusChunked : public AudioEffect {
@@ -159,7 +151,8 @@ class AudioEffectOpusChunked : public AudioEffect {
     virtual Ref<AudioEffectInstance> _instantiate() override;
 
     void createencoder();
-    void resetencoder(int Dreason=3);
+    void deleteencoder();
+    void resetencoder();
 
     bool chunk_available();
     void drop_chunk();