Skip to content

Commit

Permalink
NGC audio receiving
Browse files Browse the repository at this point in the history
  • Loading branch information
@zoff99
zoff99 committed Dec 24, 2023
1 parent bdd51b1 commit b0e8e37
Show file tree
Hide file tree
Showing 3 changed files with 239 additions and 5 deletions.
74 changes: 71 additions & 3 deletions src/main/kotlin/com/zoffcc/applications/trifa/HelperGeneric.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ import com.zoffcc.applications.trifa.HelperMessage.tox_friend_send_message_wrapp
import com.zoffcc.applications.trifa.HelperMessage.update_message_in_db_messageid
import com.zoffcc.applications.trifa.HelperMessage.update_message_in_db_resend_count
import com.zoffcc.applications.trifa.MainActivity.Companion.modify_message_with_ft
import com.zoffcc.applications.trifa.MainActivity.Companion.ngc_audio_in_queue
import com.zoffcc.applications.trifa.MainActivity.Companion.tox_file_control
import com.zoffcc.applications.trifa.MainActivity.Companion.tox_friend_by_public_key
import com.zoffcc.applications.trifa.MainActivity.Companion.tox_friend_delete
Expand All @@ -49,6 +50,7 @@ import com.zoffcc.applications.trifa.MainActivity.Companion.tox_group_leave
import com.zoffcc.applications.trifa.MainActivity.Companion.tox_group_peer_get_public_key
import com.zoffcc.applications.trifa.MainActivity.Companion.tox_messagev3_friend_send_message
import com.zoffcc.applications.trifa.MainActivity.Companion.tox_self_set_nospam
import com.zoffcc.applications.trifa.MainActivity.Companion.toxav_ngc_audio_decode
import com.zoffcc.applications.trifa.MainActivity.Companion.toxav_ngc_video_decode
import com.zoffcc.applications.trifa.MainActivity.Companion.update_savedata_file
import com.zoffcc.applications.trifa.TRIFAGlobals.VFS_FILE_DIR
Expand Down Expand Up @@ -623,6 +625,72 @@ object HelperGeneric {
update_savedata_file_wrapper()
}

fun play_ngc_incoming_audio_frame(group_number: Long,
peer_id: Long,
encoded_audio_and_header: ByteArray,
length: Long)
{
val group_id = HelperGroup.tox_group_by_groupnum__wrapper(group_number).lowercase()
if ((globalstore.isMinimized()) || (avstatestore.state.calling_state_get() == AVState.CALL_STATUS.CALL_STATUS_CALLING)
|| (!groupstore.state.visible) || (groupstore.stateFlow.value.selectedGroupId != group_id))
{
// Log.i(TAG, "play_ngc_incoming_audio_frame: audio frame not from selected group chat")
return
}

ngc_video_packet_last_incoming_ts = System.currentTimeMillis()
val ngc_incoming_video_from_peer: String = tox_group_peer_get_public_key(group_number, peer_id)!!
ngc_update_video_incoming_peer_list(ngc_incoming_video_from_peer)

if (ngc_video_showing_video_from_peer_pubkey.equals("-1"))
{
Log.i(TAG, "play_ngc_incoming_audio_frame: ngc_video_showing_video_from_peer_pubkey:2a:" + ngc_video_showing_video_from_peer_pubkey)
ngc_video_showing_video_from_peer_pubkey = ngc_incoming_video_from_peer
Log.i(TAG, "play_ngc_incoming_audio_frame: ngc_video_showing_video_from_peer_pubkey:2b:" + ngc_video_showing_video_from_peer_pubkey)
}
else if (!ngc_video_showing_video_from_peer_pubkey.equals(ngc_incoming_video_from_peer, true))
{
// we are already showing the video of a different peer in the group
// Log.i(TAG, "play_ngc_incoming_audio_frame: we are already playing the audio of a different peer in the group")
return
}
// remove header from data (10 bytes)
// remove header from data (10 bytes)
val pcm_encoded_length = (length - 10).toInt()
val pcm_encoded_buf = ByteArray(pcm_encoded_length * 10)
val pcm_decoded_buf = ByteArray(20000)
val bytes_in_40ms = 1920
val pcm_decoded_buf_delta_1 = ByteArray(bytes_in_40ms * 2)
val pcm_decoded_buf_delta_2 = ByteArray(bytes_in_40ms * 2)
val pcm_decoded_buf_delta_3 = ByteArray(bytes_in_40ms * 2)
try
{
System.arraycopy(encoded_audio_and_header, 10, pcm_encoded_buf, 0, pcm_encoded_length)
//
//Log.i(TAG, "play_ngc_incoming_audio_frame:toxav_ngc_audio_decode:"
// + pcm_encoded_buf
// + " " + pcm_encoded_length
// + " " + pcm_decoded_buf)
val decoded_samples: Int = toxav_ngc_audio_decode(
pcm_encoded_buf,
pcm_encoded_length,
pcm_decoded_buf)
//Log.i(TAG, "play_ngc_incoming_audio_frame:toxav_ngc_audio_decode:decoded_samples="
// + decoded_samples)
// put pcm data into a FIFO
System.arraycopy(pcm_decoded_buf, 0, pcm_decoded_buf_delta_1, 0, bytes_in_40ms * 2)
ngc_audio_in_queue.offer(pcm_decoded_buf_delta_1)
System.arraycopy(pcm_decoded_buf, bytes_in_40ms * 2, pcm_decoded_buf_delta_2, 0, bytes_in_40ms * 2)
ngc_audio_in_queue.offer(pcm_decoded_buf_delta_2)
System.arraycopy(pcm_decoded_buf, bytes_in_40ms * 2 * 2, pcm_decoded_buf_delta_3, 0, bytes_in_40ms * 2)
ngc_audio_in_queue.offer(pcm_decoded_buf_delta_3)
ngc_video_frame_last_incoming_ts = System.currentTimeMillis()
} catch (e: java.lang.Exception)
{
e.printStackTrace()
}
}

fun show_ngc_incoming_video_frame_v2(group_number: Long,
peer_id: Long,
encoded_video_and_header: ByteArray,
Expand Down Expand Up @@ -676,12 +744,12 @@ object HelperGeneric {
System.arraycopy(encoded_video_and_header, 12, high_seqnum, 0, 1)
System.arraycopy(encoded_video_and_header, 13, chkskum, 0, 1)
val seqnum = java.lang.Byte.toUnsignedInt(low_seqnum[0]) + Integer.toUnsignedLong((high_seqnum[0].toInt() shl 8))
//if (seqnum != (last_video_seq_num + 1)) {
// if (seqnum != (last_video_seq_num + 1)) {
// Log.i(TAG, "!!!!!!!seqnumber_missing!!!!! " + seqnum + " -> " + (last_video_seq_num + 1));
//}
last_video_seq_num = seqnum
//val crc_8 = Integer.toUnsignedLong(calc_crc_8(yuv_frame_encoded_buf))
//if (java.lang.Byte.toUnsignedInt(chkskum[0]).toLong() != crc_8) {
// val crc_8 = Integer.toUnsignedLong(calc_crc_8(yuv_frame_encoded_buf))
// if (java.lang.Byte.toUnsignedInt(chkskum[0]).toLong() != crc_8) {
// Log.i(TAG, "checksum=" + java.lang.Byte.toUnsignedInt(chkskum[0]).toLong()
// + " crc8=" + crc_8 + " seqnum=" + seqnum
// + " yuv_frame_encoded_bytes=" + (yuv_frame_encoded_bytes + 14));
Expand Down
4 changes: 3 additions & 1 deletion src/main/kotlin/com/zoffcc/applications/trifa/MainActivity.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ import com.zoffcc.applications.trifa.HelperGeneric.bytesToHex
import com.zoffcc.applications.trifa.HelperGeneric.get_friend_msgv3_capability
import com.zoffcc.applications.trifa.HelperGeneric.hexstring_to_bytebuffer
import com.zoffcc.applications.trifa.HelperGeneric.io_file_copy
import com.zoffcc.applications.trifa.HelperGeneric.play_ngc_incoming_audio_frame
import com.zoffcc.applications.trifa.HelperGeneric.read_chunk_from_SD_file
import com.zoffcc.applications.trifa.HelperGeneric.show_ngc_incoming_video_frame_v2
import com.zoffcc.applications.trifa.HelperGeneric.shrink_image_file
Expand Down Expand Up @@ -154,6 +155,7 @@ class MainActivity
@JvmStatic var video_play_last_timestamp: Long = 0
var video_play_fps_value: Int = 0
@JvmStatic val video_play_measure_after_frame = 5;
var ngc_audio_in_queue: BlockingQueue<ByteArray> = LinkedBlockingQueue(3 * 5)

//
var PREF__ngc_video_bitrate: Int = LOWER_NGC_VIDEO_BITRATE // ~600 kbits/s -> ~60 kbytes/s
Expand Down Expand Up @@ -2774,7 +2776,7 @@ class MainActivity
{
if (data[6] == 0x01.toByte() && data[7] == 0x31.toByte() && data[8] == 1.toByte() && data[9] == 48.toByte())
{
//*****//play_ngc_incoming_audio_frame(group_number, peer_id, data, length)
play_ngc_incoming_audio_frame(group_number, peer_id, data, length)
} else
{
}
Expand Down
166 changes: 165 additions & 1 deletion src/main/kotlin/com/zoffcc/applications/trifa/TrifaToxService.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,11 @@ import com.zoffcc.applications.trifa.HelperMessage.tox_friend_send_message_wrapp
import com.zoffcc.applications.trifa.HelperMessage.update_message_in_db_messageid
import com.zoffcc.applications.trifa.HelperMessage.update_message_in_db_no_read_recvedts
import com.zoffcc.applications.trifa.HelperMessage.update_message_in_db_resend_count
import com.zoffcc.applications.trifa.HelperRelay.get_relay_for_friend
import com.zoffcc.applications.trifa.MainActivity.Companion.PREF__udp_enabled
import com.zoffcc.applications.trifa.MainActivity.Companion.add_tcp_relay_single_wrapper
import com.zoffcc.applications.trifa.MainActivity.Companion.bootstrap_single_wrapper
import com.zoffcc.applications.trifa.MainActivity.Companion.init_tox_callbacks
import com.zoffcc.applications.trifa.MainActivity.Companion.ngc_audio_in_queue
import com.zoffcc.applications.trifa.MainActivity.Companion.tox_friend_by_public_key
import com.zoffcc.applications.trifa.MainActivity.Companion.tox_friend_get_connection_status
import com.zoffcc.applications.trifa.MainActivity.Companion.tox_friend_get_name
Expand Down Expand Up @@ -61,13 +61,16 @@ import contactstore
import globalstore
import grouppeerstore
import groupstore
import kotlinx.coroutines.GlobalScope
import kotlinx.coroutines.launch
import online_button_text_wrapper
import org.briarproject.briar.desktop.contact.ContactItem
import org.briarproject.briar.desktop.contact.GroupItem
import org.briarproject.briar.desktop.contact.GroupPeerItem
import set_tox_running_state
import toxdatastore
import unlock_data_dir_input
import java.io.File
import java.nio.ByteBuffer
import java.util.*

Expand Down Expand Up @@ -111,6 +114,9 @@ class TrifaToxService
// --------------- bootstrap ---------------
// --------------- bootstrap ---------------

ngc_audio_play_thread_running = true
ngc_audio_play_thread_start()

if (!old_is_tox_started)
{
TRIFAGlobals.bootstrapping = true
Expand Down Expand Up @@ -249,6 +255,16 @@ class TrifaToxService
// ------- MAIN TOX LOOP ---------------------------------------------------------------
// ------- MAIN TOX LOOP ---------------------------------------------------------------
// ------- MAIN TOX LOOP ---------------------------------------------------------------
ngc_audio_play_thread_running = false
try
{
ngc_audio_play_thread!!.join(500)
}
catch(e: Exception)
{
e.printStackTrace()
}

try
{
sleep(100) // wait a bit, for "something" to finish up in the native code
Expand All @@ -271,6 +287,7 @@ class TrifaToxService
e.printStackTrace()
}
update_savedata_file_wrapper()

is_tox_started = false
set_tox_running_state("stopped")
clear_friends()
Expand All @@ -297,6 +314,151 @@ class TrifaToxService
(ToxServiceThread as Thread).start()
}

fun ngc_audio_play_thread_start()
{
Log.i(TAG, "ngc_audio_play_thread_start:starting Thread")
ngc_audio_play_thread = object : Thread()
{
override fun run()
{
try
{
val sleep_millis: Long = 40
var sleep_millis_current: Long = sleep_millis
var d1: Long = 0
val sampling_rate = 48000
val channels = 1
val bytes_in_40ms = 1920
// val sample_count = bytes_in_40ms / 2
while (ngc_audio_play_thread_running)
{
d1 = System.currentTimeMillis()
// -- play incoming bytes --
// -- play incoming bytes --
try
{
val buf: ByteArray = ngc_audio_in_queue.poll()
if (buf != null)
{
if ((sampling_rate != AudioSelectOutBox.SAMPLE_RATE) ||
(channels != AudioSelectOutBox.CHANNELS) ||
(AudioSelectOutBox.sourceDataLine == null))
{
Log.i(TAG, "ngc_audio_play_thread_start:11:1");
AudioSelectOutBox.init()
AudioSelectOutBox.change_audio_format(sampling_rate, channels)
Log.i(TAG, "ngc_audio_play_thread_start:11:2");
}
if (sampling_rate != AudioSelectOutBox.SAMPLE_RATE ||
channels != AudioSelectOutBox.CHANNELS)
{
Log.i(TAG, "ngc_audio_play_thread_start:22:1:$sampling_rate" + " "
+ AudioSelectOutBox.SAMPLE_RATE)
AudioSelectOutBox.change_audio_format(sampling_rate, channels)
Log.i(TAG, "ngc_audio_play_thread_start:22:2")
}

try
{
val want_bytes = buf.size
val sample_count = want_bytes / 2
try
{
AudioSelectOutBox.semaphore_audio_out_convert.acquire_passthru()
if (AudioSelectOutBox.semaphore_audio_out_convert_active_threads >= AudioSelectOutBox.semaphore_audio_out_convert_max_active_threads)
{
Log.i(TAG, "ngc_audio_play_thread_start:too many threads running")
AudioSelectOutBox.semaphore_audio_out_convert.release_passthru()
return
}
AudioSelectOutBox.semaphore_audio_out_convert.release_passthru()
} catch (e: java.lang.Exception)
{
AudioSelectOutBox.semaphore_audio_out_convert.release_passthru()
}

val t_audio_pcm_play = Thread{
try
{
AudioSelectOutBox.semaphore_audio_out_convert.acquire_passthru()
AudioSelectOutBox.semaphore_audio_out_convert_active_threads++
AudioSelectOutBox.semaphore_audio_out_convert.release_passthru()
} catch (e: java.lang.Exception)
{
AudioSelectOutBox.semaphore_audio_out_convert.release_passthru()
}
// HINT: this acutally plays incoming Audio
// HINT: this may block!!
try
{
val bytes_actually_written = AudioSelectOutBox.sourceDataLine.write(buf, 0, want_bytes)
if (bytes_actually_written != want_bytes)
{
Log.i(TAG, "ngc_audio_play_thread_start:bytes_actually_written=" + bytes_actually_written + " want_bytes=" + want_bytes)
}
} catch (e: java.lang.Exception)
{
Log.i(TAG, "ngc_audio_play_thread_start:sourceDataLine.write:EE:" + e.message) // e.printStackTrace();
}
try
{
AudioSelectOutBox.semaphore_audio_out_convert.acquire_passthru()
AudioSelectOutBox.semaphore_audio_out_convert_active_threads--
AudioSelectOutBox.semaphore_audio_out_convert.release_passthru()
} catch (e: java.lang.Exception)
{
AudioSelectOutBox.semaphore_audio_out_convert.release_passthru()
}
var global_audio_out_vu: Float = MainActivity.AUDIO_VU_MIN_VALUE
if (sample_count > 0)
{
val vu_value = AudioBar.audio_vu(buf, sample_count)
global_audio_out_vu = if (vu_value > MainActivity.AUDIO_VU_MIN_VALUE)
{
vu_value
} else
{
0f
}
}
val global_audio_out_vu_ = global_audio_out_vu
AudioBar.set_cur_value(global_audio_out_vu_.toInt(), AudioBar.audio_out_bar)
}
t_audio_pcm_play.start()
}
catch(_: Exception)
{
}

}
} catch (e: java.lang.Exception)
{
}
// -- play incoming bytes --
// -- play incoming bytes --
val delta = (System.currentTimeMillis() - d1)

sleep_millis_current = sleep_millis - delta
if (sleep_millis_current < 1)
{
sleep_millis_current = 1
} else if (sleep_millis_current > sleep_millis + 5)
{
sleep_millis_current = sleep_millis + 5
}

sleep(sleep_millis_current - 1L, 1000000 - 5000) // sleep
// Log.i(TAG, "ngc_audio_play_thread_start: running ...")
}
} catch (_: Exception)
{
}
Log.i(TAG, "ngc_audio_play_thread_start: Thread ending")
}
}
(ngc_audio_play_thread as Thread).start()
}

private fun load_db_prefs()
{
MainActivity.DB_PREF__open_files_directly = false
Expand Down Expand Up @@ -371,6 +533,8 @@ class TrifaToxService
var last_resend_pending_messages3_ms: Long = -1
var last_resend_pending_messages4_ms: Long = -1
var last_start_queued_fts_ms: Long = -1
var ngc_audio_play_thread_running = false
var ngc_audio_play_thread: Thread? = null

// ------------------------------
fun bootstrap_me()
Expand Down

0 comments on commit b0e8e37

Please sign in to comment.