Skip to content

Commit b0f5508

Browse files
cloudwebrtcashkan-saeedi-mazdehNickKhalowcdga777markgrossnickle
authored
* Fixed a memory leak * Fixed the last error regarding the leak fix * fix. * ring buffer as struct * to prevent null reference * Multithread refactor (livekit#13) * disconnect support * Fix name * To handle better connection and disconnection from room * change to avoid using of Coroutines, using async methods instead * async OnAudioRead * Cleaning logs * Cancel tokens * Cancel token * remove Task.Run, since usually SendRequest is called from other task * getting rid of use YieldInstruction * adding audio filter in main thread * passing CancellationToken instead of CancellationTokenSource * added check for cancellation at beginning * Cleaning handles * fix await * remove cruft * Cancellation token cleanup * Factory async method * basic handling of remaining events * more async * wip * Compile pass * formatting * fix pending check * revert async changes * prep audio filter for threads * loggin * logs to debug publishing track * minor cleanup * added some methods to handle localtrack and SetSubscribed, Clenaing code * Cleaning up * fixes for yield instructions * macro for verbose logs * Audio data and pointer fix Still says samplerate and num of channels are incorrect * remove possibility of null * todo ref buffering * minor cleanup * cruft * todo object pooling * remove redundant Task usages on separated threads --------- Co-authored-by: cdga777 <[email protected]> Co-authored-by: nickkhalow <[email protected]> * feat: optimisations (livekit#16) * remove redundant Task usages on separated threads * FfiRequestsPool * IFFIClient interface * disposable FFIClient * FFIClient initialize simplify * PoolableFFIClient * NativeMethods pass pointer * response pooling * tests for reject manual creation of protocol objects * FfiResponseWrap, remove manual new Request calls * remove IPoolableFFIClient * FFIBridge introduction, remove closures, RAII with FfiResponseWrap and FfiRequestWrap * Clear -> generic ClearMessage * Make the Ffi Handle type consistent. * room support spans * cleanup. * feat: optimisation - memory pooling, FfiHandles (livekit#18) * fix clear message * optimise FfiNewRequest - no FfiHandle * LiveKitInitialize with IntPtr * const format * memory pool * inner message pooling * ArrayMemoryPool with thread safety * remove obsolete * thread safety * RingBuffer recycling * dangerous access for memory buffer (livekit#26) * feat: Audio/Video Support (livekit#29) * ensure channels and rate match * subscription change * fixing early subscription * track handle fix temp logs to confirm * track handle fix * streaming fixes * remix fix * resample log check * try off frame * handler fix * log cleanup * saving track for local Participant * remix potential fix * cleanup checks * testing without thread * revert * destroy cleanup and possible fix to ringbuffer * log cleanup * test * revert * disable thread for test * test * fix * adding threading back in for writing * video prep * handle fix * fix cast * removing streams from video for now * buffer fix * hack to get it to send a web texture * more consistent naming * format fix * bad merge fix * first pass on screen share and memory fix * debug and formatting * remove logs * updated protobuff and fixed compiler errors * video working * dynamic buffertype and stride * split video source and partial wip on cameravideoSource * camera support * merge cont. * wip merging * adding back stomped change * latest libs for mac * Fixes for Audio/Video Streaming * remove cruft * fix when an AudioTrack is created * mac fix * null check which can occur if you are closing app * format checks * utils * fix for metal * Subscription management * cleanup * log cleanup * cleanup * log --------- Co-authored-by: cdga777 <[email protected]> * update local metadata. * fix message truncation bug. * Update livekit_ffi.dll.meta * Update libwebrtc.jar.meta * update. * some minor improvements. * update. --------- Co-authored-by: Ashkan Saeidi Mazdeh <[email protected]> Co-authored-by: nickkhalow <[email protected]> Co-authored-by: cdga777 <[email protected]> Co-authored-by: Mark Grossnickle <[email protected]> Co-authored-by: Nick Khalow <[email protected]>
1 parent e8f771b commit b0f5508

File tree

81 files changed

+2008
-599
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

81 files changed

+2008
-599
lines changed

‎README.md.meta

+7
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Runtime/Plugins/ffi-android-arm64/liblivekit_ffi.so.meta

+1-1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Runtime/Plugins/ffi-linux-arm64/liblivekit_ffi.so.meta

+8-8
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Runtime/Plugins/ffi-windows-x86_64/livekit_ffi.dll.meta

+9-8
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Runtime/Plugins/libwebrtc.jar.meta

+26
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Runtime/Scripts/AudioFrame.cs

+7-6
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
using System;
22
using LiveKit.Proto;
3+
using LiveKit.Internal;
34
using Unity.Collections;
45
using Unity.Collections.LowLevel.Unsafe;
56

@@ -9,7 +10,7 @@ public class AudioFrame : IDisposable
910
{
1011
private AudioFrameBufferInfo _info;
1112

12-
private FfiOwnedHandle _handle;
13+
private FfiHandle _handle;
1314

1415
private bool _disposed = false;
1516

@@ -27,14 +28,14 @@ public class AudioFrame : IDisposable
2728

2829
public int Length => (int) (SamplesPerChannel * NumChannels * sizeof(short));
2930

30-
internal AudioFrame(FfiOwnedHandle handle, AudioFrameBufferInfo info)
31+
internal AudioFrame(OwnedAudioFrameBuffer info)
3132
{
32-
_handle = handle;
33-
_info = info;
33+
_handle = FfiHandle.FromOwnedHandle(info.Handle);
34+
_info = info.Info;
3435
_sampleRate = _info.SampleRate;
3536
_numChannels = _info.NumChannels;
3637
_samplesPerChannel = _info.SamplesPerChannel;
37-
_dataPtr = (IntPtr)info.DataPtr;
38+
_dataPtr = (IntPtr)_info.DataPtr;
3839
}
3940

4041
internal AudioFrame(int sampleRate, int numChannels, int samplesPerChannel) {
@@ -66,4 +67,4 @@ protected virtual void Dispose(bool disposing)
6667
}
6768
}
6869
}
69-
}
70+
}

‎Runtime/Scripts/AudioSource.cs

+111-52
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
using UnityEngine;
33
using LiveKit.Proto;
44
using LiveKit.Internal;
5+
using System.Threading;
6+
using LiveKit.Internal.FFIClients.Requests;
57

68
namespace LiveKit
79
{
@@ -10,86 +12,143 @@ public class RtcAudioSource
1012
private AudioSource _audioSource;
1113
private AudioFilter _audioFilter;
1214

13-
internal readonly FfiOwnedHandle Handle;
15+
internal readonly FfiHandle Handle;
1416
protected AudioSourceInfo _info;
1517

1618
// Used on the AudioThread
1719
private AudioFrame _frame;
20+
private Thread _readAudioThread;
21+
private object _lock = new object();
22+
private float[] _data;
23+
private int _channels;
24+
private int _sampleRate;
25+
private volatile bool _pending = false;
1826

1927
public RtcAudioSource(AudioSource source)
2028
{
21-
var newAudioSource = new NewAudioSourceRequest();
29+
using var request = FFIBridge.Instance.NewRequest<NewAudioSourceRequest>();
30+
var newAudioSource = request.request;
2231
newAudioSource.Type = AudioSourceType.AudioSourceNative;
2332
newAudioSource.NumChannels = 2;
2433
newAudioSource.SampleRate = 48000;
25-
26-
var request = new FfiRequest();
27-
request.NewAudioSource = newAudioSource;
28-
29-
var resp = FfiClient.SendRequest(request);
30-
var respSource = resp.NewAudioSource.Source;
31-
_info = respSource.Info;
32-
33-
Handle = respSource.Handle;
34+
using var response = request.Send();
35+
FfiResponse res = response;
36+
_info = res.NewAudioSource.Source.Info;
37+
//TODO pooling handles
38+
Handle = FfiHandle.FromOwnedHandle(res.NewAudioSource.Source.Handle);
3439
UpdateSource(source);
3540
}
3641

37-
private void UpdateSource(AudioSource source)
42+
public void Start()
3843
{
39-
_audioSource = source;
40-
_audioFilter = source.gameObject.AddComponent<AudioFilter>();
41-
//_audioFilter.hideFlags = HideFlags.HideInInspector;
44+
Stop();
45+
_readAudioThread = new Thread(Update);
46+
_readAudioThread.Start();
47+
4248
_audioFilter.AudioRead += OnAudioRead;
43-
source.Play();
49+
_audioSource.Play();
4450
}
4551

46-
47-
private void OnAudioRead(float[] data, int channels, int sampleRate)
52+
public void Stop()
4853
{
49-
var samplesPerChannel = data.Length / channels;
50-
if (_frame == null || _frame.NumChannels != channels
51-
|| _frame.SampleRate != sampleRate
52-
|| _frame.SamplesPerChannel != samplesPerChannel)
54+
if (_readAudioThread != null)
5355
{
54-
_frame = new AudioFrame(sampleRate, channels, samplesPerChannel);
55-
}
56-
57-
static short FloatToS16(float v) {
58-
v *= 32768f;
59-
v = Math.Min(v, 32767f);
60-
v = Math.Max(v, -32768f);
61-
return (short)(v + Math.Sign(v) * 0.5f);
56+
_readAudioThread.Abort();
57+
_readAudioThread = null;
6258
}
59+
if(_audioFilter) _audioFilter.AudioRead -= OnAudioRead;
60+
if(_audioSource) _audioSource.Stop();
61+
}
6362

64-
unsafe {
65-
var frameData = new Span<short>(_frame.Data.ToPointer(), _frame.Length / sizeof(short));
66-
for (int i = 0; i < data.Length; i++)
63+
private void Update()
64+
{
65+
while (true)
66+
{
67+
Thread.Sleep(Constants.TASK_DELAY);
68+
if (_pending)
6769
{
68-
frameData[i] = FloatToS16(data[i]);
70+
ReadAudio();
6971
}
7072
}
73+
}
7174

72-
// Don't play the audio locally
73-
Array.Clear(data, 0, data.Length);
74-
75-
var audioFrameBufferInfo = new AudioFrameBufferInfo();
76-
77-
audioFrameBufferInfo.DataPtr = (ulong)_frame.Data;
78-
audioFrameBufferInfo.NumChannels = _frame.NumChannels;
79-
audioFrameBufferInfo.SampleRate = _frame.SampleRate;
80-
audioFrameBufferInfo.SamplesPerChannel = _frame.SamplesPerChannel;
81-
82-
var pushFrame = new CaptureAudioFrameRequest();
83-
pushFrame.SourceHandle = Handle.Id;
84-
pushFrame.Buffer = audioFrameBufferInfo;
85-
86-
var request = new FfiRequest();
87-
request.CaptureAudioFrame = pushFrame;
75+
private void ReadAudio()
76+
{
77+
_pending = false;
78+
lock (_lock)
79+
{
80+
var samplesPerChannel = _data.Length / _channels;
81+
if (_frame == null
82+
|| _frame.NumChannels != _channels
83+
|| _frame.SampleRate != _sampleRate
84+
|| _frame.SamplesPerChannel != samplesPerChannel)
85+
{
86+
_frame = new AudioFrame(_sampleRate, _channels, samplesPerChannel);
87+
}
88+
try
89+
{
8890

89-
FfiClient.SendRequest(request);
91+
static short FloatToS16(float v)
92+
{
93+
v *= 32768f;
94+
v = Math.Min(v, 32767f);
95+
v = Math.Max(v, -32768f);
96+
return (short)(v + Math.Sign(v) * 0.5f);
97+
}
98+
99+
unsafe
100+
{
101+
var frameData = new Span<short>(_frame.Data.ToPointer(), _frame.Length / sizeof(short));
102+
for (int i = 0; i < _data.Length; i++)
103+
{
104+
frameData[i] = FloatToS16(_data[i]);
105+
}
106+
107+
// Don't play the audio locally
108+
Array.Clear(_data, 0, _data.Length);
109+
110+
using var request = FFIBridge.Instance.NewRequest<CaptureAudioFrameRequest>();
111+
using var audioFrameBufferInfo = request.TempResource<AudioFrameBufferInfo>();
112+
113+
var pushFrame = request.request;
114+
pushFrame.SourceHandle = (ulong)Handle.DangerousGetHandle();
115+
116+
pushFrame.Buffer = audioFrameBufferInfo;
117+
pushFrame.Buffer.DataPtr = (ulong)_frame.Data;
118+
pushFrame.Buffer.NumChannels = _frame.NumChannels;
119+
pushFrame.Buffer.SampleRate = _frame.SampleRate;
120+
pushFrame.Buffer.SamplesPerChannel = _frame.SamplesPerChannel;
121+
122+
using var response = request.Send();
123+
124+
pushFrame.Buffer.DataPtr = 0;
125+
pushFrame.Buffer.NumChannels = 0;
126+
pushFrame.Buffer.SampleRate = 0;
127+
pushFrame.Buffer.SamplesPerChannel = 0;
128+
}
129+
}
130+
catch (Exception e)
131+
{
132+
Utils.Error("Audio Framedata error: " + e.Message);
133+
}
134+
}
135+
}
90136

137+
private void UpdateSource(AudioSource source)
138+
{
139+
_audioSource = source;
140+
_audioFilter = source.gameObject.AddComponent<AudioFilter>();
141+
}
91142

92-
//Debug.Log($"Pushed audio frame with {data.Length} samples");
143+
private void OnAudioRead(float[] data, int channels, int sampleRate)
144+
{
145+
lock (_lock)
146+
{
147+
_data = data;
148+
_channels = channels;
149+
_sampleRate = sampleRate;
150+
_pending = true;
151+
}
93152
}
94153
}
95154
}

0 commit comments

Comments
 (0)