saving track for local Participant (+27 squashed commits)

Squashed commits:
[0e29f5f] log cleanup
[c537528] handler fix
[9bb1de9] try off frame
[be7bf95] resample log check
[d89b17d] remix fix
[fe7d015] streaming fixes
[0c6946d] track handle fix
[8161ace] track handle fix

temp logs to confirm
[77ed7ae] fixing early subscription
[da86881] subscription change
[4cf717e] ensure channels and rate match
[768fb01] Multithread refactor (livekit#13)

* disconnect support

* Fix name

* To handle better connection and disconnection from room

* change to avoid using of Coroutines, using async methods instead

* async OnAudioRead

* Cleaning logs

* Cancel tokens

* Cancel token

* remove Task.Run, since usually SendRequest is called from other task

* getting rid of use YieldInstruction

* adding audio filter in main thread

* passing CancellationToken instead of CancellationTokenSource

* added check for cancellation at beginning

* Cleaning handles

* fix await

* remove cruft

* Cancellation token cleanup

* Factory async method

* basic handling of remaining events

* more async

* wip

* Compile pass

* formatting

* fix pending check

* revert async changes

* prep audio filter for threads

* loggin

* logs to debug publishing track

* minor cleanup

* added some methods to handle localtrack and SetSubscribed, Clenaing code

* Cleaning up

* fixes for yield instructions

* macro for verbose logs

* Audio data and pointer fix

Still says samplerate and num of channels are incorrect

* remove possibility of null

* todo ref buffering

* minor cleanup

* cruft

* todo object pooling

* remove redundant Task usages on separated threads

---------

Co-authored-by: cdga777 <cdga777@gmail.com>
Co-authored-by: nickkhalow <nickhalow@gmail.com>
[a400ec9] enable nullables
[a5a63c4] minor cleanup
[892407a] to prevent null reference
[511ef58] Hack for now

Room Ids are different
[6456edc] Test sending data
[eca1cb5] new initialize call
[830ebd1] modification trying to fix compiler errors
[496618c] Added metas.
[2cf59c3] Updated FFIClient with the correct FfiRequest class name based on updated proto file generated classes.
[c8e3906] updated protobuf messages to latest to be able to publish data.
[bb762d2] Updated dll
[80d3dcd] Fixed the last error regarding the leak fix
[7320b31] Fixed a compile error on the distructor
[561282f] Fixed a memory leak
[db871de] Fixed building issue.

.vs/VSWorkspaceState.json

@@ -0,0 +1,7 @@
+{
+  "ExpandedNodes": [
+    ""
+  ],
+  "SelectedNode": "\\C:\\Users\\ashka\\Source\\Repos\\client-sdk-unity",
+  "PreviewInSolutionExplorer": false
+}

Runtime/Scripts/AudioFrame.cs

@@ -7,9 +7,10 @@ namespace LiveKit
     public class AudioFrame : IDisposable
     {
         private AudioFrameBufferInfo _info;
-
-        internal readonly FfiHandle Handle;
         private bool _disposed = false;
+        private FfiHandle _handle;
+
+        internal FfiHandle Handle => _handle;
 
         public uint NumChannels => _info.NumChannels;
         public uint SampleRate => _info.SampleRate;
@@ -20,7 +21,7 @@ public class AudioFrame : IDisposable
 
         internal AudioFrame(FfiHandle handle, AudioFrameBufferInfo info)
         {
-            Handle = handle;
+            _handle = handle;
             _info = info;
         }
 
@@ -30,14 +31,20 @@ internal AudioFrame(int sampleRate, int numChannels, int samplesPerChannel) {
             alloc.NumChannels = (uint) numChannels;
             alloc.SamplesPerChannel = (uint) samplesPerChannel;
 
-            var request = new FFIRequest();
+            var request = new FfiRequest();
             request.AllocAudioBuffer = alloc;
 
+            Init(request);
+        }
+
+        private void Init(FfiRequest request)
+        {
             var res = FfiClient.SendRequest(request);
-            var bufferInfo = res.AllocAudioBuffer.Buffer;
+            var bufferInfo = res.AllocAudioBuffer.Buffer.Info;
 
-            Handle = new FfiHandle((IntPtr)bufferInfo.Handle.Id);
+            _handle = new FfiHandle((IntPtr)res.AllocAudioBuffer.Buffer.Handle.Id);
             _info = bufferInfo;
+
         }
 
         ~AudioFrame()

Runtime/Scripts/AudioSource.cs

@@ -6,6 +6,8 @@
 using UnityEngine.Rendering;
 using Unity.Collections;
 using Unity.Collections.LowLevel.Unsafe;
+using System.Threading.Tasks;
+using System.Threading;
 
 namespace LiveKit
 {
@@ -14,74 +16,133 @@ public class RtcAudioSource
         private AudioSource _audioSource;
         private AudioFilter _audioFilter;
 
-        internal readonly FfiHandle Handle;
+        private FfiHandle _handle;
+
+        internal FfiHandle Handle => _handle;
+
         protected AudioSourceInfo _info;
 
         // Used on the AudioThread
         private AudioFrame _frame;
-
-        public RtcAudioSource(AudioSource source)
+        private Thread _readAudioThread;
+        private object _lock = new object();
+        private float[] _data;
+        private int _channels;
+        private int _sampleRate;
+        private volatile bool _pending = false;
+
+        public RtcAudioSource(AudioSource source, AudioFilter audioFilter)
         {
             var newAudioSource = new NewAudioSourceRequest();
             newAudioSource.Type = AudioSourceType.AudioSourceNative;
+            newAudioSource.NumChannels = 2;
+            newAudioSource.SampleRate = 48000;
 
-            var request = new FFIRequest();
+            var request = new FfiRequest();
             request.NewAudioSource = newAudioSource;
 
             var resp = FfiClient.SendRequest(request);
-            _info = resp.NewAudioSource.Source;
-            Handle = new FfiHandle((IntPtr)_info.Handle.Id);
-            UpdateSource(source);
+            _info = resp.NewAudioSource.Source.Info;
+            _handle = new FfiHandle((IntPtr)resp.NewAudioSource.Source.Handle.Id);
+            UpdateSource(source, audioFilter);
         }
 
-        private void UpdateSource(AudioSource source)
+        public void Start()
         {
-            _audioSource = source;
-            _audioFilter = source.gameObject.AddComponent<AudioFilter>();
-            //_audioFilter.hideFlags = HideFlags.HideInInspector;
-            _audioFilter.AudioRead += OnAudioRead;
-            source.Play();
+            Stop();
+            _readAudioThread = new Thread(Update);
+            _readAudioThread.Start();
         }
 
+        public void Stop()
+        {
+            _readAudioThread?.Abort();
+        }
 
-        private void OnAudioRead(float[] data, int channels, int sampleRate)
+        private void Update()
         {
-            var samplesPerChannel = data.Length / channels;
-            if (_frame == null || _frame.NumChannels != channels 
-                || _frame.SampleRate != sampleRate 
-                || _frame.SamplesPerChannel != samplesPerChannel)
+            while (true)
             {
-                _frame = new AudioFrame(sampleRate, channels, samplesPerChannel);
+                Thread.Sleep(Constants.TASK_DELAY);
+                if (_pending)
+                {
+                    ReadAudio();
+                }
             }
+        }
 
-            static short FloatToS16(float v) {
-                v *= 32768f;
-                v = Math.Min(v, 32767f);
-                v = Math.Max(v, -32768f);
-                return (short)(v + Math.Sign(v) * 0.5f);
-            }
+        private void ReadAudio()
+        {
+            _pending = false;
+            lock (_lock)
+            {
+                var samplesPerChannel = _data.Length / _channels;
+                if (_frame == null
+                    || _frame.NumChannels != _channels
+                    || _frame.SampleRate != _sampleRate
+                    || _frame.SamplesPerChannel != samplesPerChannel)
+                {
+                    _frame = new AudioFrame(_sampleRate, _channels, samplesPerChannel);
+                }
 
-            unsafe {
-                var frameData = new Span<short>(_frame.Data.ToPointer(), _frame.Length / sizeof(short));
-                for (int i = 0; i < data.Length; i++)
+                try
                 {
-                    frameData[i] = FloatToS16(data[i]);
+                    static short FloatToS16(float v)
+                    {
+                        v *= 32768f;
+                        v = Math.Min(v, 32767f);
+                        v = Math.Max(v, -32768f);
+                        return (short)(v + Math.Sign(v) * 0.5f);
+                    }
+
+                    unsafe
+                    {
+                        var frameData = new Span<short>(_frame.Data.ToPointer(), _frame.Length / sizeof(short));
+                        for (int i = 0; i < _data.Length; i++)
+                        {
+                            frameData[i] = FloatToS16(_data[i]);
+                        }
+
+                        var pushFrame = new CaptureAudioFrameRequest();
+                        pushFrame.SourceHandle = (ulong)Handle.DangerousGetHandle();
+                        pushFrame.Buffer = new AudioFrameBufferInfo()
+                        {
+                            DataPtr = (ulong)_frame.Data,
+                            NumChannels = _frame.NumChannels,
+                            SampleRate = _frame.SampleRate,
+                            SamplesPerChannel = _frame.SamplesPerChannel
+                        };
+                        Debug.Log($"Num Channels {_frame.NumChannels} Sample {_frame.SampleRate} and {_frame.SamplesPerChannel}");
+                        var request = new FfiRequest();
+                        request.CaptureAudioFrame = pushFrame;
+                        FfiClient.SendRequest(request);
+                    }
+                }
+                catch (Exception e)
+                {
+                    Utils.Error("Audio Framedata error: " + e.Message);
                 }
             }
+        }
 
-            // Don't play the audio locally
-            Array.Clear(data, 0, data.Length);
-
-            var pushFrame = new CaptureAudioFrameRequest();
-            pushFrame.SourceHandle = new FFIHandleId { Id = (ulong)Handle.DangerousGetHandle() };
-            pushFrame.BufferHandle = new FFIHandleId { Id = (ulong)_frame.Handle.DangerousGetHandle() };
-
-            var request = new FFIRequest();
-            request.CaptureAudioFrame = pushFrame;
-
-            FfiClient.SendRequest(request);
+        private void UpdateSource(AudioSource source, AudioFilter audioFilter)
+        {
+            _audioSource = source;
+            _audioFilter = audioFilter;
+            _audioFilter.AudioRead += OnAudioRead;
+            _audioSource.Play();
+        }
 
-            Debug.Log($"Pushed audio frame with {data.Length} samples");
+        private void OnAudioRead(float[] data, int channels, int sampleRate)
+        {
+            lock (_lock)
+            {
+                _data = data;
+                _channels = channels;
+                _sampleRate = sampleRate;
+                _pending = true;
+            }
         }
+
     }
-}
+}