feat: optimisations (#16)

.vs/VSWorkspaceState.json

@@ -0,0 +1,7 @@
+{
+  "ExpandedNodes": [
+    ""
+  ],
+  "SelectedNode": "\\C:\\Users\\ashka\\Source\\Repos\\client-sdk-unity",
+  "PreviewInSolutionExplorer": false
+}

Runtime/Scripts/AudioFrame.cs

@@ -1,43 +1,45 @@
 using System;
 using LiveKit.Proto;
 using LiveKit.Internal;
+using LiveKit.Internal.FFIClients.Requests;
 
 namespace LiveKit
 {
     public class AudioFrame : IDisposable
     {
         private AudioFrameBufferInfo _info;
-
-        internal readonly FfiHandle Handle;
         private bool _disposed = false;
+        private FfiHandle _handle;
+
+        internal FfiHandle Handle => _handle;
 
         public uint NumChannels => _info.NumChannels;
         public uint SampleRate => _info.SampleRate;
         public uint SamplesPerChannel => _info.SamplesPerChannel;
 
         public IntPtr Data => (IntPtr)_info.DataPtr;
-        public int Length => (int) (SamplesPerChannel * NumChannels * sizeof(short));
+        public int Length => (int)(SamplesPerChannel * NumChannels * sizeof(short));
 
         internal AudioFrame(FfiHandle handle, AudioFrameBufferInfo info)
         {
-            Handle = handle;
+            _handle = handle;
             _info = info;
         }
 
-        internal AudioFrame(int sampleRate, int numChannels, int samplesPerChannel) {
-            var alloc = new AllocAudioBufferRequest();
-            alloc.SampleRate = (uint) sampleRate;
-            alloc.NumChannels = (uint) numChannels;
-            alloc.SamplesPerChannel = (uint) samplesPerChannel;
-
-            var request = new FFIRequest();
-            request.AllocAudioBuffer = alloc;
-
-            var res = FfiClient.SendRequest(request);
-            var bufferInfo = res.AllocAudioBuffer.Buffer;
-
-            Handle = new FfiHandle((IntPtr)bufferInfo.Handle.Id);
+        internal AudioFrame(int sampleRate, int numChannels, int samplesPerChannel)
+        {
+            using var request = FFIBridge.Instance.NewRequest<AllocAudioBufferRequest>();
+            var alloc = request.request;
+            alloc.SampleRate = (uint)sampleRate;
+            alloc.NumChannels = (uint)numChannels;
+            alloc.SamplesPerChannel = (uint)samplesPerChannel;
+            using var response = request.Send();
+            FfiResponse res = response;
+            var bufferInfo = res.AllocAudioBuffer.Buffer.Info;
+            _handle = new FfiHandle((IntPtr)res.AllocAudioBuffer.Buffer.Handle.Id);
             _info = bufferInfo;
+
+            res.AllocAudioBuffer = null;
         }
 
         ~AudioFrame()
@@ -60,4 +62,4 @@ protected virtual void Dispose(bool disposing)
             }
         }
     }
-}
+}

Runtime/Scripts/AudioSource.cs

@@ -1,11 +1,9 @@
 using System;
-using System.Collections;
 using UnityEngine;
 using LiveKit.Proto;
 using LiveKit.Internal;
-using UnityEngine.Rendering;
-using Unity.Collections;
-using Unity.Collections.LowLevel.Unsafe;
+using System.Threading;
+using LiveKit.Internal.FFIClients.Requests;
 
 namespace LiveKit
 {
@@ -14,74 +12,135 @@ public class RtcAudioSource
         private AudioSource _audioSource;
         private AudioFilter _audioFilter;
 
-        internal readonly FfiHandle Handle;
+        private FfiHandle _handle;
+
+        internal FfiHandle Handle => _handle;
+
         protected AudioSourceInfo _info;
 
         // Used on the AudioThread
         private AudioFrame _frame;
-
-        public RtcAudioSource(AudioSource source)
+        private Thread _readAudioThread;
+        private object _lock = new object();
+        private float[] _data;
+        private int _channels;
+        private int _sampleRate;
+        private volatile bool _pending = false;
+
+        public RtcAudioSource(AudioSource source, AudioFilter audioFilter)
         {
-            var newAudioSource = new NewAudioSourceRequest();
+            using var request = FFIBridge.Instance.NewRequest<NewAudioSourceRequest>();
+            var newAudioSource = request.request;
             newAudioSource.Type = AudioSourceType.AudioSourceNative;
-
-            var request = new FFIRequest();
-            request.NewAudioSource = newAudioSource;
-
-            var resp = FfiClient.SendRequest(request);
-            _info = resp.NewAudioSource.Source;
-            Handle = new FfiHandle((IntPtr)_info.Handle.Id);
-            UpdateSource(source);
+            //newAudioSource.NumChannels = 2;
+            //newAudioSource.SampleRate = 48000;
+            using var response = request.Send();
+            FfiResponse res = response;
+            _info = res.NewAudioSource.Source.Info;
+            //TODO pooling handles
+            _handle = new FfiHandle((IntPtr)res.NewAudioSource.Source.Handle.Id);
+            UpdateSource(source, audioFilter);
         }
 
-        private void UpdateSource(AudioSource source)
+        public void Start()
         {
-            _audioSource = source;
-            _audioFilter = source.gameObject.AddComponent<AudioFilter>();
-            //_audioFilter.hideFlags = HideFlags.HideInInspector;
-            _audioFilter.AudioRead += OnAudioRead;
-            source.Play();
+            Stop();
+            _readAudioThread = new Thread(Update);
+            _readAudioThread.Start();
         }
 
+        public void Stop()
+        {
+            _readAudioThread?.Abort();
+        }
 
-        private void OnAudioRead(float[] data, int channels, int sampleRate)
+        private void Update()
         {
-            var samplesPerChannel = data.Length / channels;
-            if (_frame == null || _frame.NumChannels != channels 
-                || _frame.SampleRate != sampleRate 
-                || _frame.SamplesPerChannel != samplesPerChannel)
+            while (true)
             {
-                _frame = new AudioFrame(sampleRate, channels, samplesPerChannel);
+                Thread.Sleep(Constants.TASK_DELAY);
+                if (_pending)
+                {
+                    ReadAudio();
+                }
             }
+        }
 
-            static short FloatToS16(float v) {
-                v *= 32768f;
-                v = Math.Min(v, 32767f);
-                v = Math.Max(v, -32768f);
-                return (short)(v + Math.Sign(v) * 0.5f);
-            }
+        private void ReadAudio()
+        {
+            _pending = false;
+            lock (_lock)
+            {
+                var samplesPerChannel = _data.Length / _channels;
+                if (_frame == null
+                    || _frame.NumChannels != _channels
+                    || _frame.SampleRate != _sampleRate
+                    || _frame.SamplesPerChannel != samplesPerChannel)
+                {
+                    _frame = new AudioFrame(_sampleRate, _channels, samplesPerChannel);
+                }
 
-            unsafe {
-                var frameData = new Span<short>(_frame.Data.ToPointer(), _frame.Length / sizeof(short));
-                for (int i = 0; i < data.Length; i++)
+                try
+                {
+                    static short FloatToS16(float v)
+                    {
+                        v *= 32768f;
+                        v = Math.Min(v, 32767f);
+                        v = Math.Max(v, -32768f);
+                        return (short)(v + Math.Sign(v) * 0.5f);
+                    }
+
+                    unsafe
+                    {
+                        var frameData = new Span<short>(_frame.Data.ToPointer(), _frame.Length / sizeof(short));
+                        for (int i = 0; i < _data.Length; i++)
+                        {
+                            frameData[i] = FloatToS16(_data[i]);
+                        }
+
+                        using var request = FFIBridge.Instance.NewRequest<CaptureAudioFrameRequest>();
+                        var pushFrame = request.request;
+                        pushFrame.SourceHandle = (ulong)Handle.DangerousGetHandle();
+                        pushFrame.Buffer = new AudioFrameBufferInfo
+                        {
+                            DataPtr = (ulong)_frame.Data, NumChannels = _frame.NumChannels, SampleRate = _frame.SampleRate,
+                            SamplesPerChannel = _frame.SamplesPerChannel
+                        };
+                        using var response = request.Send();
+                    }
+
+                    Utils.Debug($"Pushed audio frame with {_data.Length} sample rate "
+                                + _frame.SampleRate
+                                + "  num channels "
+                                + _frame.NumChannels
+                                + " and samplers per channel "
+                                + _frame.SamplesPerChannel);
+                }
+                catch (Exception e)
                 {
-                    frameData[i] = FloatToS16(data[i]);
+                    Utils.Error("Audio Framedata error: " + e.Message);
                 }
             }
+        }
 
-            // Don't play the audio locally
-            Array.Clear(data, 0, data.Length);
-
-            var pushFrame = new CaptureAudioFrameRequest();
-            pushFrame.SourceHandle = new FFIHandleId { Id = (ulong)Handle.DangerousGetHandle() };
-            pushFrame.BufferHandle = new FFIHandleId { Id = (ulong)_frame.Handle.DangerousGetHandle() };
-
-            var request = new FFIRequest();
-            request.CaptureAudioFrame = pushFrame;
-
-            FfiClient.SendRequest(request);
+        private void UpdateSource(AudioSource source, AudioFilter audioFilter)
+        {
+            _audioSource = source;
+            _audioFilter = audioFilter;
+            _audioFilter.AudioRead += OnAudioRead;
+            _audioSource.Play();
+        }
 
-            Debug.Log($"Pushed audio frame with {data.Length} samples");
+        private void OnAudioRead(float[] data, int channels, int sampleRate)
+        {
+            lock (_lock)
+            {
+                _data = data;
+                _channels = channels;
+                _sampleRate = sampleRate;
+                _pending = true;
+            }
         }
+
     }
-}
+}