diff --git a/README.md.meta b/README.md.meta new file mode 100644 index 0000000..ac4f7ec --- /dev/null +++ b/README.md.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 8e02b2c343acee545beb7cc63d3045c8 +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Plugins/ffi-android-arm64/liblivekit_ffi.so.meta b/Runtime/Plugins/ffi-android-arm64/liblivekit_ffi.so.meta index c20e3af..af3defb 100644 --- a/Runtime/Plugins/ffi-android-arm64/liblivekit_ffi.so.meta +++ b/Runtime/Plugins/ffi-android-arm64/liblivekit_ffi.so.meta @@ -7,7 +7,7 @@ PluginImporter: executionOrder: {} defineConstraints: [] isPreloaded: 0 - isOverridable: 1 + isOverridable: 0 isExplicitlyReferenced: 0 validateReferences: 1 platformData: diff --git a/Runtime/Plugins/ffi-linux-arm64/liblivekit_ffi.so.meta b/Runtime/Plugins/ffi-linux-arm64/liblivekit_ffi.so.meta index cf3c180..5f4affc 100644 --- a/Runtime/Plugins/ffi-linux-arm64/liblivekit_ffi.so.meta +++ b/Runtime/Plugins/ffi-linux-arm64/liblivekit_ffi.so.meta @@ -17,12 +17,12 @@ PluginImporter: enabled: 0 settings: Exclude Android: 1 - Exclude Editor: 1 - Exclude Linux64: 1 + Exclude Editor: 0 + Exclude Linux64: 0 Exclude OSXUniversal: 1 Exclude WebGL: 1 - Exclude Win: 1 - Exclude Win64: 1 + Exclude Win: 0 + Exclude Win64: 0 Exclude WindowsStoreApps: 1 Exclude iOS: 1 - first: @@ -40,7 +40,7 @@ PluginImporter: - first: Editor: Editor second: - enabled: 0 + enabled: 1 settings: CPU: AnyCPU DefaultValueInitialized: true @@ -48,7 +48,7 @@ PluginImporter: - first: Standalone: Linux64 second: - enabled: 0 + enabled: 1 settings: CPU: None - first: @@ -60,13 +60,13 @@ PluginImporter: - first: Standalone: Win second: - enabled: 0 + enabled: 1 settings: CPU: None - first: Standalone: Win64 second: - enabled: 0 + enabled: 1 settings: CPU: None - first: diff --git a/Runtime/Plugins/ffi-windows-x86_64/livekit_ffi.dll.meta b/Runtime/Plugins/ffi-windows-x86_64/livekit_ffi.dll.meta index 84825fb..5c8a334 100644 --- a/Runtime/Plugins/ffi-windows-x86_64/livekit_ffi.dll.meta +++ b/Runtime/Plugins/ffi-windows-x86_64/livekit_ffi.dll.meta @@ -21,8 +21,8 @@ PluginImporter: Exclude Linux64: 1 Exclude OSXUniversal: 1 Exclude WebGL: 1 - Exclude Win: 1 - Exclude Win64: 1 + Exclude Win: 0 + Exclude Win64: 0 Exclude WindowsStoreApps: 1 Exclude iOS: 1 - first: @@ -50,13 +50,13 @@ PluginImporter: second: enabled: 0 settings: - CPU: None + CPU: AnyCPU - first: Standalone: OSXUniversal second: enabled: 0 settings: - CPU: None + CPU: x86_64 - first: Standalone: Win second: @@ -68,7 +68,7 @@ PluginImporter: second: enabled: 0 settings: - CPU: None + CPU: x86_64 - first: WebGL: WebGL second: @@ -89,10 +89,11 @@ PluginImporter: second: enabled: 0 settings: - AddToEmbeddedBinaries: false CPU: AnyCPU - CompileFlags: - FrameworkDependencies: + DontProcess: false + PlaceholderPath: + SDK: AnySDK + ScriptingBackend: AnyScriptingBackend userData: assetBundleName: assetBundleVariant: diff --git a/Runtime/Plugins/libwebrtc.jar.meta b/Runtime/Plugins/libwebrtc.jar.meta index af3efed..5fbbef1 100644 --- a/Runtime/Plugins/libwebrtc.jar.meta +++ b/Runtime/Plugins/libwebrtc.jar.meta @@ -26,7 +26,33 @@ PluginImporter: second: enabled: 0 settings: + CPU: ARM64 DefaultValueInitialized: true + OS: OSX + - first: + Standalone: Linux64 + second: + enabled: 0 + settings: + CPU: None + - first: + Standalone: OSXUniversal + second: + enabled: 1 + settings: + CPU: ARM64 + - first: + Standalone: Win + second: + enabled: 0 + settings: + CPU: None + - first: + Standalone: Win64 + second: + enabled: 0 + settings: + CPU: None userData: assetBundleName: assetBundleVariant: diff --git a/Runtime/Scripts/AudioFrame.cs b/Runtime/Scripts/AudioFrame.cs index 3344b3a..e9e5dbd 100644 --- a/Runtime/Scripts/AudioFrame.cs +++ b/Runtime/Scripts/AudioFrame.cs @@ -1,5 +1,6 @@ using System; using LiveKit.Proto; +using LiveKit.Internal; using Unity.Collections; using Unity.Collections.LowLevel.Unsafe; @@ -9,7 +10,7 @@ public class AudioFrame : IDisposable { private AudioFrameBufferInfo _info; - private FfiOwnedHandle _handle; + private FfiHandle _handle; private bool _disposed = false; @@ -27,14 +28,14 @@ public class AudioFrame : IDisposable public int Length => (int) (SamplesPerChannel * NumChannels * sizeof(short)); - internal AudioFrame(FfiOwnedHandle handle, AudioFrameBufferInfo info) + internal AudioFrame(OwnedAudioFrameBuffer info) { - _handle = handle; - _info = info; + _handle = FfiHandle.FromOwnedHandle(info.Handle); + _info = info.Info; _sampleRate = _info.SampleRate; _numChannels = _info.NumChannels; _samplesPerChannel = _info.SamplesPerChannel; - _dataPtr = (IntPtr)info.DataPtr; + _dataPtr = (IntPtr)_info.DataPtr; } internal AudioFrame(int sampleRate, int numChannels, int samplesPerChannel) { @@ -66,4 +67,4 @@ protected virtual void Dispose(bool disposing) } } } -} +} \ No newline at end of file diff --git a/Runtime/Scripts/AudioSource.cs b/Runtime/Scripts/AudioSource.cs index 79c6e33..1d71733 100644 --- a/Runtime/Scripts/AudioSource.cs +++ b/Runtime/Scripts/AudioSource.cs @@ -2,6 +2,8 @@ using UnityEngine; using LiveKit.Proto; using LiveKit.Internal; +using System.Threading; +using LiveKit.Internal.FFIClients.Requests; namespace LiveKit { @@ -10,86 +12,143 @@ public class RtcAudioSource private AudioSource _audioSource; private AudioFilter _audioFilter; - internal readonly FfiOwnedHandle Handle; + internal readonly FfiHandle Handle; protected AudioSourceInfo _info; // Used on the AudioThread private AudioFrame _frame; + private Thread _readAudioThread; + private object _lock = new object(); + private float[] _data; + private int _channels; + private int _sampleRate; + private volatile bool _pending = false; public RtcAudioSource(AudioSource source) { - var newAudioSource = new NewAudioSourceRequest(); + using var request = FFIBridge.Instance.NewRequest(); + var newAudioSource = request.request; newAudioSource.Type = AudioSourceType.AudioSourceNative; newAudioSource.NumChannels = 2; newAudioSource.SampleRate = 48000; - - var request = new FfiRequest(); - request.NewAudioSource = newAudioSource; - - var resp = FfiClient.SendRequest(request); - var respSource = resp.NewAudioSource.Source; - _info = respSource.Info; - - Handle = respSource.Handle; + using var response = request.Send(); + FfiResponse res = response; + _info = res.NewAudioSource.Source.Info; + //TODO pooling handles + Handle = FfiHandle.FromOwnedHandle(res.NewAudioSource.Source.Handle); UpdateSource(source); } - private void UpdateSource(AudioSource source) + public void Start() { - _audioSource = source; - _audioFilter = source.gameObject.AddComponent(); - //_audioFilter.hideFlags = HideFlags.HideInInspector; + Stop(); + _readAudioThread = new Thread(Update); + _readAudioThread.Start(); + _audioFilter.AudioRead += OnAudioRead; - source.Play(); + _audioSource.Play(); } - - private void OnAudioRead(float[] data, int channels, int sampleRate) + public void Stop() { - var samplesPerChannel = data.Length / channels; - if (_frame == null || _frame.NumChannels != channels - || _frame.SampleRate != sampleRate - || _frame.SamplesPerChannel != samplesPerChannel) + if (_readAudioThread != null) { - _frame = new AudioFrame(sampleRate, channels, samplesPerChannel); - } - - static short FloatToS16(float v) { - v *= 32768f; - v = Math.Min(v, 32767f); - v = Math.Max(v, -32768f); - return (short)(v + Math.Sign(v) * 0.5f); + _readAudioThread.Abort(); + _readAudioThread = null; } + if(_audioFilter) _audioFilter.AudioRead -= OnAudioRead; + if(_audioSource) _audioSource.Stop(); + } - unsafe { - var frameData = new Span(_frame.Data.ToPointer(), _frame.Length / sizeof(short)); - for (int i = 0; i < data.Length; i++) + private void Update() + { + while (true) + { + Thread.Sleep(Constants.TASK_DELAY); + if (_pending) { - frameData[i] = FloatToS16(data[i]); + ReadAudio(); } } + } - // Don't play the audio locally - Array.Clear(data, 0, data.Length); - - var audioFrameBufferInfo = new AudioFrameBufferInfo(); - - audioFrameBufferInfo.DataPtr = (ulong)_frame.Data; - audioFrameBufferInfo.NumChannels = _frame.NumChannels; - audioFrameBufferInfo.SampleRate = _frame.SampleRate; - audioFrameBufferInfo.SamplesPerChannel = _frame.SamplesPerChannel; - - var pushFrame = new CaptureAudioFrameRequest(); - pushFrame.SourceHandle = Handle.Id; - pushFrame.Buffer = audioFrameBufferInfo; - - var request = new FfiRequest(); - request.CaptureAudioFrame = pushFrame; + private void ReadAudio() + { + _pending = false; + lock (_lock) + { + var samplesPerChannel = _data.Length / _channels; + if (_frame == null + || _frame.NumChannels != _channels + || _frame.SampleRate != _sampleRate + || _frame.SamplesPerChannel != samplesPerChannel) + { + _frame = new AudioFrame(_sampleRate, _channels, samplesPerChannel); + } + try + { - FfiClient.SendRequest(request); + static short FloatToS16(float v) + { + v *= 32768f; + v = Math.Min(v, 32767f); + v = Math.Max(v, -32768f); + return (short)(v + Math.Sign(v) * 0.5f); + } + + unsafe + { + var frameData = new Span(_frame.Data.ToPointer(), _frame.Length / sizeof(short)); + for (int i = 0; i < _data.Length; i++) + { + frameData[i] = FloatToS16(_data[i]); + } + + // Don't play the audio locally + Array.Clear(_data, 0, _data.Length); + + using var request = FFIBridge.Instance.NewRequest(); + using var audioFrameBufferInfo = request.TempResource(); + + var pushFrame = request.request; + pushFrame.SourceHandle = (ulong)Handle.DangerousGetHandle(); + + pushFrame.Buffer = audioFrameBufferInfo; + pushFrame.Buffer.DataPtr = (ulong)_frame.Data; + pushFrame.Buffer.NumChannels = _frame.NumChannels; + pushFrame.Buffer.SampleRate = _frame.SampleRate; + pushFrame.Buffer.SamplesPerChannel = _frame.SamplesPerChannel; + + using var response = request.Send(); + + pushFrame.Buffer.DataPtr = 0; + pushFrame.Buffer.NumChannels = 0; + pushFrame.Buffer.SampleRate = 0; + pushFrame.Buffer.SamplesPerChannel = 0; + } + } + catch (Exception e) + { + Utils.Error("Audio Framedata error: " + e.Message); + } + } + } + private void UpdateSource(AudioSource source) + { + _audioSource = source; + _audioFilter = source.gameObject.AddComponent(); + } - //Debug.Log($"Pushed audio frame with {data.Length} samples"); + private void OnAudioRead(float[] data, int channels, int sampleRate) + { + lock (_lock) + { + _data = data; + _channels = channels; + _sampleRate = sampleRate; + _pending = true; + } } } } diff --git a/Runtime/Scripts/AudioStream.cs b/Runtime/Scripts/AudioStream.cs index 5d2fee2..ece8ca8 100644 --- a/Runtime/Scripts/AudioStream.cs +++ b/Runtime/Scripts/AudioStream.cs @@ -3,12 +3,13 @@ using LiveKit.Internal; using LiveKit.Proto; using System.Runtime.InteropServices; +using LiveKit.Internal.FFIClients.Requests; namespace LiveKit { public class AudioStream { - internal readonly FfiOwnedHandle Handle; + internal readonly FfiHandle Handle; private AudioSource _audioSource; private AudioFilter _audioFilter; private RingBuffer _buffer; @@ -26,17 +27,14 @@ public AudioStream(IAudioTrack audioTrack, AudioSource source) if (!audioTrack.Participant.TryGetTarget(out var participant)) throw new InvalidOperationException("audiotrack's participant is invalid"); - var newAudioStream = new NewAudioStreamRequest(); - newAudioStream.TrackHandle = ((Track)audioTrack).TrackHandle.Id; + using var request = FFIBridge.Instance.NewRequest(); + var newAudioStream = request.request; + newAudioStream.TrackHandle = (ulong)audioTrack.TrackHandle.DangerousGetHandle(); newAudioStream.Type = AudioStreamType.AudioStreamNative; - - var request = new FfiRequest(); - request.NewAudioStream = newAudioStream; - - var resp = FfiClient.SendRequest(request); - var streamInfo = resp.NewAudioStream.Stream; - - Handle = streamInfo.Handle; + + using var response = request.Send(); + FfiResponse res = response; + Handle = FfiHandle.FromOwnedHandle(res.NewAudioStream.Stream.Handle); FfiClient.Instance.AudioStreamEventReceived += OnAudioStreamEvent; UpdateSource(source); @@ -59,6 +57,7 @@ private void OnAudioRead(float[] data, int channels, int sampleRate) if (_buffer == null || channels != _numChannels || sampleRate != _sampleRate || data.Length != _tempBuffer.Length) { int size = (int)(channels * sampleRate * 0.2); + _buffer?.Dispose(); _buffer = new RingBuffer(size * sizeof(short)); _tempBuffer = new short[data.Length]; _numChannels = (uint)channels; @@ -86,14 +85,13 @@ static float S16ToFloat(short v) // Called on the MainThread (See FfiClient) private void OnAudioStreamEvent(AudioStreamEvent e) { - if(Handle.Id != e.StreamHandle) + if((ulong)Handle.DangerousGetHandle() != e.StreamHandle) return; if (e.MessageCase != AudioStreamEvent.MessageOneofCase.FrameReceived) return; - var newFrame = e.FrameReceived.Frame; - var frame = new AudioFrame(newFrame.Handle, newFrame.Info); + var frame = new AudioFrame(e.FrameReceived.Frame); lock (_lock) { diff --git a/Runtime/Scripts/CameraVideoSource.cs b/Runtime/Scripts/CameraVideoSource.cs new file mode 100644 index 0000000..3637110 --- /dev/null +++ b/Runtime/Scripts/CameraVideoSource.cs @@ -0,0 +1,64 @@ +using UnityEngine; +using LiveKit.Proto; +using LiveKit.Internal; +using UnityEngine.Rendering; +using Unity.Collections; + +namespace LiveKit +{ + public class CameraVideoSource : RtcVideoSource + { + public Camera Camera { get; } + + public override int GetWidth() + { + return Camera.pixelWidth; + } + + public override int GetHeight() + { + return Camera.pixelHeight; + } + + public CameraVideoSource(Camera camera, VideoBufferType bufferType = VideoBufferType.Rgba) : base(VideoStreamSource.Screen, bufferType) + { + Camera = camera; + + var targetFormat = Utils.GetSupportedGraphicsFormat(SystemInfo.graphicsDeviceType); + _dest = new RenderTexture(GetWidth(), GetHeight(), 0, targetFormat); + camera.targetTexture = _dest as RenderTexture; + _data = new NativeArray(GetWidth() * GetHeight() * GetStrideForBuffer(bufferType), Allocator.Persistent); + } + + ~CameraVideoSource() + { + Dispose(); + ClearRenderTexture(); + } + + public override void Stop() + { + base.Stop(); + ClearRenderTexture(); + } + + private void ClearRenderTexture() + { + if (_dest) + { + var renderText = _dest as RenderTexture; + renderText.Release(); // can only be done on main thread + } + } + + // Read the texture data into a native array asynchronously + protected override void ReadBuffer() + { + if (_reading) + return; + _reading = true; + AsyncGPUReadback.RequestIntoNativeArray(ref _data, _dest, 0, GetTextureFormat(_bufferType), OnReadback); + } + } +} + diff --git a/Runtime/Scripts/VideoSource.cs.meta b/Runtime/Scripts/CameraVideoSource.cs.meta similarity index 83% rename from Runtime/Scripts/VideoSource.cs.meta rename to Runtime/Scripts/CameraVideoSource.cs.meta index ac9d70c..7b80115 100644 --- a/Runtime/Scripts/VideoSource.cs.meta +++ b/Runtime/Scripts/CameraVideoSource.cs.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: b3995117cdee64fb3b81d5cf568e03c6 +guid: a687aca3b53dfe64fa4e6061a03d4f8d MonoImporter: externalObjects: {} serializedVersion: 2 diff --git a/Runtime/Scripts/E2EE.cs b/Runtime/Scripts/E2EE.cs index 9f8161b..3aba416 100644 --- a/Runtime/Scripts/E2EE.cs +++ b/Runtime/Scripts/E2EE.cs @@ -1,8 +1,8 @@ using System.Collections.Generic; using LiveKit.Internal; +using LiveKit.Internal.FFIClients.Requests; using LiveKit.Proto; - namespace LiveKit { public enum EncryptionType @@ -36,10 +36,10 @@ public Proto.E2eeOptions ToProto() public class KeyProvider { - internal FfiOwnedHandle RoomHandle; + internal FfiHandle RoomHandle; public KeyProviderOptions KeyProviderOptions; - public KeyProvider(FfiOwnedHandle roomHandle, KeyProviderOptions keyProviderOptions) + public KeyProvider(FfiHandle roomHandle, KeyProviderOptions keyProviderOptions) { RoomHandle = roomHandle; KeyProviderOptions = keyProviderOptions; @@ -47,82 +47,67 @@ public KeyProvider(FfiOwnedHandle roomHandle, KeyProviderOptions keyProviderOpti public void SetSharedKey(byte[] key, int keyIndex) { - var e2eeReq = new E2eeRequest(); - var req = new FfiRequest(); - req.E2Ee = e2eeReq; - req.E2Ee.RoomHandle = RoomHandle.Id; - req.E2Ee.SetSharedKey = new SetSharedKeyRequest(); - req.E2Ee.SetSharedKey.KeyIndex = keyIndex; - req.E2Ee.SetSharedKey.SharedKey = Google.Protobuf.ByteString.CopyFrom(key); - - FfiClient.SendRequest(req); + using var request = FFIBridge.Instance.NewRequest(); + var e2ee = request.request; + e2ee.KeyIndex = keyIndex; + e2ee.SharedKey = Google.Protobuf.ByteString.CopyFrom(key); + + using var response = request.Send(); } public byte[] GetSharedKey(int keyIndex) { - var e2eeReq = new E2eeRequest(); - var req = new FfiRequest(); - req.E2Ee = e2eeReq; - req.E2Ee.RoomHandle = RoomHandle.Id; - req.E2Ee.GetSharedKey = new GetSharedKeyRequest(); - req.E2Ee.GetSharedKey.KeyIndex = keyIndex; - var resp = FfiClient.SendRequest(req); + using var request = FFIBridge.Instance.NewRequest(); + var e2ee = request.request; + e2ee.KeyIndex = keyIndex; + using var response = request.Send(); + FfiResponse resp = response; return resp.E2Ee.GetSharedKey.Key.ToByteArray(); } public byte[] RatchetSharedKey(int keyIndex) { - var e2eeReq = new E2eeRequest(); - var req = new FfiRequest(); - req.E2Ee = e2eeReq; - req.E2Ee.RoomHandle = RoomHandle.Id; - req.E2Ee.RatchetSharedKey = new RatchetSharedKeyRequest(); - req.E2Ee.RatchetSharedKey.KeyIndex = keyIndex; - var resp = FfiClient.SendRequest(req); + using var request = FFIBridge.Instance.NewRequest(); + var e2ee = request.request; + e2ee.KeyIndex = keyIndex; + using var response = request.Send(); + FfiResponse resp = response; return resp.E2Ee.RatchetSharedKey.NewKey.ToByteArray(); } public void SetKey(string participantIdentity, byte[] key, int keyIndex) { - var e2eeReq = new E2eeRequest(); - var req = new FfiRequest(); - req.E2Ee = e2eeReq; - req.E2Ee.RoomHandle = RoomHandle.Id; - req.E2Ee.SetKey = new SetKeyRequest(); - req.E2Ee.SetKey.KeyIndex = keyIndex; - req.E2Ee.SetKey.ParticipantIdentity = participantIdentity; - req.E2Ee.SetKey.Key = Google.Protobuf.ByteString.CopyFrom(key); - - FfiClient.SendRequest(req); + using var request = FFIBridge.Instance.NewRequest(); + var e2ee = request.request; + e2ee.KeyIndex = keyIndex; + e2ee.ParticipantIdentity = participantIdentity; + + using var response = request.Send(); } public byte[] GetKey(string participantIdentity, int keyIndex) { - var e2eeReq = new E2eeRequest(); - var req = new FfiRequest(); - req.E2Ee = e2eeReq; - req.E2Ee.RoomHandle = RoomHandle.Id; - req.E2Ee.GetKey = new GetKeyRequest(); - req.E2Ee.GetKey.KeyIndex = keyIndex; - req.E2Ee.GetKey.ParticipantIdentity = participantIdentity; - var resp = FfiClient.SendRequest(req); + using var request = FFIBridge.Instance.NewRequest(); + var e2ee = request.request; + e2ee.KeyIndex = keyIndex; + e2ee.ParticipantIdentity = participantIdentity; + using var response = request.Send(); + FfiResponse resp = response; return resp.E2Ee.GetKey.Key.ToByteArray(); } public byte[] RatchetKey(string participantIdentity, int keyIndex) { - var e2eeReq = new E2eeRequest(); - var req = new FfiRequest(); - req.E2Ee = e2eeReq; - req.E2Ee.RoomHandle = RoomHandle.Id; - req.E2Ee.RatchetKey = new RatchetKeyRequest(); - req.E2Ee.RatchetKey.KeyIndex = keyIndex; - req.E2Ee.RatchetKey.ParticipantIdentity = participantIdentity; - var resp = FfiClient.SendRequest(req); + using var request = FFIBridge.Instance.NewRequest(); + var e2ee = request.request; + e2ee.KeyIndex = keyIndex; + e2ee.ParticipantIdentity = participantIdentity; + using var response = request.Send(); + FfiResponse resp = response; return resp.E2Ee.RatchetKey.NewKey.ToByteArray(); } } @@ -130,13 +115,13 @@ public byte[] RatchetKey(string participantIdentity, int keyIndex) public class FrameCryptor { - internal ulong RoomHandle; + internal FfiHandle RoomHandle; public string ParticipantIdentity; public string TrackSid; public bool Enabled; public int KeyIndex; - public FrameCryptor(ulong roomHandle, string identity, string trackSid, bool enabled, int keyIndex) + public FrameCryptor(FfiHandle roomHandle, string identity, string trackSid, bool enabled, int keyIndex) { RoomHandle = roomHandle; ParticipantIdentity = identity; @@ -148,40 +133,32 @@ public FrameCryptor(ulong roomHandle, string identity, string trackSid, bool ena public void SetKeyIndex(int keyIndex) { KeyIndex = keyIndex; - var e2eeReq = new E2eeRequest(); - var req = new FfiRequest(); - req.E2Ee = e2eeReq; - req.E2Ee.RoomHandle = RoomHandle; - req.E2Ee.CryptorSetKeyIndex = new FrameCryptorSetKeyIndexRequest(); - req.E2Ee.CryptorSetKeyIndex.KeyIndex = keyIndex; - req.E2Ee.CryptorSetKeyIndex.ParticipantIdentity = ParticipantIdentity; - - var resp = FfiClient.SendRequest(req); + using var request = FFIBridge.Instance.NewRequest(); + var e2ee = request.request; + e2ee.KeyIndex = keyIndex; + e2ee.ParticipantIdentity = ParticipantIdentity; + request.Send(); } public void SetEnabled(bool enabled) { Enabled = enabled; - var e2eeReq = new E2eeRequest(); - var req = new FfiRequest(); - req.E2Ee = e2eeReq; - req.E2Ee.RoomHandle = RoomHandle; - req.E2Ee.CryptorSetEnabled = new FrameCryptorSetEnabledRequest(); - req.E2Ee.CryptorSetEnabled.Enabled = enabled; - req.E2Ee.CryptorSetEnabled.ParticipantIdentity = ParticipantIdentity; - - var resp = FfiClient.SendRequest(req); + using var request = FFIBridge.Instance.NewRequest(); + var e2ee = request.request; + e2ee.Enabled = enabled; + e2ee.ParticipantIdentity = ParticipantIdentity; + request.Send(); } } public class E2EEManager { - internal FfiOwnedHandle RoomHandle; + internal FfiHandle RoomHandle; public KeyProvider KeyProvider; public E2EEOptions E2EEOptions; - public E2EEManager(FfiOwnedHandle roomHandle, E2EEOptions e2EEOptions) + public E2EEManager(FfiHandle roomHandle, E2EEOptions e2EEOptions) { RoomHandle = roomHandle; KeyProvider = new KeyProvider(roomHandle, e2EEOptions.KeyProviderOptions); @@ -190,30 +167,25 @@ public E2EEManager(FfiOwnedHandle roomHandle, E2EEOptions e2EEOptions) public void setEnabled(bool enabled) { - var e2eeReq = new E2eeRequest(); - var req = new FfiRequest(); - req.E2Ee = e2eeReq; - req.E2Ee.RoomHandle = RoomHandle.Id; - req.E2Ee.ManagerSetEnabled = new E2eeManagerSetEnabledRequest(); - req.E2Ee.ManagerSetEnabled.Enabled = enabled; - - var resp = FfiClient.SendRequest(req); + using var request = FFIBridge.Instance.NewRequest(); + var e2ee = request.request; + e2ee.Enabled = enabled; + request.Send(); } public List frameCryptors() { - var e2eeReq = new E2eeRequest(); - var req = new FfiRequest(); - req.E2Ee = e2eeReq; - req.E2Ee.RoomHandle = RoomHandle.Id; - req.E2Ee.ManagerGetFrameCryptors = new E2eeManagerGetFrameCryptorsRequest(); + using var request = FFIBridge.Instance.NewRequest(); + var e2ee = request.request; + + using var response = request.Send(); + FfiResponse resp = response; - var resp = FfiClient.SendRequest(req); List cryptors = new List(); foreach(var c in resp.E2Ee.ManagerGetFrameCryptors.FrameCryptors) { - cryptors.Add(new FrameCryptor(RoomHandle.Id, c.ParticipantIdentity, c.TrackSid, c.Enabled, c.KeyIndex)); + cryptors.Add(new FrameCryptor(RoomHandle, c.ParticipantIdentity, c.TrackSid, c.Enabled, c.KeyIndex)); } return cryptors; } diff --git a/Runtime/Scripts/Internal/AudioFilter.cs b/Runtime/Scripts/Internal/AudioFilter.cs index b15fe03..ab82374 100644 --- a/Runtime/Scripts/Internal/AudioFilter.cs +++ b/Runtime/Scripts/Internal/AudioFilter.cs @@ -31,5 +31,10 @@ void OnAudioFilterRead(float[] data, int channels) // Called by Unity on the Audio thread AudioRead?.Invoke(data, channels, _sampleRate); } + + private void OnDestroy() + { + AudioRead = null; + } } } diff --git a/Runtime/Scripts/Internal/AudioResampler.cs b/Runtime/Scripts/Internal/AudioResampler.cs index 8af73ea..47fa5c4 100644 --- a/Runtime/Scripts/Internal/AudioResampler.cs +++ b/Runtime/Scripts/Internal/AudioResampler.cs @@ -1,5 +1,6 @@ -using LiveKit.Internal; +using LiveKit.Internal.FFIClients.Requests; using LiveKit.Proto; +using UnityEngine; namespace LiveKit { @@ -9,30 +10,30 @@ public class AudioResampler public AudioResampler() { - var newResampler = new NewAudioResamplerRequest(); - var request = new FfiRequest(); - request.NewAudioResampler = newResampler; - - var resp = FfiClient.SendRequest(request); - resampler = resp.NewAudioResampler.Resampler; + using var request = FFIBridge.Instance.NewRequest(); + using var response = request.Send(); + FfiResponse res = response; + resampler = res.NewAudioResampler.Resampler; } - public AudioFrame RemixAndResample(AudioFrame frame, uint numChannels, uint sampleRate) { - var remix = new RemixAndResampleRequest(); + public AudioFrame RemixAndResample(AudioFrame frame, uint numChannels, uint sampleRate) + { + using var request = FFIBridge.Instance.NewRequest(); + using var audioFrameBufferInfo = request.TempResource(); + var remix = request.request; remix.ResamplerHandle = resampler.Handle.Id; remix.Buffer = frame.Info; remix.NumChannels = numChannels; remix.SampleRate = sampleRate; - var request = new FfiRequest(); - request.RemixAndResample = remix; - - var res = FfiClient.SendRequest(request); - if(res.RemixAndResample == null) { + using var response = request.Send(); + FfiResponse res = response; + if (res.RemixAndResample == null) + { return null; } var newBuffer = res.RemixAndResample.Buffer; - return new AudioFrame(newBuffer.Handle, newBuffer.Info); + return new AudioFrame(newBuffer); } } -} +} \ No newline at end of file diff --git a/Runtime/Scripts/Internal/Constants.cs b/Runtime/Scripts/Internal/Constants.cs new file mode 100644 index 0000000..edc08b3 --- /dev/null +++ b/Runtime/Scripts/Internal/Constants.cs @@ -0,0 +1,7 @@ +namespace LiveKit.Internal +{ + internal static class Constants + { + internal const int TASK_DELAY = 5; + } +} \ No newline at end of file diff --git a/Runtime/Scripts/Internal/Constants.cs.meta b/Runtime/Scripts/Internal/Constants.cs.meta new file mode 100644 index 0000000..7011ea8 --- /dev/null +++ b/Runtime/Scripts/Internal/Constants.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: aa705dec323de49189784be53276bd49 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/Internal/FFIClient.cs b/Runtime/Scripts/Internal/FFIClient.cs index 2b8f1f0..c124bd2 100644 --- a/Runtime/Scripts/Internal/FFIClient.cs +++ b/Runtime/Scripts/Internal/FFIClient.cs @@ -1,9 +1,12 @@ using System; -using System.Runtime.InteropServices; using LiveKit.Proto; using UnityEngine; using Google.Protobuf; using System.Threading; +using LiveKit.Internal.FFIClients; +using LiveKit.Internal.FFIClients.Pools; +using LiveKit.Internal.FFIClients.Pools.Memory; +using UnityEngine.Pool; #if UNITY_EDITOR using UnityEditor; @@ -11,42 +14,59 @@ namespace LiveKit.Internal { - [UnmanagedFunctionPointer(CallingConvention.Cdecl)] - delegate void FFICallbackDelegate(IntPtr data, int size); + #if UNITY_EDITOR + [InitializeOnLoad] + #endif + internal sealed class FfiClient : IFFIClient + { + private static bool initialized = false; + private static readonly Lazy instance = new(() => new FfiClient()); + public static FfiClient Instance => instance.Value; - // Callbacks - internal delegate void PublishTrackDelegate(PublishTrackCallback e); - internal delegate void ConnectReceivedDelegate(ConnectCallback e); + internal SynchronizationContext? _context; - // Events - internal delegate void RoomEventReceivedDelegate(RoomEvent e); - internal delegate void TrackEventReceivedDelegate(TrackEvent e); - internal delegate void VideoStreamEventReceivedDelegate(VideoStreamEvent e); - internal delegate void AudioStreamEventReceivedDelegate(AudioStreamEvent e); + private static bool _isDisposed = false; -#if UNITY_EDITOR - [InitializeOnLoad] -#endif - internal sealed class FfiClient - { - private static readonly Lazy _instance = new Lazy(() => new FfiClient()); - public static FfiClient Instance => _instance.Value; + private readonly IObjectPool ffiResponsePool; + private readonly MessageParser responseParser; + private readonly IMemoryPool memoryPool; - internal SynchronizationContext _context; + public event PublishTrackDelegate? PublishTrackReceived; + public event ConnectReceivedDelegate? ConnectReceived; + public event DisconnectReceivedDelegate? DisconnectReceived; + public event RoomEventReceivedDelegate? RoomEventReceived; + public event TrackEventReceivedDelegate? TrackEventReceived; + // participant events are not allowed in the fii protocol public event ParticipantEventReceivedDelegate ParticipantEventReceived; + public event VideoStreamEventReceivedDelegate? VideoStreamEventReceived; + public event AudioStreamEventReceivedDelegate? AudioStreamEventReceived; - public event PublishTrackDelegate PublishTrackReceived; - public event ConnectReceivedDelegate ConnectReceived; - public event RoomEventReceivedDelegate RoomEventReceived; - public event TrackEventReceivedDelegate TrackEventReceived; - //public event ParticipantEventReceivedDelegate ParticipantEventReceived; - public event VideoStreamEventReceivedDelegate VideoStreamEventReceived; - public event AudioStreamEventReceivedDelegate AudioStreamEventReceived; + public FfiClient() : this(Pools.NewFfiResponsePool(), new ArrayMemoryPool()) + { + } -#if UNITY_EDITOR + public FfiClient( + IObjectPool ffiResponsePool, + IMemoryPool memoryPool + ) : this( + ffiResponsePool, + new MessageParser(ffiResponsePool.Get), memoryPool) + { + } + + public FfiClient( + IObjectPool ffiResponsePool, + MessageParser responseParser, + IMemoryPool memoryPool + ) + { + this.responseParser = responseParser; + this.memoryPool = memoryPool; + this.ffiResponsePool = ffiResponsePool; + } + + #if UNITY_EDITOR static FfiClient() { - FFICallbackDelegate callback = FFICallback; - NativeMethods.FfiInitialize((ulong)Marshal.GetFunctionPointerForDelegate(callback), true); AssemblyReloadEvents.beforeAssemblyReload += OnBeforeAssemblyReload; AssemblyReloadEvents.afterAssemblyReload += OnAfterAssemblyReload; EditorApplication.quitting += Quit; @@ -55,30 +75,30 @@ static FfiClient() static void OnBeforeAssemblyReload() { - Dispose(); + Instance.Dispose(); } static void OnAfterAssemblyReload() { - + InitializeSdk(); } #else [RuntimeInitializeOnLoadMethod(RuntimeInitializeLoadType.SubsystemRegistration)] static void Init() { - FFICallbackDelegate callback = FFICallback; - NativeMethods.FfiInitialize((ulong)Marshal.GetFunctionPointerForDelegate(callback), true); Application.quitting += Quit; + InitializeSdk(); } #endif - static void Quit() + private static void Quit() { -#if UNITY_EDITOR + #if UNITY_EDITOR AssemblyReloadEvents.beforeAssemblyReload -= OnBeforeAssemblyReload; AssemblyReloadEvents.afterAssemblyReload -= OnAfterAssemblyReload; -#endif - Dispose(); + #endif + Instance.Dispose(); + } [RuntimeInitializeOnLoadMethod] @@ -86,70 +106,154 @@ static void GetMainContext() { // https://github.com/Unity-Technologies/UnityCsReference/blob/master/Runtime/Export/Scripting/UnitySynchronizationContext.cs Instance._context = SynchronizationContext.Current; + Utils.Debug("Main Context created"); + } + + private static void InitializeSdk() + { +#if NO_LIVEKIT_MODE + return; +#endif + +#if LK_VERBOSE + const bool captureLogs = true; +#else + const bool captureLogs = false; +#endif + + NativeMethods.LiveKitInitialize(FFICallback, captureLogs); + + Utils.Debug("FFIServer - Initialized"); + initialized = true; } - static void Dispose() + public void Initialize() { + InitializeSdk(); + } + + public bool Initialized() + { + return initialized; + } + + public void Dispose() + { +#if NO_LIVEKIT_MODE + return; +#endif + + _isDisposed = true; + // Stop all rooms synchronously // The rust lk implementation should also correctly dispose WebRTC - var disposeReq = new DisposeRequest(); - - var request = new FfiRequest(); - request.Dispose = disposeReq; - SendRequest(request); + SendRequest( + new FfiRequest + { + Dispose = new DisposeRequest() + } + ); Utils.Debug("FFIServer - Disposed"); } - internal static FfiResponse SendRequest(FfiRequest request) + public void Release(FfiResponse response) + { + ffiResponsePool.Release(response); + } + + public FfiResponse SendRequest(FfiRequest request) { - var data = request.ToByteArray(); - FfiResponse response; - unsafe + try { - var handle = NativeMethods.FfiNewRequest(data, data.Length, out byte* dataPtr, out int dataLen); - response = FfiResponse.Parser.ParseFrom(new Span(dataPtr, dataLen)); - handle.Dispose(); - } + unsafe + { + using var memory = memoryPool.Memory(request); + var data = memory.Span(); + request.WriteTo(data); + + fixed (byte* requestDataPtr = data) + { + var handle = NativeMethods.FfiNewRequest( + requestDataPtr, + data.Length, + out byte* dataPtr, + out int dataLen + ); - return response; + var dataSpan = new Span(dataPtr, dataLen); + var response = responseParser.ParseFrom(dataSpan)!; + NativeMethods.FfiDropHandle(handle); + return response; + } + } + } + catch (Exception e) + { + // Since we are in a thread I want to make sure we catch and log + Utils.Error(e); + // But we aren't actually handling this exception so we should re-throw here + throw new Exception("Cannot send request", e); + } } [AOT.MonoPInvokeCallback(typeof(FFICallbackDelegate))] static unsafe void FFICallback(IntPtr data, int size) - { - var respData = new Span(data.ToPointer(), size); - var response = FfiEvent.Parser.ParseFrom(respData); + { +#if NO_LIVEKIT_MODE + return; +#endif + + if (_isDisposed) return; + + var respData = new Span(data.ToPointer()!, size); + var response = FfiEvent.Parser!.ParseFrom(respData); // Run on the main thread, the order of execution is guaranteed by Unity // It uses a Queue internally - Instance._context.Post((resp) => - { - var response = resp as FfiEvent; - switch (response.MessageCase) - { - case FfiEvent.MessageOneofCase.Connect: - Instance.ConnectReceived?.Invoke(response.Connect); - break; - case FfiEvent.MessageOneofCase.PublishTrack: - Instance.PublishTrackReceived?.Invoke(response.PublishTrack); - break; - case FfiEvent.MessageOneofCase.RoomEvent: - Instance.RoomEventReceived?.Invoke(response.RoomEvent); - break; - case FfiEvent.MessageOneofCase.TrackEvent: - Instance.TrackEventReceived?.Invoke(response.TrackEvent); - break; - case FfiEvent.MessageOneofCase.VideoStreamEvent: - Instance.VideoStreamEventReceived?.Invoke(response.VideoStreamEvent); - break; - case FfiEvent.MessageOneofCase.AudioStreamEvent: - Instance.AudioStreamEventReceived?.Invoke(response.AudioStreamEvent); - break; - - } - }, response); + Instance._context?.Post((resp) => + { + var r = resp as FfiEvent; +#if LK_VERBOSE + if (r?.MessageCase != FfiEvent.MessageOneofCase.Logs) + Utils.Debug("Callback: " + r?.MessageCase); +#endif + switch (r?.MessageCase) + { + case FfiEvent.MessageOneofCase.Logs: + break; + case FfiEvent.MessageOneofCase.PublishData: + break; + case FfiEvent.MessageOneofCase.Connect: + Instance.ConnectReceived?.Invoke(r.Connect!); + break; + case FfiEvent.MessageOneofCase.PublishTrack: + Instance.PublishTrackReceived?.Invoke(r.PublishTrack!); + break; + case FfiEvent.MessageOneofCase.RoomEvent: + Instance.RoomEventReceived?.Invoke(r.RoomEvent); + break; + case FfiEvent.MessageOneofCase.TrackEvent: + Instance.TrackEventReceived?.Invoke(r.TrackEvent!); + break; + case FfiEvent.MessageOneofCase.Disconnect: + Instance.DisconnectReceived?.Invoke(r.Disconnect!); + break; + /*case FfiEvent.MessageOneofCase. ParticipantEvent: + Instance.ParticipantEventReceived?.Invoke(response.ParticipantEvent); + break;*/ + case FfiEvent.MessageOneofCase.VideoStreamEvent: + Instance.VideoStreamEventReceived?.Invoke(r.VideoStreamEvent!); + break; + case FfiEvent.MessageOneofCase.AudioStreamEvent: + Instance.AudioStreamEventReceived?.Invoke(r.AudioStreamEvent!); + break; + case FfiEvent.MessageOneofCase.CaptureAudioFrame: + break; + default: + throw new ArgumentOutOfRangeException($"Unknown message type: {r?.MessageCase.ToString() ?? "null"}"); + } + }, response); } } -} - +} \ No newline at end of file diff --git a/Runtime/Scripts/Internal/FFIClients.meta b/Runtime/Scripts/Internal/FFIClients.meta new file mode 100644 index 0000000..2060c39 --- /dev/null +++ b/Runtime/Scripts/Internal/FFIClients.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 90d3732d5c2b642f2aaa0a6455e193b3 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/Internal/FFIClients/FFIEvents.cs b/Runtime/Scripts/Internal/FFIClients/FFIEvents.cs new file mode 100644 index 0000000..ac5e978 --- /dev/null +++ b/Runtime/Scripts/Internal/FFIClients/FFIEvents.cs @@ -0,0 +1,32 @@ +using System; +using System.Runtime.InteropServices; +using LiveKit.Proto; + +namespace LiveKit.Internal +{ + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + internal delegate void FFICallbackDelegate(IntPtr data, int size); + + // Callbacks + internal delegate void PublishTrackDelegate(PublishTrackCallback e); + + + internal delegate void ConnectReceivedDelegate(ConnectCallback e); + + + internal delegate void DisconnectReceivedDelegate(DisconnectCallback e); + + + // Events + internal delegate void RoomEventReceivedDelegate(RoomEvent e); + + + internal delegate void TrackEventReceivedDelegate(TrackEvent e); + + + internal delegate void VideoStreamEventReceivedDelegate(VideoStreamEvent e); + + + internal delegate void AudioStreamEventReceivedDelegate(AudioStreamEvent e); + +} \ No newline at end of file diff --git a/Runtime/Scripts/Internal/FFIClients/FFIEvents.cs.meta b/Runtime/Scripts/Internal/FFIClients/FFIEvents.cs.meta new file mode 100644 index 0000000..6889c23 --- /dev/null +++ b/Runtime/Scripts/Internal/FFIClients/FFIEvents.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 4911336b575a4428a7aea293969a2a65 +timeCreated: 1706097194 \ No newline at end of file diff --git a/Runtime/Scripts/Internal/FFIClients/FfiRequestExtensions.cs b/Runtime/Scripts/Internal/FFIClients/FfiRequestExtensions.cs new file mode 100644 index 0000000..3044189 --- /dev/null +++ b/Runtime/Scripts/Internal/FFIClients/FfiRequestExtensions.cs @@ -0,0 +1,188 @@ +using System; +using System.Runtime.CompilerServices; +using LiveKit.Proto; + +namespace LiveKit.Internal.FFIClients +{ + public static class FfiRequestExtensions + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Inject(this FfiRequest ffiRequest, T request) + { + switch (request) + { + case DisposeRequest disposeRequest: + ffiRequest.Dispose = disposeRequest; + break; + // Room + case ConnectRequest connectRequest: + ffiRequest.Connect = connectRequest; + break; + case DisconnectRequest disconnectRequest: + ffiRequest.Disconnect = disconnectRequest; + break; + case PublishTrackRequest publishTrackRequest: + ffiRequest.PublishTrack = publishTrackRequest; + break; + case UnpublishTrackRequest unpublishTrackRequest: + ffiRequest.UnpublishTrack = unpublishTrackRequest; + break; + case PublishDataRequest publishDataRequest: + ffiRequest.PublishData = publishDataRequest; + break; + case SetSubscribedRequest setSubscribedRequest: + ffiRequest.SetSubscribed = setSubscribedRequest; + break; + case UpdateLocalMetadataRequest updateLocalMetadataRequest: + ffiRequest.UpdateLocalMetadata = updateLocalMetadataRequest; + break; + case UpdateLocalNameRequest updateLocalNameRequest: + ffiRequest.UpdateLocalName = updateLocalNameRequest; + break; + case GetSessionStatsRequest getSessionStatsRequest: + ffiRequest.GetSessionStats = getSessionStatsRequest; + break; + // Track + case CreateVideoTrackRequest createVideoTrackRequest: + ffiRequest.CreateVideoTrack = createVideoTrackRequest; + break; + case CreateAudioTrackRequest createAudioTrackRequest: + ffiRequest.CreateAudioTrack = createAudioTrackRequest; + break; + case GetStatsRequest getStatsRequest: + ffiRequest.GetStats = getStatsRequest; + break; + // Video + case NewVideoStreamRequest newVideoStreamRequest: + ffiRequest.NewVideoStream = newVideoStreamRequest; + break; + case NewVideoSourceRequest newVideoSourceRequest: + ffiRequest.NewVideoSource = newVideoSourceRequest; + break; + case CaptureVideoFrameRequest captureVideoFrameRequest: + ffiRequest.CaptureVideoFrame = captureVideoFrameRequest; + break; + case VideoConvertRequest videoConvertRequest: + ffiRequest.VideoConvert = videoConvertRequest; + break; + // Audio + case NewAudioStreamRequest wewAudioStreamRequest: + ffiRequest.NewAudioStream = wewAudioStreamRequest; + break; + case NewAudioSourceRequest newAudioSourceRequest: + ffiRequest.NewAudioSource = newAudioSourceRequest; + break; + case CaptureAudioFrameRequest captureAudioFrameRequest: + ffiRequest.CaptureAudioFrame = captureAudioFrameRequest; + break; + case NewAudioResamplerRequest newAudioResamplerRequest: + ffiRequest.NewAudioResampler = newAudioResamplerRequest; + break; + case RemixAndResampleRequest remixAndResampleRequest: + ffiRequest.RemixAndResample = remixAndResampleRequest; + break; + case E2eeRequest e2EeRequest: + ffiRequest.E2Ee = e2EeRequest; + break; + default: + throw new Exception($"Unknown request type: {request?.GetType().FullName ?? "null"}"); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void EnsureClean(this FfiRequest request) + { + // list of messages is taken from: livekit-ffi/protocol/ffi.proto + // https://github.com/livekit/rust-sdks/blob/cf34856e78892a639c4d3c1d6a27e9aba0a4a8ff/livekit-ffi/protocol/ffi.proto#L4 + + if ( + request.Dispose != null + || + + // Room + request.Connect != null + || request.Disconnect != null + || request.PublishTrack != null + || request.UnpublishTrack != null + || request.PublishData != null + || request.SetSubscribed != null + || request.UpdateLocalMetadata != null + || request.UpdateLocalName != null + || request.GetSessionStats != null + || + + // Track + request.CreateVideoTrack != null + || request.CreateAudioTrack != null + || request.GetStats != null + || + + // Video + request.NewVideoStream != null + || request.NewVideoSource != null + || request.CaptureVideoFrame != null + || request.VideoConvert != null + || + + // Audio + request.NewAudioStream != null + || request.NewAudioSource != null + || request.CaptureAudioFrame != null + || request.NewAudioResampler != null + || request.RemixAndResample != null + || request.E2Ee != null + ) + { + throw new InvalidOperationException("Request is not cleared"); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void EnsureClean(this FfiResponse response) + { + // list of messages is taken from: livekit-ffi/protocol/ffi.proto + // https://github.com/livekit/rust-sdks/blob/cf34856e78892a639c4d3c1d6a27e9aba0a4a8ff/livekit-ffi/protocol/ffi.proto#L4 + + if ( + response.Dispose != null + || + + // Room + response.Connect != null + || response.Disconnect != null + || response.PublishTrack != null + || response.UnpublishTrack != null + || response.PublishData != null + || response.SetSubscribed != null + || response.UpdateLocalMetadata != null + || response.UpdateLocalName != null + || response.GetSessionStats != null + || + + // Track + response.CreateVideoTrack != null + || response.CreateAudioTrack != null + || response.GetStats != null + || + + // Video + response.NewVideoStream != null + || response.NewVideoSource != null + || response.CaptureVideoFrame != null + || response.VideoConvert != null + || + + // Audio + response.NewAudioStream != null + || response.NewAudioSource != null + || response.CaptureAudioFrame != null + || response.NewAudioResampler != null + || response.RemixAndResample != null + || response.E2Ee != null + ) + { + throw new InvalidOperationException("Response is not cleared: "); + } + } + } +} \ No newline at end of file diff --git a/Runtime/Scripts/Internal/FFIClients/FfiRequestExtensions.cs.meta b/Runtime/Scripts/Internal/FFIClients/FfiRequestExtensions.cs.meta new file mode 100644 index 0000000..29bad76 --- /dev/null +++ b/Runtime/Scripts/Internal/FFIClients/FfiRequestExtensions.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: b9a99eef5be840a4b319890a29fb6020 +timeCreated: 1706109917 \ No newline at end of file diff --git a/Runtime/Scripts/Internal/FFIClients/FfiResponseWrap.cs b/Runtime/Scripts/Internal/FFIClients/FfiResponseWrap.cs new file mode 100644 index 0000000..4c46f95 --- /dev/null +++ b/Runtime/Scripts/Internal/FFIClients/FfiResponseWrap.cs @@ -0,0 +1,34 @@ +using System; +using LiveKit.Proto; + +namespace LiveKit.Internal.FFIClients +{ + public readonly struct FfiResponseWrap : IDisposable + { + private readonly FfiResponse response; + private readonly IFFIClient client; + + public FfiResponseWrap(FfiResponse response, IFFIClient client) + { + this.response = response; + this.client = client; + } + + public void Dispose() + { + //TODO pooling inner parts + response.ClearMessage(); + client.Release(response); + } + + public static implicit operator FfiResponse(FfiResponseWrap wrap) + { + return wrap.response; + } + + public override string ToString() + { + return response.ToString()!; + } + } +} \ No newline at end of file diff --git a/Runtime/Scripts/Internal/FFIClients/FfiResponseWrap.cs.meta b/Runtime/Scripts/Internal/FFIClients/FfiResponseWrap.cs.meta new file mode 100644 index 0000000..0c019ff --- /dev/null +++ b/Runtime/Scripts/Internal/FFIClients/FfiResponseWrap.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 19c342f416cf442aafa3d460b1778bdb +timeCreated: 1706100710 \ No newline at end of file diff --git a/Runtime/Scripts/Internal/FFIClients/IFFIClient.cs b/Runtime/Scripts/Internal/FFIClients/IFFIClient.cs new file mode 100644 index 0000000..4d0cdb3 --- /dev/null +++ b/Runtime/Scripts/Internal/FFIClients/IFFIClient.cs @@ -0,0 +1,15 @@ +using System; +using LiveKit.Proto; + +namespace LiveKit.Internal.FFIClients +{ + /// + /// Thread-safe interface for sending requests to the FFI layer + /// + public interface IFFIClient : IDisposable + { + FfiResponse SendRequest(FfiRequest request); + + void Release(FfiResponse response); + } +} \ No newline at end of file diff --git a/Runtime/Scripts/Internal/FFIClients/IFFIClient.cs.meta b/Runtime/Scripts/Internal/FFIClients/IFFIClient.cs.meta new file mode 100644 index 0000000..80476c3 --- /dev/null +++ b/Runtime/Scripts/Internal/FFIClients/IFFIClient.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 839433b266cf4936b3101082b8f81120 +timeCreated: 1706097843 \ No newline at end of file diff --git a/Runtime/Scripts/Internal/FFIClients/Pools.meta b/Runtime/Scripts/Internal/FFIClients/Pools.meta new file mode 100644 index 0000000..ea8143c --- /dev/null +++ b/Runtime/Scripts/Internal/FFIClients/Pools.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 8a3a31284b314bc29b7e2d5e86b57400 +timeCreated: 1706099007 \ No newline at end of file diff --git a/Runtime/Scripts/Internal/FFIClients/Pools/IMultiPool.cs b/Runtime/Scripts/Internal/FFIClients/Pools/IMultiPool.cs new file mode 100644 index 0000000..1209b59 --- /dev/null +++ b/Runtime/Scripts/Internal/FFIClients/Pools/IMultiPool.cs @@ -0,0 +1,9 @@ +namespace LiveKit.Internal.FFIClients.Pools +{ + public interface IMultiPool + { + T Get() where T : class, new(); + + void Release(T poolObject) where T : class, new(); + } +} \ No newline at end of file diff --git a/Runtime/Scripts/Internal/FFIClients/Pools/IMultiPool.cs.meta b/Runtime/Scripts/Internal/FFIClients/Pools/IMultiPool.cs.meta new file mode 100644 index 0000000..1f5db7e --- /dev/null +++ b/Runtime/Scripts/Internal/FFIClients/Pools/IMultiPool.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 49d742b575594b129646ca76980b5a5a +timeCreated: 1706110675 \ No newline at end of file diff --git a/Runtime/Scripts/Internal/FFIClients/Pools/Memory.meta b/Runtime/Scripts/Internal/FFIClients/Pools/Memory.meta new file mode 100644 index 0000000..c9ab028 --- /dev/null +++ b/Runtime/Scripts/Internal/FFIClients/Pools/Memory.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 191ffda93680492d9565b96d7d491250 +timeCreated: 1706702309 \ No newline at end of file diff --git a/Runtime/Scripts/Internal/FFIClients/Pools/Memory/ArrayMemoryPool.cs b/Runtime/Scripts/Internal/FFIClients/Pools/Memory/ArrayMemoryPool.cs new file mode 100644 index 0000000..cec384a --- /dev/null +++ b/Runtime/Scripts/Internal/FFIClients/Pools/Memory/ArrayMemoryPool.cs @@ -0,0 +1,28 @@ +using System.Buffers; + +namespace LiveKit.Internal.FFIClients.Pools.Memory +{ + public class ArrayMemoryPool : IMemoryPool + { + private readonly ArrayPool arrayPool; + + public ArrayMemoryPool() : this(ArrayPool.Create()!) + { + } + + public ArrayMemoryPool(ArrayPool arrayPool) + { + this.arrayPool = arrayPool; + } + + public MemoryWrap Memory(int byteSize) + { + return new MemoryWrap(arrayPool.Rent(byteSize)!, byteSize, this); + } + + public void Release(byte[] buffer) + { + arrayPool.Return(buffer); + } + } +} \ No newline at end of file diff --git a/Runtime/Scripts/Internal/FFIClients/Pools/Memory/ArrayMemoryPool.cs.meta b/Runtime/Scripts/Internal/FFIClients/Pools/Memory/ArrayMemoryPool.cs.meta new file mode 100644 index 0000000..aade6f3 --- /dev/null +++ b/Runtime/Scripts/Internal/FFIClients/Pools/Memory/ArrayMemoryPool.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: b7ed30725dc34e6cbc57c596523dae9b +timeCreated: 1706713567 \ No newline at end of file diff --git a/Runtime/Scripts/Internal/FFIClients/Pools/Memory/IMemoryPool.cs b/Runtime/Scripts/Internal/FFIClients/Pools/Memory/IMemoryPool.cs new file mode 100644 index 0000000..74efe2a --- /dev/null +++ b/Runtime/Scripts/Internal/FFIClients/Pools/Memory/IMemoryPool.cs @@ -0,0 +1,20 @@ +using Google.Protobuf; + +namespace LiveKit.Internal.FFIClients.Pools.Memory +{ + public interface IMemoryPool + { + MemoryWrap Memory(int byteSize); + + void Release(byte[] buffer); + } + + public static class MemoryPoolExtensions + { + public static MemoryWrap Memory(this IMemoryPool pool, IMessage forMessage) + { + var size = forMessage.CalculateSize(); + return pool.Memory(size); + } + } +} \ No newline at end of file diff --git a/Runtime/Scripts/Internal/FFIClients/Pools/Memory/IMemoryPool.cs.meta b/Runtime/Scripts/Internal/FFIClients/Pools/Memory/IMemoryPool.cs.meta new file mode 100644 index 0000000..9419e8d --- /dev/null +++ b/Runtime/Scripts/Internal/FFIClients/Pools/Memory/IMemoryPool.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 8911b9d55f4a4167a0c42940be059975 +timeCreated: 1706702315 \ No newline at end of file diff --git a/Runtime/Scripts/Internal/FFIClients/Pools/Memory/MemoryWrap.cs b/Runtime/Scripts/Internal/FFIClients/Pools/Memory/MemoryWrap.cs new file mode 100644 index 0000000..3c94bc2 --- /dev/null +++ b/Runtime/Scripts/Internal/FFIClients/Pools/Memory/MemoryWrap.cs @@ -0,0 +1,39 @@ +using System; + +namespace LiveKit.Internal.FFIClients.Pools.Memory +{ + public readonly struct MemoryWrap : IDisposable + { + private readonly byte[] buffer; + private readonly int length; + private readonly IMemoryPool memoryPool; + + public int Length => length; + + public MemoryWrap(byte[] buffer, int length, IMemoryPool memoryPool) + { + this.buffer = buffer; + this.length = length; + this.memoryPool = memoryPool; + } + + public Span Span() + { + return new Span(buffer, 0, length); + } + + /// + /// Gives the direct access to the buffer. Ownership remains on MemoryWrap and it can dispose it anytime. + /// You know what you are doing + /// + public byte[] DangerousBuffer() + { + return buffer; + } + + public void Dispose() + { + memoryPool.Release(buffer); + } + } +} \ No newline at end of file diff --git a/Runtime/Scripts/Internal/FFIClients/Pools/Memory/MemoryWrap.cs.meta b/Runtime/Scripts/Internal/FFIClients/Pools/Memory/MemoryWrap.cs.meta new file mode 100644 index 0000000..4fe4197 --- /dev/null +++ b/Runtime/Scripts/Internal/FFIClients/Pools/Memory/MemoryWrap.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 3c253d5d49a3460fa6ee10dd9fd409f7 +timeCreated: 1706702405 \ No newline at end of file diff --git a/Runtime/Scripts/Internal/FFIClients/Pools/ObjectPool.meta b/Runtime/Scripts/Internal/FFIClients/Pools/ObjectPool.meta new file mode 100644 index 0000000..6dc9689 --- /dev/null +++ b/Runtime/Scripts/Internal/FFIClients/Pools/ObjectPool.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: f8d30253e11243838563ec78d57565e9 +timeCreated: 1706714202 \ No newline at end of file diff --git a/Runtime/Scripts/Internal/FFIClients/Pools/ObjectPool/ThreadSafeObjectPool.cs b/Runtime/Scripts/Internal/FFIClients/Pools/ObjectPool/ThreadSafeObjectPool.cs new file mode 100644 index 0000000..013e3fe --- /dev/null +++ b/Runtime/Scripts/Internal/FFIClients/Pools/ObjectPool/ThreadSafeObjectPool.cs @@ -0,0 +1,45 @@ +using System; +using System.Collections.Concurrent; +using UnityEngine.Pool; + +namespace LiveKit.Internal.FFIClients.Pools.ObjectPool +{ + public class ThreadSafeObjectPool : IObjectPool where T : class + { + private readonly Func create; + private readonly Action? actionOnRelease; + private readonly ConcurrentBag bag = new(); + + public ThreadSafeObjectPool(Func create, Action? actionOnRelease = null) + { + this.create = create; + this.actionOnRelease = actionOnRelease; + } + + public T Get() + { + return bag.TryTake(out var result) + ? result! + : create()!; + } + + public PooledObject Get(out T v) + { + v = Get(); + return new PooledObject(); + } + + public void Release(T element) + { + actionOnRelease?.Invoke(element); + bag.Add(element); + } + + public void Clear() + { + bag.Clear(); + } + + public int CountInactive => bag.Count; + } +} \ No newline at end of file diff --git a/Runtime/Scripts/Internal/FFIClients/Pools/ObjectPool/ThreadSafeObjectPool.cs.meta b/Runtime/Scripts/Internal/FFIClients/Pools/ObjectPool/ThreadSafeObjectPool.cs.meta new file mode 100644 index 0000000..f2b5cec --- /dev/null +++ b/Runtime/Scripts/Internal/FFIClients/Pools/ObjectPool/ThreadSafeObjectPool.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 3a485030361a46af9d31444bff2c662c +timeCreated: 1706714208 \ No newline at end of file diff --git a/Runtime/Scripts/Internal/FFIClients/Pools/Pools.cs b/Runtime/Scripts/Internal/FFIClients/Pools/Pools.cs new file mode 100644 index 0000000..83f068e --- /dev/null +++ b/Runtime/Scripts/Internal/FFIClients/Pools/Pools.cs @@ -0,0 +1,23 @@ +using System; +using LiveKit.Internal.FFIClients.Pools.ObjectPool; +using LiveKit.Proto; +using UnityEngine.Pool; + +namespace LiveKit.Internal.FFIClients.Pools// +{ + public static class Pools + { + public static IObjectPool NewFfiResponsePool() + { + return NewClearablePool(FfiRequestExtensions.EnsureClean); + } + + public static IObjectPool NewClearablePool(Action ensureClean) where T : class, new() + { + return new ThreadSafeObjectPool( + () => new T(), + actionOnRelease: ensureClean + ); + } + } +} \ No newline at end of file diff --git a/Runtime/Scripts/Internal/FFIClients/Pools/Pools.cs.meta b/Runtime/Scripts/Internal/FFIClients/Pools/Pools.cs.meta new file mode 100644 index 0000000..600d87d --- /dev/null +++ b/Runtime/Scripts/Internal/FFIClients/Pools/Pools.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: a61ae23d54b44cfe806486058560ea5e +timeCreated: 1706094307 \ No newline at end of file diff --git a/Runtime/Scripts/Internal/FFIClients/Pools/ThreadSafeMultiPool.cs b/Runtime/Scripts/Internal/FFIClients/Pools/ThreadSafeMultiPool.cs new file mode 100644 index 0000000..e633064 --- /dev/null +++ b/Runtime/Scripts/Internal/FFIClients/Pools/ThreadSafeMultiPool.cs @@ -0,0 +1,33 @@ +using System; +using System.Collections.Concurrent; +using LiveKit.Internal.FFIClients.Pools.ObjectPool; +using UnityEngine.Pool; + +namespace LiveKit.Internal.FFIClients.Pools +{ + public class ThreadSafeMultiPool : IMultiPool + { + private readonly ConcurrentDictionary> pools = new(); + + public T Get() where T : class, new() + { + return (T)Pool().Get()!; + } + + public void Release(T poolObject) where T : class, new() + { + Pool().Release(poolObject); + } + + private IObjectPool Pool() where T : class, new() + { + var type = typeof(T); + if (!pools.TryGetValue(type, out var pool)) + { + pool = pools[type] = new ThreadSafeObjectPool(() => new T()); + } + + return pool!; + } + } +} \ No newline at end of file diff --git a/Runtime/Scripts/Internal/FFIClients/Pools/ThreadSafeMultiPool.cs.meta b/Runtime/Scripts/Internal/FFIClients/Pools/ThreadSafeMultiPool.cs.meta new file mode 100644 index 0000000..9bb17e7 --- /dev/null +++ b/Runtime/Scripts/Internal/FFIClients/Pools/ThreadSafeMultiPool.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 6854375865474dec87d8954ec9fd3043 +timeCreated: 1706714101 \ No newline at end of file diff --git a/Runtime/Scripts/Internal/FFIClients/Requests.meta b/Runtime/Scripts/Internal/FFIClients/Requests.meta new file mode 100644 index 0000000..7ad4adc --- /dev/null +++ b/Runtime/Scripts/Internal/FFIClients/Requests.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 47fe01e3230c4c26a2c1669ef88ff728 +timeCreated: 1706109018 \ No newline at end of file diff --git a/Runtime/Scripts/Internal/FFIClients/Requests/FFIBridge.cs b/Runtime/Scripts/Internal/FFIClients/Requests/FFIBridge.cs new file mode 100644 index 0000000..2d3f152 --- /dev/null +++ b/Runtime/Scripts/Internal/FFIClients/Requests/FFIBridge.cs @@ -0,0 +1,33 @@ +using System; +using LiveKit.Internal.FFIClients.Pools; + +namespace LiveKit.Internal.FFIClients.Requests +{ + public class FFIBridge : IFFIBridge + { + //TODO should be without singleton, remove it + private static readonly Lazy instance = new(() => + new FFIBridge( + FfiClient.Instance, + new ThreadSafeMultiPool() + ) + ); + + public static FFIBridge Instance => instance.Value; + + private readonly IFFIClient ffiClient; + private readonly IMultiPool multiPool; + + public FFIBridge(IFFIClient client, IMultiPool multiPool) + { + ffiClient = client; + this.multiPool = multiPool; + } + + + public FfiRequestWrap NewRequest() where T : class, new() + { + return new FfiRequestWrap(ffiClient, multiPool); + } + } +} \ No newline at end of file diff --git a/Runtime/Scripts/Internal/FFIClients/Requests/FFIBridge.cs.meta b/Runtime/Scripts/Internal/FFIClients/Requests/FFIBridge.cs.meta new file mode 100644 index 0000000..7cf5959 --- /dev/null +++ b/Runtime/Scripts/Internal/FFIClients/Requests/FFIBridge.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 85bf99cb2c5344c7b61ac965383c22d1 +timeCreated: 1706178188 \ No newline at end of file diff --git a/Runtime/Scripts/Internal/FFIClients/Requests/FFIBridgeExtensions.cs b/Runtime/Scripts/Internal/FFIClients/Requests/FFIBridgeExtensions.cs new file mode 100644 index 0000000..6aa82bd --- /dev/null +++ b/Runtime/Scripts/Internal/FFIClients/Requests/FFIBridgeExtensions.cs @@ -0,0 +1,32 @@ +using LiveKit.Proto; +namespace LiveKit.Internal.FFIClients.Requests +{ + public static class FFIBridgeExtensions + { + public static FfiResponseWrap SendConnectRequest(this IFFIBridge ffiBridge, string url, string authToken, RoomOptions roomOptions) + { + Utils.Debug("Connect...."); + using var request = ffiBridge.NewRequest(); + + var connect = request.request; + connect.Url = url; + connect.Token = authToken; + connect.Options = roomOptions.ToProto(); + var response = request.Send(); + Utils.Debug($"Connect response.... {response}"); + return response; + } + + public static FfiResponseWrap SendDisconnectRequest(this IFFIBridge ffiBridge, Room room) + { + using var request = ffiBridge.NewRequest(); + var disconnect = request.request; + disconnect.RoomHandle = (ulong)room.RoomHandle.DangerousGetHandle(); + Utils.Debug($"Disconnect.... {disconnect.RoomHandle}"); + var response = request.Send(); + // ReSharper disable once RedundantAssignment + Utils.Debug($"Disconnect response.... {response}"); + return response; + } + } +} \ No newline at end of file diff --git a/Runtime/Scripts/Internal/FFIClients/Requests/FFIBridgeExtensions.cs.meta b/Runtime/Scripts/Internal/FFIClients/Requests/FFIBridgeExtensions.cs.meta new file mode 100644 index 0000000..3272f34 --- /dev/null +++ b/Runtime/Scripts/Internal/FFIClients/Requests/FFIBridgeExtensions.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: d8fc4e53439544a519df7f521af58e5b +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/Internal/FFIClients/Requests/FfiRequestWrap.cs b/Runtime/Scripts/Internal/FFIClients/Requests/FfiRequestWrap.cs new file mode 100644 index 0000000..58f4145 --- /dev/null +++ b/Runtime/Scripts/Internal/FFIClients/Requests/FfiRequestWrap.cs @@ -0,0 +1,75 @@ +using System; +using Google.Protobuf; +using LiveKit.client_sdk_unity.Runtime.Scripts.Internal.FFIClients; +using LiveKit.Internal.FFIClients.Pools; +using LiveKit.Proto; + +namespace LiveKit.Internal.FFIClients.Requests +{ + public struct FfiRequestWrap : IDisposable where T : class, new() + { + public readonly T request; + private readonly IMultiPool multiPool; + private readonly IFFIClient ffiClient; + private readonly FfiRequest ffiRequest; + private readonly Action releaseFfiRequest; + private readonly Action releaseRequest; + + private bool sent; + + public FfiRequestWrap(IFFIClient ffiClient, IMultiPool multiPool) : this( + multiPool.Get(), + multiPool, + multiPool.Get(), + ffiClient, + multiPool.Release, + multiPool.Release + ) + { + } + + public FfiRequestWrap( + T request, + IMultiPool multiPool, + FfiRequest ffiRequest, + IFFIClient ffiClient, + Action releaseFfiRequest, + Action releaseRequest + ) + { + this.request = request; + this.multiPool = multiPool; + this.ffiRequest = ffiRequest; + this.ffiClient = ffiClient; + this.releaseFfiRequest = releaseFfiRequest; + this.releaseRequest = releaseRequest; + sent = false; + } + + public FfiResponseWrap Send() + { + if (sent) + { + throw new Exception("Request already sent"); + } + + sent = true; + ffiRequest.Inject(request); + var response = ffiClient.SendRequest(ffiRequest); + return new FfiResponseWrap(response, ffiClient); + } + + public SmartWrap TempResource() where TK : class, IMessage, new() + { + var resource = multiPool.Get(); + return new SmartWrap(resource, multiPool); + } + + public void Dispose() + { + ffiRequest.ClearMessage(); + releaseRequest(request); + releaseFfiRequest(ffiRequest); + } + } +} \ No newline at end of file diff --git a/Runtime/Scripts/Internal/FFIClients/Requests/FfiRequestWrap.cs.meta b/Runtime/Scripts/Internal/FFIClients/Requests/FfiRequestWrap.cs.meta new file mode 100644 index 0000000..3132cff --- /dev/null +++ b/Runtime/Scripts/Internal/FFIClients/Requests/FfiRequestWrap.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 277c3a1d8b2b4d85b1fb614c0a06407c +timeCreated: 1706109025 \ No newline at end of file diff --git a/Runtime/Scripts/Internal/FFIClients/Requests/IFFIBridge.cs b/Runtime/Scripts/Internal/FFIClients/Requests/IFFIBridge.cs new file mode 100644 index 0000000..b3ffe2e --- /dev/null +++ b/Runtime/Scripts/Internal/FFIClients/Requests/IFFIBridge.cs @@ -0,0 +1,10 @@ +namespace LiveKit.Internal.FFIClients.Requests +{ + /// + /// Thread-safe interface for requests to the FFI layer + /// + public interface IFFIBridge + { + FfiRequestWrap NewRequest() where T : class, new(); + } +} \ No newline at end of file diff --git a/Runtime/Scripts/Internal/FFIClients/Requests/IFFIBridge.cs.meta b/Runtime/Scripts/Internal/FFIClients/Requests/IFFIBridge.cs.meta new file mode 100644 index 0000000..5715c60 --- /dev/null +++ b/Runtime/Scripts/Internal/FFIClients/Requests/IFFIBridge.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 8cf66df1102943f1a1155b233de69eef +timeCreated: 1706110458 \ No newline at end of file diff --git a/Runtime/Scripts/Internal/FFIClients/SmartWrap.cs b/Runtime/Scripts/Internal/FFIClients/SmartWrap.cs new file mode 100644 index 0000000..f0ce798 --- /dev/null +++ b/Runtime/Scripts/Internal/FFIClients/SmartWrap.cs @@ -0,0 +1,27 @@ +using System; +using LiveKit.Internal.FFIClients.Pools; + +namespace LiveKit.client_sdk_unity.Runtime.Scripts.Internal.FFIClients +{ + public readonly struct SmartWrap : IDisposable where T : class, new() + { + public readonly T value; + private readonly IMultiPool pool; + + public SmartWrap(T value, IMultiPool pool) + { + this.value = value; + this.pool = pool; + } + + public void Dispose() + { + pool.Release(value); + } + + public static implicit operator T(SmartWrap wrap) + { + return wrap.value; + } + } +} \ No newline at end of file diff --git a/Runtime/Scripts/Internal/FFIClients/SmartWrap.cs.meta b/Runtime/Scripts/Internal/FFIClients/SmartWrap.cs.meta new file mode 100644 index 0000000..916aace --- /dev/null +++ b/Runtime/Scripts/Internal/FFIClients/SmartWrap.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: f8637cba4f7b433798d75f0a7d5128bf +timeCreated: 1706703753 \ No newline at end of file diff --git a/Runtime/Scripts/Internal/FFIHandle.cs b/Runtime/Scripts/Internal/FFIHandle.cs index 337fdcc..2ac1079 100644 --- a/Runtime/Scripts/Internal/FFIHandle.cs +++ b/Runtime/Scripts/Internal/FFIHandle.cs @@ -1,6 +1,7 @@ using System; using System.Runtime.InteropServices; using System.Runtime.ConstrainedExecution; +using LiveKit.Proto; namespace LiveKit.Internal { @@ -8,8 +9,6 @@ public class FfiHandle : SafeHandle { internal FfiHandle(IntPtr ptr) : base(ptr, true) { } - public FfiHandle() : base(IntPtr.Zero, true) { } - public override bool IsInvalid => handle == IntPtr.Zero || handle == new IntPtr(-1); [ReliabilityContract(Consistency.WillNotCorruptState, Cer.MayFail)] @@ -17,5 +16,11 @@ protected override bool ReleaseHandle() { return NativeMethods.FfiDropHandle(handle); } + + public static FfiHandle FromOwnedHandle(FfiOwnedHandle handle) + { + return new FfiHandle((IntPtr)handle.Id); + } } + } diff --git a/Runtime/Scripts/Internal/NativeMethods.cs b/Runtime/Scripts/Internal/NativeMethods.cs index 5da2af4..d2e58df 100644 --- a/Runtime/Scripts/Internal/NativeMethods.cs +++ b/Runtime/Scripts/Internal/NativeMethods.cs @@ -8,20 +8,21 @@ namespace LiveKit.Internal [SuppressUnmanagedCodeSecurity] internal static class NativeMethods { -#if UNITY_IOS + #if UNITY_IOS const string Lib = "__Internal"; -#else + #else const string Lib = "livekit_ffi"; -#endif + #endif [DllImport(Lib, CallingConvention = CallingConvention.Cdecl, EntryPoint = "livekit_ffi_drop_handle")] [ReliabilityContract(Consistency.WillNotCorruptState, Cer.MayFail)] - internal static extern bool FfiDropHandle(IntPtr handleId); + internal extern static bool FfiDropHandle(IntPtr handleId); [DllImport(Lib, CallingConvention = CallingConvention.Cdecl, EntryPoint = "livekit_ffi_request")] - internal static extern unsafe FfiHandle FfiNewRequest(byte[] data, int len, out byte* dataPtr, out int dataLen); + internal extern static unsafe IntPtr FfiNewRequest(byte* data, int len, out byte* dataPtr, out int dataLen); + //TODO optimise FfiHandle, can be replaced by FfiHandleId = uint64_t [DllImport(Lib, CallingConvention = CallingConvention.Cdecl, EntryPoint = "livekit_ffi_initialize")] - internal static extern unsafe FfiHandle FfiInitialize(ulong cb, bool captureLogs); + internal extern static IntPtr LiveKitInitialize(FFICallbackDelegate cb, bool captureLogs); } -} +} \ No newline at end of file diff --git a/Runtime/Scripts/Internal/RingBuffer.cs b/Runtime/Scripts/Internal/RingBuffer.cs index e8638a5..c5cf220 100644 --- a/Runtime/Scripts/Internal/RingBuffer.cs +++ b/Runtime/Scripts/Internal/RingBuffer.cs @@ -1,19 +1,23 @@ using System; +using System.Buffers; +using LiveKit.Internal.FFIClients.Pools.Memory; namespace LiveKit.Internal { // Basic RingBuffer implementation (used WebRtc_RingBuffer as reference) // The one from com.unity.collections is dealing element per element, which is not efficient when dealing with bytes - public class RingBuffer + public class RingBuffer : IDisposable { - private byte[] _buffer; + private MemoryWrap _buffer; private int _writePos; private int _readPos; private bool _sameWrap; + private static readonly IMemoryPool MemoryPool = new ArrayMemoryPool(ArrayPool.Shared!); + public RingBuffer(int size) { - _buffer = new byte[size]; + _buffer = MemoryPool.Memory(size); _sameWrap = true; } @@ -26,12 +30,12 @@ public int Write(ReadOnlySpan data) if (write > margin) { - data.Slice(0, margin).CopyTo(_buffer.AsSpan(_writePos)); + data.Slice(0, margin).CopyTo(_buffer.Span().Slice(_writePos)); _writePos = 0; n -= margin; _sameWrap = false; } - data.Slice(write - n, n).CopyTo(_buffer.AsSpan(_writePos)); + data.Slice(write - n, n).CopyTo(_buffer.Span().Slice(_writePos)); _writePos += n; return write; } @@ -41,13 +45,13 @@ public int Read(Span data) int readCount = GetBufferReadRegions(data.Length, out int dataIndex1, out int dataLen1, out int dataIndex2, out int dataLen2); if (dataLen2 > 0) { - _buffer.AsSpan().Slice(dataIndex1, dataLen1).CopyTo(data); - _buffer.AsSpan().Slice(dataIndex2, dataLen2).CopyTo(data.Slice(dataLen1)); + _buffer.Span().Slice(dataIndex1, dataLen1).CopyTo(data); + _buffer.Span().Slice(dataIndex2, dataLen2).CopyTo(data.Slice(dataLen1)); } else { // TODO(theomonnom): Don't always copy in this case? - _buffer.AsSpan().Slice(dataIndex1, dataLen1).CopyTo(data); + _buffer.Span().Slice(dataIndex1, dataLen1).CopyTo(data); } MoveReadPtr(readCount); @@ -120,5 +124,10 @@ public int AvailableWrite() { return _buffer.Length - AvailableRead(); } + + public void Dispose() + { + _buffer.Dispose(); + } } } diff --git a/Runtime/Scripts/Internal/Utils.cs b/Runtime/Scripts/Internal/Utils.cs index 679878c..b8a60da 100644 --- a/Runtime/Scripts/Internal/Utils.cs +++ b/Runtime/Scripts/Internal/Utils.cs @@ -1,5 +1,8 @@ +using System; using System.Diagnostics; using UnityEngine; +using UnityEngine.Experimental.Rendering; +using UnityEngine.Rendering; namespace LiveKit.Internal { @@ -22,5 +25,43 @@ public static void Error(object msg) { UnityEngine.Debug.unityLogger.Log(LogType.Error, $"{PREFIX}: {msg}"); } + + public static GraphicsFormat GetSupportedGraphicsFormat(GraphicsDeviceType type) + { + if (QualitySettings.activeColorSpace == ColorSpace.Linear) + { + switch (type) + { + case GraphicsDeviceType.Direct3D11: + case GraphicsDeviceType.Direct3D12: + case GraphicsDeviceType.Vulkan: + return GraphicsFormat.B8G8R8A8_SRGB; + case GraphicsDeviceType.OpenGLCore: + case GraphicsDeviceType.OpenGLES2: + case GraphicsDeviceType.OpenGLES3: + return GraphicsFormat.R8G8B8A8_SRGB; + case GraphicsDeviceType.Metal: + return GraphicsFormat.B8G8R8A8_SRGB; + } + } + else + { + switch (type) + { + case GraphicsDeviceType.Vulkan: + return GraphicsFormat.B8G8R8A8_UNorm; + case GraphicsDeviceType.Direct3D12: // Gamma and 3D12 required R8 + case GraphicsDeviceType.Direct3D11: // Gamma and 3D11 required R8 + case GraphicsDeviceType.OpenGLCore: + case GraphicsDeviceType.OpenGLES2: + case GraphicsDeviceType.OpenGLES3: + return GraphicsFormat.R8G8B8A8_UNorm; + case GraphicsDeviceType.Metal: + return GraphicsFormat.R8G8B8A8_UNorm; + } + } + + throw new ArgumentException($"Graphics device type {type} not supported"); + } } } diff --git a/Runtime/Scripts/Participant.cs b/Runtime/Scripts/Participant.cs index 991103e..8a04436 100644 --- a/Runtime/Scripts/Participant.cs +++ b/Runtime/Scripts/Participant.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using LiveKit.Internal; using LiveKit.Proto; +using LiveKit.Internal.FFIClients.Requests; namespace LiveKit { @@ -10,31 +11,38 @@ public class Participant { public delegate void PublishDelegate(RemoteTrackPublication publication); - public FfiOwnedHandle Handle; + private ParticipantInfo _info; + internal readonly Dictionary _tracks = new(); + public FfiHandle Handle; public string Sid => _info.Sid; public string Identity => _info.Identity; public string Name => _info.Name; public string Metadata => _info.Metadata; - - public bool Speaking { private set; get; } - public float AudioLevel { private set; get; } - public ConnectionQuality ConnectionQuality { private set; get; } - + public ConnectionQuality ConnectionQuality { internal set; get; } public event PublishDelegate TrackPublished; public event PublishDelegate TrackUnpublished; public readonly WeakReference Room; public IReadOnlyDictionary Tracks => _tracks; - internal readonly Dictionary _tracks = new(); protected Participant(OwnedParticipant participant, Room room) { Room = new WeakReference(room); - Handle = participant.Handle; + Handle = FfiHandle.FromOwnedHandle(participant.Handle); UpdateInfo(participant.Info); } + public void SetMeta(string meta) + { + _info.Metadata = meta; + } + + public void SetName(string name) + { + _info.Name = name; + } + internal void UpdateInfo(ParticipantInfo info) { _info = info; @@ -64,73 +72,76 @@ public PublishTrackInstruction PublishTrack(ILocalTrack localTrack, TrackPublish if (!Room.TryGetTarget(out var room)) throw new Exception("room is invalid"); - var track = (Track)localTrack; - - var publish = new PublishTrackRequest(); - publish.LocalParticipantHandle = Handle.Id; - publish.TrackHandle = (ulong)track.TrackHandle.Id; + var track = (Track)localTrack; + + using var request = FFIBridge.Instance.NewRequest(); + var publish = request.request; + publish.LocalParticipantHandle = (ulong)Handle.DangerousGetHandle(); + publish.TrackHandle = (ulong)track.Handle.DangerousGetHandle(); publish.Options = options; + using var response = request.Send(); + FfiResponse res = response; + return new PublishTrackInstruction(res.PublishTrack.AsyncId); + } - var request = new FfiRequest(); - request.PublishTrack = publish; + public void PublishData(byte[] data, IReadOnlyCollection destination_sids = null, bool reliable = true, string topic = null) + { + PublishData(new Span(data), destination_sids, reliable, topic); + } - track.Room = Room; + public void PublishData(Span data, IReadOnlyCollection destination_sids = null, bool reliable = true, string topic = null) + { + unsafe + { + fixed (byte* pointer = data) + { + PublishData(pointer, data.Length, destination_sids, reliable, topic); + } + } + } - var resp = FfiClient.SendRequest(request); - return new PublishTrackInstruction(resp.PublishTrack.AsyncId); + public void UpdateMetadata(string metadata) + { + using var request = FFIBridge.Instance.NewRequest(); + var updateReq = request.request; + updateReq.Metadata = metadata; + var resp = request.Send(); } - public PublishTrackInstruction publishData(byte[] data, string[] destination_sids = null, bool reliable = true, string topic = null) + public void UpdateName(string name) + { + using var request = FFIBridge.Instance.NewRequest(); + var updateReq = request.request; + updateReq.Name = name; + var resp = request.Send(); + } + + private unsafe void PublishData(byte* data, int len, IReadOnlyCollection destination_sids = null, bool reliable = true, string topic = null) { if (!Room.TryGetTarget(out var room)) throw new Exception("room is invalid"); - var publish = new PublishDataRequest(); - publish.LocalParticipantHandle = Handle.Id; + using var request = FFIBridge.Instance.NewRequest(); + + var publish = request.request; + publish.LocalParticipantHandle = (ulong)Handle.DangerousGetHandle(); publish.Kind = reliable ? DataPacketKind.KindReliable : DataPacketKind.KindLossy; - if (destination_sids is not null) - { + if (destination_sids is not null) { publish.DestinationSids.AddRange(destination_sids); } - if(topic is not null) + if (topic is not null) { publish.Topic = topic; } unsafe { - publish.DataLen = (ulong)data.Length; - publish.DataPtr = (ulong)System.Runtime.InteropServices.Marshal.UnsafeAddrOfPinnedArrayElement(data, 0); + publish.DataLen = (ulong)len; + publish.DataPtr = (ulong)data; } - - var request = new FfiRequest(); - request.PublishData = publish; - - var resp = FfiClient.SendRequest(request); - - return new PublishTrackInstruction(resp.PublishTrack.AsyncId); - } - - - public void UpdateMetadata(string metadata) - { - var updateReq = new UpdateLocalMetadataRequest(); - updateReq.Metadata = metadata; - var request = new FfiRequest(); - request.UpdateLocalMetadata = updateReq; - - FfiClient.SendRequest(request); - } - - public void UpdateName(string name) - { - var updateReq = new UpdateLocalNameRequest(); - updateReq.Name = name; - var request = new FfiRequest(); - request.UpdateLocalName = updateReq; - - FfiClient.SendRequest(request); + Utils.Debug("Sending message: " + topic); + var response = request.Send(); } } @@ -152,7 +163,7 @@ internal PublishTrackInstruction(ulong asyncId) FfiClient.Instance.PublishTrackReceived += OnPublish; } - void OnPublish(PublishTrackCallback e) + internal void OnPublish(PublishTrackCallback e) { if (e.AsyncId != _asyncId) return; diff --git a/Runtime/Scripts/Proto/Room.cs b/Runtime/Scripts/Proto/Room.cs index 5965275..706c71e 100644 --- a/Runtime/Scripts/Proto/Room.cs +++ b/Runtime/Scripts/Proto/Room.cs @@ -5,6 +5,8 @@ #pragma warning disable 1591, 0612, 3021, 8981 #region Designer generated code +using System; +using LiveKit.Internal.FFIClients; using pb = global::Google.Protobuf; using pbc = global::Google.Protobuf.Collections; using pbr = global::Google.Protobuf.Reflection; @@ -743,9 +745,14 @@ public void MergeFrom(pb::CodedInputStream input) { } } } - #endif - } + public static implicit operator ConnectResponse(FfiResponseWrap v) + { + throw new NotImplementedException(); + } +#endif + + } [global::System.Diagnostics.DebuggerDisplayAttribute("{ToString(),nq}")] public sealed partial class ConnectCallback : pb::IMessage diff --git a/Runtime/Scripts/Proto/Track.cs b/Runtime/Scripts/Proto/Track.cs index 6fc5b61..b9e4925 100644 --- a/Runtime/Scripts/Proto/Track.cs +++ b/Runtime/Scripts/Proto/Track.cs @@ -5,6 +5,8 @@ #pragma warning disable 1591, 0612, 3021, 8981 #region Designer generated code +using System; +using LiveKit.Internal.FFIClients; using pb = global::Google.Protobuf; using pbc = global::Google.Protobuf.Collections; using pbr = global::Google.Protobuf.Reflection; @@ -956,9 +958,14 @@ public void MergeFrom(pb::CodedInputStream input) { } } } - #endif - } + public static implicit operator CreateAudioTrackResponse(FfiResponseWrap v) + { + throw new NotImplementedException(); + } +#endif + + } [global::System.Diagnostics.DebuggerDisplayAttribute("{ToString(),nq}")] public sealed partial class GetStatsRequest : pb::IMessage diff --git a/Runtime/Scripts/Room.cs b/Runtime/Scripts/Room.cs index 3251f90..134558e 100644 --- a/Runtime/Scripts/Room.cs +++ b/Runtime/Scripts/Room.cs @@ -3,6 +3,7 @@ using LiveKit.Internal; using LiveKit.Proto; using System.Runtime.InteropServices; +using LiveKit.Internal.FFIClients.Requests; namespace LiveKit { @@ -106,7 +107,13 @@ public Proto.RoomOptions ToProto() public class Room { - public delegate void ParticipantDelegate(RemoteParticipant participant); + internal FfiHandle RoomHandle = null; + private readonly Dictionary _participants = new(); + + public delegate void MetaDelegate(string metaData); + public delegate void ParticipantDelegate(Participant participant); + public delegate void RemoteParticipantDelegate(RemoteParticipant participant); + public delegate void LocalPublishDelegate(TrackPublication publication, LocalParticipant participant); public delegate void PublishDelegate(RemoteTrackPublication publication, RemoteParticipant participant); public delegate void SubscribeDelegate(IRemoteTrack track, RemoteTrackPublication publication, RemoteParticipant participant); public delegate void MuteDelegate(TrackPublication publication, Participant participant); @@ -114,7 +121,7 @@ public class Room public delegate void ConnectionQualityChangeDelegate(ConnectionQuality quality, Participant participant); public delegate void DataDelegate(byte[] data, Participant participant, DataPacketKind kind, string topic); public delegate void ConnectionStateChangeDelegate(ConnectionState connectionState); - public delegate void ConnectionDelegate(); + public delegate void ConnectionDelegate(Room room); public delegate void E2EeStateChangedDelegate(Participant participant, EncryptionState state); public string Sid { private set; get; } @@ -123,12 +130,13 @@ public class Room public LocalParticipant LocalParticipant { private set; get; } public ConnectionState ConnectionState { private set; get; } public bool IsConnected => RoomHandle == null && ConnectionState != ConnectionState.ConnDisconnected; - - private readonly Dictionary _participants = new(); - public IReadOnlyDictionary Participants => _participants; + public E2EEManager E2EEManager { internal set; get; } + public IReadOnlyDictionary RemoteParticipants => _participants; public event ParticipantDelegate ParticipantConnected; public event ParticipantDelegate ParticipantDisconnected; + public event LocalPublishDelegate LocalTrackPublished; + public event LocalPublishDelegate LocalTrackUnpublished; public event PublishDelegate TrackPublished; public event PublishDelegate TrackUnpublished; public event SubscribeDelegate TrackSubscribed; @@ -139,48 +147,32 @@ public class Room public event ConnectionQualityChangeDelegate ConnectionQualityChanged; public event DataDelegate DataReceived; public event ConnectionStateChangeDelegate ConnectionStateChanged; + public event ConnectionDelegate Connected; public event ConnectionDelegate Disconnected; public event ConnectionDelegate Reconnecting; public event ConnectionDelegate Reconnected; public event E2EeStateChangedDelegate E2EeStateChanged; - - public E2EEManager E2EEManager { internal set; get; } - - internal FfiOwnedHandle RoomHandle = null; + public event MetaDelegate RoomMetadataChanged; + public event ParticipantDelegate ParticipantMetadataChanged; + public event ParticipantDelegate ParticipantNameChanged; public ConnectInstruction Connect(string url, string token, RoomOptions options) { - var connect = new ConnectRequest(); - connect.Url = url; - connect.Token = token; - connect.Options = options.ToProto(); - - var request = new FfiRequest(); - request.Connect = connect; - - var resp = FfiClient.SendRequest(request); - return new ConnectInstruction(resp.Connect.AsyncId, this, options); + using var response = FFIBridge.Instance.SendConnectRequest(url, token, options); + Utils.Debug("Connect...."); + FfiResponse res = response; + Utils.Debug($"Connect response.... {response}"); + return new ConnectInstruction(res.Connect.AsyncId, this, options); } - + public void Disconnect() { - if(!IsConnected) { - return; - } - - var disconnect = new DisconnectRequest(); - disconnect.RoomHandle = RoomHandle.Id; - var request = new FfiRequest(); - request.Disconnect = disconnect; - - FfiClient.SendRequest(request); - - RoomHandle = null; - - ConnectionState = ConnectionState.ConnDisconnected; + using var response = FFIBridge.Instance.SendDisconnectRequest(this); + Utils.Debug($"Disconnect.... {RoomHandle}"); + FfiResponse resp = response; + Utils.Debug($"Disconnect response.... {resp}"); } - internal void UpdateFromInfo(RoomInfo info) { Sid = info.Sid; @@ -188,13 +180,36 @@ internal void UpdateFromInfo(RoomInfo info) Metadata = info.Metadata; } + internal void OnEventReceived(RoomEvent e) { - if (e.RoomHandle != (ulong)RoomHandle.Id) + if (e.RoomHandle != (ulong)RoomHandle.DangerousGetHandle()) return; switch (e.MessageCase) { + case RoomEvent.MessageOneofCase.RoomMetadataChanged: + { + Metadata = e.RoomMetadataChanged.Metadata; + RoomMetadataChanged?.Invoke(e.RoomMetadataChanged.Metadata); + } + break; + case RoomEvent.MessageOneofCase.ParticipantMetadataChanged: + { + var participant = GetParticipant(e.ParticipantMetadataChanged.ParticipantSid); + participant.SetMeta(e.ParticipantMetadataChanged.Metadata); + if (participant != null) ParticipantMetadataChanged?.Invoke(participant); + else Utils.Debug("Unable to find participant: " + e.ParticipantMetadataChanged.ParticipantSid + " in Meta data Change Event"); + } + break; + case RoomEvent.MessageOneofCase.ParticipantNameChanged: + { + var participant = GetParticipant(e.ParticipantNameChanged.ParticipantSid); + participant.SetName(e.ParticipantNameChanged.Name); + if (participant != null) ParticipantNameChanged?.Invoke(participant); + else Utils.Debug("Unable to find participant: " + e.ParticipantNameChanged.ParticipantSid + " in Meta data Change Event"); + } + break; case RoomEvent.MessageOneofCase.ParticipantConnected: { var participant = CreateRemoteParticipant(e.ParticipantConnected.Info); @@ -204,15 +219,15 @@ internal void OnEventReceived(RoomEvent e) case RoomEvent.MessageOneofCase.ParticipantDisconnected: { var sid = e.ParticipantDisconnected.ParticipantSid; - var participant = Participants[Sid]; + var participant = RemoteParticipants[Sid]; _participants.Remove(Sid); ParticipantDisconnected?.Invoke(participant); } break; case RoomEvent.MessageOneofCase.TrackPublished: { - var participant = Participants[e.TrackPublished.ParticipantSid]; - var publication = new RemoteTrackPublication(e.TrackPublished.Publication.Info); + var participant = RemoteParticipants[e.TrackPublished.ParticipantSid]; + var publication = new RemoteTrackPublication(e.TrackPublished.Publication.Info, FfiHandle.FromOwnedHandle(e.TrackPublished.Publication.Handle)); participant._tracks.Add(publication.Sid, publication); participant.OnTrackPublished(publication); TrackPublished?.Invoke(publication, participant); @@ -220,7 +235,7 @@ internal void OnEventReceived(RoomEvent e) break; case RoomEvent.MessageOneofCase.TrackUnpublished: { - var participant = Participants[e.TrackUnpublished.ParticipantSid]; + var participant = RemoteParticipants[e.TrackUnpublished.ParticipantSid]; var publication = participant.Tracks[e.TrackUnpublished.PublicationSid]; participant._tracks.Remove(publication.Sid); participant.OnTrackUnpublished(publication); @@ -231,7 +246,7 @@ internal void OnEventReceived(RoomEvent e) { var track = e.TrackSubscribed.Track; var info = track.Info; - var participant = Participants[e.TrackSubscribed.ParticipantSid]; + var participant = RemoteParticipants[e.TrackSubscribed.ParticipantSid]; var publication = participant.Tracks[info.Sid]; if(publication == null) @@ -239,16 +254,15 @@ internal void OnEventReceived(RoomEvent e) participant._tracks.Add(publication.Sid, publication); } - if (info.Kind == TrackKind.KindVideo) { - var videoTrack = new RemoteVideoTrack(null, track, this, participant); + var videoTrack = new RemoteVideoTrack(track, this, participant); publication.UpdateTrack(videoTrack); TrackSubscribed?.Invoke(videoTrack, publication, participant); } else if (info.Kind == TrackKind.KindAudio) { - var audioTrack = new RemoteAudioTrack(null, track, this, participant); + var audioTrack = new RemoteAudioTrack(track, this, participant); publication.UpdateTrack(audioTrack); TrackSubscribed?.Invoke(audioTrack, publication, participant); } @@ -256,13 +270,39 @@ internal void OnEventReceived(RoomEvent e) break; case RoomEvent.MessageOneofCase.TrackUnsubscribed: { - var participant = Participants[e.TrackUnsubscribed.ParticipantSid]; + var participant = RemoteParticipants[e.TrackSubscribed.ParticipantSid]; var publication = participant.Tracks[e.TrackUnsubscribed.TrackSid]; var track = publication.Track; publication.UpdateTrack(null); TrackUnsubscribed?.Invoke(track, publication, participant); } break; + case RoomEvent.MessageOneofCase.LocalTrackUnpublished: + { + if (LocalParticipant._tracks.ContainsKey(e.LocalTrackUnpublished.PublicationSid)) + { + var publication = LocalParticipant._tracks[e.LocalTrackUnpublished.PublicationSid]; + LocalTrackUnpublished?.Invoke(publication, LocalParticipant); + } + else + { + Utils.Debug("Unable to find local track after unpublish: " + e.LocalTrackPublished.TrackSid); + } + } + break; + case RoomEvent.MessageOneofCase.LocalTrackPublished: + { + if (LocalParticipant._tracks.ContainsKey(e.LocalTrackPublished.TrackSid)) + { + var publication = LocalParticipant._tracks[e.LocalTrackPublished.TrackSid]; + LocalTrackPublished?.Invoke(publication, LocalParticipant); + } + else + { + Utils.Debug("Unable to find local track after publish: " + e.LocalTrackPublished.TrackSid); + } + } + break; case RoomEvent.MessageOneofCase.TrackMuted: { var participant = GetParticipant(e.TrackMuted.ParticipantSid); @@ -294,13 +334,14 @@ internal void OnEventReceived(RoomEvent e) { var participant = GetParticipant(e.ConnectionQualityChanged.ParticipantSid); var quality = e.ConnectionQualityChanged.Quality; + participant.ConnectionQuality = quality; ConnectionQualityChanged?.Invoke(quality, participant); } break; case RoomEvent.MessageOneofCase.DataReceived: { var dataInfo = e.DataReceived; - var data = new byte[dataInfo.Data.Data.CalculateSize()]; + var data = new byte[dataInfo.Data.Data.DataLen]; Marshal.Copy((IntPtr)dataInfo.Data.Data.DataPtr, data, 0, data.Length); var participant = GetParticipant(e.DataReceived.ParticipantSid); DataReceived?.Invoke(data, participant, dataInfo.Kind, dataInfo.Topic); @@ -311,14 +352,14 @@ internal void OnEventReceived(RoomEvent e) ConnectionStateChanged?.Invoke(e.ConnectionStateChanged.State); break; case RoomEvent.MessageOneofCase.Disconnected: - Disconnected?.Invoke(); + Disconnected?.Invoke(this); OnDisconnect(); break; case RoomEvent.MessageOneofCase.Reconnecting: - Reconnecting?.Invoke(); + Reconnecting?.Invoke(this); break; case RoomEvent.MessageOneofCase.Reconnected: - Reconnected?.Invoke(); + Reconnected?.Invoke(this); break; case RoomEvent.MessageOneofCase.E2EeStateChanged: { @@ -331,7 +372,7 @@ internal void OnEventReceived(RoomEvent e) internal void OnConnect(ConnectCallback info) { - RoomHandle = info.Room.Handle; + RoomHandle = FfiHandle.FromOwnedHandle(info.Room.Handle); UpdateFromInfo(info.Room.Info); LocalParticipant = new LocalParticipant(info.LocalParticipant, this); @@ -341,14 +382,22 @@ internal void OnConnect(ConnectCallback info) CreateRemoteParticipantWithTracks(p); FfiClient.Instance.RoomEventReceived += OnEventReceived; + FfiClient.Instance.DisconnectReceived += OnDisconnectReceived; + Connected?.Invoke(this); + } + + private void OnDisconnectReceived(DisconnectCallback e) + { + FfiClient.Instance.DisconnectReceived -= OnDisconnectReceived; + Utils.Debug($"OnDisconnect.... {e}"); } - internal void OnDisconnect() + private void OnDisconnect() { FfiClient.Instance.RoomEventReceived -= OnEventReceived; } - RemoteParticipant CreateRemoteParticipantWithTracks(ConnectCallback.Types.ParticipantWithTracks item) + internal RemoteParticipant CreateRemoteParticipantWithTracks(ConnectCallback.Types.ParticipantWithTracks item) { var participant = item.Participant; var publications = item.Publications; @@ -356,26 +405,26 @@ RemoteParticipant CreateRemoteParticipantWithTracks(ConnectCallback.Types.Partic _participants.Add(participant.Info.Sid, newParticipant); foreach (var pub in publications) { - var publication = new RemoteTrackPublication(pub.Info); + var publication = new RemoteTrackPublication(pub.Info, FfiHandle.FromOwnedHandle(pub.Handle)); newParticipant._tracks.Add(publication.Sid, publication); newParticipant.OnTrackPublished(publication); } return newParticipant; } - RemoteParticipant CreateRemoteParticipant(OwnedParticipant participant) + internal RemoteParticipant CreateRemoteParticipant(OwnedParticipant participant) { var newParticipant = new RemoteParticipant(participant, this); _participants.Add(participant.Info.Sid, newParticipant); return newParticipant; } - public Participant GetParticipant(string sid) + internal Participant GetParticipant(string sid) { if (sid == LocalParticipant.Sid) return LocalParticipant; - Participants.TryGetValue(sid, out var remoteParticipant); + RemoteParticipants.TryGetValue(sid, out var remoteParticipant); return remoteParticipant; } } diff --git a/Runtime/Scripts/RtcVideoSource.cs b/Runtime/Scripts/RtcVideoSource.cs new file mode 100644 index 0000000..f468463 --- /dev/null +++ b/Runtime/Scripts/RtcVideoSource.cs @@ -0,0 +1,154 @@ +using System; +using UnityEngine; +using LiveKit.Proto; +using LiveKit.Internal; +using UnityEngine.Rendering; +using Unity.Collections; +using Unity.Collections.LowLevel.Unsafe; +using LiveKit.Internal.FFIClients.Requests; +using System.Collections; + +namespace LiveKit +{ + public abstract class RtcVideoSource + { + public enum VideoStreamSource + { + Texture = 0, + Screen = 1, + Camera = 2 + } + + internal FfiHandle Handle { get; } + + public abstract int GetWidth(); + public abstract int GetHeight(); + + protected Texture _dest; + protected NativeArray _data; + protected VideoStreamSource _sourceType; + protected VideoBufferType _bufferType; + protected VideoSourceInfo _info; + protected bool _reading = false; + protected bool _requestPending = false; + protected bool isDisposed = true; + protected bool _playing = false; + + public RtcVideoSource(VideoStreamSource sourceType, VideoBufferType bufferType) + { + isDisposed = false; + _sourceType = sourceType; + _bufferType = bufferType; + using var request = FFIBridge.Instance.NewRequest(); + var newVideoSource = request.request; + newVideoSource.Type = VideoSourceType.VideoSourceNative; + using var response = request.Send(); + FfiResponse res = response; + _info = res.NewVideoSource.Source.Info; + Handle = FfiHandle.FromOwnedHandle(res.NewVideoSource.Source.Handle); + } + + protected TextureFormat GetTextureFormat(VideoBufferType type) + { + switch (type) + { + case VideoBufferType.Rgba: + return TextureFormat.RGBA32; + case VideoBufferType.Argb: + return TextureFormat.ARGB32; + default: + throw new NotImplementedException("TODO: Add TextureFormat support for type: " + type); + } + } + + protected int GetStrideForBuffer(VideoBufferType type) + { + switch (type) + { + case VideoBufferType.Rgba: + case VideoBufferType.Argb: + return 4; + default: + throw new NotImplementedException("TODO: Add stride support for type: " + type); + } + } + + public virtual void Start() + { + Stop(); + _playing = true; + } + + public virtual void Stop() + { + _playing = false; + } + + public IEnumerator Update() + { + while (_playing) + { + yield return null; + ReadBuffer(); + SendFrame(); + } + + yield break; + } + + public virtual void Dispose() + { + if (!isDisposed) + { + _data.Dispose(); + isDisposed = true; + } + } + + protected abstract void ReadBuffer(); + + protected virtual bool SendFrame() + { + var result = _requestPending && !isDisposed; + if (result) + { + var buffer = new VideoBufferInfo(); + unsafe + { + buffer.DataPtr = (ulong)NativeArrayUnsafeUtility.GetUnsafePtr(_data); + } + + buffer.Type = _bufferType; + buffer.Stride = (uint)GetWidth() * (uint)GetStrideForBuffer(_bufferType); + buffer.Width = (uint)GetWidth(); + buffer.Height = (uint)GetHeight(); + + // Send the frame to WebRTC + using var request = FFIBridge.Instance.NewRequest(); + var capture = request.request; + capture.SourceHandle = (ulong)Handle.DangerousGetHandle(); + capture.Rotation = VideoRotation._0; + capture.TimestampUs = DateTimeOffset.Now.ToUnixTimeMilliseconds(); + capture.Buffer = buffer; + using var response = request.Send(); + _reading = false; + _requestPending = false; + } + return result; + } + + protected void OnReadback(AsyncGPUReadbackRequest req) + { + if (!req.hasError) + { + _requestPending = true; + } + else + { + Utils.Error("GPU Read Back on Video Source Failed: " + req.ToString()); + _reading = false; + } + } + } +} + diff --git a/Runtime/Scripts/RtcVideoSource.cs.meta b/Runtime/Scripts/RtcVideoSource.cs.meta new file mode 100644 index 0000000..b46f607 --- /dev/null +++ b/Runtime/Scripts/RtcVideoSource.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 132c76bbe27f1e2419dcad5c7699c3e3 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/ScreenVideoSource.cs b/Runtime/Scripts/ScreenVideoSource.cs new file mode 100644 index 0000000..b164693 --- /dev/null +++ b/Runtime/Scripts/ScreenVideoSource.cs @@ -0,0 +1,80 @@ +using System; +using UnityEngine; +using LiveKit.Proto; +using LiveKit.Internal; +using UnityEngine.Rendering; +using Unity.Collections; +using Unity.Collections.LowLevel.Unsafe; +using System.Threading; +using LiveKit.Internal.FFIClients.Requests; +using UnityEngine.Experimental.Rendering; +using UnityEngine.UI; +using System.Threading.Tasks; + +namespace LiveKit +{ + public class ScreenVideoSource : RtcVideoSource + { + public override int GetWidth() + { + return Screen.width; + } + + public override int GetHeight() + { + return Screen.height; + } + + public ScreenVideoSource(VideoBufferType bufferType = VideoBufferType.Rgba) : base(VideoStreamSource.Screen, bufferType) + { + _data = new NativeArray(GetWidth() * GetHeight() * GetStrideForBuffer(bufferType), Allocator.Persistent); + } + + public override void Stop() + { + base.Stop(); + ClearRenderTexture(); + } + + ~ScreenVideoSource() + { + Dispose(); + ClearRenderTexture(); + } + + private void ClearRenderTexture() + { + if (_dest) + { + var renderText = _dest as RenderTexture; + renderText.Release(); // can only be done on main thread + } + } + + // Read the texture data into a native array asynchronously + protected override void ReadBuffer() + { + if (_reading) + return; + _reading = true; + if (_dest == null) + { + var targetFormat = Utils.GetSupportedGraphicsFormat(SystemInfo.graphicsDeviceType); + _dest = new RenderTexture(GetWidth(), GetHeight(), 0, targetFormat); + } + ScreenCapture.CaptureScreenshotIntoRenderTexture(_dest as RenderTexture); + AsyncGPUReadback.RequestIntoNativeArray(ref _data, _dest, 0, GetTextureFormat(_bufferType), OnReadback); + } + + protected override bool SendFrame() + { + var result = base.SendFrame(); + if (result) + { + ClearRenderTexture(); + } + return result; + } + } +} + diff --git a/Runtime/Scripts/ScreenVideoSource.cs.meta b/Runtime/Scripts/ScreenVideoSource.cs.meta new file mode 100644 index 0000000..4962380 --- /dev/null +++ b/Runtime/Scripts/ScreenVideoSource.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: c41c73a604498c64da7a0016babb6090 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/TextureVideoSource.cs b/Runtime/Scripts/TextureVideoSource.cs new file mode 100644 index 0000000..25cc104 --- /dev/null +++ b/Runtime/Scripts/TextureVideoSource.cs @@ -0,0 +1,61 @@ +using UnityEngine; +using LiveKit.Proto; +using UnityEngine.Rendering; +using Unity.Collections; +using UnityEngine.Experimental.Rendering; + +namespace LiveKit +{ + public class TextureVideoSource : RtcVideoSource + { + public Texture Texture { get; } + + public override int GetWidth() + { + return Texture.width; + } + + public override int GetHeight() + { + return Texture.height; + } + + public TextureVideoSource(Texture texture, VideoBufferType bufferType = VideoBufferType.Rgba) : base(VideoStreamSource.Texture, bufferType) + { + Texture = texture; + _data = new NativeArray(GetWidth() * GetHeight() * GetStrideForBuffer(bufferType), Allocator.Persistent); + } + + + ~TextureVideoSource() + { + Dispose(); + } + + // Read the texture data into a native array asynchronously + protected override void ReadBuffer() + { + if (_reading) + return; + _reading = true; + var gpuTextureFormat = GetTextureFormat(_bufferType); + if (!SystemInfo.IsFormatSupported(Texture.graphicsFormat, FormatUsage.ReadPixels)) + { + if (_dest == null || _dest.width != GetWidth() || _dest.height != GetHeight()) + { + + _data = new NativeArray(GetWidth() * GetHeight() * GetStrideForBuffer(_bufferType), Allocator.Persistent); + _dest = new Texture2D(GetWidth(), GetHeight(), gpuTextureFormat, false); + } + Graphics.CopyTexture(Texture, _dest); + } + else + { + _dest = Texture; + } + + AsyncGPUReadback.RequestIntoNativeArray(ref _data, _dest, 0, gpuTextureFormat, OnReadback); + } + } +} + diff --git a/Runtime/Scripts/TextureVideoSource.cs.meta b/Runtime/Scripts/TextureVideoSource.cs.meta new file mode 100644 index 0000000..e7faeb2 --- /dev/null +++ b/Runtime/Scripts/TextureVideoSource.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: a21dae64441674a478fe4ec708b9b059 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/Track.cs b/Runtime/Scripts/Track.cs index ad0b07b..57a3af3 100644 --- a/Runtime/Scripts/Track.cs +++ b/Runtime/Scripts/Track.cs @@ -1,6 +1,7 @@ using System; using LiveKit.Proto; using LiveKit.Internal; +using LiveKit.Internal.FFIClients.Requests; namespace LiveKit { @@ -13,6 +14,7 @@ public interface ITrack bool Muted { get; } WeakReference Room { get; } WeakReference Participant { get; } + FfiHandle TrackHandle { get; } } public interface ILocalTrack : ITrack @@ -51,12 +53,11 @@ public class Track : ITrack public readonly FfiHandle Handle; - internal FfiOwnedHandle TrackHandle { get; } + FfiHandle ITrack.TrackHandle => Handle; - internal Track(FfiHandle handle, OwnedTrack track, Room room, Participant participant) + internal Track(OwnedTrack track, Room room, Participant participant) { - Handle = handle; - TrackHandle = track.Handle; + Handle = FfiHandle.FromOwnedHandle(track.Handle); Room = new WeakReference(room); Participant = new WeakReference(participant); UpdateInfo(track.Info); @@ -75,54 +76,48 @@ internal void UpdateMuted(bool muted) public sealed class LocalAudioTrack : Track, ILocalTrack, IAudioTrack { - internal LocalAudioTrack(FfiHandle handle, OwnedTrack track, Room room) : base(handle, track, room, room?.LocalParticipant) { } + internal LocalAudioTrack(OwnedTrack track, Room room) : base(track, room, room?.LocalParticipant) { } - public static LocalAudioTrack CreateAudioTrack(string name, RtcAudioSource source) + public static LocalAudioTrack CreateAudioTrack(string name, RtcAudioSource source, Room room) { - var createTrack = new CreateAudioTrackRequest(); + using var request = FFIBridge.Instance.NewRequest(); + var createTrack = request.request; createTrack.Name = name; - createTrack.SourceHandle = (ulong)source.Handle.Id; - - var request = new FfiRequest(); - request.CreateAudioTrack = createTrack; + createTrack.SourceHandle = (ulong)source.Handle.DangerousGetHandle(); - var resp = FfiClient.SendRequest(request); - var newTrack = resp.CreateAudioTrack.Track; - var trackHandle = new FfiHandle((IntPtr)newTrack.Handle.Id); - var track = new LocalAudioTrack(trackHandle, newTrack, null); + using var resp = request.Send(); + FfiResponse res = resp; + var trackInfo = res.CreateAudioTrack.Track; + var track = new LocalAudioTrack(trackInfo, room); return track; } } public sealed class LocalVideoTrack : Track, ILocalTrack, IVideoTrack { - internal LocalVideoTrack(FfiHandle handle, OwnedTrack track, Room room) : base(handle, track, room, room?.LocalParticipant) { } + internal LocalVideoTrack(OwnedTrack track, Room room) : base(track, room, room?.LocalParticipant) { } - public static LocalVideoTrack CreateVideoTrack(string name, RtcVideoSource source) + public static LocalVideoTrack CreateVideoTrack(string name, RtcVideoSource source, Room room) { - - var createTrack = new CreateVideoTrackRequest(); + using var request = FFIBridge.Instance.NewRequest(); + var createTrack = request.request; createTrack.Name = name; createTrack.SourceHandle = (ulong)source.Handle.DangerousGetHandle(); - - var request = new FfiRequest(); - request.CreateVideoTrack = createTrack; - - var resp = FfiClient.SendRequest(request); - var newTrack = resp.CreateVideoTrack.Track; - var trackHandle = new FfiHandle((IntPtr)newTrack.Handle.Id); - var track = new LocalVideoTrack(trackHandle, newTrack, null); + using var response = request.Send(); + FfiResponse res = response; + var trackInfo = res.CreateVideoTrack.Track; + var track = new LocalVideoTrack(trackInfo, room); return track; } } public sealed class RemoteAudioTrack : Track, IRemoteTrack, IAudioTrack { - internal RemoteAudioTrack(FfiHandle handle, OwnedTrack track, Room room, RemoteParticipant participant) : base(handle, track, room, participant) { } + internal RemoteAudioTrack(OwnedTrack track, Room room, RemoteParticipant participant) : base(track, room, participant) { } } public sealed class RemoteVideoTrack : Track, IRemoteTrack, IVideoTrack { - internal RemoteVideoTrack(FfiHandle handle, OwnedTrack track, Room room, RemoteParticipant participant) : base(handle, track, room, participant) { } + internal RemoteVideoTrack(OwnedTrack track, Room room, RemoteParticipant participant) : base(track, room, participant) { } } -} +} \ No newline at end of file diff --git a/Runtime/Scripts/TrackPublication.cs b/Runtime/Scripts/TrackPublication.cs index e0a9ecc..51722c5 100644 --- a/Runtime/Scripts/TrackPublication.cs +++ b/Runtime/Scripts/TrackPublication.cs @@ -1,4 +1,5 @@ using LiveKit.Internal; +using LiveKit.Internal.FFIClients.Requests; using LiveKit.Proto; namespace LiveKit @@ -46,17 +47,21 @@ public sealed class RemoteTrackPublication : TrackPublication public new IRemoteTrack Track => base.Track as IRemoteTrack; public bool Subscribed = false; - internal RemoteTrackPublication(TrackPublicationInfo info) : base(info) { } + private FfiHandle Handle; + + internal RemoteTrackPublication(TrackPublicationInfo info, FfiHandle handle) : base(info) + { + Handle = handle; + } public void SetSubscribed(bool subscribed) { Subscribed = subscribed; - var updateReq = new SetSubscribedRequest(); - updateReq.Subscribe = subscribed; - var request = new FfiRequest(); - request.SetSubscribed = updateReq; - - FfiClient.SendRequest(request); + using var request = FFIBridge.Instance.NewRequest(); + var setSubscribed = request.request; + setSubscribed.Subscribe = subscribed; + setSubscribed.PublicationHandle = (ulong)Handle.DangerousGetHandle(); + using var response = request.Send(); } } @@ -64,6 +69,8 @@ public sealed class LocalTrackPublication : TrackPublication { public new ILocalTrack Track => base.Track as ILocalTrack; - internal LocalTrackPublication(TrackPublicationInfo info) : base(info) { } + internal LocalTrackPublication(TrackPublicationInfo info) : base(info) + { + } } -} +} \ No newline at end of file diff --git a/Runtime/Scripts/VideoFrame.cs b/Runtime/Scripts/VideoFrame.cs index 0b3c143..7d263dd 100644 --- a/Runtime/Scripts/VideoFrame.cs +++ b/Runtime/Scripts/VideoFrame.cs @@ -2,6 +2,7 @@ using LiveKit.Internal; using LiveKit.Proto; using System.Runtime.CompilerServices; +using LiveKit.Internal.FFIClients.Requests; namespace LiveKit { @@ -106,15 +107,15 @@ public I420Buffer ToI420() Handle.SetHandleAsInvalid(); - var toi420 = new VideoConvertRequest(); + using var request = FFIBridge.Instance.NewRequest(); + var toi420 = request.request; toi420.Buffer = Info; toi420.DstType = VideoBufferType.I420; - var request = new FfiRequest(); - request.VideoConvert = toi420; - - var resp = FfiClient.SendRequest(request); - var newInfo = resp.VideoConvert.Buffer; + using var response = request.Send(); + FfiResponse res = response; + + var newInfo = res.VideoConvert.Buffer; if (newInfo == null) throw new InvalidOperationException("failed to convert"); @@ -128,19 +129,16 @@ public RGBABBuffer ToRGBA() if (!IsValid) throw new InvalidOperationException("the handle is invalid"); - var handleId = new FfiOwnedHandle(); - handleId.Id = (ulong)Handle.DangerousGetHandle(); - - var toRGBA = new VideoConvertRequest(); + using var request = FFIBridge.Instance.NewRequest(); + var toRGBA = request.request; toRGBA.Buffer = Info; toRGBA.DstType = VideoBufferType.Rgba; - var request = new FfiRequest(); - request.VideoConvert = toRGBA; + using var response = request.Send(); + FfiResponse res = response; - var resp = FfiClient.SendRequest(request); - var newInfo = resp.VideoConvert.Buffer; + var newInfo = res.VideoConvert.Buffer; if (newInfo == null) throw new InvalidOperationException("failed to convert"); @@ -163,6 +161,7 @@ public abstract class PlanarYuvBuffer : VideoFrameBuffer */ internal PlanarYuvBuffer(FfiHandle handle, VideoBufferInfo info) : base(handle, info) { } } + public abstract class PlanarYuv8Buffer : PlanarYuvBuffer { internal PlanarYuv8Buffer(FfiHandle handle, VideoBufferInfo info) : base(handle, info) { } @@ -246,4 +245,4 @@ public class NV12Buffer : BiplanarYuv8Buffer internal NV12Buffer(FfiHandle handle, VideoBufferInfo info) : base(handle, info) { } } -} +} \ No newline at end of file diff --git a/Runtime/Scripts/VideoSource.cs b/Runtime/Scripts/VideoSource.cs deleted file mode 100644 index 2f24749..0000000 --- a/Runtime/Scripts/VideoSource.cs +++ /dev/null @@ -1,122 +0,0 @@ -using System; -using System.Collections; -using UnityEngine; -using LiveKit.Proto; -using LiveKit.Internal; -using UnityEngine.Rendering; -using Unity.Collections; -using Unity.Collections.LowLevel.Unsafe; -using UnityEngine.Experimental.Rendering; - -namespace LiveKit -{ - public abstract class RtcVideoSource - { - internal readonly FfiHandle Handle; - protected VideoSourceInfo _info; - - public RtcVideoSource() - { - var newVideoSource = new NewVideoSourceRequest(); - newVideoSource.Type = VideoSourceType.VideoSourceNative; - - var request = new FfiRequest(); - request.NewVideoSource = newVideoSource; - - var resp = FfiClient.SendRequest(request); - _info = resp.NewVideoSource.Source.Info; - Handle = new FfiHandle((IntPtr)resp.NewVideoSource.Source.Handle.Id); - } - } - - public class TextureVideoSource : RtcVideoSource - { - public Texture Texture { get; } - private NativeArray _data; - private bool _reading = false; - - public TextureVideoSource(Texture texture) - { - Texture = texture; - _data = new NativeArray(Texture.width * Texture.height * 4, Allocator.Persistent); - ReadbackSupport(); - } - - // Read the texture data into a native array asynchronously - internal void ReadBuffer() - { - if (_reading) - return; - - _reading = true; - AsyncGPUReadback.RequestIntoNativeArray(ref _data, Texture, 0, TextureFormat.RGBA32, OnReadback); - } - - public IEnumerator Update() - { - while (true) - { - yield return null; - ReadBuffer(); - - } - } - - private void ReadbackSupport() - { - GraphicsFormat[] read_formats = (GraphicsFormat[])System.Enum.GetValues(typeof(GraphicsFormat)); - - foreach(var f in read_formats) - { - if (SystemInfo.IsFormatSupported(f, FormatUsage.ReadPixels)) - { - Debug.Log("support + " + f); - } - } - } - - private void OnReadback(AsyncGPUReadbackRequest req) - { - _reading = false; - if (req.hasError) - { - Utils.Error("failed to read texture data"); - return; - } - - // ToI420 - var argbInfo = new VideoBufferInfo(); - unsafe - { - argbInfo.DataPtr = (ulong)NativeArrayUnsafeUtility.GetUnsafePtr(_data); - } - argbInfo.Type = VideoBufferType.Rgba; - argbInfo.Stride = (uint)Texture.width * 4; - argbInfo.Width = (uint)Texture.width; - argbInfo.Height = (uint)Texture.height; - - var toI420 = new VideoConvertRequest(); - toI420.FlipY = true; - toI420.Buffer = argbInfo; - toI420.DstType = VideoBufferType.I420; - - var request = new FfiRequest(); - request.VideoConvert = toI420; - - var resp = FfiClient.SendRequest(request); - var newBuffer = resp.VideoConvert.Buffer; - - var capture = new CaptureVideoFrameRequest(); - capture.Buffer = newBuffer.Info; - capture.TimestampUs = DateTimeOffset.Now.ToUnixTimeMilliseconds(); - capture.Rotation = VideoRotation._0; - capture.SourceHandle = (ulong)Handle.DangerousGetHandle(); - - - request = new FfiRequest(); - request.CaptureVideoFrame = capture; - - FfiClient.SendRequest(request); - } - } -} diff --git a/Runtime/Scripts/VideoStream.cs b/Runtime/Scripts/VideoStream.cs index c556676..9379b80 100644 --- a/Runtime/Scripts/VideoStream.cs +++ b/Runtime/Scripts/VideoStream.cs @@ -1,8 +1,9 @@ using System; -using System.Collections; using LiveKit.Internal; using LiveKit.Proto; using UnityEngine; +using LiveKit.Internal.FFIClients.Requests; +using System.Collections; namespace LiveKit { @@ -31,6 +32,8 @@ public class VideoStream public Texture2D Texture { private set; get; } public VideoFrameBuffer VideoBuffer { private set; get; } + protected bool _playing = false; + public VideoStream(IVideoTrack videoTrack) { if (!videoTrack.Room.TryGetTarget(out var room)) @@ -39,18 +42,15 @@ public VideoStream(IVideoTrack videoTrack) if (!videoTrack.Participant.TryGetTarget(out var participant)) throw new InvalidOperationException("videotrack's participant is invalid"); - var newVideoStream = new NewVideoStreamRequest(); - - newVideoStream.TrackHandle = ((Track)videoTrack).TrackHandle.Id; + using var request = FFIBridge.Instance.NewRequest(); + var newVideoStream = request.request; + newVideoStream.TrackHandle = (ulong)videoTrack.TrackHandle.DangerousGetHandle(); newVideoStream.Type = VideoStreamType.VideoStreamNative; - - var request = new FfiRequest(); - request.NewVideoStream = newVideoStream; - - var resp = FfiClient.SendRequest(request); - var streamInfo = resp.NewVideoStream.Stream; - - Handle = new FfiHandle((IntPtr)streamInfo.Handle.Id); + newVideoStream.Format = VideoBufferType.I420; + newVideoStream.NormalizeStride = true; + using var response = request.Send(); + FfiResponse res = response; + Handle = FfiHandle.FromOwnedHandle(res.NewVideoStream.Stream.Handle); FfiClient.Instance.VideoStreamEventReceived += OnVideoStreamEvent; } @@ -76,9 +76,20 @@ private void Dispose(bool disposing) } } + public virtual void Start() + { + Stop(); + _playing = true; + } + + public virtual void Stop() + { + _playing = false; + } + public IEnumerator Update() { - while (true) + while (_playing) { yield return null; @@ -111,6 +122,8 @@ public IEnumerator Update() TextureUploaded?.Invoke(); } + + yield break; } private void OnVideoStreamEvent(VideoStreamEvent e) diff --git a/Runtime/csc.rsp b/Runtime/csc.rsp new file mode 100644 index 0000000..a0c7129 --- /dev/null +++ b/Runtime/csc.rsp @@ -0,0 +1 @@ +-nullable:enable \ No newline at end of file diff --git a/Runtime/csc.rsp.meta b/Runtime/csc.rsp.meta new file mode 100644 index 0000000..01f8d53 --- /dev/null +++ b/Runtime/csc.rsp.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: a95cecbc34b2b47ae82436029c172416 +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Tests.meta b/Tests.meta new file mode 100644 index 0000000..5baec92 --- /dev/null +++ b/Tests.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 5322289c05210477f921eb13487a0bd1 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Tests/UsageStrict.cs b/Tests/UsageStrict.cs new file mode 100644 index 0000000..18c77d7 --- /dev/null +++ b/Tests/UsageStrict.cs @@ -0,0 +1,30 @@ +using System.Collections.Generic; +using System.IO; +using System.Linq; +using NUnit.Framework; + +namespace LiveKit.Tests +{ + public class UsageStrict + { + private readonly IReadOnlyList ignoreFiles = new[] { "FFIClient.cs", "Ffi.cs" }; + private const string RuntimePath = "Assets/client-sdk-unity/Runtime/Scripts"; + + //can be reworked with syntax tree, but current version is faster to implement + [Test] + [TestCase("FfiResponse")] + [TestCase("FfiRequest")] + public void NoManualCreateNew(string className) + { + var files = Directory.GetFiles(RuntimePath, "*.cs", SearchOption.AllDirectories); + var rejected = files.Where( + f => ignoreFiles.Any(i => Path.GetFileName(f) == i) == false + && File.ReadAllText(f!).Contains($"new {className}(") + ).ToList(); + Assert.IsEmpty( + rejected, + $"Forbidden manual creation \"new {className}\", use pools instead:\n{string.Join("\n", rejected)}" + ); + } + } +} \ No newline at end of file diff --git a/Tests/UsageStrict.cs.meta b/Tests/UsageStrict.cs.meta new file mode 100644 index 0000000..985757e --- /dev/null +++ b/Tests/UsageStrict.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: a11eb196d571407eb9cda7db1633a604 +timeCreated: 1706099546 \ No newline at end of file diff --git a/Tests/csc.rsp b/Tests/csc.rsp new file mode 100644 index 0000000..dcc377f --- /dev/null +++ b/Tests/csc.rsp @@ -0,0 +1 @@ +-nullable:enable \ No newline at end of file diff --git a/Tests/csc.rsp.meta b/Tests/csc.rsp.meta new file mode 100644 index 0000000..656f8f1 --- /dev/null +++ b/Tests/csc.rsp.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 2a1410297f63473e8d548ac62048634c +timeCreated: 1706100154 \ No newline at end of file diff --git a/Tests/livekit.unity.Tests.asmdef b/Tests/livekit.unity.Tests.asmdef new file mode 100644 index 0000000..34049a5 --- /dev/null +++ b/Tests/livekit.unity.Tests.asmdef @@ -0,0 +1,24 @@ +{ + "name": "Tests", + "rootNamespace": "LiveKit.Tests", + "references": [ + "UnityEngine.TestRunner", + "UnityEditor.TestRunner", + "LiveKit" + ], + "includePlatforms": [ + "Editor" + ], + "excludePlatforms": [], + "allowUnsafeCode": false, + "overrideReferences": true, + "precompiledReferences": [ + "nunit.framework.dll" + ], + "autoReferenced": false, + "defineConstraints": [ + "UNITY_INCLUDE_TESTS" + ], + "versionDefines": [], + "noEngineReferences": false +} \ No newline at end of file diff --git a/Tests/livekit.unity.Tests.asmdef.meta b/Tests/livekit.unity.Tests.asmdef.meta new file mode 100644 index 0000000..39eadbb --- /dev/null +++ b/Tests/livekit.unity.Tests.asmdef.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: cd2ee40bfc2b948869f90b24dba0150d +AssemblyDefinitionImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: