From 5a5de251a29ee1d36c87cd5882e1d05f8f0a1f31 Mon Sep 17 00:00:00 2001 From: bnco <33021110+bnco-dev@users.noreply.github.com> Date: Tue, 11 Jul 2023 22:50:45 +0100 Subject: [PATCH 1/2] Add stats for playback and record for Win, Mac, Linux, Android and dotnet platforms --- .../Dotnet/AudioClipVoipSource.cs | 10 + .../Dotnet/AudioSourceVoipSink.cs | 20 +- .../Voip/Implementations/Dotnet/IVoipSink.cs | 3 +- .../Implementations/Dotnet/IVoipSource.cs | 3 + .../Dotnet/MicrophoneVoipSource.cs | 9 + .../Implementations/Dotnet/NullVoipSink.cs | 6 +- .../Implementations/Dotnet/NullVoipSource.cs | 1 + .../Dotnet/PeerConnectionImpl.cs | 193 +++++----- .../Implementations/IPeerConnectionImpl.cs | 68 ++-- .../Voip/Implementations/JsonHelpers.meta | 8 - .../JsonHelpers/SignallingMessageHelper.cs | 97 ----- .../Implementations/SignalingMessageHelper.cs | 343 +++++++++++++++++ ...cs.meta => SignalingMessageHelper.cs.meta} | 0 .../Implementations/Unity/AudioStatsFilter.cs | 76 ++++ .../Unity/AudioStatsFilter.cs.meta | 11 + .../Unity/PeerConnectionImpl.cs | 231 ++++++----- .../Unity/PeerConnectionMicrophone.cs | 28 +- .../Unity/SpatialisationCacheAudioFilter.cs | 25 +- .../Unity/SpatialisationRestoreAudioFilter.cs | 25 +- .../Implementations/Web/PeerConnectionImpl.cs | 144 +++++-- .../Implementations/Web/Plugins/WebRTC.jslib | 103 ++++- Unity/Assets/Runtime/Voip/VoipAvatar.cs | 18 + .../Assets/Runtime/Voip/VoipPeerConnection.cs | 137 ++++--- .../Runtime/Voip/VoipSpeechIndicator.cs | 208 ++++++++++ .../Runtime/Voip/VoipSpeechIndicator.cs.meta | 11 + .../Samples/_Common/Avatar/Avatar.prefab | 104 ++--- .../Avatar Speech Indicator.prefab | 20 +- .../Avatar/SpeechIndicator/SpeechIndicator.cs | 360 +++++++++--------- .../_Common/UI/Scripts/PeersPanelControl.cs | 5 +- 29 files changed, 1541 insertions(+), 726 deletions(-) delete mode 100644 Unity/Assets/Runtime/Voip/Implementations/JsonHelpers.meta delete mode 100644 Unity/Assets/Runtime/Voip/Implementations/JsonHelpers/SignallingMessageHelper.cs create mode 100644 Unity/Assets/Runtime/Voip/Implementations/SignalingMessageHelper.cs rename Unity/Assets/Runtime/Voip/Implementations/{JsonHelpers/SignallingMessageHelper.cs.meta => SignalingMessageHelper.cs.meta} (100%) create mode 100644 Unity/Assets/Runtime/Voip/Implementations/Unity/AudioStatsFilter.cs create mode 100644 Unity/Assets/Runtime/Voip/Implementations/Unity/AudioStatsFilter.cs.meta create mode 100644 Unity/Assets/Runtime/Voip/VoipSpeechIndicator.cs create mode 100644 Unity/Assets/Runtime/Voip/VoipSpeechIndicator.cs.meta diff --git a/Unity/Assets/Runtime/Voip/Implementations/Dotnet/AudioClipVoipSource.cs b/Unity/Assets/Runtime/Voip/Implementations/Dotnet/AudioClipVoipSource.cs index 336771379..9ae2b70ff 100644 --- a/Unity/Assets/Runtime/Voip/Implementations/Dotnet/AudioClipVoipSource.cs +++ b/Unity/Assets/Runtime/Voip/Implementations/Dotnet/AudioClipVoipSource.cs @@ -39,6 +39,7 @@ void Start() StartAudio().Start(); } + public event Action statsPushed; public event EncodedSampleDelegate OnAudioSourceEncodedSample; public event RawAudioSampleDelegate OnAudioSourceRawSample; #pragma warning disable 67 @@ -136,12 +137,14 @@ void FixedUpdate() // This ratio re-samples the clip from one samples per second count to other var samplesRatio = (Clip.frequency / 16000f); + var volumeSum = 0.0f; for (int i = 0; i < pcmSamples.Length; i++) { var sampleIndex = (int)(i * samplesRatio); var clipSample = clipSamples[sampleIndex]; clipSample = Mathf.Clamp(clipSample * gain, -.999f, .999f); + volumeSum += clipSample; pcmSamples[i] = (short)(clipSample * short.MaxValue); } @@ -153,6 +156,13 @@ void FixedUpdate() OnAudioSourceEncodedSample((uint)duration, encoded); } + statsPushed?.Invoke(new AudioStats + ( + sampleCount:clipSamples.Length, + volumeSum:volumeSum, + sampleRate:16000 + )); + time += Time.fixedDeltaTime; } } diff --git a/Unity/Assets/Runtime/Voip/Implementations/Dotnet/AudioSourceVoipSink.cs b/Unity/Assets/Runtime/Voip/Implementations/Dotnet/AudioSourceVoipSink.cs index 012177222..7a17ecdde 100644 --- a/Unity/Assets/Runtime/Voip/Implementations/Dotnet/AudioSourceVoipSink.cs +++ b/Unity/Assets/Runtime/Voip/Implementations/Dotnet/AudioSourceVoipSink.cs @@ -5,6 +5,7 @@ using System.Threading.Tasks; using SIPSorceryMedia.Abstractions; using UnityEngine; +using Ubiq.Voip.Implementations; namespace Ubiq.Voip.Implementations.Dotnet { @@ -103,7 +104,7 @@ public RtpBufferer (int latencySamples, int syncSamples, stopwatch.Start(); } - public PlaybackStats Commit (int currentTimeSamples, AudioClip audioClip) + public AudioStats Commit (int currentTimeSamples, AudioClip audioClip) { var stats = Advance(currentTimeSamples, audioClip); Flush(audioClip); @@ -187,7 +188,7 @@ private void Flush (AudioClip audioClip) } // Step internal time trackers forward and zero out just-read samples - private PlaybackStats Advance (int timeSamples, AudioClip audioClip) + private AudioStats Advance (int timeSamples, AudioClip audioClip) { if (absTimeSamples < 0) { @@ -227,10 +228,10 @@ private PlaybackStats Advance (int timeSamples, AudioClip audioClip) lastTimeSamples = timeSamples; // Calculate stats for the advance - return new PlaybackStats { volumeSum=volume, sampleCount=deltaTimeSamples, sampleRate=audioClip.frequency}; + return new AudioStats(sampleCount:deltaTimeSamples,volumeSum:volume,sampleRate:audioClip.frequency); } - return new PlaybackStats { volumeSum=0, sampleCount=0, sampleRate=audioClip.frequency}; + return new AudioStats(sampleCount:0,volumeSum:0,sampleRate:audioClip.frequency); } public void AddRtp (AudioRtp rtp) @@ -239,6 +240,8 @@ public void AddRtp (AudioRtp rtp) } } + public event Action statsPushed; + public int sampleRate { get { return 16000; } } public int bufferSamples { get { return sampleRate * 2; } } public int latencySamples { get { return sampleRate / 5; } } @@ -246,7 +249,6 @@ public void AddRtp (AudioRtp rtp) public float gain = 1.0f; public AudioSource unityAudioSource { get; private set; } - public PlaybackStats lastFrameStats { get; private set; } public float Volume { get => unityAudioSource.volume; set => unityAudioSource.volume = value; } private G722AudioDecoder audioDecoder = new G722AudioDecoder(); @@ -320,7 +322,8 @@ private void Update() var timeSamples = unityAudioSource.timeSamples; var clip = unityAudioSource.clip; - lastFrameStats = rtpBufferer.Commit(timeSamples,clip); + var stats = rtpBufferer.Commit(timeSamples,clip); + statsPushed?.Invoke(stats); } private float PcmToFloat (short pcm) @@ -368,11 +371,6 @@ private Task GetCloseAudioTask() return new Task(() => { }); } - public PlaybackStats GetLastFramePlaybackStats() - { - return lastFrameStats; - } - public void UpdateSpatialization(Vector3 sourcePosition, Quaternion sourceRotation, Vector3 listenerPosition, Quaternion listenerRotation) diff --git a/Unity/Assets/Runtime/Voip/Implementations/Dotnet/IVoipSink.cs b/Unity/Assets/Runtime/Voip/Implementations/Dotnet/IVoipSink.cs index ceb4a00fe..b6521769a 100644 --- a/Unity/Assets/Runtime/Voip/Implementations/Dotnet/IVoipSink.cs +++ b/Unity/Assets/Runtime/Voip/Implementations/Dotnet/IVoipSink.cs @@ -1,10 +1,11 @@ +using System; using UnityEngine; namespace Ubiq.Voip.Implementations.Dotnet { public interface IVoipSink : SIPSorceryMedia.Abstractions.IAudioSink { - PlaybackStats GetLastFramePlaybackStats(); + event Action statsPushed; void UpdateSpatialization(Vector3 sourcePosition, Quaternion sourceRotation, Vector3 listenerPosition, Quaternion listenerRotation); } diff --git a/Unity/Assets/Runtime/Voip/Implementations/Dotnet/IVoipSource.cs b/Unity/Assets/Runtime/Voip/Implementations/Dotnet/IVoipSource.cs index 23d03efbb..04d6af0d1 100644 --- a/Unity/Assets/Runtime/Voip/Implementations/Dotnet/IVoipSource.cs +++ b/Unity/Assets/Runtime/Voip/Implementations/Dotnet/IVoipSource.cs @@ -1,6 +1,9 @@ +using System; + namespace Ubiq.Voip.Implementations.Dotnet { public interface IVoipSource : SIPSorceryMedia.Abstractions.IAudioSource { + event Action statsPushed; } } \ No newline at end of file diff --git a/Unity/Assets/Runtime/Voip/Implementations/Dotnet/MicrophoneVoipSource.cs b/Unity/Assets/Runtime/Voip/Implementations/Dotnet/MicrophoneVoipSource.cs index 35ac960a5..07805e82e 100644 --- a/Unity/Assets/Runtime/Voip/Implementations/Dotnet/MicrophoneVoipSource.cs +++ b/Unity/Assets/Runtime/Voip/Implementations/Dotnet/MicrophoneVoipSource.cs @@ -127,6 +127,7 @@ public void End () } } + public event Action statsPushed; public float gain = 1.0f; private MicrophoneListener microphoneListener = new MicrophoneListener(); @@ -221,15 +222,23 @@ private void Update() // Send samples if we have them while (microphoneListener.Advance()) { + var volumeSum = 0.0f; for (int i = 0; i < microphoneListener.samples.Length; i++) { var floatSample = microphoneListener.samples[i]; floatSample = Mathf.Clamp(floatSample*gain,-.999f,.999f); + volumeSum += floatSample; pcm[i] = (short)(floatSample * short.MaxValue); } audioEncoder.Encode(encoded,pcm); OnAudioSourceEncodedSample.Invoke(RTP_TIMESTAMP_PER_BUFFER,encoded); + statsPushed?.Invoke(new AudioStats + ( + sampleCount:microphoneListener.samples.Length, + volumeSum:volumeSum, + sampleRate:SAMPLE_RATE + )); } } diff --git a/Unity/Assets/Runtime/Voip/Implementations/Dotnet/NullVoipSink.cs b/Unity/Assets/Runtime/Voip/Implementations/Dotnet/NullVoipSink.cs index 26cf35f76..187b9f976 100644 --- a/Unity/Assets/Runtime/Voip/Implementations/Dotnet/NullVoipSink.cs +++ b/Unity/Assets/Runtime/Voip/Implementations/Dotnet/NullVoipSink.cs @@ -14,6 +14,7 @@ namespace Ubiq.Voip.Implementations.Dotnet public class NullVoipSink : MonoBehaviour, IVoipSink { #pragma warning disable 67 + public event Action statsPushed; public event SourceErrorDelegate OnAudioSinkError; #pragma warning restore 67 @@ -63,11 +64,6 @@ public void GotAudioRtp(IPEndPoint remoteEndPoint, uint ssrc, uint seqnum, uint { } - public PlaybackStats GetLastFramePlaybackStats() - { - return new PlaybackStats(); - } - public void UpdateSpatialization(Vector3 sourcePosition, Quaternion sourceRotation, Vector3 listenerPosition, Quaternion listenerRotation) { } diff --git a/Unity/Assets/Runtime/Voip/Implementations/Dotnet/NullVoipSource.cs b/Unity/Assets/Runtime/Voip/Implementations/Dotnet/NullVoipSource.cs index dc1b572cd..6f4fb4561 100644 --- a/Unity/Assets/Runtime/Voip/Implementations/Dotnet/NullVoipSource.cs +++ b/Unity/Assets/Runtime/Voip/Implementations/Dotnet/NullVoipSource.cs @@ -12,6 +12,7 @@ public class NullVoipSource : MonoBehaviour, IVoipSource private MediaFormatManager audioFormatManager; #pragma warning disable 0067 + public event Action statsPushed; public event EncodedSampleDelegate OnAudioSourceEncodedSample; public event RawAudioSampleDelegate OnAudioSourceRawSample; public event SourceErrorDelegate OnAudioSourceError; diff --git a/Unity/Assets/Runtime/Voip/Implementations/Dotnet/PeerConnectionImpl.cs b/Unity/Assets/Runtime/Voip/Implementations/Dotnet/PeerConnectionImpl.cs index bc7522556..4ebe19ddd 100644 --- a/Unity/Assets/Runtime/Voip/Implementations/Dotnet/PeerConnectionImpl.cs +++ b/Unity/Assets/Runtime/Voip/Implementations/Dotnet/PeerConnectionImpl.cs @@ -34,6 +34,20 @@ public Event(Type type, string json = null, RTCIceCandidate iceCandidate = null) } } + private class Context + { + public IPeerConnectionContext context; + public Action playbackStatsPushed; + public Action recordStatsPushed; + public Action iceConnectionStateChanged; + public Action peerConnectionStateChanged; + public bool polite; + + public MonoBehaviour behaviour => context.behaviour; + public GameObject gameObject => context.behaviour.gameObject; + public Transform transform => context.behaviour.transform; + } + private enum Implementation { Unknown, @@ -41,62 +55,76 @@ private enum Implementation Other } - public event IceConnectionStateChangedDelegate iceConnectionStateChanged; - public event PeerConnectionStateChangedDelegate peerConnectionStateChanged; - private ConcurrentQueue mainThreadActions = new ConcurrentQueue(); private ConcurrentQueue events = new ConcurrentQueue(); - private Queue messageQueue = new Queue(); private Task setupTask; - private IPeerConnectionContext context; - private bool polite; - private Coroutine updateCoroutine; - // SipSorcery Peer Connection private RTCPeerConnection peerConnection; private IVoipSink sink; private IVoipSource source; - private List coroutines = new List(); + private List coroutinesForCleanup = new List(); private Implementation otherPeerImplementation = Implementation.Unknown; private List bufferedIceCandidates = new List(); private bool hasSentLocalSdp; + private Context ctx; + public async void Dispose() { - if (context.behaviour) + if (sink != null) + { + sink.statsPushed -= Sink_StatsPushed; + } + + if (source != null) { - foreach(var coroutine in coroutines) + source.statsPushed -= Source_StatsPushed; + } + + if (ctx != null && ctx.behaviour) + { + foreach(var coroutine in coroutinesForCleanup) { - if (context.behaviour) - { - context.behaviour.StopCoroutine(coroutine); - } + ctx.behaviour.StopCoroutine(coroutine); } + coroutinesForCleanup.Clear(); } - coroutines.Clear(); + coroutinesForCleanup.Clear(); if (setupTask != null) { await setupTask.ConfigureAwait(false); setupTask.Result.Dispose(); } + + setupTask = null; + ctx = null; } public void Setup(IPeerConnectionContext context, - bool polite, List iceServers) + bool polite, List iceServers, + Action playbackStatsPushed, + Action recordStatsPushed, + Action iceConnectionStateChanged, + Action peerConnectionStateChanged) { - if (setupTask != null) + if (ctx != null) { - // Already setup or setup in progress + // Already setup return; } - this.context = context; - this.polite = polite; + ctx = new Context(); + ctx.context = context; + ctx.polite = polite; + ctx.playbackStatsPushed = playbackStatsPushed; + ctx.recordStatsPushed = recordStatsPushed; + ctx.iceConnectionStateChanged = iceConnectionStateChanged; + ctx.peerConnectionStateChanged = peerConnectionStateChanged; // Copy ice servers before entering multithreaded context var iceServersCopy = new List(); @@ -113,8 +141,7 @@ public void Setup(IPeerConnectionContext context, setupTask = Task.Run(() => DoSetup(polite,iceServersCopy)); - coroutines.Add(context.behaviour.StartCoroutine(DoUpdate())); - // coroutines.Add(context.behaviour.StartCoroutine(StartMicrophoneTrack())); + coroutinesForCleanup.Add(ctx.behaviour.StartCoroutine(DoUpdate())); } private void RequireSource () @@ -124,7 +151,7 @@ private void RequireSource () return; } - var manager = context.behaviour.transform.parent; + var manager = ctx.transform.parent; // First, see if an source already exists among siblings source = manager.GetComponentInChildren(); @@ -148,6 +175,8 @@ private void RequireSource () go.transform.parent = manager; source = go.AddComponent(); } + + source.statsPushed += Source_StatsPushed; } private void RequireSink () @@ -157,14 +186,14 @@ private void RequireSink () return; } - var manager = context.behaviour.transform.parent; + var manager = ctx.transform.parent; // First, check if a hint exists and use it var hint = manager.GetComponent(); if (hint && hint.prefab) { var go = GameObject.Instantiate(hint.prefab); - go.transform.parent = context.behaviour.transform; + go.transform.parent = ctx.transform; sink = go.GetComponentInChildren(); } @@ -172,17 +201,28 @@ private void RequireSink () if (sink == null) { var go = new GameObject("Dotnet Voip Output"); - go.transform.parent = context.behaviour.transform; + go.transform.parent = ctx.transform; sink = go.AddComponent(); } - } + sink.statsPushed += Sink_StatsPushed; + } public void ProcessSignalingMessage (string json) { events.Enqueue(new Event(json)); } + private void Source_StatsPushed (AudioStats stats) + { + ctx.recordStatsPushed?.Invoke(stats); + } + + private void Sink_StatsPushed (AudioStats stats) + { + ctx.playbackStatsPushed?.Invoke(stats); + } + private RTCPeerConnection DoSetup(bool polite, List iceServers) { @@ -225,14 +265,13 @@ private RTCPeerConnection DoSetup(bool polite, { switch(state) { - case RTCPeerConnectionState.closed : peerConnectionStateChanged(PeerConnectionState.closed); break; - case RTCPeerConnectionState.failed : peerConnectionStateChanged(PeerConnectionState.failed); break; - case RTCPeerConnectionState.disconnected : peerConnectionStateChanged(PeerConnectionState.disconnected); break; - case RTCPeerConnectionState.@new : peerConnectionStateChanged(PeerConnectionState.@new); break; - case RTCPeerConnectionState.connecting : peerConnectionStateChanged(PeerConnectionState.connecting); break; - case RTCPeerConnectionState.connected : peerConnectionStateChanged(PeerConnectionState.connected); break; + case RTCPeerConnectionState.closed : ctx.peerConnectionStateChanged?.Invoke(PeerConnectionState.closed); break; + case RTCPeerConnectionState.failed : ctx.peerConnectionStateChanged?.Invoke(PeerConnectionState.failed); break; + case RTCPeerConnectionState.disconnected : ctx.peerConnectionStateChanged?.Invoke(PeerConnectionState.disconnected); break; + case RTCPeerConnectionState.@new : ctx.peerConnectionStateChanged?.Invoke(PeerConnectionState.@new); break; + case RTCPeerConnectionState.connecting : ctx.peerConnectionStateChanged?.Invoke(PeerConnectionState.connecting); break; + case RTCPeerConnectionState.connected : ctx.peerConnectionStateChanged?.Invoke(PeerConnectionState.connected); break; } - Debug.Log($"Peer connection state change to {state}."); }); if (state == RTCPeerConnectionState.connected) @@ -265,30 +304,21 @@ private RTCPeerConnection DoSetup(bool polite, { if (media == SDPMediaTypesEnum.audio) { - // todo sink.GotAudioRtp(rep, rtpPkt.Header.SyncSource, rtpPkt.Header.SequenceNumber, rtpPkt.Header.Timestamp, rtpPkt.Header.PayloadType, rtpPkt.Header.MarkerBit == 1, rtpPkt.Payload); } }; - // Diagnostics. - pc.OnReceiveReport += (re, media, rr) => mainThreadActions.Enqueue( - () => Debug.Log($"RTCP Receive for {media} from {re}\n{rr.GetDebugSummary()}")); - pc.OnSendReport += (media, sr) => mainThreadActions.Enqueue( - () => Debug.Log($"RTCP Send for {media}\n{sr.GetDebugSummary()}")); - pc.GetRtpChannel().OnStunMessageReceived += (msg, ep, isRelay) => mainThreadActions.Enqueue( - () => Debug.Log($"STUN {msg.Header.MessageType} received from {ep}:{msg.ToString()}")); pc.oniceconnectionstatechange += (state) => mainThreadActions.Enqueue(() => { switch(state) { - case RTCIceConnectionState.closed : iceConnectionStateChanged(IceConnectionState.closed); break; - case RTCIceConnectionState.failed : iceConnectionStateChanged(IceConnectionState.failed); break; - case RTCIceConnectionState.disconnected : iceConnectionStateChanged(IceConnectionState.disconnected); break; - case RTCIceConnectionState.@new : iceConnectionStateChanged(IceConnectionState.@new); break; - case RTCIceConnectionState.checking : iceConnectionStateChanged(IceConnectionState.checking); break; - case RTCIceConnectionState.connected : iceConnectionStateChanged(IceConnectionState.connected); break; + case RTCIceConnectionState.closed : ctx.iceConnectionStateChanged?.Invoke(IceConnectionState.closed); break; + case RTCIceConnectionState.failed : ctx.iceConnectionStateChanged?.Invoke(IceConnectionState.failed); break; + case RTCIceConnectionState.disconnected : ctx.iceConnectionStateChanged?.Invoke(IceConnectionState.disconnected); break; + case RTCIceConnectionState.@new : ctx.iceConnectionStateChanged?.Invoke(IceConnectionState.@new); break; + case RTCIceConnectionState.checking : ctx.iceConnectionStateChanged?.Invoke(IceConnectionState.checking); break; + case RTCIceConnectionState.connected : ctx.iceConnectionStateChanged?.Invoke(IceConnectionState.connected); break; } - Debug.Log($"ICE connection state change to {state}."); }); pc.addTrack(new MediaStreamTrack(source.GetAudioSourceFormats())); @@ -321,11 +351,6 @@ private IEnumerator DoUpdate() yield return HandleSignalingEvent(ev); } - if (Time.realtimeSinceStartup > time + 5) - { - Debug.Log($"{peerConnection.connectionState} | {peerConnection.signalingState} | {peerConnection.iceConnectionState} | {peerConnection.iceGatheringState} "); - time = Time.realtimeSinceStartup; - } yield return null; } } @@ -340,7 +365,7 @@ private IEnumerator DoUpdate() private IEnumerator HandleSignalingEvent(Event e) { // e.type == Signaling message - var msg = SignalingMessageHelper.FromJson(e.json); + var msg = SignalingMessage.FromJson(e.json); // Id the other implementation as Dotnet requires special treatment if (otherPeerImplementation == Implementation.Unknown) @@ -349,7 +374,7 @@ private IEnumerator HandleSignalingEvent(Event e) ? Implementation.Dotnet : Implementation.Other; - if (otherPeerImplementation == Implementation.Dotnet && !polite) + if (otherPeerImplementation == Implementation.Dotnet && !ctx.polite) { yield return SetLocalDescription(peerConnection); Send(peerConnection.localDescription); @@ -361,13 +386,13 @@ private IEnumerator HandleSignalingEvent(Event e) // non-dotnet peer always takes on the role of polite // peer as the dotnet implementaton isn't smart enough // to handle rollback - polite = false; + ctx.polite = false; } } if (msg.type != null) { - ignoreOffer = !polite + ignoreOffer = !ctx.polite && msg.type == "offer" && !(peerConnection.signalingState == RTCSignalingState.stable || peerConnection.signalingState == RTCSignalingState.closed); @@ -398,16 +423,17 @@ private IEnumerator HandleSignalingEvent(Event e) private void Send(string implementation) { - context.Send(SignalingMessageHelper.ToJson(new SignalingMessage{ - implementation = implementation - })); + ctx.context.Send(ImplementationMessage.ToJson(new ImplementationMessage + ( + implementation: implementation + ))); } private void Send(RTCIceCandidate ic) { if (hasSentLocalSdp) { - InternalSend(context,ic); + InternalSend(ctx.context,ic); } else { @@ -417,13 +443,13 @@ private void Send(RTCIceCandidate ic) private void Send(RTCSessionDescription sd) { - InternalSend(context,sd); + InternalSend(ctx.context,sd); hasSentLocalSdp = true; while (bufferedIceCandidates.Count > 0) { var ic = bufferedIceCandidates[0]; bufferedIceCandidates.RemoveAt(0); - InternalSend(context,ic); + InternalSend(ctx.context,ic); } } @@ -489,7 +515,8 @@ private static RTCIceCandidateInit IceCandidateUbiqToPkg(SignalingMessage msg) private static RTCSessionDescriptionInit SessionDescriptionUbiqToPkg(SignalingMessage msg) { - return new RTCSessionDescriptionInit{ + return new RTCSessionDescriptionInit + { sdp = msg.sdp, type = StringToSdpType(msg.type) }; @@ -497,20 +524,22 @@ private static RTCSessionDescriptionInit SessionDescriptionUbiqToPkg(SignalingMe private static void InternalSend(IPeerConnectionContext context, RTCSessionDescription sd) { - context.Send(SignalingMessageHelper.ToJson(new SignalingMessage{ - sdp = sd.sdp.RawString(), - type = SdpTypeToString(sd.type) - })); + context.Send(SdpMessage.ToJson(new SdpMessage + ( + sdp: sd.sdp.RawString(), + type: SdpTypeToString(sd.type) + ))); } private static void InternalSend(IPeerConnectionContext context, RTCIceCandidate ic) { - context.Send(SignalingMessageHelper.ToJson(new SignalingMessage{ - candidate = $"candidate:{ic.candidate}", - sdpMid = ic.sdpMid, - sdpMLineIndex = (ushort?)ic.sdpMLineIndex, - usernameFragment = ic.usernameFragment - })); + context.Send(IceCandidateMessage.ToJson(new IceCandidateMessage + ( + candidate: $"candidate:{ic.candidate}", + sdpMid: ic.sdpMid, + sdpMLineIndex: (ushort?)ic.sdpMLineIndex, + usernameFragment: ic.usernameFragment + ))); } public void UpdateSpatialization(Vector3 sourcePosition, Quaternion sourceRotation, Vector3 listenerPosition, Quaternion listenerRotation) @@ -521,17 +550,5 @@ public void UpdateSpatialization(Vector3 sourcePosition, Quaternion sourceRotati listenerPosition,listenerRotation); } } - - public PlaybackStats GetLastFramePlaybackStats () - { - if (sink != null) - { - return sink.GetLastFramePlaybackStats(); - } - else - { - return new PlaybackStats(); - } - } } } \ No newline at end of file diff --git a/Unity/Assets/Runtime/Voip/Implementations/IPeerConnectionImpl.cs b/Unity/Assets/Runtime/Voip/Implementations/IPeerConnectionImpl.cs index 0d2d424fc..ad650c100 100644 --- a/Unity/Assets/Runtime/Voip/Implementations/IPeerConnectionImpl.cs +++ b/Unity/Assets/Runtime/Voip/Implementations/IPeerConnectionImpl.cs @@ -79,42 +79,62 @@ public interface IOutputVolume float Volume { get; set; } } - [System.Serializable] - public struct SessionDescriptionArgs + [Serializable] + public struct AudioStats : IEquatable { - public const string TYPE_ANSWER = "answer"; - public const string TYPE_OFFER = "offer"; - public const string TYPE_PRANSWER = "pranswer"; - public const string TYPE_ROLLBACK = "rollback"; + [field: SerializeField] public int sampleCount { get; private set; } + [field: SerializeField] public float volumeSum { get; private set; } + [field: SerializeField] public int sampleRate { get; private set; } - public string type; - public string sdp; - } + public AudioStats(int sampleCount, float volumeSum, int sampleRate) + { + this.sampleCount = sampleCount; + this.volumeSum = volumeSum; + this.sampleRate = sampleRate; + } - public struct PlaybackStats - { - public int sampleCount; - public float volumeSum; - public int sampleRate; - } + public override bool Equals(object obj) + { + return obj is AudioStats stats && Equals(stats); + } + + public bool Equals(AudioStats other) + { + return sampleCount == other.sampleCount && + volumeSum == other.volumeSum && + sampleRate == other.sampleRate; + } - public delegate void IceConnectionStateChangedDelegate(IceConnectionState state); - public delegate void PeerConnectionStateChangedDelegate(PeerConnectionState state); + public override int GetHashCode() + { + return HashCode.Combine(sampleCount, volumeSum, sampleRate); + } + + public static bool operator ==(AudioStats left, AudioStats right) + { + return left.Equals(right); + } + + public static bool operator !=(AudioStats left, AudioStats right) + { + return !(left == right); + } + } public interface IPeerConnectionImpl : IDisposable { - event IceConnectionStateChangedDelegate iceConnectionStateChanged; - event PeerConnectionStateChangedDelegate peerConnectionStateChanged; - - void Setup(IPeerConnectionContext context, bool polite, - List iceServers); + void Setup(IPeerConnectionContext context, + bool polite, + List iceServers, + Action playbackStatsPushed, + Action recordStatsPushed, + Action iceConnectionStateChanged, + Action peerConnectionStateChanged); void ProcessSignalingMessage (string json); void UpdateSpatialization (Vector3 sourcePosition, Quaternion sourceRotation, Vector3 listenerPosition, Quaternion listenerRotation); - - PlaybackStats GetLastFramePlaybackStats(); } } \ No newline at end of file diff --git a/Unity/Assets/Runtime/Voip/Implementations/JsonHelpers.meta b/Unity/Assets/Runtime/Voip/Implementations/JsonHelpers.meta deleted file mode 100644 index 501869d2f..000000000 --- a/Unity/Assets/Runtime/Voip/Implementations/JsonHelpers.meta +++ /dev/null @@ -1,8 +0,0 @@ -fileFormatVersion: 2 -guid: 5071148f41d85f44f96a98dcb6a452f7 -folderAsset: yes -DefaultImporter: - externalObjects: {} - userData: - assetBundleName: - assetBundleVariant: diff --git a/Unity/Assets/Runtime/Voip/Implementations/JsonHelpers/SignallingMessageHelper.cs b/Unity/Assets/Runtime/Voip/Implementations/JsonHelpers/SignallingMessageHelper.cs deleted file mode 100644 index aa7919816..000000000 --- a/Unity/Assets/Runtime/Voip/Implementations/JsonHelpers/SignallingMessageHelper.cs +++ /dev/null @@ -1,97 +0,0 @@ -using UnityEngine; - -namespace Ubiq.Voip.Implementations.JsonHelpers -{ - // Optional helper class to provide SignalingMessage structs, working - // around a deficiency in Unity's JSONUtility - - // IceCandidate and SessionDescription params work as expected for WebRTC - - // Ubiq params are not required for WebRTC and are instead used to identify - // the implementation in use. The purpose of this system is to help - // workaround some issues in the dotnet implementation. To identify itself, - // an implementation can send a message with the 'implementation' param. - // This must be the FIRST message sent in the peer connection. All other - // params in that message will be ignored. If no such message is sent, it is - // assumed the implementation requires no special workarounds - public static class SignalingMessageHelper - { - private struct SignalingMessageInternal - { - // Ubiq vars, not required for WebRTC - public string implementation; - public bool hasImplementation; - - // IceCandidate - public string candidate; - public bool hasCandidate; - public string sdpMid; - public bool hasSdpMid; - public int sdpMLineIndex; - public bool hasSdpMLineIndex; - public string usernameFragment; - public bool hasUsernameFragment; - - // SessionDescription - public string type; - public bool hasType; - public string sdp; - public bool hasSdp; - } - - public static SignalingMessage FromJson(string json) - { - var inter = JsonUtility.FromJson(json); - return new SignalingMessage - { - implementation = inter.hasImplementation ? inter.implementation : null, - candidate = inter.hasCandidate ? inter.candidate : null, - sdpMid = inter.hasSdpMid ? inter.sdpMid : null, - sdpMLineIndex = inter.hasSdpMLineIndex ? (ushort?)inter.sdpMLineIndex : null, - usernameFragment = inter.hasUsernameFragment ? inter.usernameFragment : null, - type = inter.hasType ? inter.type : null, - sdp = inter.hasSdp ? inter.sdp : null - }; - } - - public static string ToJson(SignalingMessage message) - { - return JsonUtility.ToJson(new SignalingMessageInternal - { - implementation = message.implementation, - hasImplementation = message.implementation != null, - candidate = message.candidate, - hasCandidate = message.candidate != null, - sdpMid = message.sdpMid, - hasSdpMid = message.sdpMid != null, - sdpMLineIndex = message.sdpMLineIndex ?? 0, - hasSdpMLineIndex = message.sdpMLineIndex != null, - usernameFragment = message.usernameFragment, - hasUsernameFragment = message.usernameFragment != null, - type = message.type, - hasType = message.type != null, - sdp = message.sdp, - hasSdp = message.sdp != null - }); - } - } - - // Optional helper struct to act as a JSON object, working around a - // deficiency in Unity's JSONUtility. Do not directly serialize/deserialize - // this. Use the accompanying helper class. - public struct SignalingMessage - { - // Ubiq vars, not required for WebRTC, ignored by most implementations - public string implementation; - - // IceCandidate - public string candidate; - public string sdpMid; - public ushort? sdpMLineIndex; - public string usernameFragment; - - // SessionDescription - public string type; - public string sdp; - } -} \ No newline at end of file diff --git a/Unity/Assets/Runtime/Voip/Implementations/SignalingMessageHelper.cs b/Unity/Assets/Runtime/Voip/Implementations/SignalingMessageHelper.cs new file mode 100644 index 000000000..63cd74ef8 --- /dev/null +++ b/Unity/Assets/Runtime/Voip/Implementations/SignalingMessageHelper.cs @@ -0,0 +1,343 @@ +using System; +using UnityEngine; + +namespace Ubiq.Voip.Implementations.JsonHelpers +{ + // Optional helper class to provide SignalingMessage structs, working + // around JSONUtility's inability to handle null, and to minimise + // network traffic, allocations and boxing (strings will still allocate). + + // IceCandidate and SessionDescription params work as expected for WebRTC + + // Ubiq params are not required for WebRTC and are instead used to identify + // the implementation in use. The purpose of this system is to help + // workaround some issues in the dotnet implementation. To identify itself, + // an implementation can send a message with the 'implementation' param. + // This must be the FIRST message sent in the peer connection. All other + // params in that message will be ignored. If no such message is sent, it is + // assumed the implementation requires no special workarounds + public static class SignalingMessageHelper + { + private const string NULL_MAGIC_STR = "\0"; + private const int NULL_MAGIC_USHORT = -1; + + private enum Classification + { + Implementation, + IceCandidate, + Sdp + } + + [Serializable] + private struct InternalImplementationMessage + { + public Classification cls; + public string implementation; + } + + [Serializable] + private struct InternalIceCandidateMessage + { + public Classification cls; + public string candidate; + public string sdpMid; + public int sdpMLineIndex; + public string usernameFragment; + } + + [Serializable] + private struct InternalSdpMessage + { + public Classification cls; + public string type; + public string sdp; + } + + [Serializable] + private struct InternalMessage + { + public Classification cls; + public string implementation; + public string candidate; + public string sdpMid; + public int sdpMLineIndex; + public string usernameFragment; + public string type; + public string sdp; + } + + public static SignalingMessage FromJson(string json) + { + var m = JsonUtility.FromJson(json); + if (m.cls == Classification.Implementation) + { + return new SignalingMessage + ( + implementation: m.implementation != NULL_MAGIC_STR ? m.implementation : null, + candidate: null, + sdpMid: null, + sdpMLineIndex: null, + usernameFragment: null, + type: null, + sdp: null + ); + } + else if (m.cls == Classification.Sdp) + { + return new SignalingMessage + ( + implementation: null, + candidate: null, + sdpMid: null, + sdpMLineIndex: null, + usernameFragment: null, + type: m.type != NULL_MAGIC_STR ? m.type : null, + sdp: m.sdp != NULL_MAGIC_STR ? m.sdp : null + ); + } + else + { + return new SignalingMessage + ( + implementation: null, + candidate: m.candidate != NULL_MAGIC_STR ? m.candidate : null, + sdpMid: m.sdpMid != NULL_MAGIC_STR ? m.sdpMid : null, + sdpMLineIndex: m.sdpMLineIndex != NULL_MAGIC_USHORT ? (ushort)m.sdpMLineIndex : null, + usernameFragment: m.usernameFragment != NULL_MAGIC_STR ? m.usernameFragment : null, + type: null, + sdp: null + ); + } + } + + public static string ToJson(ImplementationMessage message) + { + return JsonUtility.ToJson(new InternalImplementationMessage { + cls = Classification.Implementation, + implementation = message.implementation ?? NULL_MAGIC_STR + }); + } + + public static string ToJson(SdpMessage message) + { + return JsonUtility.ToJson(new InternalSdpMessage { + cls = Classification.Sdp, + type = message.type ?? NULL_MAGIC_STR, + sdp = message.sdp ?? NULL_MAGIC_STR + }); + } + + public static string ToJson(IceCandidateMessage message) + { + return JsonUtility.ToJson(new InternalIceCandidateMessage { + cls = Classification.IceCandidate, + candidate = message.candidate ?? NULL_MAGIC_STR, + sdpMid = message.sdpMid ?? NULL_MAGIC_STR, + sdpMLineIndex = message.sdpMLineIndex ?? NULL_MAGIC_USHORT, + usernameFragment = message.usernameFragment ?? NULL_MAGIC_STR + }); + } + } + + // Input struct. Don't directly serialize - use ToJson + public struct ImplementationMessage : IEquatable + { + public readonly string implementation; + + public ImplementationMessage(string implementation) + { + this.implementation = implementation; + } + + public static string ToJson(ImplementationMessage message) + { + return SignalingMessageHelper.ToJson(message); + } + + public override bool Equals(object obj) + { + return obj is ImplementationMessage message && Equals(message); + } + + public bool Equals(ImplementationMessage other) + { + return implementation == other.implementation; + } + + public override int GetHashCode() + { + return HashCode.Combine(implementation); + } + + public static bool operator ==(ImplementationMessage left, ImplementationMessage right) + { + return left.Equals(right); + } + + public static bool operator !=(ImplementationMessage left, ImplementationMessage right) + { + return !(left == right); + } + } + + // Input struct. Don't directly serialize - use ToJson + public struct SdpMessage : IEquatable + { + public readonly string type; + public readonly string sdp; + + public SdpMessage(string type, string sdp) + { + this.type = type; + this.sdp = sdp; + } + + public static string ToJson(SdpMessage message) + { + return SignalingMessageHelper.ToJson(message); + } + + public override bool Equals(object obj) + { + return obj is SdpMessage message && Equals(message); + } + + public bool Equals(SdpMessage other) + { + return type == other.type && + sdp == other.sdp; + } + + public override int GetHashCode() + { + return HashCode.Combine(type, sdp); + } + + public static bool operator ==(SdpMessage left, SdpMessage right) + { + return left.Equals(right); + } + + public static bool operator !=(SdpMessage left, SdpMessage right) + { + return !(left == right); + } + } + + // Input struct. Don't directly serialize - use ToJson + public struct IceCandidateMessage : IEquatable + { + public readonly string candidate; + public readonly string sdpMid; + public readonly ushort? sdpMLineIndex; + public readonly string usernameFragment; + + public IceCandidateMessage(string candidate, string sdpMid, ushort? sdpMLineIndex, string usernameFragment) + { + this.candidate = candidate; + this.sdpMid = sdpMid; + this.sdpMLineIndex = sdpMLineIndex; + this.usernameFragment = usernameFragment; + } + + public static string ToJson(IceCandidateMessage message) + { + return SignalingMessageHelper.ToJson(message); + } + + public override bool Equals(object obj) + { + return obj is IceCandidateMessage message && Equals(message); + } + + public bool Equals(IceCandidateMessage other) + { + return candidate == other.candidate && + sdpMid == other.sdpMid && + sdpMLineIndex == other.sdpMLineIndex && + usernameFragment == other.usernameFragment; + } + + public override int GetHashCode() + { + return HashCode.Combine(candidate, sdpMid, sdpMLineIndex, usernameFragment); + } + + public static bool operator ==(IceCandidateMessage left, IceCandidateMessage right) + { + return left.Equals(right); + } + + public static bool operator !=(IceCandidateMessage left, IceCandidateMessage right) + { + return !(left == right); + } + } + + // Output struct. Intended to provide javascript-like interface + public struct SignalingMessage : IEquatable + { + // Ubiq vars, not required for WebRTC, ignored by most implementations + public readonly string implementation; + + // SessionDescription + public readonly string type; + public readonly string sdp; + + // IceCandidate + public readonly string candidate; + public readonly string sdpMid; + public readonly ushort? sdpMLineIndex; + public readonly string usernameFragment; + + public SignalingMessage(string implementation, string candidate, + string sdpMid, ushort? sdpMLineIndex, string usernameFragment, + string type, string sdp) + { + this.implementation = implementation; + this.candidate = candidate; + this.sdpMid = sdpMid; + this.sdpMLineIndex = sdpMLineIndex; + this.usernameFragment = usernameFragment; + this.type = type; + this.sdp = sdp; + } + + public static SignalingMessage FromJson(string json) + { + return SignalingMessageHelper.FromJson(json); + } + + public override bool Equals(object obj) + { + return obj is SignalingMessage message && Equals(message); + } + + public bool Equals(SignalingMessage other) + { + return implementation == other.implementation && + candidate == other.candidate && + sdpMid == other.sdpMid && + sdpMLineIndex == other.sdpMLineIndex && + usernameFragment == other.usernameFragment && + type == other.type && + sdp == other.sdp; + } + + public override int GetHashCode() + { + return HashCode.Combine(implementation, candidate, sdpMid, + sdpMLineIndex, usernameFragment, type, sdp); + } + + public static bool operator ==(SignalingMessage left, SignalingMessage right) + { + return left.Equals(right); + } + + public static bool operator !=(SignalingMessage left, SignalingMessage right) + { + return !(left == right); + } + } +} \ No newline at end of file diff --git a/Unity/Assets/Runtime/Voip/Implementations/JsonHelpers/SignallingMessageHelper.cs.meta b/Unity/Assets/Runtime/Voip/Implementations/SignalingMessageHelper.cs.meta similarity index 100% rename from Unity/Assets/Runtime/Voip/Implementations/JsonHelpers/SignallingMessageHelper.cs.meta rename to Unity/Assets/Runtime/Voip/Implementations/SignalingMessageHelper.cs.meta diff --git a/Unity/Assets/Runtime/Voip/Implementations/Unity/AudioStatsFilter.cs b/Unity/Assets/Runtime/Voip/Implementations/Unity/AudioStatsFilter.cs new file mode 100644 index 000000000..20bfe8f87 --- /dev/null +++ b/Unity/Assets/Runtime/Voip/Implementations/Unity/AudioStatsFilter.cs @@ -0,0 +1,76 @@ +using System; +using UnityEngine; +using System.Collections.Concurrent; + +namespace Ubiq.Voip.Implementations.Unity +{ + /// + /// Gather audio stats from the attached AudioSource + /// + public class AudioStatsFilter : MonoBehaviour + { + private ConcurrentQueue statsQueue = new ConcurrentQueue(); + private int sampleRate; + private Action statsPushed; + + private volatile int channels; + + void OnDestroy() + { + statsPushed = null; + } + + void OnEnable() + { + OnAudioConfigurationChanged(false); + AudioSettings.OnAudioConfigurationChanged += OnAudioConfigurationChanged; + } + + void OnDisable() + { + AudioSettings.OnAudioConfigurationChanged -= OnAudioConfigurationChanged; + } + + void OnAudioConfigurationChanged(bool deviceWasChanged) + { + sampleRate = AudioSettings.outputSampleRate; + } + + private void Update() + { + while (statsQueue.TryDequeue(out var stats)) + { + statsPushed?.Invoke(new AudioStats( + stats.sampleCount,stats.volumeSum,sampleRate + )); + } + + Debug.Log(name + " " + channels); + } + + public void SetStatsPushedCallback(Action statsPushed) + { + this.statsPushed = statsPushed; + } + + /// + /// + /// + /// Called on the audio thread, not the main thread. + /// + /// + /// + void OnAudioFilterRead(float[] data, int channels) + { + this.channels = channels; + + var volumeSum = 0.0f; + for (int i = 0; i < data.Length; i+=channels) + { + volumeSum += Mathf.Abs(data[i]); + } + + statsQueue.Enqueue(new AudioStats(data.Length/channels,volumeSum,0)); + } + } +} \ No newline at end of file diff --git a/Unity/Assets/Runtime/Voip/Implementations/Unity/AudioStatsFilter.cs.meta b/Unity/Assets/Runtime/Voip/Implementations/Unity/AudioStatsFilter.cs.meta new file mode 100644 index 000000000..4955e4677 --- /dev/null +++ b/Unity/Assets/Runtime/Voip/Implementations/Unity/AudioStatsFilter.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: cd0fa0152b14ff44b9b4ade35e3d4c0d +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Unity/Assets/Runtime/Voip/Implementations/Unity/PeerConnectionImpl.cs b/Unity/Assets/Runtime/Voip/Implementations/Unity/PeerConnectionImpl.cs index 18d6bd3f6..b76546af7 100644 --- a/Unity/Assets/Runtime/Voip/Implementations/Unity/PeerConnectionImpl.cs +++ b/Unity/Assets/Runtime/Voip/Implementations/Unity/PeerConnectionImpl.cs @@ -1,3 +1,4 @@ +using System; using System.Collections; using System.Collections.Generic; using UnityEngine; @@ -32,6 +33,20 @@ public Event(Type type, string json = null, RTCIceCandidate iceCandidate = null) } } + private class Context + { + public IPeerConnectionContext context; + public Action playbackStatsPushed; + public Action recordStatsPushed; + public Action iceConnectionStateChanged; + public Action peerConnectionStateChanged; + public bool polite; + + public MonoBehaviour behaviour => context.behaviour; + public GameObject gameObject => context.behaviour.gameObject; + public Transform transform => context.behaviour.transform; + } + private enum Implementation { Unknown, @@ -39,53 +54,72 @@ private enum Implementation Other, } - public event IceConnectionStateChangedDelegate iceConnectionStateChanged; - public event PeerConnectionStateChangedDelegate peerConnectionStateChanged; - // Unity Peer Connection private RTCPeerConnection peerConnection; - private IPeerConnectionContext context; - private AudioSource receiverAudioSource; - private AudioSource senderAudioSource; + private SpatialisationCacheFilter cacheFilter; + private AudioStatsFilter statsFilter; + private SpatialisationRestoreFilter restoreFilter; private PeerConnectionMicrophone microphone; - private bool polite; - private List events = new List(); - private List coroutines = new List(); + private List coroutinesForCleanup = new List(); + private List objectsForCleanup = new List(); private Implementation otherPeerImplementation = Implementation.Unknown; + private Context ctx; + public void Dispose() { - if (context.behaviour) + if (ctx.behaviour) { - foreach(var coroutine in coroutines) + foreach(var coroutine in coroutinesForCleanup) { - if (context.behaviour) - { - context.behaviour.StopCoroutine(coroutine); - } + ctx.behaviour.StopCoroutine(coroutine); } + coroutinesForCleanup.Clear(); - microphone.RemoveUser(context.behaviour.gameObject); + if (microphone) + { + microphone.statsPushed -= PeerConnectionMicrophone_OnStats; + microphone.RemoveUser(ctx.gameObject); + } } - coroutines.Clear(); + + foreach(var obj in objectsForCleanup) + { + if (obj) + { + UnityEngine.Object.Destroy(obj); + } + } + objectsForCleanup.Clear(); + + ctx = null; } public void Setup(IPeerConnectionContext context, - bool polite, List iceServers) + bool polite, List iceServers, + Action playbackStatsPushed, + Action recordStatsPushed, + Action iceConnectionStateChanged, + Action peerConnectionStateChanged) { - if (this.context != null) + if (ctx != null) { // Already setup return; } - this.context = context; - this.polite = polite; + ctx = new Context(); + ctx.context = context; + ctx.polite = polite; + ctx.playbackStatsPushed = playbackStatsPushed; + ctx.recordStatsPushed = recordStatsPushed; + ctx.iceConnectionStateChanged = iceConnectionStateChanged; + ctx.peerConnectionStateChanged = peerConnectionStateChanged; var configuration = GetConfiguration(iceServers); @@ -93,10 +127,23 @@ public void Setup(IPeerConnectionContext context, RequireReceiverAudioSource(); receiverStream.OnAddTrack += (MediaStreamTrackEvent e) => { - RequireComponent(context.behaviour.gameObject); + if (e.Track.Kind != TrackKind.Audio) + { + return; + } + + // Restore spatialisation for Unity's WebRTC package. First, + // give the output AudioSource a clip full of 1s (elsewhere), + // then apply filters in order. + // 1. Cache the spatialised output + RequireCacheFilter(); + // 2. Play back WebRTC audio (added through SetTrack) receiverAudioSource.SetTrack(e.Track as AudioStreamTrack); - RequireComponent(context.behaviour.gameObject); - receiverAudioSource.loop = true; + // 3. (unrelated) Add a filter to gather audio stats + RequireStatsFilter(); + // 4. Restore spatialisation by multiplying output of 2 with 1 + RequireRestoreFilter(); + receiverAudioSource.Play(); }; @@ -104,13 +151,11 @@ public void Setup(IPeerConnectionContext context, { OnConnectionStateChange = (RTCPeerConnectionState state) => { - peerConnectionStateChanged(PeerConnectionStatePkgToUbiq(state)); - Debug.Log($"Peer connection state change to {state}."); + peerConnectionStateChanged?.Invoke(PeerConnectionStatePkgToUbiq(state)); }, OnIceConnectionChange = (RTCIceConnectionState state) => { - iceConnectionStateChanged(IceConnectionStatePkgToUbiq(state)); - Debug.Log($"Ice connection state change to {state}."); + iceConnectionStateChanged?.Invoke(IceConnectionStatePkgToUbiq(state)); }, OnIceCandidate = (RTCIceCandidate candidate) => { @@ -124,29 +169,15 @@ public void Setup(IPeerConnectionContext context, { events.Add(new Event(Event.Type.NegotiationNeeded)); }, - OnIceGatheringStateChange = (RTCIceGatheringState state) => - { - Debug.Log($"Ice gathering state change to {state}."); - } }; - coroutines.Add(context.behaviour.StartCoroutine(DoSignaling())); - coroutines.Add(context.behaviour.StartCoroutine(StartMicrophoneTrack())); - - // peerConnection.AddTrack(senderAudioTrack,senderStream); - - // Diagnostics. - // pc.OnReceiveReport += (re, media, rr) => mainThreadActions.Enqueue( - // () => Debug.Log($"RTCP Receive for {media} from {re}\n{rr.GetDebugSummary()}")); - // pc.OnSendReport += (media, sr) => mainThreadActions.Enqueue( - // () => Debug.Log($"RTCP Send for {media}\n{sr.GetDebugSummary()}")); - // pc.GetRtpChannel().OnStunMessageReceived += (msg, ep, isRelay) => mainThreadActions.Enqueue( - // () => Debug.Log($"STUN {msg.Header.MessageType} received from {ep}.")); + coroutinesForCleanup.Add(context.behaviour.StartCoroutine(DoSignaling())); + coroutinesForCleanup.Add(context.behaviour.StartCoroutine(StartMicrophoneTrack())); } private IEnumerator StartMicrophoneTrack() { - var manager = context.behaviour.transform.parent.gameObject; + var manager = ctx.transform.parent.gameObject; microphone = manager.GetComponent(); if (!microphone) @@ -154,10 +185,16 @@ private IEnumerator StartMicrophoneTrack() microphone = manager.AddComponent(); } - yield return microphone.AddUser(context.behaviour.gameObject); + yield return microphone.AddUser(ctx.gameObject); + microphone.statsPushed += PeerConnectionMicrophone_OnStats; peerConnection.AddTrack(microphone.audioStreamTrack); } + private void PeerConnectionMicrophone_OnStats(AudioStats stats) + { + ctx.recordStatsPushed.Invoke(stats); + } + private bool ignoreOffer; // Manage all signaling, sending and receiving offers. @@ -180,7 +217,7 @@ private IEnumerator DoSignaling() if (e.type == Event.Type.OnIceCandidate) { - Send(context,e.iceCandidate); + Send(ctx.context,e.iceCandidate); continue; } @@ -188,7 +225,7 @@ private IEnumerator DoSignaling() { var op = peerConnection.SetLocalDescription(); yield return op; - Send(context,peerConnection.LocalDescription); + Send(ctx.context,peerConnection.LocalDescription); continue; } @@ -208,13 +245,13 @@ private IEnumerator DoSignaling() // non-dotnet peer always takes on the role of polite // peer as the dotnet implementaton isn't smart enough // to handle rollback - polite = true; + ctx.polite = true; } } if (msg.type != null) { - ignoreOffer = !polite + ignoreOffer = !ctx.polite && msg.type == "offer" && peerConnection.SignalingState != RTCSignalingState.Stable; if (ignoreOffer) @@ -230,7 +267,7 @@ private IEnumerator DoSignaling() op = peerConnection.SetLocalDescription(); yield return op; - Send(context,peerConnection.LocalDescription); + Send(ctx.context,peerConnection.LocalDescription); } continue; } @@ -246,6 +283,43 @@ private IEnumerator DoSignaling() } } + private void RequireCacheFilter() + { + if (cacheFilter) + { + return; + } + + cacheFilter = ctx.gameObject.AddComponent(); + cacheFilter.hideFlags = HideFlags.HideInInspector; + objectsForCleanup.Add(cacheFilter); + } + + private void RequireStatsFilter() + { + if (statsFilter) + { + return; + } + + statsFilter = ctx.gameObject.AddComponent(); + statsFilter.SetStatsPushedCallback(ctx.playbackStatsPushed); + statsFilter.hideFlags = HideFlags.HideInInspector; + objectsForCleanup.Add(statsFilter); + } + + private void RequireRestoreFilter() + { + if (restoreFilter) + { + return; + } + + restoreFilter = ctx.gameObject.AddComponent(); + restoreFilter.hideFlags = HideFlags.HideInInspector; + objectsForCleanup.Add(restoreFilter); + } + private void RequireReceiverAudioSource () { if (receiverAudioSource) @@ -254,10 +328,11 @@ private void RequireReceiverAudioSource () } // Setup receive audio source - receiverAudioSource = context.behaviour.gameObject.AddComponent(); + receiverAudioSource = ctx.gameObject.AddComponent(); receiverAudioSource.spatialize = true; receiverAudioSource.spatialBlend = 1.0f; + receiverAudioSource.loop = true; // Use a clip filled with 1s // This helps us piggyback on Unity's spatialisation using filters @@ -272,6 +347,9 @@ private void RequireReceiverAudioSource () AudioSettings.outputSampleRate, false); receiverAudioSource.clip.SetData(samples,0); + + objectsForCleanup.Add(receiverAudioSource.clip); + objectsForCleanup.Add(receiverAudioSource); } public void ProcessSignalingMessage (string json) @@ -292,20 +370,9 @@ private static RTCConfiguration GetConfiguration(List iceServe public void UpdateSpatialization(Vector3 sourcePosition, Quaternion sourceRotation, Vector3 listenerPosition, Quaternion listenerRotation) { - context.behaviour.transform.position = sourcePosition; - context.behaviour.transform.rotation = sourceRotation; - } - - public PlaybackStats GetLastFramePlaybackStats () - { - // if (sink != null) - // { - // return sink.GetLastFramePlaybackStats(); - // } - // else - // { - return new PlaybackStats(); - // } + var t = ctx.transform; + t.position = sourcePosition; + t.rotation = sourceRotation; } private static RTCIceCandidate IceCandidateUbiqToPkg(SignalingMessage msg) @@ -401,30 +468,22 @@ private static RTCSdpType StringToSdpType(string type) private static void Send(IPeerConnectionContext context, RTCSessionDescription sd) { - context.Send(SignalingMessageHelper.ToJson(new SignalingMessage{ - sdp = sd.sdp, - type = SdpTypeToString(sd.type) - })); + context.Send(SdpMessage.ToJson(new SdpMessage + ( + type: SdpTypeToString(sd.type), + sdp: sd.sdp + ))); } private static void Send(IPeerConnectionContext context, RTCIceCandidate ic) { - context.Send(SignalingMessageHelper.ToJson(new SignalingMessage{ - candidate = ic.Candidate, - sdpMid = ic.SdpMid, - sdpMLineIndex = (ushort?)ic.SdpMLineIndex, - usernameFragment = ic.UserNameFragment - })); - } - - private static T RequireComponent(GameObject gameObject) where T : MonoBehaviour - { - var c = gameObject.GetComponent(); - if (c) - { - return c; - } - return gameObject.AddComponent(); + context.Send(IceCandidateMessage.ToJson(new IceCandidateMessage + ( + candidate: ic.Candidate, + sdpMid: ic.SdpMid, + sdpMLineIndex: (ushort?)ic.SdpMLineIndex, + usernameFragment: ic.UserNameFragment + ))); } } } \ No newline at end of file diff --git a/Unity/Assets/Runtime/Voip/Implementations/Unity/PeerConnectionMicrophone.cs b/Unity/Assets/Runtime/Voip/Implementations/Unity/PeerConnectionMicrophone.cs index 4d61682fe..86397d05a 100644 --- a/Unity/Assets/Runtime/Voip/Implementations/Unity/PeerConnectionMicrophone.cs +++ b/Unity/Assets/Runtime/Voip/Implementations/Unity/PeerConnectionMicrophone.cs @@ -35,15 +35,13 @@ public State state } } - public event Action stopping = delegate {}; - public event Action started = delegate {}; - public event Action ready = delegate {}; + public AudioStreamTrack audioStreamTrack { get; private set; } + public event Action statsPushed; + private AudioSource audioSource; private List users = new List(); private bool microphoneAuthorized; - - public AudioSource audioSource; - public AudioStreamTrack audioStreamTrack; + private AudioStatsFilter statsFilter; private void Awake() { @@ -97,6 +95,11 @@ private void Update() } } + private void StatsFilter_StatsPushed(AudioStats stats) + { + statsPushed?.Invoke(stats); + } + private void RequireAudioSource() { if(!audioSource) @@ -106,10 +109,19 @@ private void RequireAudioSource() if (!audioSource) { audioSource = gameObject.AddComponent(); + statsFilter = gameObject.AddComponent(); + statsFilter.hideFlags = HideFlags.HideInInspector; + statsFilter.SetStatsPushedCallback(StatsFilter_StatsPushed); } } } + /// + /// Indicate a new user to the microphone, with an optional callback + /// for audio stats. If run as part of a coroutine, this will complete + /// when the microphone is ready to be used. If the user has already + /// been added, the callback will be replaced. + /// public IEnumerator AddUser(GameObject user) { if (!users.Contains(user)) @@ -123,6 +135,10 @@ public IEnumerator AddUser(GameObject user) } } + /// + /// Remove a user from the microphone. If user count reaches zero, the + /// microphone will be stopped. + /// public void RemoveUser(GameObject user) { users.Remove(user); diff --git a/Unity/Assets/Runtime/Voip/Implementations/Unity/SpatialisationCacheAudioFilter.cs b/Unity/Assets/Runtime/Voip/Implementations/Unity/SpatialisationCacheAudioFilter.cs index 09b51d234..d7d737c84 100644 --- a/Unity/Assets/Runtime/Voip/Implementations/Unity/SpatialisationCacheAudioFilter.cs +++ b/Unity/Assets/Runtime/Voip/Implementations/Unity/SpatialisationCacheAudioFilter.cs @@ -3,33 +3,18 @@ namespace Ubiq.Voip.Implementations.Unity { /// - /// + /// Part of the workaround for lack of spatialisation in Unity's WebRTC + /// implementation. This part stores the raw spatialisation information. + /// Requires an AudioClip filled with 1s /// - public class SpatialisationCacheAudioFilter : MonoBehaviour + public class SpatialisationCacheFilter : MonoBehaviour { public float[] cache = new float[4096]; - private int m_sampleRate; - - void OnEnable() - { - OnAudioConfigurationChanged(false); - AudioSettings.OnAudioConfigurationChanged += OnAudioConfigurationChanged; - } - - void OnDisable() - { - AudioSettings.OnAudioConfigurationChanged -= OnAudioConfigurationChanged; - } - - void OnAudioConfigurationChanged(bool deviceWasChanged) - { - m_sampleRate = AudioSettings.outputSampleRate; - } /// /// /// - /// Call on the audio thread, not main thread. + /// Called on the audio thread, not the main thread. /// /// /// diff --git a/Unity/Assets/Runtime/Voip/Implementations/Unity/SpatialisationRestoreAudioFilter.cs b/Unity/Assets/Runtime/Voip/Implementations/Unity/SpatialisationRestoreAudioFilter.cs index e67baaf0c..0f6591d35 100644 --- a/Unity/Assets/Runtime/Voip/Implementations/Unity/SpatialisationRestoreAudioFilter.cs +++ b/Unity/Assets/Runtime/Voip/Implementations/Unity/SpatialisationRestoreAudioFilter.cs @@ -3,34 +3,23 @@ namespace Ubiq.Voip.Implementations.Unity { /// - /// + /// Part of the workaround for lack of spatialisation in Unity's WebRTC + /// implementation. This part multiplies the cache with the output of the + /// WebRTC audio filter. /// - public class SpatialisationRestoreAudioFilter : MonoBehaviour + public class SpatialisationRestoreFilter : MonoBehaviour { - private SpatialisationCacheAudioFilter cacheAudioFilter; - private int m_sampleRate; + private SpatialisationCacheFilter cacheAudioFilter; void OnEnable() { - OnAudioConfigurationChanged(false); - AudioSettings.OnAudioConfigurationChanged += OnAudioConfigurationChanged; - cacheAudioFilter = GetComponent(); - } - - void OnDisable() - { - AudioSettings.OnAudioConfigurationChanged -= OnAudioConfigurationChanged; - } - - void OnAudioConfigurationChanged(bool deviceWasChanged) - { - m_sampleRate = AudioSettings.outputSampleRate; + cacheAudioFilter = GetComponent(); } /// /// /// - /// Call on the audio thread, not main thread. + /// Called on the audio thread, not the main thread. /// /// /// diff --git a/Unity/Assets/Runtime/Voip/Implementations/Web/PeerConnectionImpl.cs b/Unity/Assets/Runtime/Voip/Implementations/Web/PeerConnectionImpl.cs index 351f01d13..5720e4ef3 100644 --- a/Unity/Assets/Runtime/Voip/Implementations/Web/PeerConnectionImpl.cs +++ b/Unity/Assets/Runtime/Voip/Implementations/Web/PeerConnectionImpl.cs @@ -1,3 +1,4 @@ +using System; using System.Collections; using System.Collections.Generic; using UnityEngine; @@ -54,11 +55,33 @@ public static extern bool JS_WebRTC_ProcessSignalingMessage(int pc, [DllImport("__Internal")] public static extern void JS_WebRTC_SetPanner(int pc, float x, float y, float z); [DllImport("__Internal")] - public static extern int JS_WebRTC_GetStatsSamples(int pc); + public static extern int JS_WebRTC_GetPlaybackStatsSampleCount(int pc); [DllImport("__Internal")] - public static extern float JS_WebRTC_GetStatsVolume(int pc); + public static extern float JS_WebRTC_GetPlaybackStatsVolumeSum(int pc); [DllImport("__Internal")] - public static extern void JS_WebRTC_EndStats(int pc); + public static extern int JS_WebRTC_GetPlaybackStatsSampleRate(int pc); + [DllImport("__Internal")] + public static extern int JS_WebRTC_GetRecordStatsSampleCount(); + [DllImport("__Internal")] + public static extern float JS_WebRTC_GetRecordStatsVolumeSum(); + [DllImport("__Internal")] + public static extern int JS_WebRTC_GetRecordStatsSampleRate(); + [DllImport("__Internal")] + public static extern void JS_WebRTC_EndStats(int pc, int frameCount); + + private class Context + { + public IPeerConnectionContext context; + public Action playbackStatsPushed; + public Action recordStatsPushed; + public Action iceConnectionStateChanged; + public Action peerConnectionStateChanged; + public bool polite; + + public MonoBehaviour behaviour => context.behaviour; + public GameObject gameObject => context.behaviour.gameObject; + public Transform transform => context.behaviour.transform; + } private enum Implementation { @@ -67,13 +90,8 @@ private enum Implementation Other } - public event IceConnectionStateChangedDelegate iceConnectionStateChanged; - public event PeerConnectionStateChangedDelegate peerConnectionStateChanged; - private Queue messageQueue = new Queue(); - private IPeerConnectionContext context; - private Coroutine updateCoroutine; private int peerConnectionId = -1; @@ -86,11 +104,16 @@ private enum Implementation // not yet set https://stackoverflow.com/questions/38198751 private bool hasRemoteDescription = false; + private Context ctx; + public void Dispose() { if (updateCoroutine != null) { - context.behaviour.StopCoroutine(updateCoroutine); + if (ctx != null && ctx.behaviour) + { + ctx.behaviour.StopCoroutine(updateCoroutine); + } updateCoroutine = null; } @@ -99,18 +122,20 @@ public void Dispose() JS_WebRTC_Close(peerConnectionId); peerConnectionId = -1; } - } - public PlaybackStats GetLastFramePlaybackStats () - { - return new PlaybackStats - { - sampleCount = JS_WebRTC_GetStatsSamples(peerConnectionId), - volumeSum = JS_WebRTC_GetStatsVolume(peerConnectionId), - sampleRate = 16000 - }; + ctx = null; } + // public PlaybackStats GetLastFramePlaybackStats () + // { + // return new PlaybackStats + // { + // sampleCount = JS_WebRTC_GetStatsSamples(peerConnectionId), + // volumeSum = JS_WebRTC_GetStatsVolume(peerConnectionId), + // sampleRate = 16000 + // }; + // } + public void UpdateSpatialization(Vector3 sourcePosition, Quaternion sourceRotation, Vector3 listenerPosition, Quaternion listenerRotation) @@ -122,16 +147,27 @@ public void UpdateSpatialization(Vector3 sourcePosition, JS_WebRTC_SetPanner(peerConnectionId,p.x,p.y,p.z); } - public void Setup(IPeerConnectionContext context, bool polite, - List iceServers) + public void Setup(IPeerConnectionContext context, + bool polite, + List iceServers, + Action playbackStatsPushed, + Action recordStatsPushed, + Action iceConnectionStateChanged, + Action peerConnectionStateChanged) { - if (this.context != null) + if (ctx != null) { - // Already setup or setup in progress + // Already setup return; } - this.context = context; + ctx = new Context(); + ctx.context = context; + ctx.polite = polite; + ctx.playbackStatsPushed = playbackStatsPushed; + ctx.recordStatsPushed = recordStatsPushed; + ctx.iceConnectionStateChanged = iceConnectionStateChanged; + ctx.peerConnectionStateChanged = peerConnectionStateChanged; peerConnectionId = JS_WebRTC_New(); for (int i = 0; i < iceServers.Count; i++) @@ -143,7 +179,7 @@ public void Setup(IPeerConnectionContext context, bool polite, JS_WebRTC_New_SetPolite(peerConnectionId,polite); JS_WebRTC_New_Start(peerConnectionId); - updateCoroutine = context.behaviour.StartCoroutine(Update()); + updateCoroutine = ctx.behaviour.StartCoroutine(Update()); } public void ProcessSignalingMessage (string json) @@ -202,12 +238,35 @@ private IEnumerator Update() ProcessSignalingMessages(); SendSignalingMessages(); - JS_WebRTC_EndStats(peerConnectionId); + JS_WebRTC_EndStats(peerConnectionId,Time.frameCount); + + PushPlaybackStats(); + PushRecordStats(); yield return null; } } + private void PushPlaybackStats() + { + ctx.playbackStatsPushed?.Invoke(new AudioStats + ( + sampleCount: JS_WebRTC_GetPlaybackStatsSampleCount(peerConnectionId), + volumeSum: JS_WebRTC_GetPlaybackStatsVolumeSum(peerConnectionId), + sampleRate: JS_WebRTC_GetPlaybackStatsSampleRate(peerConnectionId) + )); + } + + private void PushRecordStats() + { + ctx.recordStatsPushed?.Invoke(new AudioStats + ( + sampleCount: JS_WebRTC_GetRecordStatsSampleCount(), + volumeSum: JS_WebRTC_GetRecordStatsVolumeSum(), + sampleRate: JS_WebRTC_GetRecordStatsSampleRate() + )); + } + private void UpdateHasRemoteDescription() { hasRemoteDescription = hasRemoteDescription || @@ -220,8 +279,7 @@ private void UpdateIceConnectionState() JS_WebRTC_GetIceConnectionState(peerConnectionId); if (state != lastIceConnectionState) { - Debug.Log("ICE Connection State Changed: " + state); - iceConnectionStateChanged(state); + ctx.iceConnectionStateChanged?.Invoke(state); lastIceConnectionState = state; } } @@ -232,8 +290,7 @@ private void UpdatePeerConnectionState() JS_WebRTC_GetPeerConnectionState(peerConnectionId); if (state != lastPeerConnectionState) { - Debug.Log("Peer Connection State Changed: " + state); - peerConnectionStateChanged(state); + ctx.peerConnectionStateChanged?.Invoke(state); lastPeerConnectionState = state; } } @@ -243,15 +300,26 @@ private void SendSignalingMessages() // Check for new ice candidates provided by the peer connection while (JS_WebRTC_SignalingMessages_Has(peerConnectionId)) { - var sdpMLineIndex = JS_WebRTC_SignalingMessages_GetSdpMLineIndex(peerConnectionId); - context.Send(SignalingMessageHelper.ToJson(new SignalingMessage{ - candidate = JS_WebRTC_SignalingMessages_GetCandidate(peerConnectionId), - sdpMid = JS_WebRTC_SignalingMessages_GetSdpMid(peerConnectionId), - sdpMLineIndex = sdpMLineIndex < 0 ? null : (ushort?)sdpMLineIndex , - usernameFragment = JS_WebRTC_SignalingMessages_GetUsernameFragment(peerConnectionId), - type = JS_WebRTC_SignalingMessages_GetType(peerConnectionId), - sdp = JS_WebRTC_SignalingMessages_GetSdp(peerConnectionId) - })); + var sdp = JS_WebRTC_SignalingMessages_GetSdp(peerConnectionId); + if (sdp != null) + { + ctx.context.Send(SdpMessage.ToJson(new SdpMessage + ( + type:JS_WebRTC_SignalingMessages_GetType(peerConnectionId), + sdp:sdp + ))); + } + else + { + var sdpMLineIndex = JS_WebRTC_SignalingMessages_GetSdpMLineIndex(peerConnectionId); + ctx.context.Send(IceCandidateMessage.ToJson(new IceCandidateMessage + ( + candidate:JS_WebRTC_SignalingMessages_GetCandidate(peerConnectionId), + sdpMid:JS_WebRTC_SignalingMessages_GetSdpMid(peerConnectionId), + sdpMLineIndex:sdpMLineIndex < 0 ? null : (ushort?)sdpMLineIndex, + usernameFragment:JS_WebRTC_SignalingMessages_GetUsernameFragment(peerConnectionId) + ))); + } JS_WebRTC_SignalingMessages_Pop(peerConnectionId); } diff --git a/Unity/Assets/Runtime/Voip/Implementations/Web/Plugins/WebRTC.jslib b/Unity/Assets/Runtime/Voip/Implementations/Web/Plugins/WebRTC.jslib index fc7b1dd47..a11f85a0b 100644 --- a/Unity/Assets/Runtime/Voip/Implementations/Web/Plugins/WebRTC.jslib +++ b/Unity/Assets/Runtime/Voip/Implementations/Web/Plugins/WebRTC.jslib @@ -7,6 +7,21 @@ var unityWebRtcInteropLibrary = { audioContext: null, whenUserMedia: null, + // For record stats + processorNode: null, + gainNode: null, + src: null, + record: { + lastFrame: -1, + volumeSum: 0, + sampleCount: 0, + sampleRate: 0, + stats: { + sampleCount: 0, + volumeSum: 0 + } + }, + iceConnectionState: { closed: 0, failed: 1, @@ -91,13 +106,15 @@ var unityWebRtcInteropLibrary = { instance.polite = polite; instance.msgs = []; - instance.volume = 0; - instance.samples = 0; - instance.sampleRate = 1; - instance.stats = { - samples: 0, - volume: 0, - }; + instance.playback = { + volumeSum: 0, + sampleCount: 0, + sampleRate: 0, + stats: { + sampleCount: 0, + volumeSum: 0 + } + } instance.panner = new PannerNode(context.audioContext, { panningModel: 'HRTF', distanceModel: 'inverse', @@ -138,10 +155,10 @@ var unityWebRtcInteropLibrary = { let arr = inputBuffer.getChannelData(0); let length = arr.length; for (let i = 0; i < length; i++) { - instance.volume += Math.abs(arr[i]); + instance.playback.volumeSum += Math.abs(arr[i]); } - instance.samples += length; - instance.sampleRate = inputBuffer.sampleRate; + instance.playback.sampleCount += length; + instance.playback.sampleRate = inputBuffer.sampleRate; }; instance.gainNode = new GainNode(audioContext,{gain:0}); instance.src = audioContext.createMediaStreamSource(stream); @@ -185,6 +202,25 @@ var unityWebRtcInteropLibrary = { return; } + if (!context.processorNode) { + let audioContext = context.audioContext; + context.processorNode = audioContext.createScriptProcessor(4096, 1, 1); + context.processorNode.onaudioprocess = ({inputBuffer}) => { + let arr = inputBuffer.getChannelData(0); + let length = arr.length; + for (let i = 0; i < length; i++) { + context.record.volumeSum += Math.abs(arr[i]); + } + context.record.sampleCount += length; + context.record.sampleRate = inputBuffer.sampleRate; + }; + context.gainNode = new GainNode(audioContext,{gain:0}); + context.src = audioContext.createMediaStreamSource(stream); + context.src.connect(context.processorNode); + context.processorNode.connect(context.gainNode); + context.gainNode.connect(audioContext.destination); + } + for (const track of stream.getTracks()) { instance.pc.addTrack(track,stream); } @@ -471,34 +507,63 @@ var unityWebRtcInteropLibrary = { instance.panner.positionZ.value = z; }, - JS_WebRTC_GetStatsSamples: function(id) { + JS_WebRTC_GetPlaybackStatsSampleCount: function(id) { let instance = context.instances[id]; if (!instance || !instance.pc) { return 0; } - return 16000 * (instance.stats.samples / instance.sampleRate); + return instance.playback.stats.sampleCount; }, - JS_WebRTC_GetStatsVolume: function(id) { + JS_WebRTC_GetPlaybackStatsVolumeSum: function(id) { let instance = context.instances[id]; if (!instance || !instance.pc) { return 0; } - return 16000 * (instance.stats.volume / instance.sampleRate); + return instance.playback.stats.volumeSum; }, - JS_WebRTC_EndStats: function(id) { + JS_WebRTC_GetPlaybackStatsSampleRate: function(id) { + let instance = context.instances[id]; + if (!instance || !instance.pc) { + return 0; + } + + return instance.playback.sampleRate; + }, + + JS_WebRTC_GetRecordStatsSampleCount: function() { + return context.record.stats.sampleCount; + }, + + JS_WebRTC_GetRecordStatsVolumeSum: function() { + return context.record.stats.volumeSum; + }, + + JS_WebRTC_GetRecordStatsSampleRate: function() { + return context.record.sampleRate; + }, + + JS_WebRTC_EndStats: function(id, frameCount) { let instance = context.instances[id]; if (!instance || !instance.pc) { return; } - instance.stats.volume = instance.volume; - instance.stats.samples = instance.samples; - instance.volume = 0; - instance.samples = 0; + instance.playback.stats.volumeSum = instance.playback.volumeSum; + instance.playback.stats.sampleCount = instance.playback.sampleCount; + instance.playback.volumeSum = 0; + instance.playback.sampleCount = 0; + + if (frameCount != context.record.lastFrame) { + context.record.stats.volumeSum = context.record.volumeSum; + context.record.stats.sampleCount = context.record.sampleCount; + context.record.volumeSum = 0; + context.record.sampleCount = 0; + context.record.lastFrame = frameCount; + } } }; diff --git a/Unity/Assets/Runtime/Voip/VoipAvatar.cs b/Unity/Assets/Runtime/Voip/VoipAvatar.cs index 8faed4889..e60b32c46 100644 --- a/Unity/Assets/Runtime/Voip/VoipAvatar.cs +++ b/Unity/Assets/Runtime/Voip/VoipAvatar.cs @@ -9,6 +9,7 @@ namespace Ubiq.Avatars public class VoipAvatar : MonoBehaviour { public Transform audioSourcePosition; + public VoipSpeechIndicator speechIndicator; public VoipPeerConnection peerConnection { get; private set; } private AvatarManager avatarManager; @@ -17,6 +18,14 @@ public class VoipAvatar : MonoBehaviour private VoipAvatar localVoipAvatar; + private void OnDestroy() + { + if (peerConnection) + { + peerConnection.playbackStatsPushed -= PeerConnection_PlaybackStatsPushed; + } + } + private void Start() { avatarManager = GetComponentInParent(); @@ -37,6 +46,7 @@ private void OnPeerConnection(VoipPeerConnection peerConnection) } this.peerConnection = peerConnection; + peerConnection.playbackStatsPushed += PeerConnection_PlaybackStatsPushed; } private void LateUpdate() @@ -69,5 +79,13 @@ private void LateUpdate() source.position,source.rotation, listener.position,listener.rotation); } + + private void PeerConnection_PlaybackStatsPushed(VoipPeerConnection.AudioStats stats) + { + if (speechIndicator && avatar && !avatar.IsLocal) + { + speechIndicator.PushAudioStats(stats); + } + } } } diff --git a/Unity/Assets/Runtime/Voip/VoipPeerConnection.cs b/Unity/Assets/Runtime/Voip/VoipPeerConnection.cs index 4ae505d4a..de3c7b2f6 100644 --- a/Unity/Assets/Runtime/Voip/VoipPeerConnection.cs +++ b/Unity/Assets/Runtime/Voip/VoipPeerConnection.cs @@ -3,7 +3,6 @@ using UnityEngine; using UnityEngine.Events; using Ubiq.Messaging; -using Ubiq.Logging; using Ubiq.Voip.Implementations; namespace Ubiq.Voip @@ -23,7 +22,7 @@ public PeerConnectionContext (VoipPeerConnection peerConnection) } } - // Defined here as well as in Impl for external use + // Defined here as well as in IPeerConnectionImpl for external use public enum IceConnectionState { closed = 0, @@ -35,7 +34,7 @@ public enum IceConnectionState completed = 6 } - // Defined here as well as in Impl for external use + // Defined here as well as in IPeerConnectionImpl for external use public enum PeerConnectionState { closed = 0, @@ -46,30 +45,47 @@ public enum PeerConnectionState connected = 5 } - // Defined here as well as in Impl for external use - public struct PlaybackStats + // Defined here as well as in IPeerConnectionImpl for external use + [Serializable] + public struct AudioStats : IEquatable { - public int samples; - public float volume; - public int sampleRate; - } + [field: SerializeField] public int sampleCount { get; private set; } + [field: SerializeField] public float volumeSum { get; private set; } + [field: SerializeField] public int sampleRate { get; private set; } - public struct SessionStatistics - { - public uint PacketsSent; - public uint BytesSent; - public uint PacketsRecieved; - public uint BytesReceived; - } + public AudioStats(int sampleCount, float volumeSum, int sampleRate) + { + this.sampleCount = sampleCount; + this.volumeSum = volumeSum; + this.sampleRate = sampleRate; + } - /// - /// Summarises the throughput for different sessions in this connection. - /// This is returned when the statistics are polled from this peer connection. - /// - public struct TransmissionStats - { - public SessionStatistics Audio; - public SessionStatistics Video; + public override bool Equals(object obj) + { + return obj is AudioStats stats && Equals(stats); + } + + public bool Equals(AudioStats other) + { + return sampleCount == other.sampleCount && + volumeSum == other.volumeSum && + sampleRate == other.sampleRate; + } + + public override int GetHashCode() + { + return HashCode.Combine(sampleCount, volumeSum, sampleRate); + } + + public static bool operator ==(AudioStats left, AudioStats right) + { + return left.Equals(right); + } + + public static bool operator !=(AudioStats left, AudioStats right) + { + return !(left == right); + } } public string PeerUuid { get; private set; } @@ -85,6 +101,12 @@ public struct TransmissionStats public IceConnectionStateEvent OnIceConnectionStateChanged = new IceConnectionStateEvent(); public PeerConnectionStateEvent OnPeerConnectionStateChanged = new PeerConnectionStateEvent(); + // C# events rather than Unity events as these will be potentially be + // called multiple times every frame, and Unity events are slow by + // comparison (https://www.jacksondunstan.com/articles/3335) + public event Action playbackStatsPushed; + public event Action recordStatsPushed; + private NetworkId networkId; private NetworkScene networkScene; private IPeerConnectionImpl impl; @@ -128,12 +150,11 @@ public void Setup (NetworkId networkId, NetworkScene scene, this.impl = PeerConnectionImplFactory.Create(); - impl.iceConnectionStateChanged += OnImplIceConnectionStateChanged; - impl.peerConnectionStateChanged += OnImplPeerConnectionStateChanged; - networkScene.AddProcessor(networkId, ProcessMessage); - impl.Setup(new PeerConnectionContext(this),polite,iceServers); + impl.Setup(new PeerConnectionContext(this),polite,iceServers, + Impl_PlaybackStatsPushed,Impl_RecordStatsPushed, + Impl_IceConnectionStateChanged,Impl_PeerConnectionStateChanged); isSetup = true; } @@ -150,61 +171,31 @@ private void SendFromImpl(string json) networkScene.Send(networkId,json); } - private void OnImplIceConnectionStateChanged (Ubiq.Voip.Implementations.IceConnectionState state) + private static AudioStats ConvertStats(Implementations.AudioStats stats) { - iceConnectionState = (IceConnectionState)state; - OnIceConnectionStateChanged.Invoke((IceConnectionState)state); + return new AudioStats(stats.sampleCount,stats.volumeSum,stats.sampleRate); } - private void OnImplPeerConnectionStateChanged (Ubiq.Voip.Implementations.PeerConnectionState state) + private void Impl_PlaybackStatsPushed (Implementations.AudioStats stats) { - peerConnectionState = (PeerConnectionState)state; - OnPeerConnectionStateChanged.Invoke((PeerConnectionState)state); + playbackStatsPushed?.Invoke(ConvertStats(stats)); } - /// - /// Poll this PeerConnection for statistics about its bandwidth usage. - /// - /// - /// This information is also available through RTCP Reports. This method allows the statistics to be polled, - /// rather than wait for a report. If this method is not never called, there is no performance overhead. - /// - public TransmissionStats GetTransmissionStats() + private void Impl_RecordStatsPushed (Implementations.AudioStats stats) { - TransmissionStats report = new TransmissionStats(); - //todo - - // if (rtcPeerConnection != null) - // { - // if (rtcPeerConnection.AudioRtcpSession != null) - // { - // report.Audio.PacketsSent = rtcPeerConnection.AudioRtcpSession.PacketsSentCount; - // report.Audio.PacketsRecieved = rtcPeerConnection.AudioRtcpSession.PacketsReceivedCount; - // report.Audio.BytesSent = rtcPeerConnection.AudioRtcpSession.OctetsSentCount; - // report.Audio.BytesReceived = rtcPeerConnection.AudioRtcpSession.OctetsReceivedCount; - // } - // if (rtcPeerConnection.VideoRtcpSession != null) - // { - // report.Video.PacketsSent = rtcPeerConnection.VideoRtcpSession.PacketsSentCount; - // report.Video.PacketsRecieved = rtcPeerConnection.VideoRtcpSession.PacketsReceivedCount; - // report.Video.BytesSent = rtcPeerConnection.VideoRtcpSession.OctetsSentCount; - // report.Video.BytesReceived = rtcPeerConnection.VideoRtcpSession.OctetsReceivedCount; - // } - // } - return report; + recordStatsPushed?.Invoke(ConvertStats(stats)); } - public PlaybackStats GetLastFramePlaybackStats() + private void Impl_IceConnectionStateChanged (Implementations.IceConnectionState state) { - var playbackStats = new PlaybackStats(); - if (impl != null) - { - var implStats = impl.GetLastFramePlaybackStats(); - playbackStats.volume = implStats.volumeSum; - playbackStats.samples = implStats.sampleCount; - playbackStats.sampleRate = implStats.sampleRate; - } - return playbackStats; + iceConnectionState = (IceConnectionState)state; + OnIceConnectionStateChanged.Invoke((IceConnectionState)state); + } + + private void Impl_PeerConnectionStateChanged (Implementations.PeerConnectionState state) + { + peerConnectionState = (PeerConnectionState)state; + OnPeerConnectionStateChanged.Invoke((PeerConnectionState)state); } } } \ No newline at end of file diff --git a/Unity/Assets/Runtime/Voip/VoipSpeechIndicator.cs b/Unity/Assets/Runtime/Voip/VoipSpeechIndicator.cs new file mode 100644 index 000000000..94f5b0a90 --- /dev/null +++ b/Unity/Assets/Runtime/Voip/VoipSpeechIndicator.cs @@ -0,0 +1,208 @@ +using System.Collections; +using System.Collections.Generic; +using UnityEngine; + +namespace Ubiq.Voip +{ + public class VoipSpeechIndicator : MonoBehaviour + { + public List volumeIndicators; + + public Vector3 minIndicatorScale; + public Vector3 maxIndicatorScale; + + public float minVolume = 0.005f; + public float minVolumeWhenOn = 0.002f; // Should be lower for hysteresis + public float maxVolume = 0.02f; + + public float windowSeconds = 0.6f; + + private List stats = new List(); + private List volumes = new List(); + private List indicatorStates = new List(); + + void Update() + { + UpdateVolumes(); + UpdateIndicators(); + UpdatePosition(); + } + + private void UpdateVolumes() + { + if (volumeIndicators.Count == 0 ) + { + return; + } + + var secondsPerIndicator = windowSeconds / volumeIndicators.Count; + if (volumes.Count != volumeIndicators.Count) + { + volumes.Clear(); + for (int i = 0; i < volumeIndicators.Count; i++) + { + volumes.Add(0); + } + } + + for (int i = 0; i < volumes.Count; i++) + { + volumes[i] = 0; + } + + if (stats.Count == 0) + { + return; + } + + // Walk through the stats structs, calculate volume for each window + var samplesPerVolume = secondsPerIndicator * stats[0].sampleRate; + var volumeIdx = 0; + var sampleHead = 0.0f; + var volumeSum = 0.0f; + var endSamples = samplesPerVolume; + for (int i = stats.Count-1; i >= 0; i--) + { + var prevSampleHead = sampleHead; + while (sampleHead + stats[i].sampleCount > endSamples) + { + // Volume window ends in this audio stats struct + // Multiple volume windows may end in this stats struct + var samplesToUse = endSamples - sampleHead; + var proportion = samplesToUse / stats[i].sampleCount; + volumeSum += stats[i].volumeSum * proportion; + volumes[volumeIdx] = volumeSum / samplesPerVolume; + + volumeIdx++; + volumeSum = 0; + sampleHead = endSamples; + endSamples += samplesPerVolume; + + // We've fully filled all volume windows. Now remove any + // stats structs old enough to feature in no volume windows + if (volumeIdx >= volumes.Count) + { + stats.RemoveRange(0,i); + break; + } + } + + if (volumeIdx >= volumes.Count) + { + break; + } + + // Volume window stretches into next audio stats + var endSampleHead = prevSampleHead + stats[i].sampleCount; + var remainingSamples = endSampleHead - sampleHead; + volumeSum += stats[i].volumeSum * (remainingSamples / stats[i].sampleCount); + sampleHead = endSampleHead; + } + + // Zero out any unfilled volumes. Means we need a full audio stat + // window or a volume will be zero. Should always be the case after + // the first second or so. + for (int i = volumeIdx; i < volumes.Count; i++) + { + volumes[i] = 0; + } + } + + private void UpdateIndicators() + { + if (indicatorStates.Count != volumeIndicators.Count) + { + indicatorStates.Clear(); + for (int i = 0; i < volumeIndicators.Count; i++) + { + indicatorStates.Add(false); + } + } + + for(int i = 0; i < volumeIndicators.Count; i++) + { + var thresh = indicatorStates[i] ? minVolumeWhenOn : minVolume; + indicatorStates[i] = volumes[i] > thresh; + + if (indicatorStates[i]) + { + volumeIndicators[i].gameObject.SetActive(true); + var range = maxVolume - minVolumeWhenOn; + var t = (volumes[i] - minVolumeWhenOn) / range; + volumeIndicators[i].localScale = Vector3.Lerp( + minIndicatorScale,maxIndicatorScale,t); + } + else + { + volumeIndicators[i].gameObject.SetActive(false); + } + } + } + + private void UpdatePosition() + { + var cameraTransform = Camera.main.transform; + var headTransform = transform.parent; + var indicatorRootTransform = transform; + + // If no indicator is being shown currently, reset position + var indicatorVisible = false; + for (int i = 0; i < volumeIndicators.Count; i++) + { + if (volumeIndicators[i].gameObject.activeInHierarchy) + { + indicatorVisible = true; + break; + } + } + + if (!indicatorVisible) + { + indicatorRootTransform.forward = headTransform.forward; + } + + // Rotate s.t. the indicator is always 90 deg from camera + // Method - always two acceptable orientations, pick the closest + var headToCamera = cameraTransform.position - headTransform.position; + var headToCameraDir = headToCamera.normalized; + var dirA = Vector3.Cross(headToCameraDir,headTransform.up); + var dirB = Vector3.Cross(headTransform.up,headToCameraDir); + + var simA = Vector3.Dot(dirA,indicatorRootTransform.forward); + var simB = Vector3.Dot(dirB,indicatorRootTransform.forward); + + var forward = simA > simB ? dirA : dirB; + + // Deal with rare case when avatars share a position + if (forward.sqrMagnitude <= 0) + { + forward = indicatorRootTransform.forward; + } + + indicatorRootTransform.forward = forward; + } + + /// + /// Pushes a new set of audio stats to the indicator. Treats the stats + /// as a continuous stream, where these are the very latest stats. + /// + public void PushAudioStats(VoipPeerConnection.AudioStats stats) + { + if (stats.sampleCount == 0) + { + return; + } + + if (this.stats.Count > 0 && this.stats[0].sampleRate != stats.sampleRate) + { + // May happen if the audio device changes, which should be rare + // so just clear the buffer and start again. Will mean a small + // interruption in the indicator, but it ensure the entire stats + // buffer has the same sampleRate. Simplifies things a lot. + this.stats.Clear(); + } + + this.stats.Add(stats); + } + } +} diff --git a/Unity/Assets/Runtime/Voip/VoipSpeechIndicator.cs.meta b/Unity/Assets/Runtime/Voip/VoipSpeechIndicator.cs.meta new file mode 100644 index 000000000..afa794ce3 --- /dev/null +++ b/Unity/Assets/Runtime/Voip/VoipSpeechIndicator.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: a3da99f06c526484e9de044be0c25a4c +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Unity/Assets/Samples/_Common/Avatar/Avatar.prefab b/Unity/Assets/Samples/_Common/Avatar/Avatar.prefab index 57433c66a..0c415294f 100644 --- a/Unity/Assets/Samples/_Common/Avatar/Avatar.prefab +++ b/Unity/Assets/Samples/_Common/Avatar/Avatar.prefab @@ -29,6 +29,7 @@ Transform: m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} m_LocalPosition: {x: 0, y: 0, z: 0} m_LocalScale: {x: 1, y: 1, z: 1} + m_ConstrainProportionsScale: 0 m_Children: - {fileID: 6524766736450995788} m_Father: {fileID: 0} @@ -176,6 +177,18 @@ PrefabInstance: objectReference: {fileID: 0} m_RemovedComponents: [] m_SourcePrefab: {fileID: 100100000, guid: 85384dbfffb336d4b96168ccfd27db98, type: 3} +--- !u!114 &2500143219345673597 stripped +MonoBehaviour: + m_CorrespondingSourceObject: {fileID: 2818784942326102146, guid: 85384dbfffb336d4b96168ccfd27db98, + type: 3} + m_PrefabInstance: {fileID: 408735727642375679} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 0} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: a3da99f06c526484e9de044be0c25a4c, type: 3} + m_Name: + m_EditorClassIdentifier: --- !u!1001 &8448906606297741168 PrefabInstance: m_ObjectHideFlags: 0 @@ -250,33 +263,15 @@ PrefabInstance: objectReference: {fileID: 0} m_RemovedComponents: [] m_SourcePrefab: {fileID: 100100000, guid: b0d64f917d2447e4f85037ad6283d5bf, type: 3} ---- !u!1 &6433400705492442764 stripped -GameObject: - m_CorrespondingSourceObject: {fileID: 3172939939781028348, guid: b0d64f917d2447e4f85037ad6283d5bf, - type: 3} - m_PrefabInstance: {fileID: 8448906606297741168} - m_PrefabAsset: {fileID: 0} ---- !u!4 &6524766736450995788 stripped -Transform: - m_CorrespondingSourceObject: {fileID: 3444175152561107260, guid: b0d64f917d2447e4f85037ad6283d5bf, - type: 3} - m_PrefabInstance: {fileID: 8448906606297741168} - m_PrefabAsset: {fileID: 0} ---- !u!4 &7388576005872743378 stripped -Transform: - m_CorrespondingSourceObject: {fileID: 1425940620351308962, guid: b0d64f917d2447e4f85037ad6283d5bf, - type: 3} - m_PrefabInstance: {fileID: 8448906606297741168} - m_PrefabAsset: {fileID: 0} ---- !u!23 &8557492846676609213 stripped -MeshRenderer: - m_CorrespondingSourceObject: {fileID: 253003865581486029, guid: b0d64f917d2447e4f85037ad6283d5bf, +--- !u!137 &308880390316127805 stripped +SkinnedMeshRenderer: + m_CorrespondingSourceObject: {fileID: 8145277655520084301, guid: b0d64f917d2447e4f85037ad6283d5bf, type: 3} m_PrefabInstance: {fileID: 8448906606297741168} m_PrefabAsset: {fileID: 0} ---- !u!4 &8448906606779964024 stripped -Transform: - m_CorrespondingSourceObject: {fileID: 1027730696, guid: b0d64f917d2447e4f85037ad6283d5bf, +--- !u!137 &2129738114390050521 stripped +SkinnedMeshRenderer: + m_CorrespondingSourceObject: {fileID: 7552204851892433321, guid: b0d64f917d2447e4f85037ad6283d5bf, type: 3} m_PrefabInstance: {fileID: 8448906606297741168} m_PrefabAsset: {fileID: 0} @@ -292,27 +287,9 @@ MeshRenderer: type: 3} m_PrefabInstance: {fileID: 8448906606297741168} m_PrefabAsset: {fileID: 0} ---- !u!4 &8027680674902865557 stripped -Transform: - m_CorrespondingSourceObject: {fileID: 1884904610847097317, guid: b0d64f917d2447e4f85037ad6283d5bf, - type: 3} - m_PrefabInstance: {fileID: 8448906606297741168} - m_PrefabAsset: {fileID: 0} ---- !u!137 &2129738114390050521 stripped -SkinnedMeshRenderer: - m_CorrespondingSourceObject: {fileID: 7552204851892433321, guid: b0d64f917d2447e4f85037ad6283d5bf, - type: 3} - m_PrefabInstance: {fileID: 8448906606297741168} - m_PrefabAsset: {fileID: 0} ---- !u!4 &6815075459618467101 stripped -Transform: - m_CorrespondingSourceObject: {fileID: 3158299659199945325, guid: b0d64f917d2447e4f85037ad6283d5bf, - type: 3} - m_PrefabInstance: {fileID: 8448906606297741168} - m_PrefabAsset: {fileID: 0} ---- !u!137 &308880390316127805 stripped -SkinnedMeshRenderer: - m_CorrespondingSourceObject: {fileID: 8145277655520084301, guid: b0d64f917d2447e4f85037ad6283d5bf, +--- !u!1 &6433400705492442764 stripped +GameObject: + m_CorrespondingSourceObject: {fileID: 3172939939781028348, guid: b0d64f917d2447e4f85037ad6283d5bf, type: 3} m_PrefabInstance: {fileID: 8448906606297741168} m_PrefabAsset: {fileID: 0} @@ -329,6 +306,7 @@ MonoBehaviour: m_Name: m_EditorClassIdentifier: audioSourcePosition: {fileID: 7388576005872743378} + speechIndicator: {fileID: 2500143219345673597} --- !u!114 &8661746244001222169 MonoBehaviour: m_ObjectHideFlags: 0 @@ -407,3 +385,39 @@ MonoBehaviour: m_PreInfinity: 2 m_PostInfinity: 2 m_RotationOrder: 4 +--- !u!4 &6524766736450995788 stripped +Transform: + m_CorrespondingSourceObject: {fileID: 3444175152561107260, guid: b0d64f917d2447e4f85037ad6283d5bf, + type: 3} + m_PrefabInstance: {fileID: 8448906606297741168} + m_PrefabAsset: {fileID: 0} +--- !u!4 &6815075459618467101 stripped +Transform: + m_CorrespondingSourceObject: {fileID: 3158299659199945325, guid: b0d64f917d2447e4f85037ad6283d5bf, + type: 3} + m_PrefabInstance: {fileID: 8448906606297741168} + m_PrefabAsset: {fileID: 0} +--- !u!4 &7388576005872743378 stripped +Transform: + m_CorrespondingSourceObject: {fileID: 1425940620351308962, guid: b0d64f917d2447e4f85037ad6283d5bf, + type: 3} + m_PrefabInstance: {fileID: 8448906606297741168} + m_PrefabAsset: {fileID: 0} +--- !u!4 &8027680674902865557 stripped +Transform: + m_CorrespondingSourceObject: {fileID: 1884904610847097317, guid: b0d64f917d2447e4f85037ad6283d5bf, + type: 3} + m_PrefabInstance: {fileID: 8448906606297741168} + m_PrefabAsset: {fileID: 0} +--- !u!4 &8448906606779964024 stripped +Transform: + m_CorrespondingSourceObject: {fileID: 1027730696, guid: b0d64f917d2447e4f85037ad6283d5bf, + type: 3} + m_PrefabInstance: {fileID: 8448906606297741168} + m_PrefabAsset: {fileID: 0} +--- !u!23 &8557492846676609213 stripped +MeshRenderer: + m_CorrespondingSourceObject: {fileID: 253003865581486029, guid: b0d64f917d2447e4f85037ad6283d5bf, + type: 3} + m_PrefabInstance: {fileID: 8448906606297741168} + m_PrefabAsset: {fileID: 0} diff --git a/Unity/Assets/Samples/_Common/Avatar/SpeechIndicator/Avatar Speech Indicator.prefab b/Unity/Assets/Samples/_Common/Avatar/SpeechIndicator/Avatar Speech Indicator.prefab index 77f94ae69..808da8b19 100644 --- a/Unity/Assets/Samples/_Common/Avatar/SpeechIndicator/Avatar Speech Indicator.prefab +++ b/Unity/Assets/Samples/_Common/Avatar/SpeechIndicator/Avatar Speech Indicator.prefab @@ -27,6 +27,7 @@ Transform: m_LocalRotation: {x: 0.000000055879354, y: 0.7071054, z: -0.000000052154064, w: -0.7071082} m_LocalPosition: {x: 0, y: 0.00035412237, z: 0.38947874} m_LocalScale: {x: 0.03750001, y: 0.03750001, z: 0.0375} + m_ConstrainProportionsScale: 0 m_Children: [] m_Father: {fileID: 8332843025049073831} m_RootOrder: 2 @@ -42,10 +43,12 @@ SpriteRenderer: m_CastShadows: 0 m_ReceiveShadows: 0 m_DynamicOccludee: 1 + m_StaticShadowCaster: 0 m_MotionVectors: 1 m_LightProbeUsage: 1 m_ReflectionProbeUsage: 1 m_RayTracingMode: 0 + m_RayTraceProcedural: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -108,6 +111,7 @@ Transform: m_LocalRotation: {x: 0.000000055879354, y: 0.7071054, z: -0.000000052154064, w: -0.7071082} m_LocalPosition: {x: 0, y: 0.0003541112, z: 0.33447877} m_LocalScale: {x: 0.0225, y: 0.0225, z: 0.0225} + m_ConstrainProportionsScale: 0 m_Children: [] m_Father: {fileID: 8332843025049073831} m_RootOrder: 1 @@ -123,10 +127,12 @@ SpriteRenderer: m_CastShadows: 0 m_ReceiveShadows: 0 m_DynamicOccludee: 1 + m_StaticShadowCaster: 0 m_MotionVectors: 1 m_LightProbeUsage: 1 m_ReflectionProbeUsage: 1 m_RayTracingMode: 0 + m_RayTraceProcedural: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -171,7 +177,7 @@ GameObject: serializedVersion: 6 m_Component: - component: {fileID: 8332843025049073831} - - component: {fileID: 4330955028469597935} + - component: {fileID: 2818784942326102146} m_Layer: 0 m_Name: Avatar Speech Indicator m_TagString: Untagged @@ -189,6 +195,7 @@ Transform: m_LocalRotation: {x: -0.04492525, y: 0.0000005066394, z: 0.000000019537937, w: -0.9989904} m_LocalPosition: {x: 0, y: -0.062, z: 0.056} m_LocalScale: {x: 1, y: 1, z: 1} + m_ConstrainProportionsScale: 0 m_Children: - {fileID: 4024233642343254899} - {fileID: 8014688428566232123} @@ -196,7 +203,7 @@ Transform: m_Father: {fileID: 0} m_RootOrder: 0 m_LocalEulerAnglesHint: {x: 5.15, y: 0, z: 0} ---- !u!114 &4330955028469597935 +--- !u!114 &2818784942326102146 MonoBehaviour: m_ObjectHideFlags: 0 m_CorrespondingSourceObject: {fileID: 0} @@ -205,19 +212,19 @@ MonoBehaviour: m_GameObject: {fileID: 4422958779889287441} m_Enabled: 1 m_EditorHideFlags: 0 - m_Script: {fileID: 11500000, guid: 548f5a0b57b5ed64b8015809688ebfae, type: 3} + m_Script: {fileID: 11500000, guid: a3da99f06c526484e9de044be0c25a4c, type: 3} m_Name: m_EditorClassIdentifier: - mode: 1 volumeIndicators: - {fileID: 4024233642343254899} - {fileID: 8014688428566232123} - {fileID: 6408185506489093665} minIndicatorScale: {x: 0.0225, y: 0.0225, z: 0.0225} maxIndicatorScale: {x: 0.0375, y: 0.0375, z: 0.0375} - sampleSecondsPerIndicator: 0.08 minVolume: 0.005 + minVolumeWhenOn: 0.002 maxVolume: 0.02 + windowSeconds: 0.6 --- !u!1 &7907522415751529872 GameObject: m_ObjectHideFlags: 0 @@ -245,6 +252,7 @@ Transform: m_LocalRotation: {x: 0.000000055879354, y: 0.7071054, z: -0.000000052154064, w: -0.7071082} m_LocalPosition: {x: 0, y: 0.00035412423, z: 0.27947876} m_LocalScale: {x: 0.03, y: 0.03, z: 0.03} + m_ConstrainProportionsScale: 0 m_Children: [] m_Father: {fileID: 8332843025049073831} m_RootOrder: 0 @@ -260,10 +268,12 @@ SpriteRenderer: m_CastShadows: 0 m_ReceiveShadows: 0 m_DynamicOccludee: 1 + m_StaticShadowCaster: 0 m_MotionVectors: 1 m_LightProbeUsage: 1 m_ReflectionProbeUsage: 1 m_RayTracingMode: 0 + m_RayTraceProcedural: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: diff --git a/Unity/Assets/Samples/_Common/Avatar/SpeechIndicator/SpeechIndicator.cs b/Unity/Assets/Samples/_Common/Avatar/SpeechIndicator/SpeechIndicator.cs index 259af7099..8c7e47e38 100644 --- a/Unity/Assets/Samples/_Common/Avatar/SpeechIndicator/SpeechIndicator.cs +++ b/Unity/Assets/Samples/_Common/Avatar/SpeechIndicator/SpeechIndicator.cs @@ -12,185 +12,185 @@ namespace Ubiq.Samples /// public class SpeechIndicator : MonoBehaviour { - public enum Mode - { - Current, - History - } - - public Mode mode; - - public List volumeIndicators; - public Vector3 minIndicatorScale; - public Vector3 maxIndicatorScale; - public float sampleSecondsPerIndicator; - - public float minVolume; - public float maxVolume; - - private Avatars.Avatar avatar; - private VoipAvatar voipAvatar; - private int lastSampleTimeMilliseconds; - - private float currentFrameVolumeSum = 0; - private int currentFrameSampleCount = 0; - private float[] volumeFrames; - - private void Start() - { - avatar = GetComponentInParent(); - voipAvatar = GetComponentInParent(); - } - - private void LateUpdate() - { - if (!avatar || avatar.IsLocal || !voipAvatar) - { - Hide(); - enabled = false; - return; - } - - if (!voipAvatar.peerConnection) - { - Hide(); - return; - } - - UpdateSamples(); - UpdateIndicators(); - UpdatePosition(); - } - - private void UpdateSamples() - { - if (volumeFrames == null || volumeFrames.Length != volumeIndicators.Count) - { - volumeFrames = new float[volumeIndicators.Count]; - } - - var volumeWindowSampleCount = 0; - - var stats = voipAvatar.peerConnection.GetLastFramePlaybackStats(); - currentFrameVolumeSum += stats.volume; - currentFrameSampleCount += stats.samples; - volumeWindowSampleCount = (int)(sampleSecondsPerIndicator * stats.sampleRate); - - if (currentFrameSampleCount > volumeWindowSampleCount) - { - PushVolumeSample(currentFrameVolumeSum / currentFrameSampleCount); - currentFrameVolumeSum = 0; - currentFrameSampleCount = 0; - } - } - - private void PushVolumeSample(float sample) - { - for (int i = volumeFrames.Length - 1; i >= 1; i--) - { - volumeFrames[i] = volumeFrames[i-1]; - } - volumeFrames[0] = sample; - } - - private void UpdateIndicators() - { - switch(mode) - { - case Mode.Current : UpdateIndicatorsCurrent(); break; - case Mode.History : UpdateIndicatorsHistory(); break; - } - } - - private void UpdateIndicatorsCurrent() - { - if (volumeFrames.Length == 0) - { - return; - } - - var currentVolume = volumeFrames[0]; - var range = maxVolume - minVolume; - var t = (currentVolume - minVolume) / range; - var indicatorCount = Mathf.RoundToInt(t * volumeIndicators.Count); - - for (int i = 0; i < volumeIndicators.Count; i++) - { - volumeIndicators[i].gameObject.SetActive(i < indicatorCount); - var tScale = i/(float)volumeIndicators.Count; - volumeIndicators[i].localScale = Vector3.Lerp( - minIndicatorScale,maxIndicatorScale,tScale); - } - } - - private void UpdateIndicatorsHistory() - { - for (int i = 0; i < volumeFrames.Length; i++) - { - if (volumeFrames[i] > minVolume) - { - volumeIndicators[i].gameObject.SetActive(true); - var range = maxVolume - minVolume; - var t = (volumeFrames[i] - minVolume) / range; - volumeIndicators[i].localScale = Vector3.Lerp( - minIndicatorScale,maxIndicatorScale,t); - } - else - { - volumeIndicators[i].gameObject.SetActive(false); - } - } - } - - private void UpdatePosition() - { - var cameraTransform = Camera.main.transform; - var headTransform = transform.parent; - var indicatorRootTransform = transform; - - // If no indicator is being shown currently, reset position - var indicatorVisible = false; - for (int i = 0; i < volumeIndicators.Count; i++) - { - if (volumeIndicators[i].gameObject.activeInHierarchy) - { - indicatorVisible = true; - break; - } - } - - if (!indicatorVisible) - { - indicatorRootTransform.forward = headTransform.forward; - } - - // Rotate s.t. the indicator is always 90 deg from camera - // Method - always two acceptable orientations, pick the closest - var headToCamera = cameraTransform.position - headTransform.position; - var headToCameraDir = headToCamera.normalized; - var dirA = Vector3.Cross(headToCameraDir,headTransform.up); - var dirB = Vector3.Cross(headTransform.up,headToCameraDir); - - var simA = Vector3.Dot(dirA,indicatorRootTransform.forward); - var simB = Vector3.Dot(dirB,indicatorRootTransform.forward); - - var forward = simA > simB ? dirA : dirB; - - // Deal with rare case when avatars share a position - if (forward.sqrMagnitude <= 0) - { - forward = indicatorRootTransform.forward; - } - - indicatorRootTransform.forward = forward; - } - - private void Hide() - { - for (int i = 0; i < volumeIndicators.Count; i++) - { - volumeIndicators[i].gameObject.SetActive(false); - } - } + // public enum Mode + // { + // Current, + // History + // } + + // public Mode mode; + + // public List volumeIndicators; + // public Vector3 minIndicatorScale; + // public Vector3 maxIndicatorScale; + // public float sampleSecondsPerIndicator; + + // public float minVolume; + // public float maxVolume; + + // private Avatars.Avatar avatar; + // private VoipAvatar voipAvatar; + // private int lastSampleTimeMilliseconds; + + // private float currentFrameVolumeSum = 0; + // private int currentFrameSampleCount = 0; + // private float[] volumeFrames; + + // private void Start() + // { + // avatar = GetComponentInParent(); + // voipAvatar = GetComponentInParent(); + // } + + // private void LateUpdate() + // { + // if (!avatar || avatar.IsLocal || !voipAvatar) + // { + // Hide(); + // enabled = false; + // return; + // } + + // if (!voipAvatar.peerConnection) + // { + // Hide(); + // return; + // } + + // UpdateSamples(); + // UpdateIndicators(); + // UpdatePosition(); + // } + + // private void UpdateSamples() + // { + // if (volumeFrames == null || volumeFrames.Length != volumeIndicators.Count) + // { + // volumeFrames = new float[volumeIndicators.Count]; + // } + + // var volumeWindowSampleCount = 0; + + // var stats = voipAvatar.peerConnection.GetLastFramePlaybackStats(); + // currentFrameVolumeSum += stats.volume; + // currentFrameSampleCount += stats.samples; + // volumeWindowSampleCount = (int)(sampleSecondsPerIndicator * stats.sampleRate); + + // if (currentFrameSampleCount > volumeWindowSampleCount) + // { + // PushVolumeSample(currentFrameVolumeSum / currentFrameSampleCount); + // currentFrameVolumeSum = 0; + // currentFrameSampleCount = 0; + // } + // } + + // private void PushVolumeSample(float sample) + // { + // for (int i = volumeFrames.Length - 1; i >= 1; i--) + // { + // volumeFrames[i] = volumeFrames[i-1]; + // } + // volumeFrames[0] = sample; + // } + + // private void UpdateIndicators() + // { + // switch(mode) + // { + // case Mode.Current : UpdateIndicatorsCurrent(); break; + // case Mode.History : UpdateIndicatorsHistory(); break; + // } + // } + + // private void UpdateIndicatorsCurrent() + // { + // if (volumeFrames.Length == 0) + // { + // return; + // } + + // var currentVolume = volumeFrames[0]; + // var range = maxVolume - minVolume; + // var t = (currentVolume - minVolume) / range; + // var indicatorCount = Mathf.RoundToInt(t * volumeIndicators.Count); + + // for (int i = 0; i < volumeIndicators.Count; i++) + // { + // volumeIndicators[i].gameObject.SetActive(i < indicatorCount); + // var tScale = i/(float)volumeIndicators.Count; + // volumeIndicators[i].localScale = Vector3.Lerp( + // minIndicatorScale,maxIndicatorScale,tScale); + // } + // } + + // private void UpdateIndicatorsHistory() + // { + // for (int i = 0; i < volumeFrames.Length; i++) + // { + // if (volumeFrames[i] > minVolume) + // { + // volumeIndicators[i].gameObject.SetActive(true); + // var range = maxVolume - minVolume; + // var t = (volumeFrames[i] - minVolume) / range; + // volumeIndicators[i].localScale = Vector3.Lerp( + // minIndicatorScale,maxIndicatorScale,t); + // } + // else + // { + // volumeIndicators[i].gameObject.SetActive(false); + // } + // } + // } + + // private void UpdatePosition() + // { + // var cameraTransform = Camera.main.transform; + // var headTransform = transform.parent; + // var indicatorRootTransform = transform; + + // // If no indicator is being shown currently, reset position + // var indicatorVisible = false; + // for (int i = 0; i < volumeIndicators.Count; i++) + // { + // if (volumeIndicators[i].gameObject.activeInHierarchy) + // { + // indicatorVisible = true; + // break; + // } + // } + + // if (!indicatorVisible) + // { + // indicatorRootTransform.forward = headTransform.forward; + // } + + // // Rotate s.t. the indicator is always 90 deg from camera + // // Method - always two acceptable orientations, pick the closest + // var headToCamera = cameraTransform.position - headTransform.position; + // var headToCameraDir = headToCamera.normalized; + // var dirA = Vector3.Cross(headToCameraDir,headTransform.up); + // var dirB = Vector3.Cross(headTransform.up,headToCameraDir); + + // var simA = Vector3.Dot(dirA,indicatorRootTransform.forward); + // var simB = Vector3.Dot(dirB,indicatorRootTransform.forward); + + // var forward = simA > simB ? dirA : dirB; + + // // Deal with rare case when avatars share a position + // if (forward.sqrMagnitude <= 0) + // { + // forward = indicatorRootTransform.forward; + // } + + // indicatorRootTransform.forward = forward; + // } + + // private void Hide() + // { + // for (int i = 0; i < volumeIndicators.Count; i++) + // { + // volumeIndicators[i].gameObject.SetActive(false); + // } + // } } } \ No newline at end of file diff --git a/Unity/Assets/Samples/_Common/UI/Scripts/PeersPanelControl.cs b/Unity/Assets/Samples/_Common/UI/Scripts/PeersPanelControl.cs index b381f14d2..ef4033fa2 100644 --- a/Unity/Assets/Samples/_Common/UI/Scripts/PeersPanelControl.cs +++ b/Unity/Assets/Samples/_Common/UI/Scripts/PeersPanelControl.cs @@ -135,8 +135,9 @@ private void Update () { // var audioSink = peerConnection.audioSink; - var stats = peerConnection.GetLastFramePlaybackStats(); - var volume = stats.volume / stats.samples; + // var stats = new Ubiq.Voip.VoipPeerConnection.AudioStats(); + var volume = 0; + // var volume = stats.volumeSum / stats.samples; voipVolumeIndicator.Update(volume); voipConnectionIndicator.Update((int)peerConnection.peerConnectionState); From 5780ef6bd850a3bd202afab8504d7c660653877a Mon Sep 17 00:00:00 2001 From: bnco <33021110+bnco-dev@users.noreply.github.com> Date: Mon, 24 Jul 2023 12:23:59 +0100 Subject: [PATCH 2/2] Update speech indicator and add general purpose volume estimator --- .../Implementations/Unity/AudioStatsFilter.cs | 21 +- .../Implementations/Web/Plugins/WebRTC.jslib | 2 +- .../Runtime/Voip/VoipSpeechIndicator.cs | 190 +++++++++--------- Unity/Assets/Runtime/Voip/VolumeEstimator.cs | 106 ++++++++++ .../Runtime/Voip/VolumeEstimator.cs.meta | 11 + .../Avatar Speech Indicator.prefab | 13 +- 6 files changed, 235 insertions(+), 108 deletions(-) create mode 100644 Unity/Assets/Runtime/Voip/VolumeEstimator.cs create mode 100644 Unity/Assets/Runtime/Voip/VolumeEstimator.cs.meta diff --git a/Unity/Assets/Runtime/Voip/Implementations/Unity/AudioStatsFilter.cs b/Unity/Assets/Runtime/Voip/Implementations/Unity/AudioStatsFilter.cs index 20bfe8f87..651759721 100644 --- a/Unity/Assets/Runtime/Voip/Implementations/Unity/AudioStatsFilter.cs +++ b/Unity/Assets/Runtime/Voip/Implementations/Unity/AudioStatsFilter.cs @@ -13,25 +13,23 @@ public class AudioStatsFilter : MonoBehaviour private int sampleRate; private Action statsPushed; - private volatile int channels; - - void OnDestroy() + private void OnDestroy() { statsPushed = null; } - void OnEnable() + private void OnEnable() { OnAudioConfigurationChanged(false); AudioSettings.OnAudioConfigurationChanged += OnAudioConfigurationChanged; } - void OnDisable() + private void OnDisable() { AudioSettings.OnAudioConfigurationChanged -= OnAudioConfigurationChanged; } - void OnAudioConfigurationChanged(bool deviceWasChanged) + private void OnAudioConfigurationChanged(bool deviceWasChanged) { sampleRate = AudioSettings.outputSampleRate; } @@ -44,8 +42,6 @@ private void Update() stats.sampleCount,stats.volumeSum,sampleRate )); } - - Debug.Log(name + " " + channels); } public void SetStatsPushedCallback(Action statsPushed) @@ -60,17 +56,18 @@ public void SetStatsPushedCallback(Action statsPushed) /// /// /// - void OnAudioFilterRead(float[] data, int channels) + private void OnAudioFilterRead(float[] data, int channels) { - this.channels = channels; - var volumeSum = 0.0f; for (int i = 0; i < data.Length; i+=channels) { volumeSum += Mathf.Abs(data[i]); } - statsQueue.Enqueue(new AudioStats(data.Length/channels,volumeSum,0)); + var length = data.Length/channels; + volumeSum = volumeSum/channels; + + statsQueue.Enqueue(new AudioStats(length,volumeSum,0)); } } } \ No newline at end of file diff --git a/Unity/Assets/Runtime/Voip/Implementations/Web/Plugins/WebRTC.jslib b/Unity/Assets/Runtime/Voip/Implementations/Web/Plugins/WebRTC.jslib index a11f85a0b..3fa9a1654 100644 --- a/Unity/Assets/Runtime/Voip/Implementations/Web/Plugins/WebRTC.jslib +++ b/Unity/Assets/Runtime/Voip/Implementations/Web/Plugins/WebRTC.jslib @@ -204,7 +204,7 @@ var unityWebRtcInteropLibrary = { if (!context.processorNode) { let audioContext = context.audioContext; - context.processorNode = audioContext.createScriptProcessor(4096, 1, 1); + context.processorNode = audioContext.createScriptProcessor(1024, 1, 1); context.processorNode.onaudioprocess = ({inputBuffer}) => { let arr = inputBuffer.getChannelData(0); let length = arr.length; diff --git a/Unity/Assets/Runtime/Voip/VoipSpeechIndicator.cs b/Unity/Assets/Runtime/Voip/VoipSpeechIndicator.cs index 94f5b0a90..389cddd51 100644 --- a/Unity/Assets/Runtime/Voip/VoipSpeechIndicator.cs +++ b/Unity/Assets/Runtime/Voip/VoipSpeechIndicator.cs @@ -1,4 +1,3 @@ -using System.Collections; using System.Collections.Generic; using UnityEngine; @@ -6,105 +5,116 @@ namespace Ubiq.Voip { public class VoipSpeechIndicator : MonoBehaviour { - public List volumeIndicators; + private class SmoothVolumeEstimator + { + private VolumeEstimator prior; + private VolumeEstimator post; + private float priorTime; + private float postTime; + private VoipPeerConnection.AudioStats audioStats; + + public SmoothVolumeEstimator(float delaySeconds, float lengthSeconds) + { + prior = new VolumeEstimator(delaySeconds,lengthSeconds); + post = new VolumeEstimator(delaySeconds,lengthSeconds); + } + + public void Add(VoipPeerConnection.AudioStats audioStats, float time) + { + priorTime = postTime; + postTime = time; - public Vector3 minIndicatorScale; - public Vector3 maxIndicatorScale; + prior.PushAudioStats(this.audioStats); + post.PushAudioStats(audioStats); + this.audioStats = audioStats; + } - public float minVolume = 0.005f; - public float minVolumeWhenOn = 0.002f; // Should be lower for hysteresis - public float maxVolume = 0.02f; + public float GetVolume(float time) + { + var t = Mathf.InverseLerp(priorTime,postTime,time); + return Mathf.Lerp(prior.volume,post.volume,t); + } + } + [Tooltip("The indicators to enable and scale with VOIP volume. The lowest index is considered the most recent")] + public List volumeIndicators; + [Tooltip("The scale to set if the volume is at the VolumeFloor. Scales will be linearly interpolated between this and ScaleAtVolumeCeiling")] + public Vector3 scaleAtVolumeFloor; + [Tooltip("The scale to set if the volume is at the VolumeCeiling. Scales will be linearly interpolated between this and ScaleAtVolumeFloor")] + public Vector3 scaleAtVolumeCeiling; + [Tooltip("Above this volume, start showing the indicator")] + public float triggerVolume = 0.005f; + [Tooltip("Above this volume, keep showing the indicator, if we are already. Can be set lower than TriggerVolume for simple hysteresis")] + public float persistVolume = 0.002f; + [Tooltip("Lower bound of volume for size scaling. Most of the time, you'll want this to be the same as PersistVolume")] + public float volumeFloor = 0.002f; + [Tooltip("Upper bound of volume for size scaling")] + public float volumeCeiling = 0.02f; + [Tooltip("Total length of the sampling window. Samples older than this will be discarded")] public float windowSeconds = 0.6f; + [Tooltip("Proportion that a sub-window should overlap its neighbouring sub-windows, for smoother visuals. If 0, no overlap")] + public float windowOverlap = 0.3f; + [Tooltip("The max value of noise added or subtracted. Noise makes the indicators seem more dynamic at high framerates")] + public float noiseAmplitude = 0.05f; + [Tooltip("How quickly the noise appears to change. Noise makes the indicators seem more dynamic at high framerates")] + public float noiseFrequency = 1f; private List stats = new List(); - private List volumes = new List(); + private List volumeEstimators = new List(); + private List noises = new List(); private List indicatorStates = new List(); + private float time = 0; + + private const float NOISE_INIT_MULTIPLIER = 100.0f; void Update() { + time += Time.unscaledDeltaTime; UpdateVolumes(); + UpdateNoise(); UpdateIndicators(); UpdatePosition(); } private void UpdateVolumes() { - if (volumeIndicators.Count == 0 ) - { - return; - } - - var secondsPerIndicator = windowSeconds / volumeIndicators.Count; - if (volumes.Count != volumeIndicators.Count) + if (volumeEstimators.Count != volumeIndicators.Count) { - volumes.Clear(); + volumeEstimators.Clear(); + var secondsPerWindow = windowSeconds / volumeIndicators.Count; for (int i = 0; i < volumeIndicators.Count; i++) { - volumes.Add(0); - } - } - - for (int i = 0; i < volumes.Count; i++) - { - volumes[i] = 0; - } - - if (stats.Count == 0) - { - return; - } - - // Walk through the stats structs, calculate volume for each window - var samplesPerVolume = secondsPerIndicator * stats[0].sampleRate; - var volumeIdx = 0; - var sampleHead = 0.0f; - var volumeSum = 0.0f; - var endSamples = samplesPerVolume; - for (int i = stats.Count-1; i >= 0; i--) - { - var prevSampleHead = sampleHead; - while (sampleHead + stats[i].sampleCount > endSamples) - { - // Volume window ends in this audio stats struct - // Multiple volume windows may end in this stats struct - var samplesToUse = endSamples - sampleHead; - var proportion = samplesToUse / stats[i].sampleCount; - volumeSum += stats[i].volumeSum * proportion; - volumes[volumeIdx] = volumeSum / samplesPerVolume; - - volumeIdx++; - volumeSum = 0; - sampleHead = endSamples; - endSamples += samplesPerVolume; - - // We've fully filled all volume windows. Now remove any - // stats structs old enough to feature in no volume windows - if (volumeIdx >= volumes.Count) + var start = (i - windowOverlap) * secondsPerWindow; + start = Mathf.Max(0,start); + var length = (1 + windowOverlap) * secondsPerWindow; + if (start + length > windowSeconds) { - stats.RemoveRange(0,i); - break; + length = windowSeconds - start; } + volumeEstimators.Add(new SmoothVolumeEstimator(start,length)); } + } + } - if (volumeIdx >= volumes.Count) + private void RefreshNoises(bool force) + { + if (force || noises.Count != volumeIndicators.Count) + { + noises.Clear(); + for (int i = 0; i < volumeIndicators.Count; i++) { - break; + noises.Add(Random.value * noiseFrequency * NOISE_INIT_MULTIPLIER); } - - // Volume window stretches into next audio stats - var endSampleHead = prevSampleHead + stats[i].sampleCount; - var remainingSamples = endSampleHead - sampleHead; - volumeSum += stats[i].volumeSum * (remainingSamples / stats[i].sampleCount); - sampleHead = endSampleHead; } + } + + private void UpdateNoise() + { + RefreshNoises(force:false); - // Zero out any unfilled volumes. Means we need a full audio stat - // window or a volume will be zero. Should always be the case after - // the first second or so. - for (int i = volumeIdx; i < volumes.Count; i++) + for (int i = 0; i < noises.Count; i++) { - volumes[i] = 0; + noises[i] += noiseFrequency * Time.deltaTime; } } @@ -121,16 +131,18 @@ private void UpdateIndicators() for(int i = 0; i < volumeIndicators.Count; i++) { - var thresh = indicatorStates[i] ? minVolumeWhenOn : minVolume; - indicatorStates[i] = volumes[i] > thresh; + var vol = volumeEstimators[i].GetVolume(time); + var thresh = indicatorStates[i] ? persistVolume : triggerVolume; + indicatorStates[i] = vol > thresh; if (indicatorStates[i]) { + var noise = noiseAmplitude * Mathf.PerlinNoise(noises[i],0); + var noiseVec = Vector3.one * noise; volumeIndicators[i].gameObject.SetActive(true); - var range = maxVolume - minVolumeWhenOn; - var t = (volumes[i] - minVolumeWhenOn) / range; - volumeIndicators[i].localScale = Vector3.Lerp( - minIndicatorScale,maxIndicatorScale,t); + volumeIndicators[i].localScale = noiseVec + Vector3.Lerp( + scaleAtVolumeFloor,scaleAtVolumeCeiling, + Mathf.InverseLerp(volumeFloor,volumeCeiling,vol)); } else { @@ -159,6 +171,8 @@ private void UpdatePosition() if (!indicatorVisible) { indicatorRootTransform.forward = headTransform.forward; + IndicatorsInvisibleThisFrame(); + return; } // Rotate s.t. the indicator is always 90 deg from camera @@ -182,27 +196,23 @@ private void UpdatePosition() indicatorRootTransform.forward = forward; } + // Called every frame the indicators are invisible + private void IndicatorsInvisibleThisFrame() + { + RefreshNoises(force:true); + time = 0; + } + /// /// Pushes a new set of audio stats to the indicator. Treats the stats /// as a continuous stream, where these are the very latest stats. /// public void PushAudioStats(VoipPeerConnection.AudioStats stats) { - if (stats.sampleCount == 0) - { - return; - } - - if (this.stats.Count > 0 && this.stats[0].sampleRate != stats.sampleRate) + for (int i = 0; i < volumeEstimators.Count; i++) { - // May happen if the audio device changes, which should be rare - // so just clear the buffer and start again. Will mean a small - // interruption in the indicator, but it ensure the entire stats - // buffer has the same sampleRate. Simplifies things a lot. - this.stats.Clear(); + volumeEstimators[i].Add(stats,time); } - - this.stats.Add(stats); } } } diff --git a/Unity/Assets/Runtime/Voip/VolumeEstimator.cs b/Unity/Assets/Runtime/Voip/VolumeEstimator.cs new file mode 100644 index 000000000..ff44bf327 --- /dev/null +++ b/Unity/Assets/Runtime/Voip/VolumeEstimator.cs @@ -0,0 +1,106 @@ +using System; +using System.Collections.Generic; + +namespace Ubiq.Voip +{ + public class VolumeEstimator + { + public float volume { get { UpdateEstimate(); return _volume; } } + public float delaySeconds { get; private set; } + public float lengthSeconds { get; private set; } + + private List stats = new List(); + private bool isDirty = false; + private float _volume; + + public VolumeEstimator(float delaySeconds, float lengthSeconds) + { + SetWindow(delaySeconds,lengthSeconds); + } + + public void SetWindow(float delaySeconds, float lengthSeconds) + { + this.delaySeconds = delaySeconds; + this.lengthSeconds = lengthSeconds; + } + + /// + /// Pushes a new set of audio stats to the estimator. Treats the stats + /// as a continuous stream, where these are the very latest stats. + /// + public void PushAudioStats(VoipPeerConnection.AudioStats stats) + { + if (stats.sampleCount == 0) + { + return; + } + + if (this.stats.Count > 0 && this.stats[0].sampleRate != stats.sampleRate) + { + // May happen if the audio device changes, which should be rare + // so just clear the buffer and start again. Will mean a small + // interruption in the indicator, but it ensure the entire stats + // buffer has the same sampleRate. Simplifies things a lot. + this.stats.Clear(); + } + + this.stats.Add(stats); + isDirty = true; + } + + private void UpdateEstimate() + { + if (!isDirty) + { + return; + } + + if (stats.Count == 0) + { + _volume = 0; + return; + } + + var startSample = delaySeconds * stats[0].sampleRate; + var endSample = (delaySeconds + lengthSeconds) * stats[0].sampleRate; + var volumeSum = 0.0f; + var sampleCount = 0.0f; + var currentSample = 0; + var idx = stats.Count-1; + for (; idx >= 0; idx--) + { + if (currentSample + stats[idx].sampleCount > startSample) + { + var t = 1.0f; + var complete = false; + if (currentSample < startSample) + { + // First stats window + t = 1 - ((startSample - currentSample) / stats[idx].sampleCount); + } + + if (currentSample + stats[idx].sampleCount > endSample) + { + // Last stats window + t = (endSample - currentSample) / stats[idx].sampleCount; + complete = true; + } + + volumeSum += stats[idx].volumeSum * t; + sampleCount += stats[idx].sampleCount * t; + + if (complete) + { + break; + } + } + currentSample += stats[idx].sampleCount; + } + + stats.RemoveRange(0,idx > 0 ? idx : 0); + + _volume = volumeSum / sampleCount; + isDirty = false; + } + } +} \ No newline at end of file diff --git a/Unity/Assets/Runtime/Voip/VolumeEstimator.cs.meta b/Unity/Assets/Runtime/Voip/VolumeEstimator.cs.meta new file mode 100644 index 000000000..e78921802 --- /dev/null +++ b/Unity/Assets/Runtime/Voip/VolumeEstimator.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: ba59a6fe5eda2a6479de7f214075524b +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Unity/Assets/Samples/_Common/Avatar/SpeechIndicator/Avatar Speech Indicator.prefab b/Unity/Assets/Samples/_Common/Avatar/SpeechIndicator/Avatar Speech Indicator.prefab index 808da8b19..1fddc9d02 100644 --- a/Unity/Assets/Samples/_Common/Avatar/SpeechIndicator/Avatar Speech Indicator.prefab +++ b/Unity/Assets/Samples/_Common/Avatar/SpeechIndicator/Avatar Speech Indicator.prefab @@ -219,12 +219,15 @@ MonoBehaviour: - {fileID: 4024233642343254899} - {fileID: 8014688428566232123} - {fileID: 6408185506489093665} - minIndicatorScale: {x: 0.0225, y: 0.0225, z: 0.0225} - maxIndicatorScale: {x: 0.0375, y: 0.0375, z: 0.0375} - minVolume: 0.005 - minVolumeWhenOn: 0.002 - maxVolume: 0.02 + scaleAtVolumeFloor: {x: 0.0225, y: 0.0225, z: 0.0225} + scaleAtVolumeCeiling: {x: 0.0375, y: 0.0375, z: 0.0375} + triggerVolume: 0.005 + persistVolume: 0.002 + volumeFloor: 0.002 + volumeCeiling: 0.02 windowSeconds: 0.6 + noiseAmplitude: 0.0075 + noiseFrequency: 2 --- !u!1 &7907522415751529872 GameObject: m_ObjectHideFlags: 0