From c4e0646edcf4a6b53257c2ce8d521041074d20b0 Mon Sep 17 00:00:00 2001 From: luigi-rosso Date: Wed, 14 Feb 2024 13:20:46 +0000 Subject: [PATCH] Audio for Unity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This was quite the convoluted process and did not get implemented quite as expected, but it I eventually got to a solution which I think is good. - I originally looked at using the [Native Audio Plugin SDK](https://docs.unity3d.com/Manual/AudioMixerNativeAudioPlugin.html) but that was wildly incompatible with a graphics native plugin 🫠 - Then I found the [AudioClip API](https://docs.unity3d.com/ScriptReference/AudioClip.Create.html) which has a PCMReaderCallback! It worked but it had a ton of latency (like you'd hear things half a second delayed). - I decreased the sample frame size, no change. - I lowered the DSP buffer size in Unity, no change. - Finally found [this post](https://forum.unity.com/threads/audioclip-pcmreadercallback-has-insane-latency.1022065/) that confirmed what I was seeing. - That led me to the OnAudioFilterRead callback, which finally respects the DSP buffer size setting. Now there's no audible delay! Hover events with audio work great! What's cool about this is that you can inject a different AudioEngine per artboard (nested artboards inherit it) or you can share one. Separating them means that the AudioSource (if you use one instead of an AudioListener on a camera) will have spatial audio! So you'll hear the Audio from the artboard fade away as you walk away from whatever GameObject you attach the script to. Diffs= 61f553d6d Audio for Unity (#6606) Co-authored-by: Luigi Rosso --- .rive_head | 2 +- dependencies/premake5_miniaudio.lua | 1 - dependencies/premake5_miniaudio_v2.lua | 2 +- include/rive/artboard.hpp | 10 +++++++ include/rive/audio/audio_engine.hpp | 5 ++++ src/artboard.cpp | 16 +++++++++++ src/audio/audio_engine.cpp | 40 ++++++++++++++++++++++++++ src/audio_event.cpp | 9 +++++- 8 files changed, 81 insertions(+), 4 deletions(-) diff --git a/.rive_head b/.rive_head index 92d5321d..44802f14 100644 --- a/.rive_head +++ b/.rive_head @@ -1 +1 @@ -562fc5c51f6ea36b510c8403418e80569079a042 +61f553d6d079a00c264c8668c07762aee46b9720 diff --git a/dependencies/premake5_miniaudio.lua b/dependencies/premake5_miniaudio.lua index 183cfdce..a83cc41a 100644 --- a/dependencies/premake5_miniaudio.lua +++ b/dependencies/premake5_miniaudio.lua @@ -1,3 +1,2 @@ local dependency = require('dependency') --- miniaudio = dependency.github('rive-app/miniaudio', 'rive') miniaudio = dependency.github('rive-app/miniaudio', 'rive_changes') diff --git a/dependencies/premake5_miniaudio_v2.lua b/dependencies/premake5_miniaudio_v2.lua index c2661349..b193cb87 100644 --- a/dependencies/premake5_miniaudio_v2.lua +++ b/dependencies/premake5_miniaudio_v2.lua @@ -1,3 +1,3 @@ dofile('rive_build_config.lua') local dependency = require('dependency') -miniaudio = dependency.github('rive-app/miniaudio', 'rive') +miniaudio = dependency.github('rive-app/miniaudio', 'rive_changes') diff --git a/include/rive/artboard.hpp b/include/rive/artboard.hpp index 6fe3baa1..a110e0f7 100644 --- a/include/rive/artboard.hpp +++ b/include/rive/artboard.hpp @@ -11,6 +11,7 @@ #include "rive/shapes/shape_paint_container.hpp" #include "rive/text/text_value_run.hpp" #include "rive/event.hpp" +#include "rive/audio/audio_engine.hpp" #include #include @@ -57,6 +58,10 @@ class Artboard : public ArtboardBase, public CoreContext, public ShapePaintConta bool m_IsInstance = false; bool m_FrameOrigin = true; +#ifdef EXTERNAL_RIVE_AUDIO_ENGINE + rcp m_audioEngine; +#endif + void sortDependencies(); void sortDrawOrder(); @@ -251,6 +256,11 @@ class Artboard : public ArtboardBase, public CoreContext, public ShapePaintConta void frameOrigin(bool value); StatusCode import(ImportStack& importStack) override; + +#ifdef EXTERNAL_RIVE_AUDIO_ENGINE + rcp audioEngine() const; + void audioEngine(rcp audioEngine); +#endif }; class ArtboardInstance : public Artboard diff --git a/include/rive/audio/audio_engine.hpp b/include/rive/audio/audio_engine.hpp index b84a13fa..de7b21d4 100644 --- a/include/rive/audio/audio_engine.hpp +++ b/include/rive/audio/audio_engine.hpp @@ -45,6 +45,7 @@ class AudioEngine : public RefCnt #ifdef EXTERNAL_RIVE_AUDIO_ENGINE bool readAudioFrames(float* frames, uint64_t numFrames, uint64_t* framesRead = nullptr); + bool sumAudioFrames(float* frames, uint64_t numFrames); #endif private: @@ -57,6 +58,10 @@ class AudioEngine : public RefCnt void completeSound(rcp sound); void purgeCompletedSounds(); static void SoundCompleted(void* pUserData, ma_sound* pSound); + +#ifdef EXTERNAL_RIVE_AUDIO_ENGINE + std::vector m_readFrames; +#endif }; } // namespace rive diff --git a/src/artboard.cpp b/src/artboard.cpp index 0aa29a35..6f18718b 100644 --- a/src/artboard.cpp +++ b/src/artboard.cpp @@ -846,3 +846,19 @@ std::unique_ptr ArtboardInstance::defaultScene() } return scene; } + +#ifdef EXTERNAL_RIVE_AUDIO_ENGINE +rcp Artboard::audioEngine() const { return m_audioEngine; } +void Artboard::audioEngine(rcp audioEngine) +{ + m_audioEngine = audioEngine; + for (auto nestedArtboard : m_NestedArtboards) + { + auto artboard = nestedArtboard->artboard(); + if (artboard != nullptr) + { + artboard->audioEngine(audioEngine); + } + } +} +#endif \ No newline at end of file diff --git a/src/audio/audio_engine.cpp b/src/audio/audio_engine.cpp index d99fb617..c85eee17 100644 --- a/src/audio/audio_engine.cpp +++ b/src/audio/audio_engine.cpp @@ -1,4 +1,5 @@ #ifdef WITH_RIVE_AUDIO +#include "rive/math/simd.hpp" #ifdef __APPLE__ #include #if TARGET_IPHONE_SIMULATOR || TARGET_OS_MACCATALYST || TARGET_OS_IPHONE @@ -172,6 +173,45 @@ bool AudioEngine::readAudioFrames(float* frames, uint64_t numFrames, uint64_t* f (ma_uint64)numFrames, (ma_uint64*)framesRead) == MA_SUCCESS; } +bool AudioEngine::sumAudioFrames(float* frames, uint64_t numFrames) +{ + size_t numChannels = (size_t)channels(); + size_t count = (size_t)numFrames * numChannels; + + if (m_readFrames.size() < count) + { + m_readFrames.resize(count); + } + ma_uint64 framesRead = 0; + if (ma_engine_read_pcm_frames(m_engine, + (void*)m_readFrames.data(), + (ma_uint64)numFrames, + &framesRead) != MA_SUCCESS) + { + return false; + } + + count = framesRead * numChannels; + + const size_t alignedCount = count - count % 4; + float* src = m_readFrames.data(); + float* dst = frames; + float* srcEnd = src + alignedCount; + + while (src != srcEnd) + { + float4 sum = simd::load4f(src) + simd::load4f(dst); + simd::store(dst, sum); + + src += 4; + dst += 4; + } + for (size_t i = alignedCount; i < count; i++) + { + frames[i] += m_readFrames[i]; + } + return true; +} #endif #endif \ No newline at end of file diff --git a/src/audio_event.cpp b/src/audio_event.cpp index 3a6ead53..8ea32c67 100644 --- a/src/audio_event.cpp +++ b/src/audio_event.cpp @@ -2,6 +2,7 @@ #include "rive/assets/audio_asset.hpp" #include "rive/audio/audio_engine.hpp" #include "rive/audio/audio_sound.hpp" +#include "rive/artboard.hpp" using namespace rive; @@ -20,7 +21,13 @@ void AudioEvent::trigger(const CallbackData& value) { return; } - auto engine = AudioEngine::RuntimeEngine(); + + auto engine = +#ifdef EXTERNAL_RIVE_AUDIO_ENGINE + artboard()->audioEngine() != nullptr ? artboard()->audioEngine() : +#endif + AudioEngine::RuntimeEngine(); + engine->play(audioSource, engine->timeInFrames(), 0, 0); #endif }