expose soxr on the ffi (#444)

Cargo.toml

@@ -5,6 +5,7 @@ members = [
     "livekit-protocol",
     "livekit-ffi",
     "libwebrtc",
+    "soxr-sys",
     "webrtc-sys",
     "webrtc-sys/build",
 ]

livekit-ffi/Cargo.toml

@@ -19,6 +19,7 @@ tracing = ["tokio/tracing", "console-subscriber"]
 
 [dependencies]
 livekit = { path = "../livekit", version = "0.6.0" }
+soxr-sys = { path = "../soxr-sys" }
 livekit-protocol = { path = "../livekit-protocol", version = "0.3.5" }
 tokio = { version = "1", features = ["full", "parking_lot"] }
 futures-util = { version = "0.3", default-features = false, features = ["sink"] }

livekit-ffi/protocol/audio_frame.proto

@@ -87,6 +87,70 @@ message RemixAndResampleResponse {
   OwnedAudioFrameBuffer buffer = 1;
 }
 
+
+// New resampler using SoX (much better quality)
+
+message NewSoxResamplerRequest {
+  double input_rate = 1;
+  double output_rate  = 2;
+  uint32 num_channels = 3;
+  SoxResamplerDataType input_data_type = 4;
+  SoxResamplerDataType output_data_type = 5;
+  SoxQualityRecipe quality_recipe = 6;
+  uint32 flags = 7;
+}
+message NewSoxResamplerResponse {
+  OwnedSoxResampler resampler = 1;
+  optional string error = 2;
+}
+
+message PushSoxResamplerRequest {
+  uint64 resampler_handle = 1;
+  uint64 data_ptr = 2; // *const i16
+  uint32 size = 3; // in bytes
+}
+
+message PushSoxResamplerResponse {
+  uint64 output_ptr = 1; // *const i16 (could be null)
+  uint32 size = 2; // in bytes
+  optional string error = 3;
+}
+
+message FlushSoxResamplerRequest {
+  uint64 resampler_handle = 1;
+}
+
+message FlushSoxResamplerResponse {
+  uint64 output_ptr = 1; // *const i16 (could be null)
+  uint32 size = 2; // in bytes
+  optional string error = 3;
+}
+
+enum SoxResamplerDataType {
+  // TODO(theomonnom): support other datatypes (shouldn't really be needed)
+  SOXR_DATATYPE_INT16I = 0;
+  SOXR_DATATYPE_INT16S = 1;
+}
+
+enum SoxQualityRecipe {
+  SOXR_QUALITY_QUICK = 0;
+  SOXR_QUALITY_LOW = 1;
+  SOXR_QUALITY_MEDIUM = 2;
+  SOXR_QUALITY_HIGH = 3;
+  SOXR_QUALITY_VERYHIGH = 4;
+}
+
+enum SoxFlagBits {
+  SOXR_ROLLOFF_SMALL = 0;  // 1 << 0
+  SOXR_ROLLOFF_MEDIUM = 1; // 1 << 1
+  SOXR_ROLLOFF_NONE = 2;   // 1 << 2
+  SOXR_HIGH_PREC_CLOCK = 3; // 1 << 3
+  SOXR_DOUBLE_PRECISION = 4; // 1 << 4
+  SOXR_VR = 5; // 1 << 5
+}
+
+
+
 //
 // AudioFrame buffer
 //
@@ -168,3 +232,17 @@ message OwnedAudioResampler {
   FfiOwnedHandle handle = 1;
   AudioResamplerInfo info = 2;
 }
+
+
+
+//
+// Sox AudioResampler
+//
+
+
+message SoxResamplerInfo {}
+
+message OwnedSoxResampler {
+  FfiOwnedHandle handle = 1;
+  SoxResamplerInfo info = 2;
+}

livekit-ffi/protocol/ffi.proto

@@ -77,7 +77,7 @@ message FfiRequest {
     EnableRemoteTrackRequest enable_remote_track = 18;
     GetStatsRequest get_stats = 19;
 
-    // Video
+// Video
     NewVideoStreamRequest new_video_stream = 20;
     NewVideoSourceRequest new_video_source = 21;
     CaptureVideoFrameRequest capture_video_frame = 22;
@@ -93,6 +93,9 @@ message FfiRequest {
     RemixAndResampleRequest remix_and_resample = 30;
     E2eeRequest e2ee = 31;
     AudioStreamFromParticipantRequest audio_stream_from_participant = 32;
+    NewSoxResamplerRequest new_sox_resampler = 33;
+    PushSoxResamplerRequest push_sox_resampler = 34;
+    FlushSoxResamplerRequest flush_sox_resampler = 35;
   }
 }
 
@@ -138,6 +141,9 @@ message FfiResponse {
     RemixAndResampleResponse remix_and_resample = 30;
     AudioStreamFromParticipantResponse audio_stream_from_participant = 31;
     E2eeResponse e2ee = 32;
+    NewSoxResamplerResponse new_sox_resampler = 33;
+    PushSoxResamplerResponse push_sox_resampler = 34;
+    FlushSoxResamplerResponse flush_sox_resampler = 35;
   }
 }
 

livekit-ffi/src/conversion/mod.rs

@@ -14,6 +14,7 @@
 
 pub mod audio_frame;
 pub mod participant;
+pub mod resampler;
 pub mod room;
 pub mod stats;
 pub mod track;

livekit-ffi/src/conversion/resampler.rs

@@ -0,0 +1 @@
+