From 288e2c66cf4924031b55edf57e354a0393963236 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Fri, 8 Dec 2023 18:17:07 +0100 Subject: [PATCH] feat: add svc codecs (#263) --- examples/wgpu_room/src/app.rs | 4 +-- examples/wgpu_room/src/logo_track.rs | 1 + examples/wgpu_room/src/service.rs | 27 +++++++++++++++++++++ livekit-ffi/protocol/video_frame.proto | 1 + livekit-ffi/src/conversion/video_frame.rs | 1 + livekit-ffi/src/livekit.proto.rs | 3 +++ livekit/src/room/options.rs | 2 ++ livekit/src/room/track/local_track.rs | 7 ++++++ livekit/src/room/track/remote_track.rs | 7 ++++++ webrtc-sys/libwebrtc/build_linux.sh | 1 + webrtc-sys/libwebrtc/build_macos.sh | 2 +- webrtc-sys/libwebrtc/build_windows.cmd | 2 +- webrtc-sys/libwebrtc/patches/add_deps.patch | 25 +++++++++++++++++-- webrtc-sys/src/video_decoder_factory.cpp | 27 +++++++++++++++++++++ webrtc-sys/src/video_encoder_factory.cpp | 19 +++++++++------ 15 files changed, 116 insertions(+), 13 deletions(-) diff --git a/examples/wgpu_room/src/app.rs b/examples/wgpu_room/src/app.rs index 7191d4a86..159614861 100644 --- a/examples/wgpu_room/src/app.rs +++ b/examples/wgpu_room/src/app.rs @@ -162,8 +162,8 @@ impl LkApp { }); ui.menu_button("Debug", |ui| { - if ui.button("Refresh stats").clicked() { - // TODO + if ui.button("Log stats").clicked() { + let _ = self.service.send(AsyncCmd::LogStats); } }); }); diff --git a/examples/wgpu_room/src/logo_track.rs b/examples/wgpu_room/src/logo_track.rs index aac58a6a9..bcd348521 100644 --- a/examples/wgpu_room/src/logo_track.rs +++ b/examples/wgpu_room/src/logo_track.rs @@ -1,6 +1,7 @@ use image::ImageFormat; use image::RgbaImage; use livekit::options::TrackPublishOptions; +use livekit::options::VideoCodec; use livekit::prelude::*; use livekit::webrtc::video_source::RtcVideoSource; use livekit::webrtc::video_source::VideoResolution; diff --git a/examples/wgpu_room/src/service.rs b/examples/wgpu_room/src/service.rs index e7bd6dfef..1d1d3653c 100644 --- a/examples/wgpu_room/src/service.rs +++ b/examples/wgpu_room/src/service.rs @@ -33,6 +33,7 @@ pub enum AsyncCmd { publication: RemoteTrackPublication, }, E2eeKeyRatchet, + LogStats, } #[derive(Debug)] @@ -199,6 +200,32 @@ async fn service_task(inner: Arc, mut cmd_rx: mpsc::UnboundedRecei } } } + AsyncCmd::LogStats => { + if let Some(state) = running_state.as_ref() { + for (_, publication) in state.room.local_participant().tracks() { + if let Some(track) = publication.track() { + log::info!( + "track stats: LOCAL {:?} {:?}", + track.sid(), + track.get_stats().await, + ); + } + } + + for (_, participant) in state.room.participants() { + for (_, publication) in participant.tracks() { + if let Some(track) = publication.track() { + log::info!( + "track stats: {:?} {:?} {:?}", + participant.identity(), + track.sid(), + track.get_stats().await, + ); + } + } + } + } + } } } } diff --git a/livekit-ffi/protocol/video_frame.proto b/livekit-ffi/protocol/video_frame.proto index 45faf9c7e..6734c299a 100644 --- a/livekit-ffi/protocol/video_frame.proto +++ b/livekit-ffi/protocol/video_frame.proto @@ -97,6 +97,7 @@ enum VideoCodec { VP8 = 0; H264 = 1; AV1 = 2; + VP9 = 3; } enum VideoRotation { diff --git a/livekit-ffi/src/conversion/video_frame.rs b/livekit-ffi/src/conversion/video_frame.rs index d5adb4d46..1fe56ab7e 100644 --- a/livekit-ffi/src/conversion/video_frame.rs +++ b/livekit-ffi/src/conversion/video_frame.rs @@ -132,6 +132,7 @@ impl From for VideoCodec { proto::VideoCodec::Vp8 => Self::VP8, proto::VideoCodec::H264 => Self::H264, proto::VideoCodec::Av1 => Self::AV1, + proto::VideoCodec::Vp9 => Self::VP9, } } } diff --git a/livekit-ffi/src/livekit.proto.rs b/livekit-ffi/src/livekit.proto.rs index 7f809d6bc..4dcc97290 100644 --- a/livekit-ffi/src/livekit.proto.rs +++ b/livekit-ffi/src/livekit.proto.rs @@ -1843,6 +1843,7 @@ pub enum VideoCodec { Vp8 = 0, H264 = 1, Av1 = 2, + Vp9 = 3, } impl VideoCodec { /// String value of the enum field names used in the ProtoBuf definition. @@ -1854,6 +1855,7 @@ impl VideoCodec { VideoCodec::Vp8 => "VP8", VideoCodec::H264 => "H264", VideoCodec::Av1 => "AV1", + VideoCodec::Vp9 => "VP9", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -1862,6 +1864,7 @@ impl VideoCodec { "VP8" => Some(Self::Vp8), "H264" => Some(Self::H264), "AV1" => Some(Self::Av1), + "VP9" => Some(Self::Vp9), _ => None, } } diff --git a/livekit/src/room/options.rs b/livekit/src/room/options.rs index 01e034019..ed7b93538 100644 --- a/livekit/src/room/options.rs +++ b/livekit/src/room/options.rs @@ -20,6 +20,7 @@ use livekit_protocol as proto; pub enum VideoCodec { VP8, H264, + VP9, AV1, } @@ -28,6 +29,7 @@ impl VideoCodec { match self { VideoCodec::VP8 => "vp8", VideoCodec::H264 => "h264", + VideoCodec::VP9 => "vp9", VideoCodec::AV1 => "av1", } } diff --git a/livekit/src/room/track/local_track.rs b/livekit/src/room/track/local_track.rs index 1fed93559..79c85d301 100644 --- a/livekit/src/room/track/local_track.rs +++ b/livekit/src/room/track/local_track.rs @@ -42,6 +42,13 @@ impl LocalTrack { Self::Video(track) => track.rtc_track().into(), } } + + pub async fn get_stats(&self) -> RoomResult> { + match self { + Self::Audio(track) => track.get_stats().await, + Self::Video(track) => track.get_stats().await, + } + } } impl From for Track { diff --git a/livekit/src/room/track/remote_track.rs b/livekit/src/room/track/remote_track.rs index 62fb3955f..75fd877e6 100644 --- a/livekit/src/room/track/remote_track.rs +++ b/livekit/src/room/track/remote_track.rs @@ -37,6 +37,13 @@ impl RemoteTrack { Self::Video(track) => track.rtc_track().into(), } } + + pub async fn get_stats(&self) -> RoomResult> { + match self { + Self::Audio(track) => track.get_stats().await, + Self::Video(track) => track.get_stats().await, + } + } } pub(super) async fn get_stats(inner: &Arc) -> RoomResult> { diff --git a/webrtc-sys/libwebrtc/build_linux.sh b/webrtc-sys/libwebrtc/build_linux.sh index c445fa6a4..1322a828f 100755 --- a/webrtc-sys/libwebrtc/build_linux.sh +++ b/webrtc-sys/libwebrtc/build_linux.sh @@ -91,6 +91,7 @@ args="is_debug=$debug \ rtc_build_tools=false \ rtc_build_examples=false \ rtc_libvpx_build_vp9=true \ + enable_libaom=true \ is_component_build=false \ enable_stripping=true \ use_goma=false \ diff --git a/webrtc-sys/libwebrtc/build_macos.sh b/webrtc-sys/libwebrtc/build_macos.sh index d5d4723a6..4498b2eb0 100755 --- a/webrtc-sys/libwebrtc/build_macos.sh +++ b/webrtc-sys/libwebrtc/build_macos.sh @@ -92,11 +92,11 @@ gn gen "$OUTPUT_DIR" --root="src" \ rtc_build_examples=false \ rtc_build_tools=false \ rtc_libvpx_build_vp9=true \ + enable_libaom=true \ is_component_build=false \ enable_stripping=true \ rtc_enable_symbol_export=true \ rtc_enable_objc_symbol_export=false \ - enable_libaom = true \ rtc_include_dav1d_in_internal_decoder_factory = true \ rtc_use_h264=true \ use_custom_libcxx=false \ diff --git a/webrtc-sys/libwebrtc/build_windows.cmd b/webrtc-sys/libwebrtc/build_windows.cmd index 77bf8925e..a1ecc33a6 100644 --- a/webrtc-sys/libwebrtc/build_windows.cmd +++ b/webrtc-sys/libwebrtc/build_windows.cmd @@ -65,7 +65,7 @@ if "!profile!" == "debug" ( rem generate ninja for release call gn.bat gen %OUTPUT_DIR% --root="src" ^ - --args="is_debug=!debug! is_clang=true target_cpu=\"!arch!\" use_custom_libcxx=false rtc_disable_check_msg=true rtc_include_tests=false rtc_build_examples=false rtc_build_tools=false is_component_build=false rtc_enable_protobuf=false rtc_use_h264=true ffmpeg_branding=\"Chrome\" symbol_level=0 enable_iterator_debugging=false" + --args="is_debug=!debug! is_clang=true target_cpu=\"!arch!\" use_custom_libcxx=false rtc_libvpx_build_vp9=true enable_libaom=true rtc_disable_check_msg=true rtc_include_tests=false rtc_build_examples=false rtc_build_tools=false is_component_build=false rtc_enable_protobuf=false rtc_use_h264=true ffmpeg_branding=\"Chrome\" symbol_level=0 enable_iterator_debugging=false" rem build ninja.exe -C %OUTPUT_DIR% :default diff --git a/webrtc-sys/libwebrtc/patches/add_deps.patch b/webrtc-sys/libwebrtc/patches/add_deps.patch index ab4ffefa2..1b383de4e 100644 --- a/webrtc-sys/libwebrtc/patches/add_deps.patch +++ b/webrtc-sys/libwebrtc/patches/add_deps.patch @@ -1,8 +1,29 @@ diff --git a/BUILD.gn b/BUILD.gn -index d5289b8..12685d1 100644 +index d5289b85d7..76823f6cff 100644 --- a/BUILD.gn +++ b/BUILD.gn -@@ -517,6 +517,10 @@ if (!build_with_chromium) { +@@ -24,6 +24,9 @@ + import("//build/config/linux/pkg_config.gni") + import("//build/config/sanitizers/sanitizers.gni") + import("webrtc.gni") ++import("//third_party/libaom/options.gni") ++ ++ + if (rtc_enable_protobuf) { + import("//third_party/protobuf/proto_library.gni") + } +@@ -292,6 +295,10 @@ config("common_config") { + defines += [ "WEBRTC_INCLUDE_INTERNAL_AUDIO_DEVICE" ] + } + ++ if (enable_libaom) { ++ defines += [ "RTC_USE_LIBAOM_AV1_ENCODER" ] ++ } ++ + if (rtc_libvpx_build_vp9) { + defines += [ "RTC_ENABLE_VP9" ] + } +@@ -517,6 +524,10 @@ if (!build_with_chromium) { "pc:rtc_pc", "sdk", "video", diff --git a/webrtc-sys/src/video_decoder_factory.cpp b/webrtc-sys/src/video_decoder_factory.cpp index a988231ab..40e742897 100644 --- a/webrtc-sys/src/video_decoder_factory.cpp +++ b/webrtc-sys/src/video_decoder_factory.cpp @@ -16,13 +16,19 @@ #include "livekit/video_decoder_factory.h" +#include "api/video_codecs/av1_profile.h" #include "api/video_codecs/sdp_video_format.h" #include "livekit/objc_video_factory.h" #include "media/base/media_constants.h" #include "modules/video_coding/codecs/h264/include/h264.h" #include "modules/video_coding/codecs/vp8/include/vp8.h" +#include "modules/video_coding/codecs/vp9/include/vp9.h" #include "rtc_base/logging.h" +#if defined(RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY) +#include "modules/video_coding/codecs/av1/dav1d_decoder.h" // nogncheck +#endif + #ifdef WEBRTC_ANDROID #include "livekit/android.h" #endif @@ -52,10 +58,21 @@ std::vector VideoDecoderFactory::GetSupportedFormats() } formats.push_back(webrtc::SdpVideoFormat(cricket::kVp8CodecName)); + for (const webrtc::SdpVideoFormat& format : + webrtc::SupportedVP9DecoderCodecs()) + formats.push_back(format); for (const webrtc::SdpVideoFormat& h264_format : webrtc::SupportedH264DecoderCodecs()) formats.push_back(h264_format); +#if defined(RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY) + formats.push_back(webrtc::SdpVideoFormat(cricket::kAv1CodecName)); + formats.push_back(webrtc::SdpVideoFormat( + cricket::kAv1CodecName, + {{webrtc::kAV1FmtpProfile, + AV1ProfileToString(webrtc::AV1Profile::kProfile1).data()}})); +#endif + return formats; } @@ -86,9 +103,19 @@ std::unique_ptr VideoDecoderFactory::CreateVideoDecoder( if (absl::EqualsIgnoreCase(format.name, cricket::kVp8CodecName)) return webrtc::VP8Decoder::Create(); + if (absl::EqualsIgnoreCase(format.name, cricket::kVp9CodecName)) + return webrtc::VP9Decoder::Create(); if (absl::EqualsIgnoreCase(format.name, cricket::kH264CodecName)) return webrtc::H264Decoder::Create(); + +#if defined(RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY) + if (absl::EqualsIgnoreCase(format.name, cricket::kAv1CodecName)) { + return webrtc::CreateDav1dDecoder(); + } +#endif + + RTC_LOG(LS_ERROR) << "No VideoDecoder found for " << format.name; return nullptr; } diff --git a/webrtc-sys/src/video_encoder_factory.cpp b/webrtc-sys/src/video_encoder_factory.cpp index 6d293ee99..b2005867c 100644 --- a/webrtc-sys/src/video_encoder_factory.cpp +++ b/webrtc-sys/src/video_encoder_factory.cpp @@ -19,15 +19,18 @@ #include "api/video_codecs/sdp_video_format.h" #include "api/video_codecs/video_encoder.h" #include "api/video_codecs/video_encoder_factory_template.h" -#include "api/video_codecs/video_encoder_factory_template_libvpx_vp8_adapter.h" #include "livekit/objc_video_factory.h" #include "media/base/media_constants.h" #include "media/engine/simulcast_encoder_adapter.h" #include "rtc_base/logging.h" - +#if defined(RTC_USE_LIBAOM_AV1_ENCODER) +#include "api/video_codecs/video_encoder_factory_template_libaom_av1_adapter.h" +#endif #if defined(WEBRTC_USE_H264) #include "api/video_codecs/video_encoder_factory_template_open_h264_adapter.h" #endif +#include "api/video_codecs/video_encoder_factory_template_libvpx_vp8_adapter.h" +#include "api/video_codecs/video_encoder_factory_template_libvpx_vp9_adapter.h" #ifdef WEBRTC_ANDROID #include "livekit/android.h" @@ -35,13 +38,15 @@ namespace livekit { -using Factory = - webrtc::VideoEncoderFactoryTemplate; + webrtc::LibvpxVp9EncoderTemplateAdapter>; VideoEncoderFactory::InternalFactory::InternalFactory() { #ifdef __APPLE__