From 42aac20207aa6f12b297083c55353a44f3d2cda4 Mon Sep 17 00:00:00 2001 From: Iason Paraskevopoulos Date: Wed, 23 Oct 2024 09:52:54 +0100 Subject: [PATCH] Enable buffer scaling Exports VideoFrameBuffer::Scale for every buffer by extending the livekit::*Buffer types. --- libwebrtc/src/native/video_frame.rs | 48 ++++++++++++ libwebrtc/src/video_frame.rs | 24 ++++++ .../include/livekit/video_frame_buffer.h | 27 +++++++ webrtc-sys/src/video_frame_buffer.cpp | 74 +++++++++++++++++++ webrtc-sys/src/video_frame_buffer.rs | 16 ++++ 5 files changed, 189 insertions(+) diff --git a/libwebrtc/src/native/video_frame.rs b/libwebrtc/src/native/video_frame.rs index 4f665ad04..0f2250fb0 100644 --- a/libwebrtc/src/native/video_frame.rs +++ b/libwebrtc/src/native/video_frame.rs @@ -312,6 +312,14 @@ impl I420Buffer { ) } } + + pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> vf::I420Buffer { + vf::I420Buffer { + handle: I420Buffer { + sys_handle: self.sys_handle.pin_mut().scale(scaled_width, scaled_height), + }, + } + } } impl I420ABuffer { @@ -410,6 +418,14 @@ impl I420ABuffer { ) } } + + pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> vf::I420ABuffer { + vf::I420ABuffer { + handle: I420ABuffer { + sys_handle: self.sys_handle.pin_mut().scale(scaled_width, scaled_height), + }, + } + } } impl I422Buffer { @@ -516,6 +532,14 @@ impl I422Buffer { ) } } + + pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> vf::I422Buffer { + vf::I422Buffer { + handle: I422Buffer { + sys_handle: self.sys_handle.pin_mut().scale(scaled_width, scaled_height), + }, + } + } } impl I444Buffer { pub fn new( @@ -621,6 +645,14 @@ impl I444Buffer { ) } } + + pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> vf::I444Buffer { + vf::I444Buffer { + handle: I444Buffer { + sys_handle: self.sys_handle.pin_mut().scale(scaled_width, scaled_height), + }, + } + } } impl I010Buffer { @@ -738,6 +770,14 @@ impl I010Buffer { ) } } + + pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> vf::I010Buffer { + vf::I010Buffer { + handle: I010Buffer { + sys_handle: self.sys_handle.pin_mut().scale(scaled_width, scaled_height), + }, + } + } } impl NV12Buffer { @@ -843,4 +883,12 @@ impl NV12Buffer { ) } } + + pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> vf::NV12Buffer { + vf::NV12Buffer { + handle: NV12Buffer { + sys_handle: self.sys_handle.pin_mut().scale(scaled_width, scaled_height), + }, + } + } } diff --git a/libwebrtc/src/video_frame.rs b/libwebrtc/src/video_frame.rs index a2732a137..1b77c51d4 100644 --- a/libwebrtc/src/video_frame.rs +++ b/libwebrtc/src/video_frame.rs @@ -235,6 +235,10 @@ impl I420Buffer { ) } } + + pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> I420Buffer { + self.handle.scale(scaled_width, scaled_height) + } } impl I420ABuffer { @@ -274,6 +278,10 @@ impl I420ABuffer { ) } } + + pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> I420ABuffer { + self.handle.scale(scaled_width, scaled_height) + } } impl I422Buffer { @@ -317,6 +325,10 @@ impl I422Buffer { ) } } + + pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> I422Buffer { + self.handle.scale(scaled_width, scaled_height) + } } impl I444Buffer { @@ -360,6 +372,10 @@ impl I444Buffer { ) } } + + pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> I444Buffer { + self.handle.scale(scaled_width, scaled_height) + } } impl I010Buffer { @@ -403,6 +419,10 @@ impl I010Buffer { ) } } + + pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> I010Buffer { + self.handle.scale(scaled_width, scaled_height) + } } impl NV12Buffer { @@ -439,6 +459,10 @@ impl NV12Buffer { ) } } + + pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> NV12Buffer { + self.handle.scale(scaled_width, scaled_height) + } } #[cfg(not(target_arch = "wasm32"))] diff --git a/webrtc-sys/include/livekit/video_frame_buffer.h b/webrtc-sys/include/livekit/video_frame_buffer.h index a7e25fdf5..5fc31395d 100644 --- a/webrtc-sys/include/livekit/video_frame_buffer.h +++ b/webrtc-sys/include/livekit/video_frame_buffer.h @@ -151,6 +151,11 @@ class BiplanarYuv8Buffer : public BiplanarYuvBuffer { class I420Buffer : public PlanarYuv8Buffer { public: explicit I420Buffer(rtc::scoped_refptr buffer); + + std::unique_ptr scale(int scaled_width, int scaled_height) const; + + private: + webrtc::I420BufferInterface* buffer() const; }; class I420ABuffer : public I420Buffer { @@ -160,6 +165,8 @@ class I420ABuffer : public I420Buffer { unsigned int stride_a() const; const uint8_t* data_a() const; + std::unique_ptr scale(int scaled_width, int scaled_height) const; + private: webrtc::I420ABufferInterface* buffer() const; }; @@ -167,21 +174,41 @@ class I420ABuffer : public I420Buffer { class I422Buffer : public PlanarYuv8Buffer { public: explicit I422Buffer(rtc::scoped_refptr buffer); + + std::unique_ptr scale(int scaled_width, int scaled_height) const; + + private: + webrtc::I422BufferInterface* buffer() const; }; class I444Buffer : public PlanarYuv8Buffer { public: explicit I444Buffer(rtc::scoped_refptr buffer); + + std::unique_ptr scale(int scaled_width, int scaled_height) const; + + private: + webrtc::I444BufferInterface* buffer() const; }; class I010Buffer : public PlanarYuv16BBuffer { public: explicit I010Buffer(rtc::scoped_refptr buffer); + + std::unique_ptr scale(int scaled_width, int scaled_height) const; + + private: + webrtc::I010BufferInterface* buffer() const; }; class NV12Buffer : public BiplanarYuv8Buffer { public: explicit NV12Buffer(rtc::scoped_refptr buffer); + + std::unique_ptr scale(int scaled_width, int scaled_height) const; + + private: + webrtc::NV12BufferInterface* buffer() const; }; std::unique_ptr copy_i420_buffer( diff --git a/webrtc-sys/src/video_frame_buffer.cpp b/webrtc-sys/src/video_frame_buffer.cpp index b33ccf18a..990e9d761 100644 --- a/webrtc-sys/src/video_frame_buffer.cpp +++ b/webrtc-sys/src/video_frame_buffer.cpp @@ -192,6 +192,19 @@ webrtc::BiplanarYuv8Buffer* BiplanarYuv8Buffer::buffer() const { I420Buffer::I420Buffer(rtc::scoped_refptr buffer) : PlanarYuv8Buffer(buffer) {} +webrtc::I420BufferInterface* I420Buffer::buffer() const { + return static_cast(buffer_.get()); +} + +std::unique_ptr I420Buffer::scale(int scaled_width, + int scaled_height) const { + rtc::scoped_refptr result = + buffer()->Scale(scaled_width, scaled_height); + return std::make_unique( + rtc::scoped_refptr( + const_cast(result->GetI420()))); +} + I420ABuffer::I420ABuffer( rtc::scoped_refptr buffer) : I420Buffer(buffer) {} @@ -208,18 +221,79 @@ webrtc::I420ABufferInterface* I420ABuffer::buffer() const { return static_cast(buffer_.get()); } +std::unique_ptr I420ABuffer::scale(int scaled_width, + int scaled_height) const { + rtc::scoped_refptr result = + buffer()->Scale(scaled_width, scaled_height); + return std::make_unique( + rtc::scoped_refptr( + const_cast(result->GetI420A()))); +} + I422Buffer::I422Buffer(rtc::scoped_refptr buffer) : PlanarYuv8Buffer(buffer) {} +webrtc::I422BufferInterface* I422Buffer::buffer() const { + return static_cast(buffer_.get()); +} + +std::unique_ptr I422Buffer::scale(int scaled_width, + int scaled_height) const { + rtc::scoped_refptr result = + buffer()->Scale(scaled_width, scaled_height); + return std::make_unique( + rtc::scoped_refptr( + const_cast(result->GetI422()))); +} + I444Buffer::I444Buffer(rtc::scoped_refptr buffer) : PlanarYuv8Buffer(buffer) {} +webrtc::I444BufferInterface* I444Buffer::buffer() const { + return static_cast(buffer_.get()); +} + +std::unique_ptr I444Buffer::scale(int scaled_width, + int scaled_height) const { + rtc::scoped_refptr result = + buffer()->Scale(scaled_width, scaled_height); + return std::make_unique( + rtc::scoped_refptr( + const_cast(result->GetI444()))); +} + I010Buffer::I010Buffer(rtc::scoped_refptr buffer) : PlanarYuv16BBuffer(buffer) {} +webrtc::I010BufferInterface* I010Buffer::buffer() const { + return static_cast(buffer_.get()); +} + +std::unique_ptr I010Buffer::scale(int scaled_width, + int scaled_height) const { + rtc::scoped_refptr result = + buffer()->Scale(scaled_width, scaled_height); + return std::make_unique( + rtc::scoped_refptr( + const_cast(result->GetI010()))); +} + NV12Buffer::NV12Buffer(rtc::scoped_refptr buffer) : BiplanarYuv8Buffer(buffer) {} +webrtc::NV12BufferInterface* NV12Buffer::buffer() const { + return static_cast(buffer_.get()); +} + +std::unique_ptr NV12Buffer::scale(int scaled_width, + int scaled_height) const { + rtc::scoped_refptr result = + buffer()->Scale(scaled_width, scaled_height); + return std::make_unique( + rtc::scoped_refptr( + const_cast(result->GetNV12()))); +} + std::unique_ptr copy_i420_buffer( const std::unique_ptr& i420) { return std::make_unique(webrtc::I420Buffer::Copy(*i420->get())); diff --git a/webrtc-sys/src/video_frame_buffer.rs b/webrtc-sys/src/video_frame_buffer.rs index 4a202aad0..8f0cadae1 100644 --- a/webrtc-sys/src/video_frame_buffer.rs +++ b/webrtc-sys/src/video_frame_buffer.rs @@ -87,6 +87,22 @@ pub mod ffi { fn stride_a(self: &I420ABuffer) -> u32; fn data_a(self: &I420ABuffer) -> *const u8; + fn scale(self: &I420Buffer, scaled_width: i32, scaled_height: i32) + -> UniquePtr; + fn scale( + self: &I420ABuffer, + scaled_width: i32, + scaled_height: i32, + ) -> UniquePtr; + fn scale(self: &I422Buffer, scaled_width: i32, scaled_height: i32) + -> UniquePtr; + fn scale(self: &I444Buffer, scaled_width: i32, scaled_height: i32) + -> UniquePtr; + fn scale(self: &I010Buffer, scaled_width: i32, scaled_height: i32) + -> UniquePtr; + fn scale(self: &NV12Buffer, scaled_width: i32, scaled_height: i32) + -> UniquePtr; + fn copy_i420_buffer(i420: &UniquePtr) -> UniquePtr; fn new_i420_buffer( width: i32,