Skip to content

Commit

Permalink
Set is_screencast in publish_track
Browse files Browse the repository at this point in the history
  • Loading branch information
iparaskev committed Jan 24, 2025
1 parent d276130 commit 6d49d4c
Show file tree
Hide file tree
Showing 12 changed files with 98 additions and 61 deletions.
8 changes: 4 additions & 4 deletions .nanpa/properly_set_is_screencast.kdl
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
patch type="added" "Set is_screencast on VideoTrackSource creation" package="webrtc-sys"
patch type="added" "Set is_screencast on NativeVideoSource creation" package="libwebrtc"
patch type="added" "Extend NewVideoSourceRequest with is_screencast" package="livekit-ffi"
patch type="changed" "Use new NativeVideoSource constructor in examples" package="examples"
patch type="added" "Add a setter for is_screencast in InternalSource" package="webrtc-sys"
patch type="added" "Expose set_is_screencast in NativeVideoSource" package="libwebrtc"
patch type="added" "Extend enum_dispatch to create functions that don't have a return type" package="livekit-protocol"
patch type="added" "Set is_screencast in publish_track when the source is Screenshare" package="livekit"
19 changes: 13 additions & 6 deletions examples/wgpu_room/src/logo_track.rs
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,10 @@ pub struct LogoTrack {
impl LogoTrack {
pub fn new(room: Arc<Room>) -> Self {
Self {
rtc_source: NativeVideoSource::new(
VideoResolution { width: FB_WIDTH as u32, height: FB_HEIGHT as u32 },
false,
),
rtc_source: NativeVideoSource::new(VideoResolution {
width: FB_WIDTH as u32,
height: FB_HEIGHT as u32,
}),
room,
handle: None,
}
Expand Down Expand Up @@ -83,7 +83,11 @@ impl LogoTrack {
)
.await?;

let handle = TrackHandle { close_tx, task, track };
let handle = TrackHandle {
close_tx,
task,
track,
};

self.handle = Some(handle);
Ok(())
Expand All @@ -94,7 +98,10 @@ impl LogoTrack {
let _ = handle.close_tx.send(());
let _ = handle.task.await;

self.room.local_participant().unpublish_track(&handle.track.sid()).await?;
self.room
.local_participant()
.unpublish_track(&handle.track.sid())
.await?;
}
Ok(())
}
Expand Down
13 changes: 8 additions & 5 deletions libwebrtc/src/native/video_source.rs
Original file line number Diff line number Diff line change
Expand Up @@ -50,12 +50,11 @@ struct VideoSourceInner {
}

impl NativeVideoSource {
pub fn new(resolution: VideoResolution, is_screencast: bool) -> NativeVideoSource {
pub fn new(resolution: VideoResolution) -> NativeVideoSource {
let source = Self {
sys_handle: vt_sys::ffi::new_video_track_source(
&vt_sys::ffi::VideoResolution::from(resolution.clone()),
is_screencast,
),
sys_handle: vt_sys::ffi::new_video_track_source(&vt_sys::ffi::VideoResolution::from(
resolution.clone(),
)),
inner: Arc::new(Mutex::new(VideoSourceInner { captured_frames: 0 })),
};

Expand Down Expand Up @@ -114,4 +113,8 @@ impl NativeVideoSource {
pub fn video_resolution(&self) -> VideoResolution {
self.sys_handle.video_resolution().into()
}

pub fn set_is_screencast(&self, is_screencast: bool) {
self.sys_handle.set_is_screencast(is_screencast);
}
}
14 changes: 11 additions & 3 deletions libwebrtc/src/video_source.rs
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,10 @@ impl RtcVideoSource {
[Native];
pub fn video_resolution(self: &Self) -> VideoResolution;
);
enum_dispatch!(
[Native];
pub fn set_is_screencast(self: &Self, is_screencast: bool);
);
}

#[cfg(not(target_arch = "wasm32"))]
Expand All @@ -58,13 +62,13 @@ pub mod native {

impl Default for NativeVideoSource {
fn default() -> Self {
Self::new(VideoResolution::default(), false)
Self::new(VideoResolution::default())
}
}

impl NativeVideoSource {
pub fn new(resolution: VideoResolution, is_screencast: bool) -> Self {
Self { handle: vs_imp::NativeVideoSource::new(resolution, is_screencast) }
pub fn new(resolution: VideoResolution) -> Self {
Self { handle: vs_imp::NativeVideoSource::new(resolution) }
}

pub fn capture_frame<T: AsRef<dyn VideoBuffer>>(&self, frame: &VideoFrame<T>) {
Expand All @@ -74,6 +78,10 @@ pub mod native {
pub fn video_resolution(&self) -> VideoResolution {
self.handle.video_resolution()
}

pub fn set_is_screencast(&self, is_screencast: bool) {
self.handle.set_is_screencast(is_screencast)
}
}
}

Expand Down
13 changes: 6 additions & 7 deletions livekit-ffi/protocol/video_frame.proto
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023-2025 LiveKit, Inc.
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -44,12 +44,11 @@ message VideoStreamFromParticipantResponse { required OwnedVideoStream stream =

// Create a new VideoSource
// VideoSource is used to send video frame to a track
message NewVideoSourceRequest {
required VideoSourceType type = 1;
message NewVideoSourceRequest {
required VideoSourceType type = 1;
// Used to determine which encodings to use + simulcast layers
// Most of the time it corresponds to the source resolution
// Most of the time it corresponds to the source resolution
required VideoSourceResolution resolution = 2;
optional bool is_screencast = 3;
}
message NewVideoSourceResponse { required OwnedVideoSource source = 1; }

Expand All @@ -71,7 +70,7 @@ message VideoConvertRequest {
message VideoConvertResponse {
oneof message {
string error = 1;
OwnedVideoBuffer buffer = 2;
OwnedVideoBuffer buffer = 2;
}
}

Expand Down Expand Up @@ -155,7 +154,7 @@ message OwnedVideoStream {

message VideoStreamEvent {
required uint64 stream_handle = 1;
oneof message {
oneof message {
VideoFrameReceived frame_received = 2;
VideoStreamEOS eos = 3;
}
Expand Down
16 changes: 6 additions & 10 deletions livekit-ffi/src/livekit.proto.rs
Original file line number Diff line number Diff line change
Expand Up @@ -302,11 +302,11 @@ impl EncryptionState {
/// # Safety
/// The foreign language is responsable for disposing handles
/// Forgetting to dispose the handle may lead to memory leaks
///
///
/// Dropping a handle doesn't necessarily mean that the object is destroyed if it is still used
/// on the FfiServer (Atomic reference counting)
///
/// When refering to a handle without owning it, we just use a uint32 without this message.
///
/// When refering to a handle without owning it, we just use a uint32 without this message.
/// (the variable name is suffixed with "_handle")
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
Expand Down Expand Up @@ -1762,11 +1762,9 @@ pub struct NewVideoSourceRequest {
#[prost(enumeration="VideoSourceType", required, tag="1")]
pub r#type: i32,
/// Used to determine which encodings to use + simulcast layers
/// Most of the time it corresponds to the source resolution
/// Most of the time it corresponds to the source resolution
#[prost(message, required, tag="2")]
pub resolution: VideoSourceResolution,
#[prost(bool, optional, tag="3")]
pub is_screencast: ::core::option::Option<bool>,
}
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
Expand Down Expand Up @@ -2009,8 +2007,6 @@ impl VideoRotation {
}
}
}
/// Values of this enum must not be changed
/// It is used to serialize a rtc.VideoFrame on Python
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
#[repr(i32)]
pub enum VideoBufferType {
Expand Down Expand Up @@ -3455,7 +3451,7 @@ pub struct NewAudioSourceResponse {
#[prost(message, required, tag="1")]
pub source: OwnedAudioSource,
}
/// Push a frame to an AudioSource
/// Push a frame to an AudioSource
/// The data provided must be available as long as the client receive the callback.
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
Expand Down Expand Up @@ -4009,7 +4005,7 @@ pub struct RpcMethodInvocationEvent {
// that it receives from the server.
//
// Therefore, the ffi client is easier to implement if there is less handles to manage.
//
//
// - We are mainly using FfiHandle on info messages (e.g: RoomInfo, TrackInfo, etc...)
// For this reason, info are only sent once, at creation (We're not using them for updates, we can infer them from
// events on the client implementation).
Expand Down
8 changes: 3 additions & 5 deletions livekit-ffi/src/server/video_source.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023-2025 LiveKit, Inc.
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -35,10 +35,8 @@ impl FfiVideoSource {
#[cfg(not(target_arch = "wasm32"))]
proto::VideoSourceType::VideoSourceNative => {
use livekit::webrtc::video_source::native::NativeVideoSource;
let video_source = NativeVideoSource::new(
new_source.resolution.into(),
new_source.is_screencast(),
);

let video_source = NativeVideoSource::new(new_source.resolution.into());
RtcVideoSource::Native(video_source)
}
_ => return Err(FfiError::InvalidRequest("unsupported video source type".into())),
Expand Down
16 changes: 15 additions & 1 deletion livekit-protocol/src/enum_dispatch.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 LiveKit, Inc.
// Copyright 2023-2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -33,9 +33,23 @@ macro_rules! enum_dispatch {
}
};

// Handle functions without a return type
(@fnc [$($variant:ident),+]: $vis:vis fn $fnc:ident($self:ident: $sty:ty $(, $arg:ident: $t:ty)*)) => {
#[inline]
$vis fn $fnc($self: $sty, $($arg: $t),*) {
enum_dispatch!(@match [$($variant),+]: $fnc, $self, ($($arg,)*))
}
};

($variants:tt; $($vis:vis fn $fnc:ident$args:tt -> $ret:ty;)+) => {
$(
enum_dispatch!(@fnc $variants: $vis fn $fnc$args -> $ret);
)+
};

($variants:tt; $($vis:vis fn $fnc:ident$args:tt;)+) => {
$(
enum_dispatch!(@fnc $variants: $vis fn $fnc$args);
)+
};
}
9 changes: 8 additions & 1 deletion livekit/src/room/participant/local_participant.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 LiveKit, Inc.
// Copyright 2023-2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -232,6 +232,13 @@ impl LocalParticipant {

encodings = compute_video_encodings(req.width, req.height, &options);
req.layers = video_layers_from_encodings(req.width, req.height, &encodings);

match options.source {
TrackSource::Screenshare => {
video_track.rtc_source().set_is_screencast(true);
}
_ => {}
}
}
LocalTrack::Audio(_audio_track) => {
// Setup audio encoding
Expand Down
10 changes: 6 additions & 4 deletions webrtc-sys/include/livekit/video_track.h
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ class VideoTrackSource {
* resolution set to (0, 0) means no resolution/optional, the source will
* guess the resolution at the first captured frame.
*/
InternalSource(const VideoResolution& resolution, bool is_screencast);
InternalSource(const VideoResolution& resolution);
~InternalSource() override;

bool is_screencast() const override;
Expand All @@ -100,6 +100,7 @@ class VideoTrackSource {
bool remote() const override;
VideoResolution video_resolution() const;
bool on_captured_frame(const webrtc::VideoFrame& frame);
void set_is_screencast(bool is_screencast);

private:
mutable webrtc::Mutex mutex_;
Expand All @@ -109,7 +110,7 @@ class VideoTrackSource {
};

public:
VideoTrackSource(const VideoResolution& resolution, bool is_screencast);
VideoTrackSource(const VideoResolution& resolution);

VideoResolution video_resolution() const;

Expand All @@ -118,13 +119,14 @@ class VideoTrackSource {

rtc::scoped_refptr<InternalSource> get() const;

void set_is_screencast(bool is_screencast) const;

private:
rtc::scoped_refptr<InternalSource> source_;
};

std::shared_ptr<VideoTrackSource> new_video_track_source(
const VideoResolution& resolution,
bool is_screencast);
const VideoResolution& resolution);

static std::shared_ptr<MediaStreamTrack> video_to_media(
std::shared_ptr<VideoTrack> track) {
Expand Down
27 changes: 16 additions & 11 deletions webrtc-sys/src/video_track.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -105,18 +105,21 @@ std::shared_ptr<NativeVideoSink> new_native_video_sink(
}

VideoTrackSource::InternalSource::InternalSource(
const VideoResolution& resolution,
bool is_screencast)
: rtc::AdaptedVideoTrackSource(4),
resolution_(resolution),
is_screencast_(is_screencast) {}
const VideoResolution& resolution)
: rtc::AdaptedVideoTrackSource(4), resolution_(resolution) {}

VideoTrackSource::InternalSource::~InternalSource() {}

bool VideoTrackSource::InternalSource::is_screencast() const {
webrtc::MutexLock lock(&mutex_);
return is_screencast_;
}

void VideoTrackSource::InternalSource::set_is_screencast(bool is_screencast) {
webrtc::MutexLock lock(&mutex_);
is_screencast_ = is_screencast;
}

absl::optional<bool> VideoTrackSource::InternalSource::needs_denoising() const {
return false;
}
Expand Down Expand Up @@ -178,9 +181,8 @@ bool VideoTrackSource::InternalSource::on_captured_frame(
return true;
}

VideoTrackSource::VideoTrackSource(const VideoResolution& resolution,
bool is_screencast) {
source_ = rtc::make_ref_counted<InternalSource>(resolution, is_screencast);
VideoTrackSource::VideoTrackSource(const VideoResolution& resolution) {
source_ = rtc::make_ref_counted<InternalSource>(resolution);
}

VideoResolution VideoTrackSource::video_resolution() const {
Expand All @@ -193,15 +195,18 @@ bool VideoTrackSource::on_captured_frame(
return source_->on_captured_frame(rtc_frame);
}

void VideoTrackSource::set_is_screencast(bool is_screencast) const {
source_->set_is_screencast(is_screencast);
}

rtc::scoped_refptr<VideoTrackSource::InternalSource> VideoTrackSource::get()
const {
return source_;
}

std::shared_ptr<VideoTrackSource> new_video_track_source(
const VideoResolution& resolution,
bool is_screencast) {
return std::make_shared<VideoTrackSource>(resolution, is_screencast);
const VideoResolution& resolution) {
return std::make_shared<VideoTrackSource>(resolution);
}

} // namespace livekit
6 changes: 2 additions & 4 deletions webrtc-sys/src/video_track.rs
Original file line number Diff line number Diff line change
Expand Up @@ -67,10 +67,8 @@ pub mod ffi {

fn video_resolution(self: &VideoTrackSource) -> VideoResolution;
fn on_captured_frame(self: &VideoTrackSource, frame: &UniquePtr<VideoFrame>) -> bool;
fn new_video_track_source(
resolution: &VideoResolution,
is_screencast: bool,
) -> SharedPtr<VideoTrackSource>;
fn set_is_screencast(self: &VideoTrackSource, is_screencast: bool);
fn new_video_track_source(resolution: &VideoResolution) -> SharedPtr<VideoTrackSource>;
fn video_to_media(track: SharedPtr<VideoTrack>) -> SharedPtr<MediaStreamTrack>;
unsafe fn media_to_video(track: SharedPtr<MediaStreamTrack>) -> SharedPtr<VideoTrack>;
fn _shared_video_track() -> SharedPtr<VideoTrack>;
Expand Down

0 comments on commit 6d49d4c

Please sign in to comment.