Skip to content

Commit

Permalink
feat(multi-stream-support) Adjust presence and videoType bridge messa…
Browse files Browse the repository at this point in the history 
…ge. (#1887)

* feat(multi-stream-support) Adjust presence and videoType message.
Adjust the sourceInfo for all the available local tracks. Do not add handlers for videoType changes on the peer since the videoType doesn't change when multiple stream support is enabled.

* fix(presence) Send the presence in old format for backwards compat.
Send the presence messages in both the old and new format when source name signaling is enabled and sending multiple streams is disabled.

* fix(JitsiConference) Send the videoType bridge msg after mute/unmute op is complete.

* ref(multi-stream-support) Throw if a track is being replaced with a track of different videoType.
This is not supported in the multi sender enabled mode.

* Address review comments.
  • Loading branch information
@jallamsetty1
jallamsetty1 authored Feb 16, 2022
1 parent 79afa71 commit 55131be
Show file tree
Hide file tree
Showing 4 changed files with 94 additions and 65 deletions.
122 changes: 64 additions & 58 deletions JitsiConference.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -778,12 +778,9 @@ JitsiConference.prototype._sendBridgeVideoTypeMessage = function(localtrack) {
videoType = BridgeVideoType.DESKTOP_HIGH_FPS;
}

if (FeatureFlags.isSourceNameSignalingEnabled()) {
this.rtc.sendSourceVideoType(
getSourceNameForJitsiTrack(this.myUserId(), MediaType.VIDEO, 0),
videoType
);
} else {
if (FeatureFlags.isSourceNameSignalingEnabled() && localtrack) {
this.rtc.sendSourceVideoType(localtrack.getSourceName(), videoType);
} else if (!FeatureFlags.isSourceNameSignalingEnabled()) {
this.rtc.setVideoType(videoType);
}
};
Expand Down Expand Up @@ -1252,6 +1249,13 @@ JitsiConference.prototype.removeTrack = function(track) {
* @returns {Promise} resolves when the replacement is finished
*/
JitsiConference.prototype.replaceTrack = function(oldTrack, newTrack) {
const oldVideoType = oldTrack?.getVideoType();
const newVideoType = newTrack?.getVideoType();

if (FeatureFlags.isMultiStreamSupportEnabled() && oldTrack && newTrack && oldVideoType !== newVideoType) {
throw new Error(`Replacing a track of videoType=${oldVideoType} with a track of videoType=${newVideoType} is`
+ ' not supported in this mode.');
}
const oldTrackBelongsToConference = this === oldTrack?.conference;

if (oldTrackBelongsToConference && oldTrack.disposed) {
Expand Down Expand Up @@ -1395,29 +1399,30 @@ JitsiConference.prototype._setupNewTrack = function(newTrack) {
* @private
*/
JitsiConference.prototype._setNewVideoType = function(track) {
if (FeatureFlags.isSourceNameSignalingEnabled() && track) {
// FIXME once legacy signaling using 'sendCommand' is removed, signalingLayer.setTrackVideoType must be adjusted
// to send the presence (not just modify it).
this._signalingLayer.setTrackVideoType(track.getSourceName(), track.videoType);
let videoTypeChanged = false;

// TODO: Optimize to detect whether presence was changed, for now always report changed to send presence
return true;
if (FeatureFlags.isSourceNameSignalingEnabled() && track) {
videoTypeChanged = this._signalingLayer.setTrackVideoType(track.getSourceName(), track.videoType);
}

const videoTypeTagName = 'videoType';
if (!FeatureFlags.isMultiStreamSupportEnabled()) {
const videoTypeTagName = 'videoType';

// if track is missing we revert to default type Camera, the case where we screenshare and
// we return to be video muted
const trackVideoType = track ? track.videoType : VideoType.CAMERA;
// If track is missing we revert to default type Camera, the case where we screenshare and
// we return to be video muted.
const trackVideoType = track ? track.videoType : VideoType.CAMERA;

// if video type is camera and there is no videoType in presence, we skip adding it, as this is the default one
if (trackVideoType !== VideoType.CAMERA || this.room.getFromPresence(videoTypeTagName)) {
// we will not use this.sendCommand here to avoid sending the presence immediately, as later we may also set
// and the mute status
return this.room.addOrReplaceInPresence(videoTypeTagName, { value: trackVideoType });
// If video type is camera and there is no videoType in presence, we skip adding it, as this is the default one
if (trackVideoType !== VideoType.CAMERA || this.room.getFromPresence(videoTypeTagName)) {
// We will not use this.sendCommand here to avoid sending the presence immediately, as later we may also
// set the mute status.
const legacyTypeChanged = this.room.addOrReplaceInPresence(videoTypeTagName, { value: trackVideoType });

videoTypeChanged = videoTypeChanged || legacyTypeChanged;
}
}

return false;
return videoTypeChanged;
};

/**
Expand All @@ -1429,21 +1434,31 @@ JitsiConference.prototype._setNewVideoType = function(track) {
* @private
*/
JitsiConference.prototype._setTrackMuteStatus = function(mediaType, localTrack, isMuted) {
if (FeatureFlags.isSourceNameSignalingEnabled()) {
// TODO When legacy signaling part is removed, remember to adjust signalingLayer.setTrackMuteStatus, so that
// it triggers sending the presence (it only updates it for now, because the legacy code below sends).
this._signalingLayer.setTrackMuteStatus(localTrack?.getSourceName(), isMuted);
}
let presenceChanged = false;

if (!this.room) {
return false;
if (FeatureFlags.isSourceNameSignalingEnabled() && localTrack) {
presenceChanged = this._signalingLayer.setTrackMuteStatus(localTrack.getSourceName(), isMuted);
}

if (mediaType === MediaType.AUDIO) {
return this.room.addAudioInfoToPresence(isMuted);
// Add the 'audioMuted' and 'videoMuted' tags when source name signaling is enabled for backward compatibility.
// It won't be used anymore when multiple stream support is enabled.
if (!FeatureFlags.isMultiStreamSupportEnabled()) {
let audioMuteChanged, videoMuteChanged;

if (!this.room) {
return false;
}

if (mediaType === MediaType.AUDIO) {
audioMuteChanged = this.room.addAudioInfoToPresence(isMuted);
} else {
videoMuteChanged = this.room.addVideoInfoToPresence(isMuted);
}

presenceChanged = presenceChanged || audioMuteChanged || videoMuteChanged;
}

return this.room.addVideoInfoToPresence(isMuted);
return presenceChanged;
};

/**
Expand All @@ -1470,11 +1485,7 @@ JitsiConference.prototype._addLocalTrackAsUnmute = function(track) {
logger.debug('Add local MediaStream as unmute - no P2P Jingle session started yet');
}

return Promise.allSettled(addAsUnmutePromises)
.then(() => {
// Signal the video type to the bridge.
track.isVideoTrack() && this._sendBridgeVideoTypeMessage(track);
});
return Promise.allSettled(addAsUnmutePromises);
};

/**
Expand All @@ -1498,11 +1509,7 @@ JitsiConference.prototype._removeLocalTrackAsMute = function(track) {
logger.debug('Remove local MediaStream - no P2P JingleSession started yet');
}

return Promise.allSettled(removeAsMutePromises)
.then(() => {
// Signal the video type to the bridge.
track.isVideoTrack() && this._sendBridgeVideoTypeMessage();
});
return Promise.allSettled(removeAsMutePromises);
};

/**
Expand Down Expand Up @@ -3608,27 +3615,26 @@ JitsiConference.prototype._updateRoomPresence = function(jingleSession, ctx) {
ctx.skip = true;
}

const localAudioTracks = jingleSession.peerconnection.getLocalTracks(MediaType.AUDIO);
const localVideoTracks = jingleSession.peerconnection.getLocalTracks(MediaType.VIDEO);
let presenceChanged = false;
let muteStatusChanged, videoTypeChanged;
const localTracks = this.getLocalTracks();

if (localAudioTracks && localAudioTracks.length) {
presenceChanged = this._setTrackMuteStatus(MediaType.AUDIO, localAudioTracks[0], localAudioTracks[0].isMuted());
} else if (this._setTrackMuteStatus(MediaType.AUDIO, undefined, true)) {
presenceChanged = true;
// Set presence for all the available local tracks.
for (const track of localTracks) {
muteStatusChanged = this._setTrackMuteStatus(track.getType(), track, track.isMuted());
if (track.getType() === MediaType.VIDEO) {
videoTypeChanged = this._setNewVideoType(track);
}
presenceChanged = presenceChanged || muteStatusChanged || videoTypeChanged;
}

if (localVideoTracks && localVideoTracks.length) {
const muteStatusChanged = this._setTrackMuteStatus(
MediaType.VIDEO, localVideoTracks[0], localVideoTracks[0].isMuted());
const videoTypeChanged = this._setNewVideoType(localVideoTracks[0]);
// Set the presence in the legacy format if there are no local tracks and multi stream support is not enabled.
if (!localTracks.length && !FeatureFlags.isMultiStreamSupportEnabled()) {
const audioMuteStatusChanged = this._setTrackMuteStatus(MediaType.AUDIO, undefined, true);
const videoMuteStatusChanged = this._setTrackMuteStatus(MediaType.VIDEO, undefined, true);

presenceChanged = presenceChanged || muteStatusChanged || videoTypeChanged;
} else {
const muteStatusChanged = this._setTrackMuteStatus(MediaType.VIDEO, undefined, true);
const videoTypeChanged = this._setNewVideoType(); // set back to default video type

presenceChanged = presenceChanged || muteStatusChanged || videoTypeChanged;
videoTypeChanged = this._setNewVideoType();
presenceChanged = audioMuteStatusChanged || videoMuteStatusChanged || videoTypeChanged;
}

presenceChanged && this.room.sendPresence();
Expand Down
4 changes: 4 additions & 0 deletions modules/RTC/JitsiLocalTrack.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -395,6 +395,7 @@ export default class JitsiLocalTrack extends JitsiTrack {
this._unregisterHandlers();
this.stopStream();
this._setStream(null);

resolve();
},
reject);
Expand Down Expand Up @@ -451,6 +452,9 @@ export default class JitsiLocalTrack extends JitsiTrack {
return promise
.then(() => {
this._sendMuteStatus(muted);

// Send the videoType message to the bridge.
this.isVideoTrack() && this.conference && this.conference._sendBridgeVideoTypeMessage(this);
this.emit(TRACK_MUTE_CHANGED, this);
});
}
Expand Down
30 changes: 24 additions & 6 deletions modules/xmpp/SignalingLayerImpl.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,10 +72,12 @@ export default class SignalingLayerImpl extends SignalingLayer {
*/
_addLocalSourceInfoToPresence() {
if (this.chatRoom) {
this.chatRoom.addOrReplaceInPresence(
return this.chatRoom.addOrReplaceInPresence(
SOURCE_INFO_PRESENCE_ELEMENT,
{ value: JSON.stringify(this._localSourceState) });
}

return false;
}

/**
Expand Down Expand Up @@ -195,7 +197,10 @@ export default class SignalingLayerImpl extends SignalingLayer {
emitVideoTypeEvent(from, node.value);
}
};
room.addPresenceListener('videoType', this._videoTypeHandler);

if (!FeatureFlags.isMultiStreamSupportEnabled()) {
room.addPresenceListener('videoType', this._videoTypeHandler);
}

this._sourceInfoHandler = (node, mucNick) => {
const endpointId = mucNick;
Expand All @@ -220,11 +225,19 @@ export default class SignalingLayerImpl extends SignalingLayer {
}
}

const newVideoType = sourceInfoJSON[sourceName].videoType;
// Assume a default videoType of 'camera' for video sources.
const newVideoType = mediaType === MediaType.VIDEO
? sourceInfoJSON[sourceName].videoType ?? VideoType.CAMERA
: undefined;

if (oldSourceState.videoType !== newVideoType) {
oldSourceState.videoType = newVideoType;
emitEventsFromHere && emitVideoTypeEvent(endpointId, newVideoType);

// videoType is not allowed to change on a given JitsiLocalTrack when multi stream support is
// enabled.
emitEventsFromHere
&& !FeatureFlags.isMultiStreamSupportEnabled()
&& emitVideoTypeEvent(endpointId, newVideoType);
}
}

Expand Down Expand Up @@ -387,14 +400,17 @@ export default class SignalingLayerImpl extends SignalingLayer {
// FIXME This only adjusts the presence, but doesn't actually send it. Here we temporarily rely on
// the legacy signaling part to send the presence. Remember to add "send presence" here when the legacy
// signaling is removed.
this._addLocalSourceInfoToPresence();
return this._addLocalSourceInfoToPresence();
}

return false;
}

/**
* Sets track's video type.
* @param {SourceName} sourceName - the track's source name.
* @param {VideoType} videoType - the new video type.
* @returns {boolean}
*/
setTrackVideoType(sourceName, videoType) {
if (!this._localSourceState[sourceName]) {
Expand All @@ -408,8 +424,10 @@ export default class SignalingLayerImpl extends SignalingLayer {
// NOTE this doesn't send the actual presence, because is called from the same place where the legacy video
// type is emitted which does the actual sending. A send presence statement needs to be added when
// the legacy part is removed.
this._addLocalSourceInfoToPresence();
return this._addLocalSourceInfoToPresence();
}

return false;
}

/**
Expand Down
3 changes: 2 additions & 1 deletion types/auto/modules/xmpp/SignalingLayerImpl.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,11 +100,12 @@ export default class SignalingLayerImpl extends SignalingLayer {
* Sets track's video type.
* @param {SourceName} sourceName - the track's source name.
* @param {VideoType} videoType - the new video type.
* @returns {boolean}
*/
setTrackVideoType(sourceName: any, videoType: {
CAMERA: string;
DESKTOP: string;
}): void;
}): boolean;
/**
* Saves the source name for a track identified by it's ssrc.
* @param {number} ssrc the ssrc of the target track.
Expand Down

0 comments on commit 55131be

Please sign in to comment.