diff --git a/modules/xmpp/JingleSessionPC.js b/modules/xmpp/JingleSessionPC.js
index 35feb6734e..62cacb617b 100644
--- a/modules/xmpp/JingleSessionPC.js
+++ b/modules/xmpp/JingleSessionPC.js
@@ -1868,8 +1868,6 @@ export default class JingleSessionPC extends JingleSession {
                     const mid = remoteSdp.media.findIndex(mLine => mLine.includes(line));
 
                     if (mid > -1) {
-                        const mediaType = SDPUtil.parseMLine(remoteSdp.media[mid].split('\r\n')[0])?.media;
-
                         if (this.isP2P) {
                             // Do not remove ssrcs from m-line in p2p mode. If the ssrc is removed and added back to
                             // the same m-line (on source-add), Chrome/Safari do not render the media even if it is
@@ -1879,6 +1877,7 @@ export default class JingleSessionPC extends JingleSession {
                             // fire the "removetrack" event on the associated MediaStream. Also, the current direction
                             // of the transceiver for p2p will depend on whether a local sources is added or not. It
                             // will be 'sendrecv' if the local source is present, 'sendonly' otherwise.
+                            const mediaType = SDPUtil.parseMLine(remoteSdp.media[mid].split('\r\n')[0])?.media;
                             const desiredDirection = this.peerconnection.getDesiredMediaDirection(mediaType, false);
 
                             [ MediaDirection.SENDRECV, MediaDirection.SENDONLY ].forEach(direction => {
@@ -1886,14 +1885,10 @@ export default class JingleSessionPC extends JingleSession {
                                     .replace(`a=${direction}`, `a=${desiredDirection}`);
                             });
                         } else {
-                            // Change the port to 0 to reject the m-line associated with the source. The rejected
-                            // m-lines are recycled when new ssrcs need to be added to the remote description.
-                            const port = SDPUtil.parseMLine(remoteSdp.media[mid].split('\r\n')[0])?.port;
-
+                            // Jvb connections will have direction set to 'sendonly' for the remote sources.
                             remoteSdp.media[mid] = remoteSdp.media[mid].replace(`${line}\r\n`, '');
-                            remoteSdp.media[mid] = remoteSdp.media[mid].replace(
-                                `m=${mediaType} ${port}`,
-                                `m=${mediaType} 0`);
+                            remoteSdp.media[mid] = remoteSdp.media[mid]
+                                .replace(`a=${MediaDirection.SENDONLY}`, `a=${MediaDirection.INACTIVE}`);
                         }
                     }
                 });