From 23035268ba0cf366414246f32083b337bd5a3fe3 Mon Sep 17 00:00:00 2001 From: Jaya Allamsetty Date: Tue, 17 Oct 2023 12:35:56 -0400 Subject: [PATCH] fix: Negotiate AV1 DD header exts only for AV1 and H.264. --- modules/RTC/TPCUtils.js | 1 - modules/RTC/TraceablePeerConnection.js | 69 +++++++++++++++++++++++++- modules/xmpp/JingleSessionPC.js | 17 ++----- 3 files changed, 70 insertions(+), 17 deletions(-) diff --git a/modules/RTC/TPCUtils.js b/modules/RTC/TPCUtils.js index 82cc651a12..0c8c262e9e 100644 --- a/modules/RTC/TPCUtils.js +++ b/modules/RTC/TPCUtils.js @@ -62,7 +62,6 @@ export class TPCUtils { }); } - // TODO - Check if AV1 DD extension headers are negotiated and add that check here for AV1 and H.264. const scalabilityModeEnabled = this.codecSettings[codec].scalabilityModeEnabled && (typeof codecConfig.scalabilityModeEnabled === 'undefined' || codecConfig.scalabilityModeEnabled); diff --git a/modules/RTC/TraceablePeerConnection.js b/modules/RTC/TraceablePeerConnection.js index c06fc6dfb1..b41c14701e 100644 --- a/modules/RTC/TraceablePeerConnection.js +++ b/modules/RTC/TraceablePeerConnection.js @@ -334,6 +334,13 @@ export default function TraceablePeerConnection( */ this._senderMaxHeights = new Map(); + /** + * Flag indicating bridge support for AV1 codec. On the bridge connection, it is supported only when support for + * AV1 Dependency Descriptor header extensions is offered by Jicofo. H.264 simulcast is also possible when these + * header extensions are negotiated. + */ + this._supportsAv1HeaderExts = false; + /** * Holds the RTCRtpTransceiver mids that the local tracks are attached to, mapped per their * {@link JitsiLocalTrack.rtcId}. @@ -576,7 +583,11 @@ TraceablePeerConnection.prototype._getReceiversByEndpointIds = function(endpoint * false if it's turned off. */ TraceablePeerConnection.prototype.isSpatialScalabilityOn = function() { - return !this.options.disableSimulcast; + const h264SimulcastEnabled = this.tpcUtils.codecSettings[CodecMimeType.H264].scalabilityModeEnabled + && this._supportsAv1HeaderExts; + + return !this.options.disableSimulcast + && (this.codecSettings.codecList[0] !== CodecMimeType.H264 || h264SimulcastEnabled); }; /** @@ -1719,6 +1730,57 @@ TraceablePeerConnection.prototype._mungeCodecOrder = function(description) { }); }; +/** + * Checks if the AV1 Dependency descriptors are negotiated on the bridge peerconnection and disables them when the + * codec selected is VP8 or VP9. + * + * @param {RTCSessionDescription} description that needs to be munged. + * @returns {RTCSessionDescription} the munged description. + */ +TraceablePeerConnection.prototype._updateAv1DdHeaders = function(description) { + const parsedSdp = transform.parse(description.sdp); + const mLines = parsedSdp.media.filter(m => m.type === MediaType.VIDEO); + const extUri = 'https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension'; + + if (!mLines.length) { + return description; + } + + mLines.forEach((mLine, idx) => { + const senderMids = Array.from(this._localTrackTransceiverMids.values()); + const isSender = senderMids.length + ? senderMids.find(mid => mLine.mid.toString() === mid.toString()) + : idx === 0; + const payload = mLine.payloads.split(' ')[0]; + let { codec } = mLine.rtp.find(rtp => rtp.payload === Number(payload)); + + codec = codec.toLowerCase(); + + if (isSender && mLine.ext?.length) { + const headerIndex = mLine.ext.findIndex(ext => ext.uri === extUri); + const shouldNegotiateHeaderExts = codec === CodecMimeType.AV1 || codec === CodecMimeType.H264; + + if (!this._supportsAv1HeaderExts && headerIndex >= 0) { + this._supportsAv1HeaderExts = true; + } + + if (this._supportsAv1HeaderExts && shouldNegotiateHeaderExts && headerIndex < 0) { + mLine.ext.push({ + value: 11, + uri: extUri + }); + } else if (!shouldNegotiateHeaderExts && headerIndex >= 0) { + mLine.ext.splice(headerIndex, 1); + } + } + }); + + return new RTCSessionDescription({ + type: description.type, + sdp: transform.write(parsedSdp) + }); +}; + /** * Add {@link JitsiLocalTrack} to this TPC. * @param {JitsiLocalTrack} track @@ -1906,7 +1968,8 @@ TraceablePeerConnection.prototype.getConfiguredVideoCodec = function() { } const parsedSdp = transform.parse(sdp); const mLine = parsedSdp.media.find(m => m.type === MediaType.VIDEO); - const codec = mLine.rtp[0].codec; + const payload = mLine.payloads.split(' ')[0]; + const { codec } = mLine.rtp.find(rtp => rtp.payload === Number(payload)); if (codec) { return Object.values(CodecMimeType).find(value => value === codec.toLowerCase()); @@ -2558,6 +2621,7 @@ TraceablePeerConnection.prototype.setLocalDescription = function(description) { // Munge the order of the codecs based on the preferences set through config.js. localDescription = this._mungeCodecOrder(localDescription); localDescription = this._setMaxBitrates(localDescription, true); + localDescription = this._updateAv1DdHeaders(localDescription); this.trace('setLocalDescription::postTransform', dumpSDP(localDescription)); @@ -2619,6 +2683,7 @@ TraceablePeerConnection.prototype.setRemoteDescription = function(description) { // Munge the order of the codecs based on the preferences set through config.js. remoteDescription = this._mungeCodecOrder(remoteDescription); remoteDescription = this._setMaxBitrates(remoteDescription); + remoteDescription = this._updateAv1DdHeaders(remoteDescription); this.trace('setRemoteDescription::postTransform (munge codec order)', dumpSDP(remoteDescription)); return new Promise((resolve, reject) => { diff --git a/modules/xmpp/JingleSessionPC.js b/modules/xmpp/JingleSessionPC.js index 895950620c..2c5d4f173f 100644 --- a/modules/xmpp/JingleSessionPC.js +++ b/modules/xmpp/JingleSessionPC.js @@ -3,7 +3,6 @@ import $ from 'jquery'; import { $build, $iq, Strophe } from 'strophe.js'; import { JitsiTrackEvents } from '../../JitsiTrackEvents'; -import CodecMimeType from '../../service/RTC/CodecMimeType'; import { MediaDirection } from '../../service/RTC/MediaDirection'; import { MediaType } from '../../service/RTC/MediaType'; import { VideoType } from '../../service/RTC/VideoType'; @@ -402,7 +401,6 @@ export default class JingleSessionPC extends JingleSession { pcOptions.capScreenshareBitrate = false; pcOptions.codecSettings = options.codecSettings; pcOptions.enableInsertableStreams = options.enableInsertableStreams; - let h264SimulcastEnabled = browser.supportsScalabilityModeAPI(); if (options.videoQuality) { const settings = Object.entries(options.videoQuality) @@ -413,20 +411,11 @@ export default class JingleSessionPC extends JingleSession { }); pcOptions.videoQuality = Object.fromEntries(settings); - const h264Settings = pcOptions.videoQuality[CodecMimeType.H264]; - - h264SimulcastEnabled = h264SimulcastEnabled - && (typeof h264Settings === 'undefined' || h264Settings.scalabilityModeEnabled); } pcOptions.forceTurnRelay = options.forceTurnRelay; pcOptions.audioQuality = options.audioQuality; pcOptions.usesUnifiedPlan = this.usesUnifiedPlan = browser.supportsUnifiedPlan(); - const preferredJvbCodec = options.codecSettings?.codecList[0]; - - pcOptions.disableSimulcast = this.isP2P - ? true - : options.disableSimulcast - ?? (preferredJvbCodec?.toLowerCase() === CodecMimeType.H264 && !h264SimulcastEnabled); + pcOptions.disableSimulcast = this.isP2P ? true : options.disableSimulcast; if (!this.isP2P) { // Do not send lower spatial layers for low fps screenshare and enable them only for high fps screenshare. @@ -1008,7 +997,7 @@ export default class JingleSessionPC extends JingleSession { // modify sendSessionAccept method to do that this.sendSessionAccept(() => { // Start processing tasks on the modification queue. - logger.debug('Resuming the modification queue after session is established!'); + logger.debug(`${this} Resuming the modification queue after session is established!`); this.modificationQueue.resume(); success(); @@ -1128,7 +1117,7 @@ export default class JingleSessionPC extends JingleSession { this.state = JingleSessionState.ACTIVE; // Start processing tasks on the modification queue. - logger.debug('Resuming the modification queue after session is established!'); + logger.debug(`${this} Resuming the modification queue after session is established!`); this.modificationQueue.resume(); const newLocalSdp = new SDP(this.peerconnection.localDescription.sdp);