Add sender controlled playout delay limits

webrtc/modules/rtp_rtcp/source/rtp_sender.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #include "webrtc/modules/rtp_rtcp/source/rtp_sender.h"
 
 #include <stdlib.h>  // srand
 #include <algorithm>
 #include <utility>
 
 #include "webrtc/base/checks.h"
 #include "webrtc/base/logging.h"
 #include "webrtc/base/trace_event.h"
 #include "webrtc/base/timeutils.h"
 #include "webrtc/call.h"
 #include "webrtc/call/rtc_event_log.h"
 #include "webrtc/modules/rtp_rtcp/include/rtp_cvo.h"
 #include "webrtc/modules/rtp_rtcp/source/byte_io.h"
+#include "webrtc/modules/rtp_rtcp/source/playout_delay_oracle.h"
 #include "webrtc/modules/rtp_rtcp/source/rtp_sender_audio.h"
 #include "webrtc/modules/rtp_rtcp/source/rtp_sender_video.h"
 #include "webrtc/modules/rtp_rtcp/source/time_util.h"
 
 namespace webrtc {
 
 // Max in the RFC 3550 is 255 bytes, we limit it to be modulus 32 for SRTP.
 static const size_t kMaxPaddingLength = 224;
 static const int kSendSideDelayWindowMs = 1000;
 static const uint32_t kAbsSendTimeFraction = 18;
@@ skipping 246 matching lines @@
 bool RTPSender::IsRtpHeaderExtensionRegistered(RTPExtensionType type) {
   rtc::CritScope lock(&send_critsect_);
   return rtp_header_extension_map_.IsRegistered(type);
 }
 
 int32_t RTPSender::DeregisterRtpHeaderExtension(RTPExtensionType type) {
   rtc::CritScope lock(&send_critsect_);
   return rtp_header_extension_map_.Deregister(type);
 }
 
-size_t RTPSender::RtpHeaderExtensionTotalLength() const {
+size_t RTPSender::RtpHeaderExtensionMaxLength() const {
   rtc::CritScope lock(&send_critsect_);
   return rtp_header_extension_map_.GetTotalLengthInBytes();
 }
 
 int32_t RTPSender::RegisterPayload(
     const char payload_name[RTP_PAYLOAD_NAME_SIZE],
     int8_t payload_number,
     uint32_t frequency,
     size_t channels,
     uint32_t rate) {
@@ skipping 77 matching lines @@
   max_payload_length_ = max_payload_length;
 }
 
 size_t RTPSender::MaxDataPayloadLength() const {
   int rtx;
   {
     rtc::CritScope lock(&send_critsect_);
     rtx = rtx_;
   }
   if (audio_configured_) {
-    return max_payload_length_ - RTPHeaderLength();
+    return max_payload_length_ - RtpHeaderMaxLength();
   } else {
-    return max_payload_length_ - RTPHeaderLength()  // RTP overhead.
-           - video_->FECPacketOverhead()            // FEC/ULP/RED overhead.
-           - ((rtx) ? 2 : 0);                       // RTX overhead.
+    return max_payload_length_ - RtpHeaderMaxLength()  // RTP overhead.
+           - video_->FECPacketOverhead()               // FEC/ULP/RED overhead.
+           - ((rtx) ? 2 : 0);                          // RTX overhead.
   }
 }
 
 size_t RTPSender::MaxPayloadLength() const {
   return max_payload_length_;
 }
 
 void RTPSender::SetRtxStatus(int mode) {
   rtc::CritScope lock(&send_critsect_);
   rtx_ = mode;
@@ skipping 80 matching lines @@
 
 int32_t RTPSender::SendOutgoingData(FrameType frame_type,
                                     int8_t payload_type,
                                     uint32_t capture_timestamp,
                                     int64_t capture_time_ms,
                                     const uint8_t* payload_data,
                                     size_t payload_size,
                                     const RTPFragmentationHeader* fragmentation,
                                     const RTPVideoHeader* rtp_hdr) {
   uint32_t ssrc;
+  uint16_t sequence_number;
   {
     // Drop this packet if we're not sending media packets.
     rtc::CritScope lock(&send_critsect_);
     ssrc = ssrc_;
+    sequence_number = sequence_number_;
     if (!sending_media_) {
       return 0;
     }
   }
   RtpVideoCodecTypes video_type = kRtpVideoGeneric;
   if (CheckPayloadType(payload_type, &video_type) != 0) {
     LOG(LS_ERROR) << "Don't send data with unknown payload type: "
                   << static_cast<int>(payload_type) << ".";
     return -1;
   }
 
   int32_t ret_val;
   if (audio_configured_) {
     TRACE_EVENT_ASYNC_STEP1("webrtc", "Audio", capture_timestamp,
                             "Send", "type", FrameTypeToString(frame_type));
     assert(frame_type == kAudioFrameSpeech || frame_type == kAudioFrameCN ||
            frame_type == kEmptyFrame);
 
     ret_val = audio_->SendAudio(frame_type, payload_type, capture_timestamp,
                                 payload_data, payload_size, fragmentation);
   } else {
     TRACE_EVENT_ASYNC_STEP1("webrtc", "Video", capture_time_ms,
                             "Send", "type", FrameTypeToString(frame_type));
     assert(frame_type != kAudioFrameSpeech && frame_type != kAudioFrameCN);
 
     if (frame_type == kEmptyFrame)
       return 0;
 
-    ret_val =
-        video_->SendVideo(video_type, frame_type, payload_type,
-                          capture_timestamp, capture_time_ms, payload_data,
-                          payload_size, fragmentation, rtp_hdr);
+    if (rtp_hdr) {
+      playout_delay_oracle_.UpdateRequest(ssrc, rtp_hdr->playout_delay,
+                                          sequence_number);
+    }
+
+    ret_val = video_->SendVideo(
+        video_type, frame_type, payload_type, capture_timestamp,
+        capture_time_ms, payload_data, payload_size, fragmentation, rtp_hdr);
   }
 
   rtc::CritScope cs(&statistics_crit_);
   // Note: This is currently only counting for video.
   if (frame_type == kVideoFrameKey) {
     ++frame_counts_.key_frames;
   } else if (frame_type == kVideoFrameDelta) {
     ++frame_counts_.delta_frames;
   }
   if (frame_count_observer_) {
@@ skipping 274 matching lines @@
       if (bytes_re_sent > target_bytes) {
         break;  // Ignore the rest of the packets in the list.
       }
     }
   }
   if (bytes_re_sent > 0) {
     UpdateNACKBitRate(bytes_re_sent, now);
   }
 }
 
+void RTPSender::OnReceivedRtcpReceiverReport(
+    const ReportBlockList& report_blocks) {
+  playout_delay_oracle_.OnReceivedRtcpReceiverReport(report_blocks);
+}
+
 bool RTPSender::ProcessNACKBitRate(uint32_t now) {
   uint32_t num = 0;
   size_t byte_count = 0;
   const uint32_t kAvgIntervalMs = 1000;
   uint32_t target_bitrate = GetTargetBitrate();
 
   rtc::CritScope lock(&send_critsect_);
 
   if (target_bitrate == 0) {
     return true;
@@ skipping 298 matching lines @@
 void RTPSender::ProcessBitrate() {
   rtc::CritScope lock(&send_critsect_);
   total_bitrate_sent_.Process();
   nack_bitrate_.Process();
   if (audio_configured_) {
     return;
   }
   video_->ProcessBitrate();
 }
 
-size_t RTPSender::RTPHeaderLength() const {
+size_t RTPSender::RtpHeaderCurrentLength() const {
   rtc::CritScope lock(&send_critsect_);
   size_t rtp_header_length = kRtpHeaderLength;
   rtp_header_length += sizeof(uint32_t) * csrcs_.size();
-  rtp_header_length += RtpHeaderExtensionTotalLength();
+  rtp_header_length += RtpHeaderExtensionCurrentLength();
   return rtp_header_length;
 }
 
+size_t RTPSender::RtpHeaderMaxLength() const {
+  rtc::CritScope lock(&send_critsect_);
+  size_t rtp_header_length = kRtpHeaderLength;
+  rtp_header_length += sizeof(uint32_t) * csrcs_.size();
+  rtp_header_length += RtpHeaderExtensionMaxLength();
+  return rtp_header_length;
+}
+
 uint16_t RTPSender::AllocateSequenceNumber(uint16_t packets_to_send) {
   rtc::CritScope lock(&send_critsect_);
   uint16_t first_allocated_sequence_number = sequence_number_;
   sequence_number_ += packets_to_send;
   return first_allocated_sequence_number;
 }
 
 void RTPSender::GetDataCounters(StreamDataCounters* rtp_stats,
                                 StreamDataCounters* rtx_stats) const {
   rtc::CritScope lock(&statistics_crit_);
@@ skipping 103 matching lines @@
       case kRtpExtensionAbsoluteSendTime:
         block_length = BuildAbsoluteSendTimeExtension(extension_data);
         break;
       case kRtpExtensionVideoRotation:
         block_length = BuildVideoRotationExtension(extension_data);
         break;
       case kRtpExtensionTransportSequenceNumber:
         block_length = BuildTransportSequenceNumberExtension(
             extension_data, transport_sequence_number_);
         break;
+      case kRtpExtensionPlayoutDelay:
+        if (playout_delay_oracle_.send_playout_delay()) {
+          block_length = BuildPlayoutDelayExtension(
+              extension_data, playout_delay_oracle_.min_playout_delay_ms(),
+              playout_delay_oracle_.max_playout_delay_ms());
+        }
+        break;
       default:
         assert(false);
     }
     total_block_length += block_length;
     type = rtp_header_extension_map_.Next(type);
   }
   if (total_block_length == 0) {
     // No extension added.
     return 0;
   }
   // Add padding elements until we've filled a 32 bit block.
   size_t padding_bytes =
       RtpUtility::Word32Align(total_block_length) - total_block_length;
   if (padding_bytes > 0) {
     memset(&data_buffer[kHeaderLength + total_block_length], 0, padding_bytes);
     total_block_length += padding_bytes;
   }
   // Set header length (in number of Word32, header excluded).
   ByteWriter<uint16_t>::WriteBigEndian(data_buffer + kPosLength,
                                        total_block_length / 4);
   // Total added length.
   return kHeaderLength + total_block_length;
 }
 
+size_t RTPSender::RtpHeaderExtensionCurrentLength() const {

[danilchap, 2016/06/02 12:12:34]

this function sums size of all extensions, including the inactive, while
MaxLength skips inactive extension.
so RtpHeaderCurrentLength may be > RtpHeaderMaxLength
that is confusing.

[Irfan, 2016/06/02 18:15:53]

I did not realize TotalLength was already taking into consideration whether an
extension is active or inactive.

I have now removed the notion of "current length" since it makes sense to let
playout delay also be treated as active or inactive. This brings the behavior
along the lines of video rotation with the only difference being video rotation
only gets active once while playout delay can be inactive again.

At some point, it makes sense to let video rotation also be inactive when
unneeded to avoid sending unnecessary data.

+  if (rtp_header_extension_map_.Size() <= 0)
+    return 0;
+
+  int header_length = 0;
+  uint8_t id;
+
+  RTPExtensionType type = rtp_header_extension_map_.First();
+  while (type != kRtpExtensionNone) {
+    switch (type) {
+      case kRtpExtensionTransmissionTimeOffset:
+        if (rtp_header_extension_map_.GetId(kRtpExtensionTransmissionTimeOffset,

[Stefan, 2016/06/02 08:07:09]

Would have been nice with a method which simply checked if the extension was
registered and not reuse GetId for that.

[danilchap, 2016/06/02 12:12:34]

rtp_header_extension_map_ has method IsRegistered for that.

[Irfan, 2016/06/02 18:15:53]

This is now removed in favour of just treating playout delay extension as
active/inactive

+                                            &id) == 0) {
+          header_length += kTransmissionTimeOffsetLength;
+        }
+        break;
+      case kRtpExtensionAudioLevel:
+        if (rtp_header_extension_map_.GetId(kRtpExtensionAudioLevel, &id) ==
+            0) {
+          header_length += kAudioLevelLength;
+        }
+        break;
+      case kRtpExtensionAbsoluteSendTime:
+        if (rtp_header_extension_map_.GetId(kRtpExtensionAbsoluteSendTime,
+                                            &id) == 0) {
+          header_length += kAbsoluteSendTimeLength;
+        }
+        break;
+      case kRtpExtensionVideoRotation:
+        if (rtp_header_extension_map_.GetId(kRtpExtensionVideoRotation, &id) ==

[danilchap, 2016/06/02 12:12:34]

this means video rotation extension would be counted even when inactive.
This way it might happen RtpHeaderCurrentLength > RtpHeaderMaxLength

[Irfan, 2016/06/02 18:15:53]

see above

+            0) {
+          header_length += kVideoRotationLength;
+        }
+        break;
+      case kRtpExtensionTransportSequenceNumber:
+        if (rtp_header_extension_map_.GetId(
+                kRtpExtensionTransportSequenceNumber, &id) == 0) {
+          header_length += kTransportSequenceNumberLength;
+        }
+        break;
+      case kRtpExtensionPlayoutDelay:
+        if (playout_delay_oracle_.send_playout_delay()) {
+          if (rtp_header_extension_map_.GetId(kRtpExtensionPlayoutDelay, &id) ==
+              0) {
+            header_length += kPlayoutDelayLength;
+          }
+        }
+        break;
+      default:

[danilchap, 2016/06/02 12:12:34]

do not use default, this way compiler would warn when new value added to the
enum.

[Irfan, 2016/06/02 18:15:52]

removed

+        assert(false);
+    }
+    type = rtp_header_extension_map_.Next(type);
+  }
+  if (header_length == 0)
+    return 0;
+  // Add padding to fill a 32 bit block.
+  size_t padding_bytes = RtpUtility::Word32Align(header_length) - header_length;

[danilchap, 2016/06/02 12:12:34]

this three lines could be replaced with header_length =
RtpUtility::Word32Align(header_length);
or may be better to just
return RtpUtility::Word32Align(header_length) + kRtpOneByteHeaderLength;

[Irfan, 2016/06/02 18:15:53]

removed

+  if (padding_bytes > 0)
+    header_length += padding_bytes;
+
+  return header_length + kRtpOneByteHeaderLength;
+}
+
 uint8_t RTPSender::BuildTransmissionTimeOffsetExtension(
     uint8_t* data_buffer) const {
   // From RFC 5450: Transmission Time Offsets in RTP Streams.
   //
   // The transmission time is signaled to the receiver in-band using the
   // general mechanism for RTP header extensions [RFC5285]. The payload
   // of this extension (the transmitted value) is a 24-bit signed integer.
   // When added to the RTP timestamp of the packet, it represents the
   // "effective" RTP transmission time of the packet, on the RTP
   // timescale.
@@ skipping 131 matching lines @@
   }
   size_t pos = 0;
   const uint8_t len = 1;
   data_buffer[pos++] = (id << 4) + len;
   ByteWriter<uint16_t>::WriteBigEndian(data_buffer + pos, sequence_number);
   pos += 2;
   assert(pos == kTransportSequenceNumberLength);
   return kTransportSequenceNumberLength;
 }
 
+uint8_t RTPSender::BuildPlayoutDelayExtension(
+    uint8_t* data_buffer,
+    uint16_t min_playout_delay_ms,
+    uint16_t max_playout_delay_ms) const {
+  RTC_DCHECK_LE(min_playout_delay_ms, kPlayoutDelayMaxMs);
+  RTC_DCHECK_LE(max_playout_delay_ms, kPlayoutDelayMaxMs);
+  RTC_DCHECK_LE(min_playout_delay_ms, max_playout_delay_ms);
+  //   0                   1                   2                   3
+  //   0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+  //  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+  //  |  ID   | len=2 |   MIN delay           |   MAX delay           |
+  //  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+  uint8_t id;
+  if (rtp_header_extension_map_.GetId(kRtpExtensionPlayoutDelay, &id) != 0) {
+    // Not registered.
+    return 0;
+  }
+  size_t pos = 0;
+  const uint8_t len = 2;
+  // Convert MS to value to be sent on extension header.
+  uint16_t min_playout = min_playout_delay_ms / kPlayoutDelayGranularityMs;
+  uint16_t max_playout = max_playout_delay_ms / kPlayoutDelayGranularityMs;
+
+  data_buffer[pos++] = (id << 4) + len;
+  data_buffer[pos++] = min_playout >> 4;
+  data_buffer[pos++] = ((min_playout & 0xf) << 4) | (max_playout >> 8);
+  data_buffer[pos++] = max_playout & 0xff;
+  assert(pos == kPlayoutDelayLength);
+  return kPlayoutDelayLength;
+}
+
 bool RTPSender::FindHeaderExtensionPosition(RTPExtensionType type,
                                             const uint8_t* rtp_packet,
                                             size_t rtp_packet_length,
                                             const RTPHeader& rtp_header,
                                             size_t* position) const {
   // Get length until start of header extension block.
   int extension_block_pos =
       rtp_header_extension_map_.GetLengthUntilBlockStartInBytes(type);
   if (extension_block_pos < 0) {
     LOG(LS_WARNING) << "Failed to find extension position for " << type
@@ skipping 453 matching lines @@
   rtc::CritScope lock(&send_critsect_);
 
   RtpState state;
   state.sequence_number = sequence_number_rtx_;
   state.start_timestamp = start_timestamp_;
 
   return state;
 }
 
 }  // namespace webrtc