Implement FrameMarking header extension support

webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 283 matching lines @@
                                const RTPFragmentationHeader* fragmentation,
                                const RTPVideoHeader* video_header) {
   if (payload_size == 0)
     return false;
 
   // Create header that will be reused in all packets.
   std::unique_ptr<RtpPacketToSend> rtp_header = rtp_sender_->AllocatePacket();
   rtp_header->SetPayloadType(payload_type);
   rtp_header->SetTimestamp(rtp_timestamp);
   rtp_header->set_capture_time_ms(capture_time_ms);
+
+  // Set Frame Marks.
+  FrameMarks frame_marks;
+  bool frame_marking_enabled = true;
+
+  // Common info
+  frame_marks.start_of_frame = true;
+  frame_marks.end_of_frame = false;
+  frame_marks.independent = (frame_type == kVideoFrameKey);
+
+  // Codec specific.
+  switch (video_type) {

[nisse-webrtc, 2017/08/22 07:44:46]

Should we really add more codec-specific logic in this layer?

Since the information represented by frame marks are supposed to be
codec-independent, they ought to be made available by the encoder classes in
some way so that the information can be used without caring about which encoder
was used.

[sergio.garcia.murillo, 2017/08/25 10:49:38]

The information is codec specific and varies within layers of the same frame, so
it can't be added to the image information (it is only available after the rtp
packetization). This logic could be moved to a framemarking helper function
instead.

+    case kRtpVideoH264:
+      // Nothing to add
+      frame_marks.discardable = false;
+      frame_marks.temporal_layer_id = kNoTemporalIdx;
+      frame_marks.layer_id = kNoSpatialIdx;
+      frame_marks.tl0_pic_idx = static_cast<uint8_t>(kNoTl0PicIdx);
+      break;
+    case kRtpVideoVp8:
+      frame_marks.discardable = video_header->codecHeader.VP8.nonReference;
+      frame_marks.base_layer_sync = video_header->codecHeader.VP8.layerSync;
+      frame_marks.temporal_layer_id = video_header->codecHeader.VP8.temporalIdx;
+      frame_marks.layer_id = kNoSpatialIdx;
+      frame_marks.tl0_pic_idx = video_header->codecHeader.VP8.tl0PicIdx;
+      break;
+    case kRtpVideoVp9:
+      frame_marks.discardable = false;
+      // Layer id format is codec dependant.
+      frame_marks.temporal_layer_id =
+          video_header->codecHeader.VP9.temporal_idx;
+      frame_marks.layer_id =
+          FrameMarking::CreateLayerId(video_header->codecHeader.VP9);
+      frame_marks.tl0_pic_idx = video_header->codecHeader.VP9.tl0_pic_idx;
+      break;
+    default:
+      // Do not use frame marking.
+      frame_marking_enabled = false;
+  }
+  // Only add frame marking for known codecs.
+  if (frame_marking_enabled)
+    // Add extension header for frame marking.
+    rtp_header->SetExtension<FrameMarking>(frame_marks);

[danilchap, 2017/07/17 15:14:39]

[ RUN      ] VideoSendStreamTest.Vp9FlexModeRefCount
==10472==WARNING: MemorySanitizer: use-of-uninitialized-value
    #0 0x1ffddb0 in webrtc::FrameMarking::ValueSize(webrtc::FrameMarks const&)
webrtc/modules/rtp_rtcp/source/rtp_header_extensions.cc:451:10
    #1 0x20411d8 in SetExtension<webrtc::FrameMarking, webrtc::FrameMarks>
webrtc/modules/rtp_rtcp/source/rtp_packet.h:186:29
    #2 0x20411d8 in
webrtc::RTPSenderVideo::SendVideo(webrtc::RtpVideoCodecTypes, webrtc::FrameType,
signed char, unsigned int, long, unsigned char const*, unsigned long,
webrtc::RTPFragmentationHeader const*, webrtc::RTPVideoHeader const*)
webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc:346:0

[sergio.garcia.murillo, 2017/08/25 10:49:38]

Done. Created default constructor to initialize members.

+
   auto last_packet = rtc::MakeUnique<RtpPacketToSend>(*rtp_header);
 
   size_t fec_packet_overhead;
   bool red_enabled;
   int32_t retransmission_settings;
   {
     rtc::CritScope cs(&crit_);
     // According to
     // http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/
     // ts_126114v120700p.pdf Section 7.4.5:
@@ skipping 60 matching lines @@
   const RTPFragmentationHeader* frag =
       (video_type == kRtpVideoVp8) ? nullptr : fragmentation;
   size_t num_packets =
       packetizer->SetPayloadData(payload_data, payload_size, frag);
 
   if (num_packets == 0)
     return false;
 
   bool first_frame = first_frame_sent_();
   for (size_t i = 0; i < num_packets; ++i) {
+    bool first = (i == 0);
     bool last = (i + 1) == num_packets;
     auto packet = last ? std::move(last_packet)
                        : rtc::MakeUnique<RtpPacketToSend>(*rtp_header);
     if (!packetizer->NextPacket(packet.get()))
       return false;
     RTC_DCHECK_LE(packet->payload_size(),
                   last ? max_data_payload_length - last_packet_reduction_len
                        : max_data_payload_length);
+
+    // Only add frame marking for known codecs.
+    if (frame_marking_enabled) {
+      // Update start and end marks.
+      frame_marks.start_of_frame = first;
+      frame_marks.end_of_frame = last;
+      // Update extension header for frame marking.
+      packet->SetExtension<FrameMarking>(frame_marks);
+    }
+
     if (!rtp_sender_->AssignSequenceNumber(packet.get()))
       return false;
 
     bool protect_packet = (packetizer->GetProtectionType() == kProtectedPacket);
     // Put packetization finish timestamp into extension.
     if (packet->HasExtension<VideoTimingExtension>()) {
       packet->set_packetization_finish_time_ms(clock_->TimeInMilliseconds());
       // TODO(ilnik): Due to webrtc:7859, packets with timing extensions are not
       // protected by FEC. It reduces FEC efficiency a bit. When FEC is moved
       // below the pacer, it can be re-enabled for these packets.
@@ skipping 45 matching lines @@
   rtc::CritScope cs(&crit_);
   return retransmission_settings_;
 }
 
 void RTPSenderVideo::SetSelectiveRetransmissions(uint8_t settings) {
   rtc::CritScope cs(&crit_);
   retransmission_settings_ = settings;
 }
 
 }  // namespace webrtc