Cast: Make vp8 3-buffer mode work

media/cast/sender/vp8_encoder.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/cast/sender/vp8_encoder.h"
 
 #include <vector>
 
 #include "base/logging.h"
 #include "media/base/video_frame.h"
 #include "media/cast/cast_defines.h"
 #include "media/cast/net/cast_transport_config.h"
 #include "third_party/libvpx/source/libvpx/vpx/vp8cx.h"
 
 namespace media {
 namespace cast {
 
 static const uint32 kMinIntra = 300;
 
-static int ComputeMaxNumOfRepeatedBuffes(int max_unacked_frames) {
-  if (max_unacked_frames > kNumberOfVp8VideoBuffers)
-    return (max_unacked_frames - 1) / kNumberOfVp8VideoBuffers;
-
-  return 0;
-}
-
 Vp8Encoder::Vp8Encoder(const VideoSenderConfig& video_config,
                        int max_unacked_frames)
     : cast_config_(video_config),
       use_multiple_video_buffers_(
           cast_config_.max_number_of_video_buffers_used ==
           kNumberOfVp8VideoBuffers),
-      max_number_of_repeated_buffers_in_a_row_(
-          ComputeMaxNumOfRepeatedBuffes(max_unacked_frames)),
       key_frame_requested_(true),
       first_frame_received_(false),
       last_encoded_frame_id_(kStartFrameId),
-      number_of_repeated_buffers_(0) {
-  // TODO(pwestin): we need to figure out how to synchronize the acking with the
-  // internal state of the encoder, ideally the encoder will tell if we can
-  // send another frame.
-  DCHECK(!use_multiple_video_buffers_ ||
-         max_number_of_repeated_buffers_in_a_row_ == 0)
-      << "Invalid config";
-
+      last_acked_frame_id_(kStartFrameId),
+      frame_id_to_reference_(kStartFrameId - 1),
+      undroppable_frames_(0) {
   // VP8 have 3 buffers available for prediction, with
   // max_number_of_video_buffers_used set to 1 we maximize the coding efficiency
   // however in this mode we can not skip frames in the receiver to catch up
   // after a temporary network outage; with max_number_of_video_buffers_used
   // set to 3 we allow 2 frames to be skipped by the receiver without error
   // propagation.
   DCHECK(cast_config_.max_number_of_video_buffers_used == 1 ||
          cast_config_.max_number_of_video_buffers_used ==
              kNumberOfVp8VideoBuffers)
       << "Invalid argument";
@@ skipping 11 matching lines @@
   config_.reset(new vpx_codec_enc_cfg_t());
   encoder_.reset(new vpx_codec_ctx_t());
 
   // Creating a wrapper to the image - setting image data to NULL. Actual
   // pointer will be set during encode. Setting align to 1, as it is
   // meaningless (actual memory is not allocated).
   raw_image_ = vpx_img_wrap(
       NULL, IMG_FMT_I420, cast_config_.width, cast_config_.height, 1, NULL);
 
   for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) {
-    acked_frame_buffers_[i] = true;
-    used_buffers_frame_id_[i] = kStartFrameId;
+    buffer_state_[i].frame_id = kStartFrameId;
+    buffer_state_[i].state = kBufferStartState;
   }
   InitEncode(cast_config_.number_of_encode_threads);
 }
 
 void Vp8Encoder::InitEncode(int number_of_encode_threads) {
   DCHECK(thread_checker_.CalledOnValidThread());
   // Populate encoder configuration with default values.
   if (vpx_codec_enc_config_default(vpx_codec_vp8_cx(), config_.get(), 0)) {
     DCHECK(false) << "Invalid return value";
   }
@@ skipping 64 matching lines @@
   vpx_codec_flags_t flags = 0;
   if (key_frame_requested_) {
     flags = VPX_EFLAG_FORCE_KF;
     // Self reference.
     latest_frame_id_to_reference = last_encoded_frame_id_ + 1;
     // We can pick any buffer as buffer_to_update since we update
     // them all.
     buffer_to_update = kLastBuffer;
   } else {
     // Reference all acked frames (buffers).
-    latest_frame_id_to_reference = GetLatestFrameIdToReference();
-    GetCodecReferenceFlags(&flags);
+    latest_frame_id_to_reference = GetCodecReferenceFlags(&flags);
     buffer_to_update = GetNextBufferToUpdate();
     GetCodecUpdateFlags(buffer_to_update, &flags);
   }
 
   // Note: The duration does not reflect the real time between frames. This is
   // done to keep the encoder happy.
   //
   // TODO(miu): This is a semi-hack.  We should consider using
   // |video_frame->timestamp()| instead.
   uint32 duration = kVideoFrequency / cast_config_.max_frame_rate;
@@ skipping 46 matching lines @@
     encoded_image->referenced_frame_id = latest_frame_id_to_reference;
   }
 
   DVLOG(1) << "VP8 encoded frame_id " << encoded_image->frame_id
            << ", sized:" << encoded_image->data.size();
 
   if (is_key_frame) {
     key_frame_requested_ = false;
 
     for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) {
-      used_buffers_frame_id_[i] = encoded_image->frame_id;
+      buffer_state_[i].state = kBufferSent;
+      buffer_state_[i].frame_id = encoded_image->frame_id;
     }
-    // We can pick any buffer as last_used_vp8_buffer_ since we update
-    // them all.
-    last_used_vp8_buffer_ = buffer_to_update;
   } else {
     if (buffer_to_update != kNoBuffer) {
-      acked_frame_buffers_[buffer_to_update] = false;
-      used_buffers_frame_id_[buffer_to_update] = encoded_image->frame_id;
-      last_used_vp8_buffer_ = buffer_to_update;
+      buffer_state_[buffer_to_update].state = kBufferSent;
+      buffer_state_[buffer_to_update].frame_id = encoded_image->frame_id;
     }
   }
   return true;
 }
 
-void Vp8Encoder::GetCodecReferenceFlags(vpx_codec_flags_t* flags) {
+uint32 Vp8Encoder::GetCodecReferenceFlags(vpx_codec_flags_t* flags) {
   if (!use_multiple_video_buffers_)
-    return;
+    return last_encoded_frame_id_ + 1;
 
-  // We need to reference something.
-  DCHECK(acked_frame_buffers_[kAltRefBuffer] ||
-         acked_frame_buffers_[kGoldenBuffer] ||
-         acked_frame_buffers_[kLastBuffer])
-      << "Invalid state";
-
-  if (!acked_frame_buffers_[kAltRefBuffer]) {
-    *flags |= VP8_EFLAG_NO_REF_ARF;
-  }
-  if (!acked_frame_buffers_[kGoldenBuffer]) {
-    *flags |= VP8_EFLAG_NO_REF_GF;
-  }
-  if (!acked_frame_buffers_[kLastBuffer]) {
-    *flags |= VP8_EFLAG_NO_REF_LAST;
-  }
-}
-
-uint32 Vp8Encoder::GetLatestFrameIdToReference() {
-  if (!use_multiple_video_buffers_)
-    return last_encoded_frame_id_;
-
-  int latest_frame_id_to_reference = -1;
-  if (acked_frame_buffers_[kAltRefBuffer]) {
-    latest_frame_id_to_reference = used_buffers_frame_id_[kAltRefBuffer];
-  }
-  if (acked_frame_buffers_[kGoldenBuffer]) {
-    if (latest_frame_id_to_reference == -1) {
-      latest_frame_id_to_reference = used_buffers_frame_id_[kGoldenBuffer];
-    } else {
-      if (IsNewerFrameId(used_buffers_frame_id_[kGoldenBuffer],
-                         latest_frame_id_to_reference)) {
-        latest_frame_id_to_reference = used_buffers_frame_id_[kGoldenBuffer];
+  uint32 latest_frame_to_reference = last_encoded_frame_id_ - 512;

[Alpha Left Google, 2014/08/25 23:23:44]

Why 512? Seems like this should be a constant and documented.

[hubbe, 2014/08/27 04:14:03]

Any number larger than 256 will work.

+  // Reference all acked frames.
+  // Probably only referencing the last one would be enough.
+  // We also allow referencing the previous frame, even it has not
+  // been acked yet, but only if it was assigned to some buffer.
+  for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) {
+    if (buffer_state_[i].state == kBufferAcked) {

[Alpha Left Google, 2014/08/25 23:23:44]

This code reads to me that:

If buffer_state_[kAltRefBuffer].state == kBufferAcked then do not reference alt
ref frame.
If buffer_state_[kGoldenBuffer].state == kBufferAcked then do not reference
golden frame.
If buffer_state_[kLastBuffer].state == kBufferAcked then do not reference last
frame.

Is this correct? This seems to contradict the above comment that says we only
reference ACKed frame.

[hubbe, 2014/08/27 04:14:03]

Looks like I misplaced an else somewhere...

+      if (IsNewerFrameId(buffer_state_[i].frame_id,
+                         latest_frame_to_reference)) {
+        latest_frame_to_reference = buffer_state_[i].frame_id;
+      }
+      switch (i) {
+        case kAltRefBuffer:
+          *flags |= VP8_EFLAG_NO_REF_ARF;
+          break;
+        case kGoldenBuffer:
+          *flags |= VP8_EFLAG_NO_REF_GF;
+          break;
+        case kLastBuffer:
+          *flags |= VP8_EFLAG_NO_REF_LAST;
+          break;
       }
     }
   }
-  if (acked_frame_buffers_[kLastBuffer]) {
-    if (latest_frame_id_to_reference == -1) {
-      latest_frame_id_to_reference = used_buffers_frame_id_[kLastBuffer];
-    } else {
-      if (IsNewerFrameId(used_buffers_frame_id_[kLastBuffer],
-                         latest_frame_id_to_reference)) {
-        latest_frame_id_to_reference = used_buffers_frame_id_[kLastBuffer];
-      }
-    }
+
+  if (latest_frame_to_reference == last_encoded_frame_id_ - 512) {

[Alpha Left Google, 2014/08/25 23:23:44]

Why 512? Please add a comment.

[hubbe, 2014/08/27 04:14:03]

It just needs to be the same as the 512 above, made a constant.

+    // We have nothing to reference, it's kind of like a key frame,
+    // but doesn't reset buffers.
+    latest_frame_to_reference = last_encoded_frame_id_ + 1;
   }
-  DCHECK(latest_frame_id_to_reference != -1) << "Invalid state";
-  return static_cast<uint32>(latest_frame_id_to_reference);
+
+  return latest_frame_to_reference;
 }
 
 Vp8Encoder::Vp8Buffers Vp8Encoder::GetNextBufferToUpdate() {
   if (!use_multiple_video_buffers_)
     return kNoBuffer;
 
-  // Update at most one buffer, except for key-frames.
+  // Here are the rules for which buffer to select for update:
+  // 1. If there is a buffer in state kStartState, use it.
+  // 2. If there is a buffer other than the oldest buffer

[Alpha Left Google, 2014/08/25 23:23:44]

Please document the rationale for these conditions.

[hubbe, 2014/08/27 04:14:03]

Done.

+  //    which is Acked, use the oldest buffer.
+  // 3. If there are Sent buffers which are older than
+  //    latest_acked_frame_, use the oldest one.
+  // 4. If all else fails, just overwrite the newest buffer,
+  //    but no more than 3 times in a row.
 
-  Vp8Buffers buffer_to_update = kNoBuffer;
-  if (number_of_repeated_buffers_ < max_number_of_repeated_buffers_in_a_row_) {
-    // TODO(pwestin): experiment with this. The issue with only this change is
-    // that we can end up with only 4 frames in flight when we expect 6.
-    // buffer_to_update = last_used_vp8_buffer_;
-    buffer_to_update = kNoBuffer;
-    ++number_of_repeated_buffers_;
-  } else {
-    number_of_repeated_buffers_ = 0;
-    switch (last_used_vp8_buffer_) {
-      case kAltRefBuffer:
-        buffer_to_update = kLastBuffer;
-        VLOG(1) << "VP8 update last buffer";
-        break;
-      case kLastBuffer:
-        buffer_to_update = kGoldenBuffer;
-        VLOG(1) << "VP8 update golden buffer";
-        break;
-      case kGoldenBuffer:
-        buffer_to_update = kAltRefBuffer;
-        VLOG(1) << "VP8 update alt-ref buffer";
-        break;
-      case kNoBuffer:
-        DCHECK(false) << "Invalid state";
-        break;
+  // Buffers, sorted from oldest frame to newest.
+  Vp8Encoder::Vp8Buffers buffers[kNumberOfVp8VideoBuffers];
+
+  for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) {
+    Vp8Encoder::Vp8Buffers buffer = static_cast<Vp8Encoder::Vp8Buffers>(i);
+
+    // Rule 1
+    if (buffer_state_[buffer].state == kBufferStartState) {
+      undroppable_frames_ = 0;
+      return buffer;
+    }
+    buffers[buffer] = buffer;
+  }
+
+  // Sorting three elements with selection sort.
+  for (int i = 0; i < kNumberOfVp8VideoBuffers - 1; i++) {
+    for (int j = i + 1; j < kNumberOfVp8VideoBuffers; j++) {
+      if (IsOlderFrameId(buffer_state_[buffers[j]].frame_id,
+                         buffer_state_[buffers[i]].frame_id)) {
+        std::swap(buffers[i], buffers[j]);
+      }
     }
   }
-  return buffer_to_update;
+
+  // Rule 2
+  if (buffer_state_[buffers[1]].state == kBufferAcked ||
+      buffer_state_[buffers[2]].state == kBufferAcked) {
+    undroppable_frames_ = 0;
+    return buffers[0];
+  }
+
+  // Rule 3
+  for (int i = 0; i < kNumberOfVp8VideoBuffers; i++) {
+    if (buffer_state_[buffers[i]].state == kBufferSent &&
+        IsOlderFrameId(buffer_state_[buffers[i]].frame_id,
+                       last_acked_frame_id_)) {
+      undroppable_frames_ = 0;
+      return buffers[i];
+    }
+  }
+
+  // Rule 4
+  if (undroppable_frames_ >= 3) {
+    undroppable_frames_ = 0;
+    return kNoBuffer;
+  } else {
+    undroppable_frames_++;
+    return buffers[kNumberOfVp8VideoBuffers - 1];
+  }
 }
 
 void Vp8Encoder::GetCodecUpdateFlags(Vp8Buffers buffer_to_update,
                                      vpx_codec_flags_t* flags) {
   if (!use_multiple_video_buffers_)
     return;
 
   // Update at most one buffer, except for key-frames.
   switch (buffer_to_update) {
     case kAltRefBuffer:
@@ skipping 31 matching lines @@
   }
 }
 
 void Vp8Encoder::LatestFrameIdToReference(uint32 frame_id) {
   DCHECK(thread_checker_.CalledOnValidThread());
   if (!use_multiple_video_buffers_)
     return;
 
   VLOG(1) << "VP8 ok to reference frame:" << static_cast<int>(frame_id);
   for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) {
-    if (frame_id == used_buffers_frame_id_[i]) {
-      acked_frame_buffers_[i] = true;
+    if (frame_id == buffer_state_[i].frame_id) {
+      buffer_state_[i].state = kBufferAcked;
+      break;
+    }
+    if (IsOlderFrameId(last_acked_frame_id_, frame_id)) {

[Alpha Left Google, 2014/08/25 23:23:44]

Why is this inside the for loop? I don't understand what does this if statement
do.

[hubbe, 2014/08/27 04:14:03]

Messed up merge, fixed.

+      last_acked_frame_id_ = frame_id;
     }
   }
 }
 
 void Vp8Encoder::GenerateKeyFrame() {
   DCHECK(thread_checker_.CalledOnValidThread());
   key_frame_requested_ = true;
 }
 
 // Calculate the max size of the key frame relative to a normal delta frame.
 uint32 Vp8Encoder::MaxIntraTarget(uint32 optimal_buffer_size_ms) const {
   // Set max to the optimal buffer level (normalized by target BR),
   // and scaled by a scale_parameter.
   // Max target size = scalePar * optimalBufferSize * targetBR[Kbps].
   // This values is presented in percentage of perFrameBw:
   // perFrameBw = targetBR[Kbps] * 1000 / frameRate.
   // The target in % is as follows:
 
   float scale_parameter = 0.5;
   uint32 target_pct = optimal_buffer_size_ms * scale_parameter *
                       cast_config_.max_frame_rate / 10;
 
   // Don't go below 3 times the per frame bandwidth.
   return std::max(target_pct, kMinIntra);
 }
 
 }  // namespace cast
 }  // namespace media