Memory copy the VideoFrame to match the requirement for HW encoders.

media/cast/sender/external_video_encoder.cc

Index: media/cast/sender/external_video_encoder.cc
diff --git a/media/cast/sender/external_video_encoder.cc b/media/cast/sender/external_video_encoder.cc
index 3eb7740a3c5ba7621b48054c4c7bd1a171a7ef8e..5f59030160f2b87966e8669b07b8e50212171d97 100644
--- a/media/cast/sender/external_video_encoder.cc
+++ b/media/cast/sender/external_video_encoder.cc
@@ -13,12 +13,12 @@
 #include "base/format_macros.h"
 #include "base/logging.h"
 #include "base/macros.h"
-#include "base/memory/scoped_vector.h"
 #include "base/memory/shared_memory.h"
 #include "base/message_loop/message_loop.h"
 #include "base/metrics/histogram.h"
 #include "base/strings/stringprintf.h"
 #include "build/build_config.h"
+#include "media/base/bind_to_current_loop.h"
 #include "media/base/video_frame.h"
 #include "media/base/video_types.h"
 #include "media/base/video_util.h"
@@ -33,8 +33,14 @@
 namespace {
 
 enum { MAX_H264_QUANTIZER = 51 };
+
+// Number of buffers for encoded bit stream.
 static const size_t kOutputBufferCount = 3;
 
+// Maximum number of extra input buffers for encoder. The input buffers are only
+// used when copy is needed to match the required coded size.
+static const size_t kExtraInputBufferCount = 2;

[miu, 2016/05/03 19:05:43]

Note: They've just allowed the use of constexpr, a new C++11 construct.  So,
please use that for these constants:

  constexpr size_t kOutputBufferCount = 3;
  constexpr size_t kExtraInputBufferCount = 2;

[xjz, 2016/05/03 23:30:15]

Done.

+
 }  // namespace
 
 namespace media {
@@ -97,7 +103,8 @@ class ExternalVideoEncoder::VEAClientImpl
         codec_profile_(media::VIDEO_CODEC_PROFILE_UNKNOWN),
         key_frame_quantizer_parsable_(false),
         requested_bit_rate_(-1),
-        has_seen_zero_length_encoded_frame_(false) {}
+        has_seen_zero_length_encoded_frame_(false),
+        max_allowed_input_buffers_(0) {}
 
   base::SingleThreadTaskRunner* task_runner() const {
     return task_runner_.get();
@@ -135,6 +142,15 @@ class ExternalVideoEncoder::VEAClientImpl
                                                                max_frame_rate_);
   }
 
+  // The destruction call back of the copied video frame to free its use of
+  // the input buffer.
+  void ReturnInputBufferToPool(int index) {
+    DCHECK(task_runner_->RunsTasksOnCurrentThread());
+    DCHECK_GE(index, 0);
+    DCHECK_LT(index, static_cast<int>(input_buffers_.size()));
+    free_input_buffer_index_.push_back(index);
+  }
+
   void EncodeVideoFrame(
       const scoped_refptr<media::VideoFrame>& video_frame,
       const base::TimeTicks& reference_time,
@@ -149,8 +165,45 @@ class ExternalVideoEncoder::VEAClientImpl
         video_frame, reference_time, frame_encoded_callback,
         requested_bit_rate_));
 
+    scoped_refptr<media::VideoFrame> frame = video_frame;
+    if (video_frame->coded_size() != frame_coded_size_) {
+      DCHECK_GE(frame_coded_size_.width(), video_frame->visible_rect().width());
+      DCHECK_GE(frame_coded_size_.height(),
+                video_frame->visible_rect().height());
+
+      if (free_input_buffer_index_.empty()) {
+        if (input_buffers_.size() < max_allowed_input_buffers_) {

[miu, 2016/05/03 19:05:43]

It's unlikely, but possible, for this method to be called more than once before
we get more input buffers created. This might cause the code to create more than
the max allowed input buffers.

[xjz, 2016/05/03 23:30:15]

Done. Changed to only allocate one buffer at one time.

+          create_video_encode_memory_cb_.Run(
+              media::VideoFrame::AllocationSize(media::PIXEL_FORMAT_I420,
+                                                frame_coded_size_),
+              base::Bind(&VEAClientImpl::OnCreateInputSharedMemory, this));
+        }
+        ExitEncodingWithErrors();
+        return;
+      }
+
+      int index = free_input_buffer_index_.back();
+      base::SharedMemory* input_buffer = input_buffers_[index].get();
+      frame = VideoFrame::WrapExternalSharedMemory(
+          video_frame->format(), frame_coded_size_, video_frame->visible_rect(),
+          video_frame->visible_rect().size(),
+          static_cast<uint8_t*>(input_buffer->memory()),
+          input_buffer->mapped_size(), input_buffer->handle(), 0,
+          video_frame->timestamp());
+      if (!frame || !media::I420CopyWithPadding(*video_frame, frame.get())) {
+        LOG(DFATAL) << "Error: ExternalVideoEncoder: copy failed.";
+        ExitEncodingWithErrors();
+        return;
+      }
+
+      frame->AddDestructionObserver(media::BindToCurrentLoop(base::Bind(
+          &ExternalVideoEncoder::VEAClientImpl::ReturnInputBufferToPool, this,
+          index)));
+      free_input_buffer_index_.pop_back();
+    }
+
     // BitstreamBufferReady will be called once the encoder is done.
-    video_encode_accelerator_->Encode(video_frame, key_frame_requested);
+    video_encode_accelerator_->Encode(frame, key_frame_requested);
   }
 
  protected:
@@ -177,8 +230,10 @@ class ExternalVideoEncoder::VEAClientImpl
                                size_t output_buffer_size) final {
     DCHECK(task_runner_->RunsTasksOnCurrentThread());
 
-    // TODO(miu): Investigate why we are ignoring |input_count| (4) and instead
-    // using |kOutputBufferCount| (3) here.
+    frame_coded_size_ = input_coded_size;
+
+    max_allowed_input_buffers_ = input_count + kExtraInputBufferCount;
+
     for (size_t j = 0; j < kOutputBufferCount; ++j) {
       create_video_encode_memory_cb_.Run(
           output_buffer_size,
@@ -201,7 +256,8 @@ class ExternalVideoEncoder::VEAClientImpl
       NotifyError(media::VideoEncodeAccelerator::kPlatformFailureError);
       return;
     }
-    base::SharedMemory* output_buffer = output_buffers_[bitstream_buffer_id];
+    base::SharedMemory* output_buffer =
+        output_buffers_[bitstream_buffer_id].get();
     if (payload_size > output_buffer->mapped_size()) {
       NOTREACHED();
       VLOG(1) << "BitstreamBufferReady(): invalid payload_size = "
@@ -223,7 +279,8 @@ class ExternalVideoEncoder::VEAClientImpl
     } else if (!in_progress_frame_encodes_.empty()) {
       const InProgressFrameEncode& request = in_progress_frame_encodes_.front();
 
-      scoped_ptr<SenderEncodedFrame> encoded_frame(new SenderEncodedFrame());
+      std::unique_ptr<SenderEncodedFrame> encoded_frame(
+          new SenderEncodedFrame());
       encoded_frame->dependency = key_frame ? EncodedFrame::KEY :
           EncodedFrame::DEPENDENT;
       encoded_frame->frame_id = next_frame_id_++;
@@ -361,14 +418,20 @@ class ExternalVideoEncoder::VEAClientImpl
   }
 
   // Note: This method can be called on any thread.
-  void OnCreateSharedMemory(scoped_ptr<base::SharedMemory> memory) {
+  void OnCreateSharedMemory(std::unique_ptr<base::SharedMemory> memory) {
     task_runner_->PostTask(FROM_HERE,
                            base::Bind(&VEAClientImpl::OnReceivedSharedMemory,
                                       this,
                                       base::Passed(&memory)));
   }
 
-  void OnReceivedSharedMemory(scoped_ptr<base::SharedMemory> memory) {
+  void OnCreateInputSharedMemory(std::unique_ptr<base::SharedMemory> memory) {
+    task_runner_->PostTask(
+        FROM_HERE, base::Bind(&VEAClientImpl::OnReceivedInputSharedMemory, this,
+                              base::Passed(&memory)));
+  }
+
+  void OnReceivedSharedMemory(std::unique_ptr<base::SharedMemory> memory) {
     DCHECK(task_runner_->RunsTasksOnCurrentThread());
 
     output_buffers_.push_back(std::move(memory));
@@ -386,6 +449,26 @@ class ExternalVideoEncoder::VEAClientImpl
     }
   }
 
+  void OnReceivedInputSharedMemory(std::unique_ptr<base::SharedMemory> memory) {
+    DCHECK(task_runner_->RunsTasksOnCurrentThread());
+
+    input_buffers_.push_back(std::move(memory));
+    free_input_buffer_index_.push_back(input_buffers_.size() - 1);
+  }
+
+  // This is called when copy errors occur in encoding process when there is
+  // need to copy the VideoFrames to match the required coded size for encoder.
+  void ExitEncodingWithErrors() {
+    DCHECK(task_runner_->RunsTasksOnCurrentThread());
+
+    std::unique_ptr<SenderEncodedFrame> no_result(nullptr);
+    cast_environment_->PostTask(
+        CastEnvironment::MAIN, FROM_HERE,
+        base::Bind(in_progress_frame_encodes_.back().frame_encoded_callback,
+                   base::Passed(&no_result)));
+    in_progress_frame_encodes_.pop_back();
+  }
+
   // Parse H264 SPS, PPS, and Slice header, and return the averaged frame
   // quantizer in the range of [0, 51], or -1 on parse error.
   double GetH264FrameQuantizer(const uint8_t* encoded_data, off_t size) {
@@ -450,7 +533,7 @@ class ExternalVideoEncoder::VEAClientImpl
   const int max_frame_rate_;
   const StatusChangeCallback status_change_cb_;  // Must be run on MAIN thread.
   const CreateVideoEncodeMemoryCallback create_video_encode_memory_cb_;
-  scoped_ptr<media::VideoEncodeAccelerator> video_encode_accelerator_;
+  std::unique_ptr<media::VideoEncodeAccelerator> video_encode_accelerator_;
   bool encoder_active_;
   uint32_t next_frame_id_;
   bool key_frame_encountered_;
@@ -460,7 +543,16 @@ class ExternalVideoEncoder::VEAClientImpl
   H264Parser h264_parser_;
 
   // Shared memory buffers for output with the VideoAccelerator.
-  ScopedVector<base::SharedMemory> output_buffers_;
+  std::vector<std::unique_ptr<base::SharedMemory>> output_buffers_;
+
+  // Shared memory buffers for input video frames with the VideoAccelerator.
+  // These buffers will be allocated only when copy is needed to match the
+  // required coded size for encoder. They are allocated on-demand, up to
+  // |max_allowed_input_buffers_|.
+  std::vector<std::unique_ptr<base::SharedMemory>> input_buffers_;
+
+  // Available input buffer index. These buffers are used in FILO order.
+  std::vector<int> free_input_buffer_index_;
 
   // FIFO list.
   std::list<InProgressFrameEncode> in_progress_frame_encodes_;
@@ -476,6 +568,14 @@ class ExternalVideoEncoder::VEAClientImpl
   // TODO(miu): Remove after discovering cause.  http://crbug.com/519022
   bool has_seen_zero_length_encoded_frame_;
 
+  // The coded size of the video frame required by Encoder. This size is
+  // obtained from VEA through |RequireBitstreamBuffers()|.
+  gfx::Size frame_coded_size_;
+
+  // The maximum number of input buffers. These buffers are used to copy
+  // VideoFrames in order to match the required coded size for encoder.
+  uint32_t max_allowed_input_buffers_;

[miu, 2016/05/03 19:05:43]

s/uint32_t/size_t/

[xjz, 2016/05/03 23:30:15]

Done.

+
   DISALLOW_COPY_AND_ASSIGN(VEAClientImpl);
 };
 
@@ -627,14 +727,11 @@ SizeAdaptableExternalVideoEncoder::SizeAdaptableExternalVideoEncoder(
 
 SizeAdaptableExternalVideoEncoder::~SizeAdaptableExternalVideoEncoder() {}
 
-scoped_ptr<VideoEncoder> SizeAdaptableExternalVideoEncoder::CreateEncoder() {
-  return scoped_ptr<VideoEncoder>(new ExternalVideoEncoder(
-      cast_environment(),
-      video_config(),
-      frame_size(),
-      last_frame_id() + 1,
-      CreateEncoderStatusChangeCallback(),
-      create_vea_cb_,
+std::unique_ptr<VideoEncoder>
+SizeAdaptableExternalVideoEncoder::CreateEncoder() {
+  return std::unique_ptr<VideoEncoder>(new ExternalVideoEncoder(
+      cast_environment(), video_config(), frame_size(), last_frame_id() + 1,
+      CreateEncoderStatusChangeCallback(), create_vea_cb_,
       create_video_encode_memory_cb_));
 }