Memory copy the VideoFrame to match the requirement for HW encoders.

media/cast/sender/external_video_encoder.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/cast/sender/external_video_encoder.h"
 
 #include <cmath>
 #include <utility>
 
 #include "base/bind.h"
 #include "base/debug/crash_logging.h"
 #include "base/debug/dump_without_crashing.h"
 #include "base/format_macros.h"
 #include "base/logging.h"
 #include "base/macros.h"
-#include "base/memory/scoped_vector.h"
 #include "base/memory/shared_memory.h"
 #include "base/message_loop/message_loop.h"
 #include "base/metrics/histogram.h"
 #include "base/strings/stringprintf.h"
 #include "build/build_config.h"
+#include "media/base/bind_to_current_loop.h"
 #include "media/base/video_frame.h"
 #include "media/base/video_types.h"
 #include "media/base/video_util.h"
 #include "media/cast/cast_config.h"
 #include "media/cast/cast_defines.h"
 #include "media/cast/common/rtp_time.h"
 #include "media/cast/logging/logging_defines.h"
 #include "media/cast/net/cast_transport_config.h"
 #include "media/cast/sender/vp8_quantizer_parser.h"
 #include "media/filters/h264_parser.h"
 
 namespace {
 
 enum { MAX_H264_QUANTIZER = 51 };
+
+// Number of buffers for encoded bit stream.
 static const size_t kOutputBufferCount = 3;
 
+// Maximum number of extra input buffers for encoder. The input buffers are only
+// used when copy is needed to match the required coded size.
+static const size_t kExtraInputBufferCount = 2;

[miu, 2016/05/03 19:05:43]

Note: They've just allowed the use of constexpr, a new C++11 construct.  So,
please use that for these constants:

  constexpr size_t kOutputBufferCount = 3;
  constexpr size_t kExtraInputBufferCount = 2;

[xjz, 2016/05/03 23:30:15]

Done.

+
 }  // namespace
 
 namespace media {
 namespace cast {
 
 // Container for the associated data of a video frame being processed.
 struct InProgressFrameEncode {
   // The source content to encode.
   const scoped_refptr<VideoFrame> video_frame;
 
@@ skipping 42 matching lines @@
         max_frame_rate_(max_frame_rate),
         status_change_cb_(status_change_cb),
         create_video_encode_memory_cb_(create_video_encode_memory_cb),
         video_encode_accelerator_(std::move(vea)),
         encoder_active_(false),
         next_frame_id_(0u),
         key_frame_encountered_(false),
         codec_profile_(media::VIDEO_CODEC_PROFILE_UNKNOWN),
         key_frame_quantizer_parsable_(false),
         requested_bit_rate_(-1),
-        has_seen_zero_length_encoded_frame_(false) {}
+        has_seen_zero_length_encoded_frame_(false),
+        max_allowed_input_buffers_(0) {}
 
   base::SingleThreadTaskRunner* task_runner() const {
     return task_runner_.get();
   }
 
   void Initialize(const gfx::Size& frame_size,
                   VideoCodecProfile codec_profile,
                   int start_bit_rate,
                   uint32_t first_frame_id) {
     DCHECK(task_runner_->RunsTasksOnCurrentThread());
@@ skipping 17 matching lines @@
   }
 
   void SetBitRate(int bit_rate) {
     DCHECK(task_runner_->RunsTasksOnCurrentThread());
 
     requested_bit_rate_ = bit_rate;
     video_encode_accelerator_->RequestEncodingParametersChange(bit_rate,
                                                                max_frame_rate_);
   }
 
+  // The destruction call back of the copied video frame to free its use of
+  // the input buffer.
+  void ReturnInputBufferToPool(int index) {
+    DCHECK(task_runner_->RunsTasksOnCurrentThread());
+    DCHECK_GE(index, 0);
+    DCHECK_LT(index, static_cast<int>(input_buffers_.size()));
+    free_input_buffer_index_.push_back(index);
+  }
+
   void EncodeVideoFrame(
       const scoped_refptr<media::VideoFrame>& video_frame,
       const base::TimeTicks& reference_time,
       bool key_frame_requested,
       const VideoEncoder::FrameEncodedCallback& frame_encoded_callback) {
     DCHECK(task_runner_->RunsTasksOnCurrentThread());
 
     if (!encoder_active_)
       return;
 
     in_progress_frame_encodes_.push_back(InProgressFrameEncode(
         video_frame, reference_time, frame_encoded_callback,
         requested_bit_rate_));
 
+    scoped_refptr<media::VideoFrame> frame = video_frame;
+    if (video_frame->coded_size() != frame_coded_size_) {
+      DCHECK_GE(frame_coded_size_.width(), video_frame->visible_rect().width());
+      DCHECK_GE(frame_coded_size_.height(),
+                video_frame->visible_rect().height());
+
+      if (free_input_buffer_index_.empty()) {
+        if (input_buffers_.size() < max_allowed_input_buffers_) {

[miu, 2016/05/03 19:05:43]

It's unlikely, but possible, for this method to be called more than once before
we get more input buffers created. This might cause the code to create more than
the max allowed input buffers.

[xjz, 2016/05/03 23:30:15]

Done. Changed to only allocate one buffer at one time.

+          create_video_encode_memory_cb_.Run(
+              media::VideoFrame::AllocationSize(media::PIXEL_FORMAT_I420,
+                                                frame_coded_size_),
+              base::Bind(&VEAClientImpl::OnCreateInputSharedMemory, this));
+        }
+        ExitEncodingWithErrors();
+        return;
+      }
+
+      int index = free_input_buffer_index_.back();
+      base::SharedMemory* input_buffer = input_buffers_[index].get();
+      frame = VideoFrame::WrapExternalSharedMemory(
+          video_frame->format(), frame_coded_size_, video_frame->visible_rect(),
+          video_frame->visible_rect().size(),
+          static_cast<uint8_t*>(input_buffer->memory()),
+          input_buffer->mapped_size(), input_buffer->handle(), 0,
+          video_frame->timestamp());
+      if (!frame || !media::I420CopyWithPadding(*video_frame, frame.get())) {
+        LOG(DFATAL) << "Error: ExternalVideoEncoder: copy failed.";
+        ExitEncodingWithErrors();
+        return;
+      }
+
+      frame->AddDestructionObserver(media::BindToCurrentLoop(base::Bind(
+          &ExternalVideoEncoder::VEAClientImpl::ReturnInputBufferToPool, this,
+          index)));
+      free_input_buffer_index_.pop_back();
+    }
+
     // BitstreamBufferReady will be called once the encoder is done.
-    video_encode_accelerator_->Encode(video_frame, key_frame_requested);
+    video_encode_accelerator_->Encode(frame, key_frame_requested);
   }
 
  protected:
   void NotifyError(VideoEncodeAccelerator::Error error) final {
     DCHECK(task_runner_->RunsTasksOnCurrentThread());
 
     DCHECK(error != VideoEncodeAccelerator::kInvalidArgumentError &&
            error != VideoEncodeAccelerator::kIllegalStateError);
 
     encoder_active_ = false;
 
     cast_environment_->PostTask(
         CastEnvironment::MAIN,
         FROM_HERE,
         base::Bind(status_change_cb_, STATUS_CODEC_RUNTIME_ERROR));
 
     // TODO(miu): Force-flush all |in_progress_frame_encodes_| immediately so
     // pending frames do not become stuck, freezing VideoSender.
   }
 
   // Called to allocate the input and output buffers.
   void RequireBitstreamBuffers(unsigned int input_count,
                                const gfx::Size& input_coded_size,
                                size_t output_buffer_size) final {
     DCHECK(task_runner_->RunsTasksOnCurrentThread());
 
-    // TODO(miu): Investigate why we are ignoring |input_count| (4) and instead
-    // using |kOutputBufferCount| (3) here.
+    frame_coded_size_ = input_coded_size;
+
+    max_allowed_input_buffers_ = input_count + kExtraInputBufferCount;
+
     for (size_t j = 0; j < kOutputBufferCount; ++j) {
       create_video_encode_memory_cb_.Run(
           output_buffer_size,
           base::Bind(&VEAClientImpl::OnCreateSharedMemory, this));
     }
   }
 
   // Encoder has encoded a frame and it's available in one of the output
   // buffers.  Package the result in a media::cast::EncodedFrame and post it
   // to the Cast MAIN thread via the supplied callback.
   void BitstreamBufferReady(int32_t bitstream_buffer_id,
                             size_t payload_size,
                             bool key_frame) final {
     DCHECK(task_runner_->RunsTasksOnCurrentThread());
     if (bitstream_buffer_id < 0 ||
         bitstream_buffer_id >= static_cast<int32_t>(output_buffers_.size())) {
       NOTREACHED();
       VLOG(1) << "BitstreamBufferReady(): invalid bitstream_buffer_id="
               << bitstream_buffer_id;
       NotifyError(media::VideoEncodeAccelerator::kPlatformFailureError);
       return;
     }
-    base::SharedMemory* output_buffer = output_buffers_[bitstream_buffer_id];
+    base::SharedMemory* output_buffer =
+        output_buffers_[bitstream_buffer_id].get();
     if (payload_size > output_buffer->mapped_size()) {
       NOTREACHED();
       VLOG(1) << "BitstreamBufferReady(): invalid payload_size = "
               << payload_size;
       NotifyError(media::VideoEncodeAccelerator::kPlatformFailureError);
       return;
     }
     if (key_frame)
       key_frame_encountered_ = true;
     if (!key_frame_encountered_) {
       // Do not send video until we have encountered the first key frame.
       // Save the bitstream buffer in |stream_header_| to be sent later along
       // with the first key frame.
       //
       // TODO(miu): Should |stream_header_| be an std::ostringstream for
       // performance reasons?
       stream_header_.append(static_cast<const char*>(output_buffer->memory()),
                             payload_size);
     } else if (!in_progress_frame_encodes_.empty()) {
       const InProgressFrameEncode& request = in_progress_frame_encodes_.front();
 
-      scoped_ptr<SenderEncodedFrame> encoded_frame(new SenderEncodedFrame());
+      std::unique_ptr<SenderEncodedFrame> encoded_frame(
+          new SenderEncodedFrame());
       encoded_frame->dependency = key_frame ? EncodedFrame::KEY :
           EncodedFrame::DEPENDENT;
       encoded_frame->frame_id = next_frame_id_++;
       if (key_frame)
         encoded_frame->referenced_frame_id = encoded_frame->frame_id;
       else
         encoded_frame->referenced_frame_id = encoded_frame->frame_id - 1;
       encoded_frame->rtp_timestamp = RtpTimeTicks::FromTimeDelta(
           request.video_frame->timestamp(), kVideoFrequency);
       encoded_frame->reference_time = request.reference_time;
@@ skipping 117 matching lines @@
   ~VEAClientImpl() final {
     // According to the media::VideoEncodeAccelerator interface, Destroy()
     // should be called instead of invoking its private destructor.
     task_runner_->PostTask(
         FROM_HERE,
         base::Bind(&media::VideoEncodeAccelerator::Destroy,
                    base::Unretained(video_encode_accelerator_.release())));
   }
 
   // Note: This method can be called on any thread.
-  void OnCreateSharedMemory(scoped_ptr<base::SharedMemory> memory) {
+  void OnCreateSharedMemory(std::unique_ptr<base::SharedMemory> memory) {
     task_runner_->PostTask(FROM_HERE,
                            base::Bind(&VEAClientImpl::OnReceivedSharedMemory,
                                       this,
                                       base::Passed(&memory)));
   }
 
-  void OnReceivedSharedMemory(scoped_ptr<base::SharedMemory> memory) {
+  void OnCreateInputSharedMemory(std::unique_ptr<base::SharedMemory> memory) {
+    task_runner_->PostTask(
+        FROM_HERE, base::Bind(&VEAClientImpl::OnReceivedInputSharedMemory, this,
+                              base::Passed(&memory)));
+  }
+
+  void OnReceivedSharedMemory(std::unique_ptr<base::SharedMemory> memory) {
     DCHECK(task_runner_->RunsTasksOnCurrentThread());
 
     output_buffers_.push_back(std::move(memory));
 
     // Wait until all requested buffers are received.
     if (output_buffers_.size() < kOutputBufferCount)
       return;
 
     // Immediately provide all output buffers to the VEA.
     for (size_t i = 0; i < output_buffers_.size(); ++i) {
       video_encode_accelerator_->UseOutputBitstreamBuffer(
           media::BitstreamBuffer(static_cast<int32_t>(i),
                                  output_buffers_[i]->handle(),
                                  output_buffers_[i]->mapped_size()));
     }
   }
 
+  void OnReceivedInputSharedMemory(std::unique_ptr<base::SharedMemory> memory) {
+    DCHECK(task_runner_->RunsTasksOnCurrentThread());
+
+    input_buffers_.push_back(std::move(memory));
+    free_input_buffer_index_.push_back(input_buffers_.size() - 1);
+  }
+
+  // This is called when copy errors occur in encoding process when there is
+  // need to copy the VideoFrames to match the required coded size for encoder.
+  void ExitEncodingWithErrors() {
+    DCHECK(task_runner_->RunsTasksOnCurrentThread());
+
+    std::unique_ptr<SenderEncodedFrame> no_result(nullptr);
+    cast_environment_->PostTask(
+        CastEnvironment::MAIN, FROM_HERE,
+        base::Bind(in_progress_frame_encodes_.back().frame_encoded_callback,
+                   base::Passed(&no_result)));
+    in_progress_frame_encodes_.pop_back();
+  }
+
   // Parse H264 SPS, PPS, and Slice header, and return the averaged frame
   // quantizer in the range of [0, 51], or -1 on parse error.
   double GetH264FrameQuantizer(const uint8_t* encoded_data, off_t size) {
     DCHECK(encoded_data);
     if (!size)
       return -1;
     h264_parser_.SetStream(encoded_data, size);
     double total_quantizer = 0;
     int num_slices = 0;
 
@@ skipping 44 matching lines @@
       }
     }
     return (num_slices == 0) ? -1 : (total_quantizer / num_slices);
   }
 
   const scoped_refptr<CastEnvironment> cast_environment_;
   const scoped_refptr<base::SingleThreadTaskRunner> task_runner_;
   const int max_frame_rate_;
   const StatusChangeCallback status_change_cb_;  // Must be run on MAIN thread.
   const CreateVideoEncodeMemoryCallback create_video_encode_memory_cb_;
-  scoped_ptr<media::VideoEncodeAccelerator> video_encode_accelerator_;
+  std::unique_ptr<media::VideoEncodeAccelerator> video_encode_accelerator_;
   bool encoder_active_;
   uint32_t next_frame_id_;
   bool key_frame_encountered_;
   std::string stream_header_;
   VideoCodecProfile codec_profile_;
   bool key_frame_quantizer_parsable_;
   H264Parser h264_parser_;
 
   // Shared memory buffers for output with the VideoAccelerator.
-  ScopedVector<base::SharedMemory> output_buffers_;
+  std::vector<std::unique_ptr<base::SharedMemory>> output_buffers_;
+
+  // Shared memory buffers for input video frames with the VideoAccelerator.
+  // These buffers will be allocated only when copy is needed to match the
+  // required coded size for encoder. They are allocated on-demand, up to
+  // |max_allowed_input_buffers_|.
+  std::vector<std::unique_ptr<base::SharedMemory>> input_buffers_;
+
+  // Available input buffer index. These buffers are used in FILO order.
+  std::vector<int> free_input_buffer_index_;
 
   // FIFO list.
   std::list<InProgressFrameEncode> in_progress_frame_encodes_;
 
   // The requested encode bit rate for the next frame.
   int requested_bit_rate_;
 
   // Used to compute utilization metrics for each frame.
   QuantizerEstimator quantizer_estimator_;
 
   // Set to true once a frame with zero-length encoded data has been
   // encountered.
   // TODO(miu): Remove after discovering cause.  http://crbug.com/519022
   bool has_seen_zero_length_encoded_frame_;
 
+  // The coded size of the video frame required by Encoder. This size is
+  // obtained from VEA through |RequireBitstreamBuffers()|.
+  gfx::Size frame_coded_size_;
+
+  // The maximum number of input buffers. These buffers are used to copy
+  // VideoFrames in order to match the required coded size for encoder.
+  uint32_t max_allowed_input_buffers_;

[miu, 2016/05/03 19:05:43]

s/uint32_t/size_t/

[xjz, 2016/05/03 23:30:15]

Done.

+
   DISALLOW_COPY_AND_ASSIGN(VEAClientImpl);
 };
 
 // static
 bool ExternalVideoEncoder::IsSupported(const VideoSenderConfig& video_config) {
   if (video_config.codec != CODEC_VIDEO_VP8 &&
       video_config.codec != CODEC_VIDEO_H264)
     return false;
 
   // TODO(miu): "Layering hooks" are needed to be able to query outside of
@@ skipping 131 matching lines @@
     const CreateVideoEncodeAcceleratorCallback& create_vea_cb,
     const CreateVideoEncodeMemoryCallback& create_video_encode_memory_cb)
     : SizeAdaptableVideoEncoderBase(cast_environment,
                                     video_config,
                                     status_change_cb),
       create_vea_cb_(create_vea_cb),
       create_video_encode_memory_cb_(create_video_encode_memory_cb) {}
 
 SizeAdaptableExternalVideoEncoder::~SizeAdaptableExternalVideoEncoder() {}
 
-scoped_ptr<VideoEncoder> SizeAdaptableExternalVideoEncoder::CreateEncoder() {
-  return scoped_ptr<VideoEncoder>(new ExternalVideoEncoder(
-      cast_environment(),
-      video_config(),
-      frame_size(),
-      last_frame_id() + 1,
-      CreateEncoderStatusChangeCallback(),
-      create_vea_cb_,
+std::unique_ptr<VideoEncoder>
+SizeAdaptableExternalVideoEncoder::CreateEncoder() {
+  return std::unique_ptr<VideoEncoder>(new ExternalVideoEncoder(
+      cast_environment(), video_config(), frame_size(), last_frame_id() + 1,
+      CreateEncoderStatusChangeCallback(), create_vea_cb_,
       create_video_encode_memory_cb_));
 }
 
 QuantizerEstimator::QuantizerEstimator() {}
 
 QuantizerEstimator::~QuantizerEstimator() {}
 
 void QuantizerEstimator::Reset() {
   last_frame_pixel_buffer_.reset();
 }
@@ skipping 139 matching lines @@
   const double kEntropyAtMaxQuantizer = 7.5;
   const double slope =
       (MAX_VP8_QUANTIZER - MIN_VP8_QUANTIZER) / kEntropyAtMaxQuantizer;
   const double quantizer = std::min<double>(
       MAX_VP8_QUANTIZER, MIN_VP8_QUANTIZER + slope * shannon_entropy);
   return quantizer;
 }
 
 }  //  namespace cast
 }  //  namespace media