MJPEG acceleration for video capture using VAAPI

content/browser/renderer_host/media/video_capture_device_client.cc

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/renderer_host/media/video_capture_device_client.h"
 
 #include "base/bind.h"
 #include "base/strings/stringprintf.h"
+#include "base/synchronization/waitable_event.h"
 #include "base/trace_event/trace_event.h"
+#include "content/browser/gpu/browser_gpu_channel_host_factory.h"
 #include "content/browser/renderer_host/media/video_capture_buffer_pool.h"
 #include "content/browser/renderer_host/media/video_capture_controller.h"
+#include "content/common/gpu/client/gpu_jpeg_decode_accelerator_host.h"
 #include "content/public/browser/browser_thread.h"
 #include "media/base/bind_to_current_loop.h"
 #include "media/base/video_capture_types.h"
 #include "media/base/video_frame.h"
+#include "media/base/video_util.h"
+#include "media/base/yuv_convert.h"
 #include "third_party/libyuv/include/libyuv.h"
 
 using media::VideoCaptureFormat;
 using media::VideoFrame;
 
 namespace content {
 
 // Class combining a Client::Buffer interface implementation and a pool buffer
 // implementation to guarantee proper cleanup on destruction on our side.
 class AutoReleaseBuffer : public media::VideoCaptureDevice::Client::Buffer {
  public:
   AutoReleaseBuffer(const scoped_refptr<VideoCaptureBufferPool>& pool,
                     int buffer_id,
                     void* data,
-                    size_t size)
+                    size_t size,
+                    base::SharedMemoryHandle handle)
       : pool_(pool),
         id_(buffer_id),
         data_(data),
-        size_(size) {
+        size_(size),
+        handle_(handle) {
     DCHECK(pool_.get());
   }
   int id() const override { return id_; }
   void* data() const override { return data_; }
   size_t size() const override { return size_; }
+  base::SharedMemoryHandle handle() const override { return handle_; }
 
  private:
   ~AutoReleaseBuffer() override { pool_->RelinquishProducerReservation(id_); }
 
   const scoped_refptr<VideoCaptureBufferPool> pool_;
   const int id_;
   void* const data_;
   const size_t size_;
+  const base::SharedMemoryHandle handle_;
 };
 
+JpegDecodeTask::JpegDecodeTask()
+    : decoding(false) {}
+
 VideoCaptureDeviceClient::VideoCaptureDeviceClient(
     const base::WeakPtr<VideoCaptureController>& controller,
     const scoped_refptr<VideoCaptureBufferPool>& buffer_pool)
     : controller_(controller),
+      jpeg_failed_(false),
       buffer_pool_(buffer_pool),
-      last_captured_pixel_format_(media::PIXEL_FORMAT_UNKNOWN) {}
+      last_captured_pixel_format_(media::PIXEL_FORMAT_UNKNOWN),
+      next_bitstream_buffer_id_(0) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+}
 
-VideoCaptureDeviceClient::~VideoCaptureDeviceClient() {}
+VideoCaptureDeviceClient::~VideoCaptureDeviceClient() {
+}
+
+bool VideoCaptureDeviceClient::InitializeJpegDecoder() {
+  // called from device thread
+  //DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  DVLOG(3) << __func__;
+  GpuChannelHost* const host =
+      BrowserGpuChannelHostFactory::instance()->GetGpuChannel();
+  GpuJpegDecodeAcceleratorHost* const decoder = host->CreateJpegDecoder();
+  if (!decoder->Initialize(this)) {
+    decoder->Destroy();
+    return false;
+  }
+  jpeg_decoder_.reset(decoder);
+
+  return true;
+}
+
+void VideoCaptureDeviceClient::VideoFrameReady(
+    int32_t bitstream_buffer_id) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  DVLOG(3) << __func__ << " " << bitstream_buffer_id;
+  base::AutoLock lock(jpeg_decode_task_.lock_);
+  DCHECK(jpeg_decode_task_.decoding);
+
+  DCHECK(jpeg_decode_task_.in_buffer.get());
+  if (bitstream_buffer_id != jpeg_decode_task_.in_buffer->id()) {
+    DLOG(ERROR) << "bitstream_buffer_id mismatched, expected "
+                << jpeg_decode_task_.in_buffer->id() << ", but got "
+                << bitstream_buffer_id;
+    return;
+  }
+
+  auto& captured_data = jpeg_decode_task_.captured_data;
+
+  BrowserThread::PostTask(
+      BrowserThread::IO,
+      FROM_HERE,
+      base::Bind(
+          &VideoCaptureController::DoIncomingCapturedVideoFrameOnIOThread,
+          controller_,
+          jpeg_decode_task_.out_buffer,
+          jpeg_decode_task_.out_frame,
+          captured_data.timestamp));
+
+  jpeg_decode_task_.out_frame = nullptr;
+  jpeg_decode_task_.out_buffer = nullptr;
+  jpeg_decode_task_.decoding = false;
+}
+
+void VideoCaptureDeviceClient::NotifyError(
+    int32_t bitstream_buffer_id,
+    media::JpegDecodeAccelerator::Error error) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  DVLOG(3) << __func__;
+  base::AutoLock lock(jpeg_decode_task_.lock_);
+  DCHECK(jpeg_decode_task_.decoding);
+
+  // TODO check bitstream_buffer_id
+  OnIncomingCapturedData2(jpeg_decode_task_.captured_data);

[wuchengli, 2015/04/14 09:41:44]

Add a LOG(ERROR) and call VideoCaptureDeviceClient::OnError should be enough. We
want to know why JPEG HW encoder fails. In the future if this stabilizes, we can
set |external_jpeg_decoder_| to null and the next frame will fall back to
software.

[kcwu, 2015/04/16 14:38:27]

Done.

+
+  jpeg_decode_task_.out_frame = nullptr;
+  jpeg_decode_task_.out_buffer = nullptr;
+  jpeg_decode_task_.decoding = false;
+}
+
+bool VideoCaptureDeviceClient::PrepareJpegDecode(
+    const CapturedData& captured_data,
+    const scoped_refptr<Buffer> out_buffer) {
+  DVLOG(3) << __func__;
+  DCHECK(!jpeg_decode_task_.decoding);
+
+  jpeg_decode_task_.captured_data = captured_data;
+
+  // enlarge input buffer if necessary
+  size_t in_buffer_size = captured_data.length;
+  if (!jpeg_decode_task_.in_shared_memory.get() ||
+      in_buffer_size > jpeg_decode_task_.in_shared_memory->mapped_size()) {
+    jpeg_decode_task_.in_shared_memory.reset(new base::SharedMemory);
+
+
+    if (!jpeg_decode_task_.in_shared_memory->CreateAnonymous(in_buffer_size)) {
+      DLOG(ERROR) << "CreateAnonymous failed";
+      return false;
+    }
+
+    if (!jpeg_decode_task_.in_shared_memory->Map(in_buffer_size)) {
+      DLOG(ERROR) << "Map failed";
+      return false;
+    }
+
+    jpeg_decode_task_.captured_data.data = reinterpret_cast<uint8*>(
+        jpeg_decode_task_.in_shared_memory->memory());
+  }
+
+  jpeg_decode_task_.in_buffer.reset(new media::BitstreamBuffer(
+      next_bitstream_buffer_id_,
+      jpeg_decode_task_.in_shared_memory->handle(),
+      in_buffer_size));
+  // Mask against 30 bits, to avoid (undefined) wraparound on signed integer.
+  next_bitstream_buffer_id_ = (next_bitstream_buffer_id_ + 1) & 0x3FFFFFFF;
+
+  gfx::Size dimensions = captured_data.frame_format.frame_size;
+
+  scoped_refptr<media::VideoFrame> out_frame =
+      media::VideoFrame::WrapExternalPackedMemory(
+          media::VideoFrame::I420,
+          dimensions,
+          gfx::Rect(dimensions),
+          dimensions,
+          reinterpret_cast<uint8*>(out_buffer->data()),
+          out_buffer->size(),
+          out_buffer->handle(),
+          0,
+          base::TimeDelta(),
+          base::Closure());
+  DCHECK(out_frame.get());
+
+  out_frame->metadata()->SetDouble(media::VideoFrameMetadata::FRAME_RATE,
+                                   captured_data.frame_format.frame_rate);
+
+  jpeg_decode_task_.out_frame = out_frame;
+  jpeg_decode_task_.out_buffer = out_buffer;
+
+  memcpy(jpeg_decode_task_.in_shared_memory->memory(),
+         captured_data.data,
+         captured_data.length);
+
+  return true;
+}
 
 void VideoCaptureDeviceClient::OnIncomingCapturedData(
     const uint8* data,
     int length,
     const VideoCaptureFormat& frame_format,
     int rotation,
     const base::TimeTicks& timestamp) {
   TRACE_EVENT0("video", "VideoCaptureController::OnIncomingCapturedData");
+  DVLOG(3) << __func__;
+
+  CapturedData captured_data;
+  captured_data.data = const_cast<uint8*>(data);
+  captured_data.length = length;
+  captured_data.frame_format = frame_format;
+  captured_data.rotation = rotation;
+  captured_data.timestamp = timestamp;
 
   if (last_captured_pixel_format_ != frame_format.pixel_format) {
     OnLog("Pixel format: " + media::VideoCaptureFormat::PixelFormatToString(
                                  frame_format.pixel_format));
     last_captured_pixel_format_ = frame_format.pixel_format;
   }
 
   if (!frame_format.IsValid())
     return;
 
+  const gfx::Size size = frame_format.frame_size;
+  if (rotation != 0 || frame_format.pixel_format != media::PIXEL_FORMAT_MJPEG ||
+      size.width() % 2 != 0 || size.height() % 2 != 0 || jpeg_failed_) {
+    OnIncomingCapturedData2(captured_data);
+    return;
+  }
+
+  if (!jpeg_decoder_) {
+    if (!InitializeJpegDecoder()) {
+      // TODO fallback to software decode

[wuchengli, 2015/04/14 09:41:44]

Add a LOG(ERROR) here. If we have GpuJpegDecoder external_jpeg_decoder_, we can
set external_jpeg_decoder_ to null and it will fall back to software.
|jpeg_failed_| won't be needed.

[kcwu, 2015/04/16 14:38:27]

Done.

+      jpeg_failed_ = true;
+      return;
+    }
+  }
+
+  gfx::Size dimensions = frame_format.frame_size;
+  scoped_refptr<Buffer> out_buffer =
+      ReserveOutputBuffer(media::VideoFrame::I420, dimensions);
+  if (!out_buffer.get()) {
+    DVLOG(1) << "Drop captured frame. No available buffers"
+             << " (all buffers are still queued to display)";
+    return;
+  }
+
+  base::AutoLock lock(jpeg_decode_task_.lock_);

[wuchengli, 2015/04/14 09:41:44]

Try to hold the lock as minimum as possible so IO thread can be run faster.

[kcwu, 2015/04/14 20:02:34]

Done.

+  if (jpeg_decode_task_.decoding) {

[wuchengli, 2015/04/14 09:41:44]

Checking out_frame == nullptr should be enough. Remove |decoding|.

[kcwu, 2015/04/16 14:38:27]

Done.

+    DVLOG(1) << "Drop captured frame. Previous jpeg frame is still decoding";
+    return;
+  }
+
+  if (!PrepareJpegDecode(captured_data, out_buffer)) {
+    // TODO(kcwu): fallback to software decode

[wuchengli, 2015/04/14 09:41:44]

Add a LOG(ERROR), call VideoCaptureDeviceClient::OnError, and remove TODO.

[kcwu, 2015/04/16 14:38:27]

Done.

+    return;
+  }
+
+  jpeg_decode_task_.decoding = true;
+  jpeg_decoder_->Decode(
+      *jpeg_decode_task_.in_buffer,
+      jpeg_decode_task_.out_frame);
+}
+
+// TODO(kcwu): need better name
+void VideoCaptureDeviceClient::OnIncomingCapturedData2(

[wuchengli, 2015/04/14 09:41:44]

The name OnIncomingCapturedData should be used for the original code. How about
moving all the code about Jpeg to a new file
content/browser/renderer_host/gpu_jpeg_decoder.cc (GpuJpegDecoder)? You can
initialize a |external_jpeg_decoder_| in the constructor. Then you can call
external_jpeg_decoder_->Decode in OnIncomingCapturedData.

Add a bool GpuJpegDecoder::Supported, which returns true only for OS_CHROMEOS
and ARCH_CPU_X86_FAMILY. So we don't add latency for those platforms that do not
support JPEG acceleration.

[kcwu, 2015/04/16 14:38:27]

Done.

+    const CapturedData& captured_data) {
+  DVLOG(3) << __func__;
+
+  const uint8* data = captured_data.data;
+  int length = captured_data.length;
+  const VideoCaptureFormat& frame_format = captured_data.frame_format;
+  int rotation = captured_data.rotation;
+  base::TimeTicks timestamp = captured_data.timestamp;
+
+  if (last_captured_pixel_format_ != frame_format.pixel_format) {
+    OnLog("Pixel format: " + media::VideoCaptureFormat::PixelFormatToString(
+                                 frame_format.pixel_format));
+    last_captured_pixel_format_ = frame_format.pixel_format;
+  }
+
   // Chopped pixels in width/height in case video capture device has odd
   // numbers for width/height.
+  // XXX kcwu: why crop??

[wuchengli, 2015/04/14 09:41:44]

remove

[kcwu, 2015/04/14 20:02:34]

Done.

   int chopped_width = 0;
   int chopped_height = 0;
   int new_unrotated_width = frame_format.frame_size.width();
   int new_unrotated_height = frame_format.frame_size.height();
 
   if (new_unrotated_width & 1) {
     --new_unrotated_width;
     chopped_width = 1;
   }
   if (new_unrotated_height & 1) {
@@ skipping 10 matching lines @@
   const gfx::Size dimensions(destination_width, destination_height);
   if (!VideoFrame::IsValidConfig(VideoFrame::I420,
                                  dimensions,
                                  gfx::Rect(dimensions),
                                  dimensions)) {
     return;
   }
 
   scoped_refptr<Buffer> buffer = ReserveOutputBuffer(VideoFrame::I420,
                                                      dimensions);
+  {
+    static int drop_count = 0;
+    static base::Time start_time = base::Time::Now();
+    if (!buffer.get())
+      drop_count++;
+    LOG(ERROR) << "drop fps = " << drop_count / (base::Time::Now() - start_time).InSecondsF();

[wuchengli, 2015/04/14 09:41:44]

remove

[kcwu, 2015/04/14 20:02:34]

Done.

+  }
   if (!buffer.get())
     return;
 
   uint8* const yplane = reinterpret_cast<uint8*>(buffer->data());
   uint8* const uplane =
       yplane + VideoFrame::PlaneAllocationSize(VideoFrame::I420,
                                                VideoFrame::kYPlane, dimensions);
   uint8* const vplane =
       uplane + VideoFrame::PlaneAllocationSize(VideoFrame::I420,
                                                VideoFrame::kUPlane, dimensions);
@@ skipping 122 matching lines @@
     DLOG_IF(ERROR, frame_bytes == 0) << "Error calculating allocation size";
   }
 
   int buffer_id_to_drop = VideoCaptureBufferPool::kInvalidId;
   const int buffer_id =
       buffer_pool_->ReserveForProducer(frame_bytes, &buffer_id_to_drop);
   if (buffer_id == VideoCaptureBufferPool::kInvalidId)
     return NULL;
   void* data;
   size_t size;
-  buffer_pool_->GetBufferInfo(buffer_id, &data, &size);
+  base::SharedMemoryHandle handle;
+  buffer_pool_->GetBufferInfo(buffer_id, &data, &size, &handle);
 
   scoped_refptr<media::VideoCaptureDevice::Client::Buffer> output_buffer(
-      new AutoReleaseBuffer(buffer_pool_, buffer_id, data, size));
+      new AutoReleaseBuffer(buffer_pool_, buffer_id, data, size, handle));
 
   if (buffer_id_to_drop != VideoCaptureBufferPool::kInvalidId) {
     BrowserThread::PostTask(BrowserThread::IO,
         FROM_HERE,
         base::Bind(&VideoCaptureController::DoBufferDestroyedOnIOThread,
                    controller_, buffer_id_to_drop));
   }
 
   return output_buffer;
 }
@@ skipping 29 matching lines @@
 }
 
 void VideoCaptureDeviceClient::OnLog(
     const std::string& message) {
   BrowserThread::PostTask(BrowserThread::IO, FROM_HERE,
                           base::Bind(&VideoCaptureController::DoLogOnIOThread,
                                      controller_, message));
 }
 
 }  // namespace content