MJPEG acceleration for video capture using VAAPI

content/browser/renderer_host/media/video_capture_device_client.cc

Index: content/browser/renderer_host/media/video_capture_device_client.cc
diff --git a/content/browser/renderer_host/media/video_capture_device_client.cc b/content/browser/renderer_host/media/video_capture_device_client.cc
index e9900a5810f57cd2567a369ea3e00bf547e0f5ac..5cb91f6b476762420c675cd1e3dfbb9c2261abc3 100644
--- a/content/browser/renderer_host/media/video_capture_device_client.cc
+++ b/content/browser/renderer_host/media/video_capture_device_client.cc
@@ -6,13 +6,18 @@
 
 #include "base/bind.h"
 #include "base/strings/stringprintf.h"
+#include "base/synchronization/waitable_event.h"
 #include "base/trace_event/trace_event.h"
+#include "content/browser/gpu/browser_gpu_channel_host_factory.h"
 #include "content/browser/renderer_host/media/video_capture_buffer_pool.h"
 #include "content/browser/renderer_host/media/video_capture_controller.h"
+#include "content/common/gpu/client/gpu_jpeg_decode_accelerator_host.h"
 #include "content/public/browser/browser_thread.h"
 #include "media/base/bind_to_current_loop.h"
 #include "media/base/video_capture_types.h"
 #include "media/base/video_frame.h"
+#include "media/base/video_util.h"
+#include "media/base/yuv_convert.h"
 #include "third_party/libyuv/include/libyuv.h"
 
 using media::VideoCaptureFormat;
@@ -51,10 +56,79 @@ VideoCaptureDeviceClient::VideoCaptureDeviceClient(
     const base::WeakPtr<VideoCaptureController>& controller,
     const scoped_refptr<VideoCaptureBufferPool>& buffer_pool)
     : controller_(controller),
+      jpeg_decode_event_(nullptr),
       buffer_pool_(buffer_pool),
-      last_captured_pixel_format_(media::PIXEL_FORMAT_UNKNOWN) {}
+      last_captured_pixel_format_(media::PIXEL_FORMAT_UNKNOWN),
+      next_bitstream_buffer_id_(0) {}
 
-VideoCaptureDeviceClient::~VideoCaptureDeviceClient() {}
+VideoCaptureDeviceClient::~VideoCaptureDeviceClient() {
+  if (jpeg_decoder_.get())

[mcasas, 2015/03/23 19:29:51]

If |jpeg_decoder_| is created from the VCD, then it
belongs to either Capture Thread or some other thread
(in Linux/CrOS case, "V4L2 thread"). It cannot be
destroyed here, which is IO Thread. You might need to
go through some extra hoops and hold a ref to the
actual capture thread.

[kcwu, 2015/03/30 18:12:14]

jpeg_decoder_ is scoped_ptr(with DefaultDeleter), so no need to call Destroy
explicitly here. These two lines are removed.

+    jpeg_decoder_->Destroy();
+}
+
+bool VideoCaptureDeviceClient::InitializeJpegDecoder() {

[mcasas, 2015/03/23 19:29:51]

This method looks convoluted and exposes too much 
info to VCD, plus creates a lot of state on VCDClient, 
which is supposed to be a shallow, near-stateless
 translation rig.

Propose adding a parameter to the constructor, 
where the result of host->CreateJpegDecoder();
is plugged in at ctor time in  VCManager. That 
way this decoder can be destructed properly
on ~VideoCaptureDeviceClient() and on IO thread.

[kcwu, 2015/03/30 18:12:14]

Using MJPEG or YUV formats is determined in
VideoCaptureDeviceLinux::V4L2CaptureDelegate::AllocateAndStart
This information is not yet available at ctor time.

I modified GpuJpegDecodeAcceleratorHost, make it work on device thread. And
InitializeJpegDecoder() is called inside OnIncomingCapturedData()

+  LOG(ERROR) << __func__;

[mcasas, 2015/03/23 19:29:51]

Suggest s/LOG(ERROR)/DVLOG(1)/

[kcwu, 2015/04/16 14:38:26]

Done.

+  GpuChannelHost* host =

[mcasas, 2015/03/23 19:29:51]

GpuChannelHost* const

[kcwu, 2015/04/14 20:02:34]

Done.

+      BrowserGpuChannelHostFactory::instance()->GetGpuChannel();
+  LOG(ERROR) << "host =" << host;
+  GpuJpegDecodeAcceleratorHost* decoder = host->CreateJpegDecoder();

[mcasas, 2015/03/23 19:29:50]

* const

[kcwu, 2015/04/14 20:02:33]

Done.

+  if (!decoder->Initialize(this)) {
+    decoder->Destroy();
+    return false;
+  }
+  jpeg_decoder_.reset(decoder);
+
+  return true;
+}
+
+void VideoCaptureDeviceClient::VideoFrameReady(
+    int32_t bitstream_buffer_id) {

[wuchengli, 2015/03/23 06:30:14]

Add DCHECK to make sure this runs on IO thread. Then you can call
DoIncomingCapturedVideoFrameOnIOThread directly.

[kcwu, 2015/03/30 18:12:14]

DCHECK added. But still PostTask because I want to release lock earlier.

+  auto map_it = jpeg_decode_task_map_.find(bitstream_buffer_id);

[wuchengli, 2015/03/23 06:30:14]

Like you said, since you are blocking the v4l2 thread, you don't need a map.
Let's assume the decode latency is low and we can block the v4l2 thread.

[kcwu, 2015/03/30 18:12:14]

This is refactored to async code.

+  if (map_it == jpeg_decode_task_map_.end()) {
+    LOG(ERROR) << "VideoFrameReady unknown bitstream_buffer_id "
+               << bitstream_buffer_id;
+    return;
+  }
+  auto captured_data = map_it->second;
+  auto frame_format = captured_data.frame_format;
+
+  BrowserThread::PostTask(
+      BrowserThread::IO,
+      FROM_HERE,
+      base::Bind(
+          &VideoCaptureController::DoIncomingCapturedVideoFrameOnIOThread,
+          controller_,
+          captured_data.buffer,
+          captured_data.frame,
+          captured_data.timestamp));
+
+  jpeg_decode_task_map_.erase(map_it);
+
+  CHECK(jpeg_decode_event_);
+  jpeg_decode_event_->Signal();
+}
+
+void VideoCaptureDeviceClient::NotifyError(
+    int32_t bitstream_buffer_id,
+    media::JpegDecodeAccelerator::Error error) {
+  LOG(ERROR) << "NotifyError";
+
+  auto map_it = jpeg_decode_task_map_.find(bitstream_buffer_id);
+  if (map_it != jpeg_decode_task_map_.end()) {
+    auto captured_data = map_it->second;
+    LOG(ERROR) << "fallback to software decode *******************************";
+    OnIncomingCapturedData2(captured_data);
+    jpeg_decode_task_map_.erase(map_it);
+    CHECK(jpeg_decode_event_);
+    jpeg_decode_event_->Signal();
+    return;
+  }
+
+  // TODO(kcwu)
+  LOG(ERROR) << "give up???";
+  CHECK(jpeg_decode_event_);
+  jpeg_decode_event_->Signal();
+}
 
 void VideoCaptureDeviceClient::OnIncomingCapturedData(
     const uint8* data,
@@ -64,6 +138,13 @@ void VideoCaptureDeviceClient::OnIncomingCapturedData(
     const base::TimeTicks& timestamp) {
   TRACE_EVENT0("video", "VideoCaptureController::OnIncomingCapturedData");
 
+  CapturedData captured_data;
+  captured_data.data = data;
+  captured_data.length = length;
+  captured_data.frame_format = frame_format;
+  captured_data.rotation = rotation;
+  captured_data.timestamp = timestamp;
+
   if (last_captured_pixel_format_ != frame_format.pixel_format) {
     OnLog("Pixel format: " + media::VideoCaptureFormat::PixelFormatToString(
                                  frame_format.pixel_format));
@@ -73,8 +154,96 @@ void VideoCaptureDeviceClient::OnIncomingCapturedData(
   if (!frame_format.IsValid())
     return;
 
+  const gfx::Size size = frame_format.frame_size;
+  if (rotation != 0 || frame_format.pixel_format != media::PIXEL_FORMAT_MJPEG ||
+      size.width() % 2 != 0 || size.height() % 2 != 0) {
+    OnIncomingCapturedData2(captured_data);

[wuchengli, 2015/03/23 06:30:14]

Please measure the decode time of both cases and put it in the change
description. The time between here and before calling
DoIncomingCapturedVideoFrameOnIOThread.

[kcwu, 2015/04/14 20:02:34]

Acknowledged.

+    return;
+  }
+
+  base::SharedMemory in_memory;
+  gfx::Size dimensions = frame_format.frame_size;
+  size_t in_buffer_size = length;
+  scoped_refptr<Buffer> out_buffer =
+      ReserveOutputBuffer(media::VideoFrame::I420, dimensions);
+  captured_data.buffer = out_buffer;
+
+  if (in_memory.CreateAnonymous(in_buffer_size) &&
+      in_memory.Map(in_buffer_size)) {
+    media::BitstreamBuffer in_buffer(next_bitstream_buffer_id_,
+                                     in_memory.handle(),
+                                     in_buffer_size);
+    // Mask against 30 bits, to avoid (undefined) wraparound on signed integer.
+    next_bitstream_buffer_id_ = (next_bitstream_buffer_id_ + 1) & 0x3FFFFFFF;
+
+    base::SharedMemoryHandle buffer_handle =
+        buffer_pool_->GetBufferSharedMemory(out_buffer->id());
+
+    uint8* shm_memory = reinterpret_cast<uint8*>(in_memory.memory());
+    memcpy(shm_memory, data, length);
+

[mcasas, 2015/03/23 19:29:50]

There's a lot of extra code to just maintain the illusion
of synchronicity for the VCD. I particularly dislike having
to Wait() on a lock (l.214).

Since anyway you're already dealing with a pool Buffer, 
what you could do is:

- allocate one Buffer here, with the exact size to hold the JPEG
frame, and VideoFrame::JPEG type.
- copy |data| inside it (using |out_buffer.data()|), 
- PostTask() out_buffer to the JPEG decoder on IO thread
- return to VCD.

After decode, on VideoFrameReady(), you use
WrapExternalPackedMemory() to make a VideoFrame.
The JPEG Buffer goes out of scope and is recycled.
You might need _another_ Buffer, with I420 format + size. 

Since JPEG and I420 have quite different sizes, the recycling
won't be optimal. You might need two pools.

[kcwu, 2015/03/30 18:12:14]

Done.

+    scoped_refptr<media::VideoFrame> out_frame =
+        media::VideoFrame::WrapExternalPackedMemory(
+            media::VideoFrame::I420,
+            dimensions,
+            gfx::Rect(dimensions),
+            dimensions,
+            reinterpret_cast<uint8*>(out_buffer->data()),
+            out_buffer->size(),
+            buffer_handle,
+            0,
+            base::TimeDelta(),
+            base::Closure());

[wuchengli, 2015/03/23 06:30:14]

Set framerate like line 387.
frame->metadata()->SetDouble(media::VideoFrameMetadata::FRAME_RATE,

[kcwu, 2015/03/30 18:12:14]

Done.

+    DCHECK(out_frame.get());
+
+    captured_data.frame = out_frame;
+    jpeg_decode_task_map_[in_buffer.id()] = captured_data;
+
+    CHECK(!jpeg_decode_event_);
+    jpeg_decode_event_ = new base::WaitableEvent(false, false);

[wuchengli, 2015/03/23 06:30:14]

Use scoped_ptr

[kcwu, 2015/03/30 18:12:14]

Code removed.

+
+    BrowserThread::PostTask(
+        BrowserThread::IO,
+        FROM_HERE,
+        base::Bind(
+            &VideoCaptureDeviceClient::DecodeJpegOnIOThread,

[wuchengli, 2015/03/23 06:30:14]

Post to JpegDecodeAccelerator::Decode directly.

[kcwu, 2015/03/30 18:12:14]

Modified to call Decode() directly.

+            base::Unretained(this),
+            in_buffer,
+            out_frame));
+
+    jpeg_decode_event_->Wait();
+    delete jpeg_decode_event_;
+    jpeg_decode_event_ = nullptr;
+  }
+}
+
+void VideoCaptureDeviceClient::DecodeJpegOnIOThread(
+    media::BitstreamBuffer in_buffer,
+    scoped_refptr<media::VideoFrame> out_frame) {
+  DCHECK(jpeg_decoder_);
+
+  jpeg_decoder_->Decode(in_buffer, out_frame);
+}
+
+// TODO(kcwu): need better name
+void VideoCaptureDeviceClient::OnIncomingCapturedData2(
+    const CapturedData& captured_data) {
+
+    const uint8* data = captured_data.data;
+    int length = captured_data.length;
+    const VideoCaptureFormat& frame_format = captured_data.frame_format;
+    int rotation = captured_data.rotation;
+    base::TimeTicks timestamp = captured_data.timestamp;
+
+  if (last_captured_pixel_format_ != frame_format.pixel_format) {
+    OnLog("Pixel format: " + media::VideoCaptureFormat::PixelFormatToString(
+                                 frame_format.pixel_format));
+    last_captured_pixel_format_ = frame_format.pixel_format;
+  }
+
   // Chopped pixels in width/height in case video capture device has odd
   // numbers for width/height.
+  // XXX kcwu: why crop??
   int chopped_width = 0;
   int chopped_height = 0;
   int new_unrotated_width = frame_format.frame_size.width();