MJPEG acceleration for video capture using VAAPI

content/browser/renderer_host/media/video_capture_device_client.cc

Index: content/browser/renderer_host/media/video_capture_device_client.cc
diff --git a/content/browser/renderer_host/media/video_capture_device_client.cc b/content/browser/renderer_host/media/video_capture_device_client.cc
index 66b7fdb3d5d8064233e13585558b4a3894daac4e..deea30dc34e9bb27f4ef5d80e7147c9d42fabcb0 100644
--- a/content/browser/renderer_host/media/video_capture_device_client.cc
+++ b/content/browser/renderer_host/media/video_capture_device_client.cc
@@ -7,8 +7,10 @@
 #include "base/bind.h"
 #include "base/strings/stringprintf.h"
 #include "base/trace_event/trace_event.h"
+#include "content/browser/gpu/browser_gpu_channel_host_factory.h"
 #include "content/browser/renderer_host/media/video_capture_buffer_pool.h"
 #include "content/browser/renderer_host/media/video_capture_controller.h"
+#include "content/common/gpu/client/gpu_jpeg_decode_accelerator_host.h"
 #include "content/public/browser/browser_thread.h"
 #include "media/base/bind_to_current_loop.h"
 #include "media/base/video_capture_types.h"
@@ -20,6 +22,140 @@ using media::VideoFrame;
 
 namespace content {
 
+struct CapturedData {
+  uint8* data;
+  int length;
+  media::VideoCaptureFormat frame_format;
+  int rotation;
+  base::TimeTicks timestamp;
+};
+
+// This class may be accessed by device thread and IO thread. The access is
+// exclusive: device thread can only access members if |decoding_| is false.
+// IO thread can only access if |decoding_| is true.
+class JpegDecodeData {
+ public:
+  typedef media::VideoCaptureDevice::Client::Buffer ClientBuffer;
+  JpegDecodeData()
+      : decoding_(false),
+        next_bitstream_buffer_id_(0),
+        in_buffer_(media::JpegDecodeAccelerator::kInvalidBitstreamBufferId,
+                   base::SharedMemory::NULLHandle(),
+                   0) {}
+
+  bool decoding() {
+    base::AutoLock lock(lock_);
+    return decoding_;
+  }
+  void set_decoding(bool decoding) {
+    base::AutoLock lock(lock_);
+    DCHECK(decoding_ != decoding);
+    decoding_ = decoding;
+  }
+
+  media::BitstreamBuffer in_buffer() { return in_buffer_; }
+  const CapturedData& captured_data() { return captured_data_; }
+  void set_captured_data(const CapturedData& data) {
+    DCHECK(!decoding());
+    captured_data_ = data;
+  }
+  scoped_refptr<ClientBuffer> out_buffer() {
+    DCHECK(decoding());
+    return out_buffer_;
+  }
+  scoped_refptr<media::VideoFrame> out_frame() { return out_frame_; }
+
+  // Adapt |captured_data| and |out_buffer| to BitstreamBuffer and
+  // VideoFrame, which JDA expects.
+  bool PrepareDecode(const CapturedData& captured_data,
+                     const scoped_refptr<ClientBuffer> out_buffer);
+  void DecodeDone();
+
+ private:
+  base::Lock lock_;
+  bool decoding_;  // protected by lock_;
+
+  CapturedData captured_data_;
+  int32 next_bitstream_buffer_id_;
+
+  scoped_ptr<base::SharedMemory> in_shared_memory_;
+  // |in_buffer_| is backed by |in_shared_memory_|.
+  media::BitstreamBuffer in_buffer_;
+
+  scoped_refptr<ClientBuffer> out_buffer_;
+  // |out_frame_| is backed by |out_buffer_|.
+  scoped_refptr<media::VideoFrame> out_frame_;
+
+  DISALLOW_COPY_AND_ASSIGN(JpegDecodeData);
+};
+
+bool JpegDecodeData::PrepareDecode(
+    const CapturedData& captured_data,
+    const scoped_refptr<ClientBuffer> out_buffer) {
+  DVLOG(3) << __func__;
+  DCHECK(!decoding());
+
+  captured_data_ = captured_data;
+
+  // Enlarge input buffer if necessary.
+  size_t in_buffer_size = captured_data.length;
+  if (!in_shared_memory_.get() ||
+      in_buffer_size > in_shared_memory_->mapped_size()) {
+    in_shared_memory_.reset(new base::SharedMemory);
+
+    if (!in_shared_memory_->CreateAnonymous(in_buffer_size)) {
+      DLOG(ERROR) << "CreateAnonymous failed";
+      return false;
+    }
+
+    if (!in_shared_memory_->Map(in_buffer_size)) {
+      DLOG(ERROR) << "Map failed";
+      return false;
+    }
+
+    captured_data_.data = reinterpret_cast<uint8*>(in_shared_memory_->memory());
+  }
+
+  in_buffer_ = media::BitstreamBuffer(
+      next_bitstream_buffer_id_, in_shared_memory_->handle(), in_buffer_size);
+  // Mask against 30 bits, to avoid (undefined) wraparound on signed integer.
+  next_bitstream_buffer_id_ = (next_bitstream_buffer_id_ + 1) & 0x3FFFFFFF;
+
+  gfx::Size dimensions = captured_data.frame_format.frame_size;
+
+  scoped_refptr<media::VideoFrame> out_frame =
+      media::VideoFrame::WrapExternalPackedMemory(
+          media::VideoFrame::I420,
+          dimensions,
+          gfx::Rect(dimensions),
+          dimensions,
+          reinterpret_cast<uint8*>(out_buffer->data()),
+          out_buffer->size(),
+          out_buffer->handle(),
+          0,
+          base::TimeDelta(),
+          base::Closure());
+  DCHECK(out_frame.get());
+
+  out_frame->metadata()->SetDouble(media::VideoFrameMetadata::FRAME_RATE,
+                                   captured_data.frame_format.frame_rate);
+
+  out_frame_ = out_frame;
+  out_buffer_ = out_buffer;
+
+  memcpy(in_shared_memory_->memory(), captured_data.data, captured_data.length);
+
+  return true;
+}
+
+void JpegDecodeData::DecodeDone() {
+  base::AutoLock lock(lock_);
+  DCHECK(decoding_);
+  out_frame_ = nullptr;
+  out_buffer_ = nullptr;
+  decoding_ = false;
+}
+
 // Class combining a Client::Buffer interface implementation and a pool buffer
 // implementation to guarantee proper cleanup on destruction on our side.
 class AutoReleaseBuffer : public media::VideoCaptureDevice::Client::Buffer {
@@ -27,16 +163,15 @@ class AutoReleaseBuffer : public media::VideoCaptureDevice::Client::Buffer {
   AutoReleaseBuffer(const scoped_refptr<VideoCaptureBufferPool>& pool,
                     int buffer_id,
                     void* data,
-                    size_t size)
-      : pool_(pool),
-        id_(buffer_id),
-        data_(data),
-        size_(size) {
+                    size_t size,
+                    base::SharedMemoryHandle handle)
+      : pool_(pool), id_(buffer_id), data_(data), size_(size), handle_(handle) {
     DCHECK(pool_.get());
   }
   int id() const override { return id_; }
   void* data() const override { return data_; }
   size_t size() const override { return size_; }
+  base::SharedMemoryHandle handle() const override { return handle_; }
 
  private:
   ~AutoReleaseBuffer() override { pool_->RelinquishProducerReservation(id_); }
@@ -45,16 +180,84 @@ class AutoReleaseBuffer : public media::VideoCaptureDevice::Client::Buffer {
   const int id_;
   void* const data_;
   const size_t size_;
+  const base::SharedMemoryHandle handle_;
 };
 
 VideoCaptureDeviceClient::VideoCaptureDeviceClient(
     const base::WeakPtr<VideoCaptureController>& controller,
     const scoped_refptr<VideoCaptureBufferPool>& buffer_pool)
     : controller_(controller),
+      jpeg_failed_(false),
       buffer_pool_(buffer_pool),
-      last_captured_pixel_format_(media::PIXEL_FORMAT_UNKNOWN) {}
+      last_captured_pixel_format_(media::PIXEL_FORMAT_UNKNOWN) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+}
+
+VideoCaptureDeviceClient::~VideoCaptureDeviceClient() {
+}
+
+bool VideoCaptureDeviceClient::InitializeJpegDecoder() {
+  // called from device thread
+  DVLOG(3) << __func__;
+  GpuChannelHost* const host =
+      BrowserGpuChannelHostFactory::instance()->GetGpuChannel();
+  jpeg_decoder_ = host->CreateJpegDecoder();
+  if (!jpeg_decoder_->Initialize(this)) {
+    jpeg_decoder_.reset();
+    return false;
+  }
+  return true;
+}
 
-VideoCaptureDeviceClient::~VideoCaptureDeviceClient() {}
+void VideoCaptureDeviceClient::VideoFrameReady(int32_t bitstream_buffer_id) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  DVLOG(3) << __func__;
+
+  // Verify parameters from GPU thread.

[wuchengli, 2015/04/15 07:11:57]

s/thread/process/

[kcwu, 2015/04/16 14:38:27]

Done.

+  if (!jpeg_decode_data_ || !jpeg_decode_data_->decoding()) {
+    DLOG(ERROR) << "got VideoFrameReady call but not decoding";
+    return;
+  }
+  if (bitstream_buffer_id != jpeg_decode_data_->in_buffer().id()) {
+    DLOG(ERROR) << "bitstream_buffer_id mismatched, expected "
+                << jpeg_decode_data_->in_buffer().id() << ", but got "
+                << bitstream_buffer_id;
+    return;
+  }
+
+  const auto& captured_data = jpeg_decode_data_->captured_data();
+  BrowserThread::PostTask(
+      BrowserThread::IO, FROM_HERE,
+      base::Bind(
+          &VideoCaptureController::DoIncomingCapturedVideoFrameOnIOThread,
+          controller_, jpeg_decode_data_->out_buffer(),
+          jpeg_decode_data_->out_frame(), captured_data.timestamp));
+
+  jpeg_decode_data_->DecodeDone();
+}
+
+void VideoCaptureDeviceClient::NotifyError(
+    int32_t bitstream_buffer_id,
+    media::JpegDecodeAccelerator::Error error) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  DVLOG(3) << __func__;
+
+  // Verify parameters from GPU thread.
+  if (!jpeg_decode_data_->decoding()) {
+    DLOG(ERROR) << "got VideoFrameReady call but not decoding";
+    return;
+  }
+  if (bitstream_buffer_id !=
+          media::JpegDecodeAccelerator::kInvalidBitstreamBufferId &&
+      bitstream_buffer_id != jpeg_decode_data_->in_buffer().id()) {
+    DLOG(ERROR) << "invalid bitstream_buffer_id " << bitstream_buffer_id;
+    return;
+  }
+
+  OnIncomingCapturedData2(jpeg_decode_data_->captured_data());
+
+  jpeg_decode_data_->DecodeDone();
+}
 
 void VideoCaptureDeviceClient::OnIncomingCapturedData(
     const uint8* data,
@@ -63,6 +266,14 @@ void VideoCaptureDeviceClient::OnIncomingCapturedData(
     int rotation,
     const base::TimeTicks& timestamp) {
   TRACE_EVENT0("video", "VideoCaptureDeviceClient::OnIncomingCapturedData");
+  DVLOG(3) << __func__;
+
+  CapturedData captured_data;
+  captured_data.data = const_cast<uint8*>(data);
+  captured_data.length = length;
+  captured_data.frame_format = frame_format;
+  captured_data.rotation = rotation;
+  captured_data.timestamp = timestamp;
 
   if (last_captured_pixel_format_ != frame_format.pixel_format) {
     OnLog("Pixel format: " + media::VideoCaptureFormat::PixelFormatToString(
@@ -73,6 +284,63 @@ void VideoCaptureDeviceClient::OnIncomingCapturedData(
   if (!frame_format.IsValid())
     return;
 
+  const gfx::Size size = frame_format.frame_size;
+  if (rotation != 0 || frame_format.pixel_format != media::PIXEL_FORMAT_MJPEG ||
+      size.width() % 2 != 0 || size.height() % 2 != 0 || jpeg_failed_) {
+    OnIncomingCapturedData2(captured_data);
+    return;
+  }
+
+  if (!jpeg_decoder_) {
+    if (!InitializeJpegDecoder()) {
+      // TODO(kcwu): fallback to software decode
+      jpeg_failed_ = true;
+      return;
+    }
+    jpeg_decode_data_.reset(new JpegDecodeData);
+  }
+
+  gfx::Size dimensions = frame_format.frame_size;
+  scoped_refptr<Buffer> out_buffer =
+      ReserveOutputBuffer(media::PIXEL_FORMAT_I420, dimensions);
+  if (!out_buffer.get()) {
+    DVLOG(1) << "Drop captured frame. No available buffers"
+             << " (all buffers are still queued to display)";
+    return;
+  }
+
+  if (jpeg_decode_data_->decoding()) {
+    DVLOG(1) << "Drop captured frame. Previous jpeg frame is still decoding";
+    return;
+  }
+
+  if (!jpeg_decode_data_->PrepareDecode(captured_data, out_buffer)) {
+    // TODO(kcwu): fallback to software decode
+    return;
+  }
+
+  jpeg_decode_data_->set_decoding(true);
+  jpeg_decoder_->Decode(jpeg_decode_data_->in_buffer(),
+                        jpeg_decode_data_->out_frame());
+}
+
+// TODO(kcwu): need better name
+void VideoCaptureDeviceClient::OnIncomingCapturedData2(
+    const CapturedData& captured_data) {
+  DVLOG(3) << __func__;
+
+  const uint8* data = captured_data.data;
+  int length = captured_data.length;
+  const VideoCaptureFormat& frame_format = captured_data.frame_format;
+  int rotation = captured_data.rotation;
+  base::TimeTicks timestamp = captured_data.timestamp;
+
+  if (last_captured_pixel_format_ != frame_format.pixel_format) {
+    OnLog("Pixel format: " + media::VideoCaptureFormat::PixelFormatToString(
+                                 frame_format.pixel_format));
+    last_captured_pixel_format_ = frame_format.pixel_format;
+  }
+
   // |chopped_{width,height} and |new_unrotated_{width,height}| are the lowest
   // bit decomposition of {width, height}, grabbing the odd and even parts.
   const int chopped_width = frame_format.frame_size.width() & 1;
@@ -316,10 +584,11 @@ VideoCaptureDeviceClient::ReserveOutputBuffer(media::VideoPixelFormat format,
     return NULL;
   void* data;
   size_t size;
-  buffer_pool_->GetBufferInfo(buffer_id, &data, &size);
+  base::SharedMemoryHandle handle;
+  buffer_pool_->GetBufferInfo(buffer_id, &data, &size, &handle);
 
   scoped_refptr<media::VideoCaptureDevice::Client::Buffer> output_buffer(
-      new AutoReleaseBuffer(buffer_pool_, buffer_id, data, size));
+      new AutoReleaseBuffer(buffer_pool_, buffer_id, data, size, handle));
 
   if (buffer_id_to_drop != VideoCaptureBufferPool::kInvalidId) {
     BrowserThread::PostTask(BrowserThread::IO,