MJPEG acceleration for video capture using VAAPI

content/browser/renderer_host/media/video_capture_device_client.cc

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/renderer_host/media/video_capture_device_client.h"
 
 #include "base/bind.h"
 #include "base/strings/stringprintf.h"
 #include "base/trace_event/trace_event.h"
+#include "content/browser/renderer_host/gpu_jpeg_decoder.h"
 #include "content/browser/renderer_host/media/video_capture_buffer_pool.h"
 #include "content/browser/renderer_host/media/video_capture_controller.h"
 #include "content/public/browser/browser_thread.h"
 #include "media/base/bind_to_current_loop.h"
 #include "media/base/video_capture_types.h"
 #include "media/base/video_frame.h"
 #include "third_party/libyuv/include/libyuv.h"
 
 using media::VideoCaptureFormat;
 using media::VideoFrame;
 
 namespace content {
 
 // Class combining a Client::Buffer interface implementation and a pool buffer
 // implementation to guarantee proper cleanup on destruction on our side.
 class AutoReleaseBuffer : public media::VideoCaptureDevice::Client::Buffer {
  public:
   AutoReleaseBuffer(const scoped_refptr<VideoCaptureBufferPool>& pool,
                     int buffer_id,
                     void* data,
-                    size_t size)
-      : pool_(pool),
-        id_(buffer_id),
-        data_(data),
-        size_(size) {
+                    size_t size,
+                    base::SharedMemoryHandle handle)
+      : pool_(pool), id_(buffer_id), data_(data), size_(size), handle_(handle) {
     DCHECK(pool_.get());
   }
   int id() const override { return id_; }
   void* data() const override { return data_; }
   size_t size() const override { return size_; }
+  base::SharedMemoryHandle handle() const override { return handle_; }
 
  private:
   ~AutoReleaseBuffer() override { pool_->RelinquishProducerReservation(id_); }
 
   const scoped_refptr<VideoCaptureBufferPool> pool_;
   const int id_;
   void* const data_;
   const size_t size_;
+  const base::SharedMemoryHandle handle_;
 };
 
 VideoCaptureDeviceClient::VideoCaptureDeviceClient(
     const base::WeakPtr<VideoCaptureController>& controller,
     const scoped_refptr<VideoCaptureBufferPool>& buffer_pool)
     : controller_(controller),
       buffer_pool_(buffer_pool),
-      last_captured_pixel_format_(media::PIXEL_FORMAT_UNKNOWN) {}
+      last_captured_pixel_format_(media::PIXEL_FORMAT_UNKNOWN) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+}
 
-VideoCaptureDeviceClient::~VideoCaptureDeviceClient() {}
+VideoCaptureDeviceClient::~VideoCaptureDeviceClient() {
+}
 
 void VideoCaptureDeviceClient::OnIncomingCapturedData(
     const uint8* data,
     int length,
     const VideoCaptureFormat& frame_format,
     int rotation,
     const base::TimeTicks& timestamp) {
   TRACE_EVENT0("video", "VideoCaptureDeviceClient::OnIncomingCapturedData");
 
   if (last_captured_pixel_format_ != frame_format.pixel_format) {
     OnLog("Pixel format: " + media::VideoCaptureFormat::PixelFormatToString(
                                  frame_format.pixel_format));
     last_captured_pixel_format_ = frame_format.pixel_format;
+
+    if (frame_format.pixel_format == media::PIXEL_FORMAT_MJPEG &&
+        !external_jpeg_decoder_ &&
+        GpuJpegDecoder::Supported()) {
+      external_jpeg_decoder_.reset(new GpuJpegDecoder(this));
+      if (!external_jpeg_decoder_->Initialize()) {

[wuchengli, 2015/04/24 05:54:23]

We'll try this for every frame if this fails. Can we do
external_jpeg_decoder_.reset(new GpuJpegDecoder(this)); in the constructor if
GpuJpegDecoder::Supported() is true?

[kcwu, 2015/04/30 19:25:39]

Done.

+        // Reset external_jpeg_decoder_ to fallback to software decode.
+        external_jpeg_decoder_.reset();
+      }
+    }
   }
 
   if (!frame_format.IsValid())
     return;
 
+  if (frame_format.pixel_format == media::PIXEL_FORMAT_MJPEG &&
+      external_jpeg_decoder_ && rotation == 0 &&
+      !external_jpeg_decoder_->IsFailed()) {

[wuchengli, 2015/04/24 05:54:23]

Let's have a failure callback from GpuJpegDecoder and clears
|external_jpeg_decoder_| in the callback. A simple NotifyError or
base::Callback<void(Status status)> DecodeCB; will be enough.
FallbackToSoftwareDecode should also be handled there.

[kcwu, 2015/04/30 19:25:39]

Since |external_jpeg_decoder_| is created in the constructor (IO thread). To
make things simpler, |external_jpeg_decoder_| will only enter fail state, never 
delete during capturing.

+    external_jpeg_decoder_->DecodeCapturedData(data, length, frame_format,

[wuchengli, 2015/04/24 05:54:23]

DecodeCapturedData can return a value to indicate the failure. If failed, just
continue OnIncomingCapturedData and it's falling back to software.

[kcwu, 2015/04/30 19:25:39]

Done.

+                                               rotation, timestamp);
+    return;
+  }
+
   // |chopped_{width,height} and |new_unrotated_{width,height}| are the lowest
   // bit decomposition of {width, height}, grabbing the odd and even parts.
   const int chopped_width = frame_format.frame_size.width() & 1;
   const int chopped_height = frame_format.frame_size.height() & 1;
   const int new_unrotated_width = frame_format.frame_size.width() & ~1;
   const int new_unrotated_height = frame_format.frame_size.height() & ~1;
 
   int destination_width = new_unrotated_width;
   int destination_height = new_unrotated_height;
   if (rotation == 90 || rotation == 270) {
@@ skipping 223 matching lines @@
   DCHECK_GT(dimensions.width(), 0);
   DCHECK_GT(dimensions.height(), 0);
 
   int buffer_id_to_drop = VideoCaptureBufferPool::kInvalidId;
   const int buffer_id =
       buffer_pool_->ReserveForProducer(format, dimensions, &buffer_id_to_drop);
   if (buffer_id == VideoCaptureBufferPool::kInvalidId)
     return NULL;
   void* data;
   size_t size;
-  buffer_pool_->GetBufferInfo(buffer_id, &data, &size);
+  base::SharedMemoryHandle handle;
+  buffer_pool_->GetBufferInfo(buffer_id, &data, &size, &handle);
 
   scoped_refptr<media::VideoCaptureDevice::Client::Buffer> output_buffer(
-      new AutoReleaseBuffer(buffer_pool_, buffer_id, data, size));
+      new AutoReleaseBuffer(buffer_pool_, buffer_id, data, size, handle));
 
   if (buffer_id_to_drop != VideoCaptureBufferPool::kInvalidId) {
     BrowserThread::PostTask(BrowserThread::IO,
         FROM_HERE,
         base::Bind(&VideoCaptureController::DoBufferDestroyedOnIOThread,
                    controller_, buffer_id_to_drop));
   }
 
   return output_buffer;
 }
@@ skipping 29 matching lines @@
 }
 
 void VideoCaptureDeviceClient::OnLog(
     const std::string& message) {
   BrowserThread::PostTask(BrowserThread::IO, FROM_HERE,
                           base::Bind(&VideoCaptureController::DoLogOnIOThread,
                                      controller_, message));
 }
 
 }  // namespace content