MJPEG acceleration for video capture, browser part

content/browser/renderer_host/media/video_capture_device_client.cc

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/renderer_host/media/video_capture_device_client.h"
 
 #include "base/bind.h"
 #include "base/strings/stringprintf.h"
 #include "base/trace_event/trace_event.h"
 #include "content/browser/compositor/image_transport_factory.h"
 #include "content/browser/gpu/browser_gpu_channel_host_factory.h"
 #include "content/browser/gpu/browser_gpu_memory_buffer_manager.h"
 #include "content/browser/gpu/gpu_data_manager_impl.h"
+#include "content/browser/renderer_host/gpu_jpeg_decoder.h"
 #include "content/browser/renderer_host/media/video_capture_buffer_pool.h"
 #include "content/browser/renderer_host/media/video_capture_controller.h"
 #include "content/common/gpu/client/context_provider_command_buffer.h"
 #include "content/common/gpu/client/gl_helper.h"
 #include "content/common/gpu/client/gpu_channel_host.h"
 #include "content/common/gpu/client/webgraphicscontext3d_command_buffer_impl.h"
 #include "content/common/gpu/gpu_process_launch_causes.h"
 #include "content/public/browser/browser_thread.h"
 #include "gpu/command_buffer/common/mailbox_holder.h"
 #include "media/base/bind_to_current_loop.h"
@@ skipping 92 matching lines @@
   }
   int id() const override { return id_; }
   size_t size() const override { return buffer_handle_->size(); }
   void* data() override { return buffer_handle_->data(); }
   gfx::GpuMemoryBufferType GetType() override {
     return buffer_handle_->GetType();
   }
   ClientBuffer AsClientBuffer() override {
     return buffer_handle_->AsClientBuffer();
   }
+  base::PlatformFile AsPlatformHandle() override {
+    return buffer_handle_->AsPlatformHandle();
+  }
 
  private:
   ~AutoReleaseBuffer() override { pool_->RelinquishProducerReservation(id_); }
 
   const int id_;
   const scoped_refptr<VideoCaptureBufferPool> pool_;
   const scoped_ptr<VideoCaptureBufferPool::BufferHandle> buffer_handle_;
 };
 
 // Internal ref-counted class wrapping an incoming GpuMemoryBuffer into a
@@ skipping 53 matching lines @@
   scoped_ptr<GLHelper> gl_helper_;
 
   DISALLOW_COPY_AND_ASSIGN(TextureWrapHelper);
 };
 
 VideoCaptureDeviceClient::VideoCaptureDeviceClient(
     const base::WeakPtr<VideoCaptureController>& controller,
     const scoped_refptr<VideoCaptureBufferPool>& buffer_pool,
     const scoped_refptr<base::SingleThreadTaskRunner>& capture_task_runner)
     : controller_(controller),
+      external_jpeg_decoder_initialized_(false),
       buffer_pool_(buffer_pool),
       capture_task_runner_(capture_task_runner),
       last_captured_pixel_format_(media::PIXEL_FORMAT_UNKNOWN) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 }
 
-VideoCaptureDeviceClient::~VideoCaptureDeviceClient() {}
+VideoCaptureDeviceClient::~VideoCaptureDeviceClient() {
+  DCHECK(capture_task_runner_->BelongsToCurrentThread());
+  external_jpeg_decoder_.reset();
+}
 
 void VideoCaptureDeviceClient::OnIncomingCapturedData(
     const uint8* data,
     int length,
     const VideoCaptureFormat& frame_format,
     int rotation,
     const base::TimeTicks& timestamp) {
   TRACE_EVENT0("video", "VideoCaptureDeviceClient::OnIncomingCapturedData");
 
   if (last_captured_pixel_format_ != frame_format.pixel_format) {
     OnLog("Pixel format: " + media::VideoCaptureFormat::PixelFormatToString(
                                  frame_format.pixel_format));
     last_captured_pixel_format_ = frame_format.pixel_format;
+
+    if (frame_format.pixel_format == media::PIXEL_FORMAT_MJPEG &&
+        !external_jpeg_decoder_initialized_) {
+      external_jpeg_decoder_initialized_ = true;
+      if (GpuJpegDecoder::Supported()) {
+        // base::Unretained is safe because |this| outlives
+        // |external_jpeg_decoder_|.
+        external_jpeg_decoder_.reset(new GpuJpegDecoder(
+            base::Bind(&VideoCaptureDeviceClient::OnIncomingCapturedVideoFrame,

[piman, 2015/06/05 23:07:51]

Maybe you can bind to DoIncomingCapturedVideoFrameOnIOThread directly to avoid
the extra PostTask?

That way you don't need Unretained(this) - you can pass the controller_ WeakPtr
directly.

[kcwu, 2015/06/17 14:35:37]

Done.

+                       base::Unretained(this)),
+            // TODO(kcwu): fallback to software decode if error.
+            base::Bind(&VideoCaptureDeviceClient::OnError,

[piman, 2015/06/05 23:07:51]

Same here, bind to DoErrorOnIOThread?

[kcwu, 2015/06/17 14:35:37]

Not exactly. VideoCaptureDeviceClient::OnError = OnLog + DoErrorOnIOThread.

Since error callback is not on the critical path, I'd prefer less code with the
cost of extra PostTask. What do you think?

+                       base::Unretained(this))));
+        if (!external_jpeg_decoder_->Initialize())
+          external_jpeg_decoder_.reset();
+      }
+    }
   }
 
   if (!frame_format.IsValid())
     return;
 
   // |chopped_{width,height} and |new_unrotated_{width,height}| are the lowest
   // bit decomposition of {width, height}, grabbing the odd and even parts.
   const int chopped_width = frame_format.frame_size.width() & 1;
   const int chopped_height = frame_format.frame_size.height() & 1;
   const int new_unrotated_width = frame_format.frame_size.width() & ~1;
@@ skipping 93 matching lines @@
       origin_colorspace = libyuv::FOURCC_MJPG;
       break;
     default:
       NOTREACHED();
   }
 
   // The input |length| can be greater than the required buffer size because of
   // paddings and/or alignments, but it cannot be smaller.
   DCHECK_GE(static_cast<size_t>(length), frame_format.ImageAllocationSize());
 
+  if (external_jpeg_decoder_ &&
+      frame_format.pixel_format == media::PIXEL_FORMAT_MJPEG && rotation == 0 &&
+      !flip) {
+    external_jpeg_decoder_->DecodeCapturedData(data, length, frame_format,
+                                               timestamp, buffer.Pass());
+    return;
+  }
+
   if (libyuv::ConvertToI420(data,
                             length,
                             yplane,
                             yplane_stride,
                             uplane,
                             uv_plane_stride,
                             vplane,
                             uv_plane_stride,
                             crop_x,
                             crop_y,
@@ skipping 320 matching lines @@
 void VideoCaptureDeviceClient::TextureWrapHelper::OnError(
     const std::string& message) {
   DCHECK(capture_task_runner_->BelongsToCurrentThread());
   DLOG(ERROR) << message;
   BrowserThread::PostTask(
       BrowserThread::IO, FROM_HERE,
       base::Bind(&VideoCaptureController::DoErrorOnIOThread, controller_));
 }
 
 }  // namespace content