Tab Capture: Backing store readbacks to YV12 VideoFrames.

content/browser/renderer_host/media/video_capture_controller.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/renderer_host/media/video_capture_controller.h"
 
 #include <set>
 
 #include "base/bind.h"
 #include "base/memory/scoped_ptr.h"
 #include "base/stl_util.h"
 #include "content/browser/renderer_host/media/media_stream_manager.h"
 #include "content/browser/renderer_host/media/video_capture_manager.h"
 #include "content/public/browser/browser_thread.h"
+#include "media/base/video_frame.h"
+#include "media/base/video_util.h"
 #include "media/base/yuv_convert.h"
 
 #if !defined(OS_IOS) && !defined(OS_ANDROID)
 #include "third_party/libyuv/include/libyuv.h"
 #endif
 
 namespace content {
 
 // The number of DIBs VideoCaptureController allocate.
 static const size_t kNoOfDIBS = 3;
@@ skipping 213 matching lines @@
   }
 
   // When all buffers have been returned by clients and device has been
   // called to stop, check if restart is needed. This could happen when
   // capture needs to be restarted due to resolution change.
   if (!ClientHasDIB() && state_ == VIDEO_CAPTURE_STATE_STOPPING) {
     PostStopping();
   }
 }
 
-///////////////////////////////////////////////////////////////////////////////
-// Implements VideoCaptureDevice::EventHandler.
-// OnIncomingCapturedFrame is called the thread running the capture device.
-// I.e.- DirectShow thread on windows and v4l2_thread on Linux.
-void VideoCaptureController::OnIncomingCapturedFrame(const uint8* data,
-                                                     int length,
-                                                     base::Time timestamp) {
+bool VideoCaptureController::ReserveSharedMemory(int* buffer_id_out,
+                                                 uint8** yplane,
+                                                 uint8** uplane,
+                                                 uint8** vplane) {
   int buffer_id = 0;
   base::SharedMemory* dib = NULL;
   {
     base::AutoLock lock(lock_);
     for (DIBMap::iterator dib_it = owned_dibs_.begin();
          dib_it != owned_dibs_.end(); dib_it++) {
       if (dib_it->second->references == 0) {
         buffer_id = dib_it->first;
         // Use special value "-1" in order to not be treated as buffer at
         // renderer side.
         dib_it->second->references = -1;
         dib = dib_it->second->shared_memory.get();
         break;
       }
     }
   }
 
-  if (!dib) {
+  if (!dib)
+    return false;
+
+  *buffer_id_out = buffer_id;
+  CHECK(dib->created_size() >= static_cast<size_t>(frame_info_.width *

[scherkus (not reviewing), 2013/02/05 22:40:44]

nit: CHECK_GE()?

[ncarter (slow), 2013/02/06 23:54:44]

Done.

+                                                   frame_info_.height * 3) / 2);
+  uint8* target = static_cast<uint8*>(dib->memory());
+  *yplane = target;
+  *uplane = *yplane + frame_info_.width * frame_info_.height;
+  *vplane = *uplane + (frame_info_.width * frame_info_.height) / 4;
+  return true;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// Implements VideoCaptureDevice::EventHandler.
+// OnIncomingCapturedFrame is called the thread running the capture device.
+// I.e.- DirectShow thread on windows and v4l2_thread on Linux.
+void VideoCaptureController::OnIncomingCapturedFrame(const uint8* data,
+                                                     int length,
+                                                     base::Time timestamp) {
+  int buffer_id = 0;
+  uint8* yplane = NULL;
+  uint8* uplane = NULL;
+  uint8* vplane = NULL;
+  if (!ReserveSharedMemory(&buffer_id, &yplane, &uplane, &vplane))
     return;
-  }
-
-  uint8* target = static_cast<uint8*>(dib->memory());
-  CHECK(dib->created_size() >= static_cast<size_t> (frame_info_.width *
-                                                    frame_info_.height * 3) /
-                                                    2);
-  uint8* yplane = target;
-  uint8* uplane = target + frame_info_.width * frame_info_.height;
-  uint8* vplane = uplane + (frame_info_.width * frame_info_.height) / 4;
 
   // Do color conversion from the camera format to I420.
   switch (frame_info_.color) {
     case media::VideoCaptureCapability::kColorUnknown:  // Color format not set.
       break;
     case media::VideoCaptureCapability::kI420: {
       DCHECK(!chopped_width_ && !chopped_height_);
-      memcpy(target, data, (frame_info_.width * frame_info_.height * 3) / 2);
+      memcpy(yplane, data, (frame_info_.width * frame_info_.height * 3) / 2);
       break;
     }
     case media::VideoCaptureCapability::kYV12: {
       DCHECK(!chopped_width_ && !chopped_height_);
       const uint8* ptr = data;
       memcpy(yplane, ptr, (frame_info_.width * frame_info_.height));
       ptr += frame_info_.width * frame_info_.height;
       memcpy(vplane, ptr, (frame_info_.width * frame_info_.height) >> 2);
       ptr += (frame_info_.width * frame_info_.height) >> 2;
       memcpy(uplane, ptr, (frame_info_.width * frame_info_.height) >> 2);
@@ skipping 51 matching lines @@
     default:
       NOTREACHED();
   }
 
   BrowserThread::PostTask(BrowserThread::IO,
       FROM_HERE,
       base::Bind(&VideoCaptureController::DoIncomingCapturedFrameOnIOThread,
                  this, buffer_id, timestamp));
 }
 
+// OnIncomingCapturedVideoFrame is called the thread running the capture device.
+void VideoCaptureController::OnIncomingCapturedVideoFrame(
+    media::VideoFrame* frame,
+    base::Time timestamp) {
+  int buffer_id = 0;
+  uint8* yplane = NULL;
+  uint8* uplane = NULL;
+  uint8* vplane = NULL;
+  if (!ReserveSharedMemory(&buffer_id, &yplane, &uplane, &vplane))
+    return;
+
+  gfx::Size target_size = gfx::Size(frame_info_.width, frame_info_.height);
+
+  if (frame->coded_size() != target_size)
+    return;  // Only exact copies are supported.

[scherkus (not reviewing), 2013/02/05 22:40:44]

if you reserve a DIB but return here, does the DIB stay permanently reserved?

[ncarter (slow), 2013/02/06 23:54:44]

Good catch! Reordered these operations.

+
+  scoped_refptr<media::VideoFrame> target_as_frame(
+      media::VideoFrame::WrapExternalYuvData(
+          media::VideoFrame::YV12,  // Actually I420, but it's equivalent here.
+          target_size, gfx::Rect(target_size), target_size,
+          frame_info_.width,        // y stride
+          frame_info_.width / 2,    // v stride
+          frame_info_.width / 2,    // u stride
+          yplane,
+          uplane,
+          vplane,
+          base::TimeDelta(),
+          base::Bind(&base::DoNothing)));
+
+  const int kYPlane = media::VideoFrame::kYPlane;
+  const int kUPlane = media::VideoFrame::kUPlane;
+  const int kVPlane = media::VideoFrame::kVPlane;
+  const int kRGBPlane = media::VideoFrame::kRGBPlane;
+
+  // Do color conversion from the camera format to I420.
+  switch (frame->format()) {
+    case media::VideoFrame::EMPTY:
+    case media::VideoFrame::INVALID:  // Color format not set.
+    case media::VideoFrame::NATIVE_TEXTURE:
+      // These types will never work.

[scherkus (not reviewing), 2013/02/05 22:40:44]

if possible, my preference would be to CHECK()since the caller is in the wrong

the runtime behaviour would be send an uninitialized target_as_frame pointing at
whatever used to be in the DIB, which doesn't seem very useful

[ncarter (slow), 2013/02/06 23:54:44]

CHECK() isn't as much fun for the user when it happens in the browser process.
As a compromise, I added handling to the pre-ReserveSharedMemory validation
step, and do a "NOTREACHED()/return;".

+      break;
+    case media::VideoFrame::I420:
+    case media::VideoFrame::YV12: {
+      DCHECK(!chopped_width_ && !chopped_height_);
+      media::CopyYPlane(frame->data(kYPlane),
+                        frame->stride(kYPlane),
+                        frame->rows(kYPlane),
+                        target_as_frame);
+      media::CopyUPlane(frame->data(kUPlane),
+                        frame->stride(kUPlane),
+                        frame->rows(kUPlane),
+                        target_as_frame);
+      media::CopyVPlane(frame->data(kVPlane),
+                        frame->stride(kVPlane),
+                        frame->rows(kVPlane),
+                        target_as_frame);
+      break;
+    }
+    case media::VideoFrame::RGB32: {
+      media::ConvertRGB32ToYUV(frame->data(kRGBPlane),
+                               target_as_frame->data(kYPlane),
+                               target_as_frame->data(kUPlane),
+                               target_as_frame->data(kVPlane),
+                               target_size.width(),
+                               target_size.height(),
+                               frame->stride(kRGBPlane),
+                               target_as_frame->stride(kYPlane),
+                               target_as_frame->stride(kUPlane));
+      break;
+    }
+    default:

[scherkus (not reviewing), 2013/02/05 22:40:44]

I like having the compiler break when enums are added/removed

[ncarter (slow), 2013/02/06 23:54:44]

See above.

+      NOTREACHED();
+  }
+
+  BrowserThread::PostTask(BrowserThread::IO,
+      FROM_HERE,
+      base::Bind(&VideoCaptureController::DoIncomingCapturedFrameOnIOThread,
+                 this, buffer_id, timestamp));
+}
+
 void VideoCaptureController::OnError() {
   BrowserThread::PostTask(BrowserThread::IO,
       FROM_HERE,
       base::Bind(&VideoCaptureController::DoErrorOnIOThread, this));
 }
 
 void VideoCaptureController::OnFrameInfo(
     const media::VideoCaptureCapability& info) {
   frame_info_= info;
   // Handle cases when |info| has odd numbers for width/height.
@@ skipping 221 matching lines @@
   base::AutoLock lock(lock_);
   for (DIBMap::iterator dib_it = owned_dibs_.begin();
        dib_it != owned_dibs_.end(); dib_it++) {
     if (dib_it->second->references > 0)
       return true;
   }
   return false;
 }
 
 }  // namespace content