From Video Capture, abolish OnFrameInfo and enable resolution changes

content/browser/renderer_host/media/video_capture_controller.cc

Index: content/browser/renderer_host/media/video_capture_controller.cc
diff --git a/content/browser/renderer_host/media/video_capture_controller.cc b/content/browser/renderer_host/media/video_capture_controller.cc
index 07986fa0f788417986bd28b4b45be4eeb2702338..d2978130a226fed68583a0952294ad89f2eac89a 100644
--- a/content/browser/renderer_host/media/video_capture_controller.cc
+++ b/content/browser/renderer_host/media/video_capture_controller.cc
@@ -77,8 +77,11 @@ struct VideoCaptureController::ControllerClient {
   base::ProcessHandle render_process_handle;
   media::VideoCaptureParams parameters;
 
-  // Buffers used by this client.
-  std::set<int> buffers;
+  // Buffers that are currently known to this client.
+  std::set<int> known_buffers;
+
+  // Buffers currently held by this client.
+  std::set<int> active_buffers;
 
   // State of capture session, controlled by VideoCaptureManager directly. This
   // transitions to true as soon as StopSession() occurs, at which point the
@@ -105,11 +108,13 @@ class VideoCaptureController::VideoCaptureDeviceClient
     : public media::VideoCaptureDevice::Client {
  public:
   explicit VideoCaptureDeviceClient(
-      const base::WeakPtr<VideoCaptureController>& controller);
+      const base::WeakPtr<VideoCaptureController>& controller,
+      const scoped_refptr<VideoCaptureBufferPool>& buffer_pool);
   virtual ~VideoCaptureDeviceClient();
 
   // VideoCaptureDevice::Client implementation.
-  virtual scoped_refptr<media::VideoFrame> ReserveOutputBuffer() OVERRIDE;
+  virtual scoped_refptr<media::VideoFrame> ReserveOutputBuffer(
+      const gfx::Size& size) OVERRIDE;
   virtual void OnIncomingCapturedFrame(const uint8* data,
                                        int length,
                                        base::Time timestamp,
@@ -126,11 +131,13 @@ class VideoCaptureController::VideoCaptureDeviceClient
       const media::VideoCaptureCapability& info) OVERRIDE;
 
  private:
+  void HandleDroppedBufferId(int buffer_id_to_drop);
+
   // The controller to which we post events.
   const base::WeakPtr<VideoCaptureController> controller_;
 
   // The pool of shared-memory buffers used for capturing.
-  scoped_refptr<VideoCaptureBufferPool> buffer_pool_;
+  const scoped_refptr<VideoCaptureBufferPool> buffer_pool_;
 
   // Chopped pixels in width/height in case video capture device has odd
   // numbers for width/height.
@@ -149,14 +156,16 @@ class VideoCaptureController::VideoCaptureDeviceClient
 };
 
 VideoCaptureController::VideoCaptureController()
-    : state_(VIDEO_CAPTURE_STATE_STARTED),
+    : buffer_pool_(new VideoCaptureBufferPool(kNoOfBuffers)),
+      state_(VIDEO_CAPTURE_STATE_STARTED),
       weak_ptr_factory_(this) {
-  memset(&current_params_, 0, sizeof(current_params_));
 }
 
 VideoCaptureController::VideoCaptureDeviceClient::VideoCaptureDeviceClient(
-    const base::WeakPtr<VideoCaptureController>& controller)
+    const base::WeakPtr<VideoCaptureController>& controller,
+    const scoped_refptr<VideoCaptureBufferPool>& buffer_pool)
     : controller_(controller),
+      buffer_pool_(buffer_pool),
       chopped_width_(0),
       chopped_height_(0) {}
 
@@ -169,7 +178,7 @@ base::WeakPtr<VideoCaptureController> VideoCaptureController::GetWeakPtr() {
 scoped_ptr<media::VideoCaptureDevice::Client>
 VideoCaptureController::NewDeviceClient() {
   scoped_ptr<media::VideoCaptureDevice::Client> result(
-      new VideoCaptureDeviceClient(this->GetWeakPtr()));
+      new VideoCaptureDeviceClient(this->GetWeakPtr(), buffer_pool_));
   return result.Pass();
 }
 
@@ -201,9 +210,6 @@ void VideoCaptureController::AddClient(
   // If we already have gotten frame_info from the device, repeat it to the new
   // client.
   if (state_ == VIDEO_CAPTURE_STATE_STARTED) {
-    if (frame_info_.IsValid()) {
-      SendFrameInfoAndBuffers(client);
-    }
     controller_clients_.push_back(client);
     return;
   }
@@ -220,15 +226,13 @@ int VideoCaptureController::RemoveClient(
     return kInvalidMediaCaptureSessionId;
 
   // Take back all buffers held by the |client|.
-  if (buffer_pool_.get()) {
-    for (std::set<int>::iterator buffer_it = client->buffers.begin();
-         buffer_it != client->buffers.end();
-         ++buffer_it) {
-      int buffer_id = *buffer_it;
-      buffer_pool_->RelinquishConsumerHold(buffer_id, 1);
-    }
+  for (std::set<int>::iterator buffer_it = client->active_buffers.begin();
+       buffer_it != client->active_buffers.end();
+       ++buffer_it) {
+    int buffer_id = *buffer_it;
+    buffer_pool_->RelinquishConsumerHold(buffer_id, 1);
   }
-  client->buffers.clear();
+  client->active_buffers.clear();
 
   int session_id = client->parameters.session_id;
   controller_clients_.remove(client);
@@ -260,20 +264,23 @@ void VideoCaptureController::ReturnBuffer(
   // If this buffer is not held by this client, or this client doesn't exist
   // in controller, do nothing.
   if (!client ||
-      client->buffers.find(buffer_id) == client->buffers.end()) {
+      client->active_buffers.find(buffer_id) == client->active_buffers.end()) {
     NOTREACHED();
     return;
   }
 
-  client->buffers.erase(buffer_id);
+  client->active_buffers.erase(buffer_id);

[Ami GONE FROM CHROMIUM, 2013/10/04 00:24:15]

FWIW 
if (myset.find(val) == myset.end()) {
  ...
}
myset.erase(val);

is equivalent to:

if (!myset.erase(val)) {
  ...
}

but does one search instead of two.

[ncarter (slow), 2013/10/16 02:08:40]

Done.

   buffer_pool_->RelinquishConsumerHold(buffer_id, 1);
 }
 
 scoped_refptr<media::VideoFrame>
-VideoCaptureController::VideoCaptureDeviceClient::ReserveOutputBuffer() {
-  return buffer_pool_->ReserveI420VideoFrame(gfx::Size(frame_info_.width,
-                                                       frame_info_.height),
-                                             0);
+VideoCaptureController::VideoCaptureDeviceClient::ReserveOutputBuffer(
+    const gfx::Size& size) {
+  int buffer_id_to_drop = VideoCaptureBufferPool::kInvalidId;
+  scoped_refptr<media::VideoFrame> frame =
+      buffer_pool_->ReserveI420VideoFrame(size, 0, &buffer_id_to_drop);
+  HandleDroppedBufferId(buffer_id_to_drop);
+  return frame;
 }
 
 #if !defined(OS_IOS) && !defined(OS_ANDROID)
@@ -286,10 +293,14 @@ void VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedFrame(
     bool flip_horiz) {
   TRACE_EVENT0("video", "VideoCaptureController::OnIncomingCapturedFrame");
 
-  if (!buffer_pool_.get())
+  if (!frame_info_.IsValid())
     return;
+
+  int buffer_id_to_drop = VideoCaptureBufferPool::kInvalidId;
   scoped_refptr<media::VideoFrame> dst = buffer_pool_->ReserveI420VideoFrame(
-      gfx::Size(frame_info_.width, frame_info_.height), rotation);
+      gfx::Size(frame_info_.width, frame_info_.height), rotation,
+      &buffer_id_to_drop);
+  HandleDroppedBufferId(buffer_id_to_drop);

[Ami GONE FROM CHROMIUM, 2013/10/04 00:24:15]

Worth extracting the duplication from here & l.279-282?

[ncarter (slow), 2013/10/16 02:08:40]

Ah, yes. You're right. Done, and nixed the HandleDroppedBufferId helper.

 
   if (!dst.get())
     return;
@@ -400,6 +411,7 @@ void VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedFrame(
       base::Bind(&VideoCaptureController::DoIncomingCapturedFrameOnIOThread,
                  controller_,
                  dst,
+                 frame_info_.frame_rate,
                  timestamp));
 }
 #else
@@ -410,6 +422,10 @@ void VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedFrame(
     int rotation,
     bool flip_vert,
     bool flip_horiz) {
+
+  if (!frame_info_.IsValid())
+    return;
+
   DCHECK(frame_info_.color == media::PIXEL_FORMAT_I420 ||
          frame_info_.color == media::PIXEL_FORMAT_YV12 ||
          frame_info_.color == media::PIXEL_FORMAT_NV21 ||
@@ -417,13 +433,13 @@ void VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedFrame(
 
   TRACE_EVENT0("video", "VideoCaptureController::OnIncomingCapturedFrame");
 
-  if (!buffer_pool_)
-    return;
+  int buffer_id_to_drop = VideoCaptureBufferPool::kInvalidId;
   scoped_refptr<media::VideoFrame> dst =
       buffer_pool_->ReserveI420VideoFrame(gfx::Size(frame_info_.width,
                                                     frame_info_.height),
-                                          rotation);
-
+                                          rotation,
+                                          &buffer_id_to_drop);
+  HandleDroppedBufferId(buffer_id_to_drop);

[Ami GONE FROM CHROMIUM, 2013/10/04 00:24:15]

and here

[ncarter (slow), 2013/10/16 02:08:40]

Done.

   if (!dst.get())
     return;
 
@@ -496,7 +512,7 @@ void VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedFrame(
   BrowserThread::PostTask(BrowserThread::IO,
       FROM_HERE,
       base::Bind(&VideoCaptureController::DoIncomingCapturedFrameOnIOThread,
-                 controller_, dst, timestamp));
+                 controller_, dst, frame_info_.frame_rate, timestamp));
 }
 #endif  // #if !defined(OS_IOS) && !defined(OS_ANDROID)
 
@@ -504,8 +520,6 @@ void
 VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedVideoFrame(
     const scoped_refptr<media::VideoFrame>& frame,
     base::Time timestamp) {
-  if (!buffer_pool_)
-    return;
 
   // If this is a frame that belongs to the buffer pool, we can forward it
   // directly to the IO thread and be done.
@@ -514,104 +528,11 @@ VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedVideoFrame(
     BrowserThread::PostTask(BrowserThread::IO,
         FROM_HERE,
         base::Bind(&VideoCaptureController::DoIncomingCapturedFrameOnIOThread,
-                   controller_, frame, timestamp));
-    return;
-  }
-
-  // Otherwise, this is a frame that belongs to the caller, and we must copy
-  // it to a frame from the buffer pool.
-  scoped_refptr<media::VideoFrame> target =
-      buffer_pool_->ReserveI420VideoFrame(gfx::Size(frame_info_.width,
-                                                    frame_info_.height),
-                                          0);
-
-  if (!target.get())
+                   controller_, frame, frame_info_.frame_rate, timestamp));
     return;
-
-  // Validate the inputs.
-  if (frame->coded_size() != target->coded_size())
-    return;  // Only exact copies are supported.
-  if (!(frame->format() == media::VideoFrame::I420 ||
-        frame->format() == media::VideoFrame::YV12 ||
-        frame->format() == media::VideoFrame::RGB32)) {
-    NOTREACHED() << "Unsupported format passed to OnIncomingCapturedVideoFrame";
-    return;
-  }
-
-  const int kYPlane = media::VideoFrame::kYPlane;
-  const int kUPlane = media::VideoFrame::kUPlane;
-  const int kVPlane = media::VideoFrame::kVPlane;
-  const int kAPlane = media::VideoFrame::kAPlane;
-  const int kRGBPlane = media::VideoFrame::kRGBPlane;
-
-  // Do color conversion from the camera format to I420.
-  switch (frame->format()) {
-#if defined(GOOGLE_TV)
-    case media::VideoFrame::HOLE:
-      // Fall-through to NOTREACHED() block.
-#endif
-    case media::VideoFrame::INVALID:
-    case media::VideoFrame::YV16:
-    case media::VideoFrame::EMPTY:
-    case media::VideoFrame::NATIVE_TEXTURE: {
-      NOTREACHED();
-      break;
-    }
-    case media::VideoFrame::I420:
-    case media::VideoFrame::YV12: {
-      DCHECK(!chopped_width_ && !chopped_height_);
-      media::CopyYPlane(frame->data(kYPlane),
-                        frame->stride(kYPlane),
-                        frame->rows(kYPlane),
-                        target.get());
-      media::CopyUPlane(frame->data(kUPlane),
-                        frame->stride(kUPlane),
-                        frame->rows(kUPlane),
-                        target.get());
-      media::CopyVPlane(frame->data(kVPlane),
-                        frame->stride(kVPlane),
-                        frame->rows(kVPlane),
-                        target.get());
-      break;
-    }
-    case media::VideoFrame::YV12A: {
-      DCHECK(!chopped_width_ && !chopped_height_);
-      media::CopyYPlane(frame->data(kYPlane),
-                        frame->stride(kYPlane),
-                        frame->rows(kYPlane),
-                        target.get());
-      media::CopyUPlane(frame->data(kUPlane),
-                        frame->stride(kUPlane),
-                        frame->rows(kUPlane),
-                        target.get());
-      media::CopyVPlane(frame->data(kVPlane),
-                        frame->stride(kVPlane),
-                        frame->rows(kVPlane),
-                        target.get());
-      media::CopyAPlane(frame->data(kAPlane),
-                        frame->stride(kAPlane),
-                        frame->rows(kAPlane),
-                        target.get());
-      break;
-    }
-    case media::VideoFrame::RGB32: {
-      media::ConvertRGB32ToYUV(frame->data(kRGBPlane),
-                               target->data(kYPlane),
-                               target->data(kUPlane),
-                               target->data(kVPlane),
-                               target->coded_size().width(),
-                               target->coded_size().height(),
-                               frame->stride(kRGBPlane),
-                               target->stride(kYPlane),
-                               target->stride(kUPlane));
-      break;
-    }
   }
 
-  BrowserThread::PostTask(BrowserThread::IO,
-      FROM_HERE,
-      base::Bind(&VideoCaptureController::DoIncomingCapturedFrameOnIOThread,
-                 controller_, target, timestamp));
+  NOTREACHED() << "Frames should always belong to the buffer pool.";
 }
 
 void VideoCaptureController::VideoCaptureDeviceClient::OnError() {
@@ -637,60 +558,39 @@ void VideoCaptureController::VideoCaptureDeviceClient::OnFrameInfo(
     chopped_height_ = 0;
   }
 #if defined(OS_IOS) || defined(OS_ANDROID)
-  if (frame_info_.color == media::PIXEL_FORMAT_NV21 &&
-      !i420_intermediate_buffer_) {
+  if (frame_info_.color == media::PIXEL_FORMAT_NV21) {
     i420_intermediate_buffer_.reset(

[Ami GONE FROM CHROMIUM, 2013/10/04 00:26:46]

Is this (unconditionally realloc'ing the buffer) a form of laziness in that it
could be saved if the resolution shrank?

[ncarter (slow), 2013/10/16 02:08:40]

This issue is moot -- this code's been rewritten at TOT. You'll see this in the
"sync with TOT" patch set.

         new uint8[frame_info_.width * frame_info_.height * 12 / 8]);
   }
 #endif  // #if defined(OS_IOS) || defined(OS_ANDROID)
-
-  DCHECK(!buffer_pool_.get());
-
-  // TODO(nick): Give BufferPool the same lifetime as the controller, have it
-  // support frame size changes, and stop checking it for NULL everywhere.
-  // http://crbug.com/266082
-  buffer_pool_ = new VideoCaptureBufferPool(
-      media::VideoFrame::AllocationSize(
-            media::VideoFrame::I420,
-            gfx::Size(frame_info_.width, frame_info_.height)),
-      kNoOfBuffers);
-
-  // Check whether all buffers were created successfully.
-  if (!buffer_pool_->Allocate()) {
-    // Transition to the error state.
-    buffer_pool_ = NULL;
-    OnError();
-    return;
-  }
-
-  BrowserThread::PostTask(BrowserThread::IO,
-      FROM_HERE,
-      base::Bind(&VideoCaptureController::DoFrameInfoOnIOThread, controller_,
-                 frame_info_, buffer_pool_));
 }
 
 void VideoCaptureController::VideoCaptureDeviceClient::OnFrameInfoChanged(
     const media::VideoCaptureCapability& info) {
-  BrowserThread::PostTask(BrowserThread::IO,
-      FROM_HERE,
-      base::Bind(&VideoCaptureController::DoFrameInfoChangedOnIOThread,
-                 controller_, info));
+  OnFrameInfo(info);
+}
+
+void VideoCaptureController::VideoCaptureDeviceClient::HandleDroppedBufferId(
+    int buffer_id_to_drop) {
+  if (buffer_id_to_drop != VideoCaptureBufferPool::kInvalidId) {
+    BrowserThread::PostTask(BrowserThread::IO,
+        FROM_HERE,
+        base::Bind(&VideoCaptureController::DoBufferDestroyedOnIOThread,
+                   controller_, buffer_id_to_drop));
+  }
 }
 
 VideoCaptureController::~VideoCaptureController() {
-  buffer_pool_ = NULL;  // Release all buffers.
   STLDeleteContainerPointers(controller_clients_.begin(),
                              controller_clients_.end());
 }
 
 void VideoCaptureController::DoIncomingCapturedFrameOnIOThread(
     const scoped_refptr<media::VideoFrame>& reserved_frame,
+    int frame_rate,
     base::Time timestamp) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 
-  if (!buffer_pool_.get())
-    return;
-
   int buffer_id = buffer_pool_->RecognizeReservedBuffer(
       reserved_frame->shared_memory_handle());
   if (buffer_id < 0) {
@@ -698,16 +598,36 @@ void VideoCaptureController::DoIncomingCapturedFrameOnIOThread(
     return;
   }
 
+  media::VideoCaptureFormat frame_format(
+      reserved_frame->coded_size().width(),
+      reserved_frame->coded_size().height(),
+      frame_rate,
+      media::VariableResolutionVideoCaptureDevice);
+
   int count = 0;
   if (state_ == VIDEO_CAPTURE_STATE_STARTED) {
     for (ControllerClients::iterator client_it = controller_clients_.begin();
          client_it != controller_clients_.end(); ++client_it) {
-      if ((*client_it)->session_closed)
+      ControllerClient* client = *client_it;
+      if (client->session_closed)
         continue;
 
-      (*client_it)->event_handler->OnBufferReady((*client_it)->controller_id,
-                                                 buffer_id, timestamp);
-      (*client_it)->buffers.insert(buffer_id);
+      if (!client->known_buffers.count(buffer_id)) {

[Ami GONE FROM CHROMIUM, 2013/10/04 00:24:15]

if (!myset.count(val)) {
  ...
  myset.insert(val);
}

is equiv to

if (myset.insert(val).second) {
  ...
}
but involves only one search instead of two.

[ncarter (slow), 2013/10/16 02:08:40]

Done.

+        // On the first use of a buffer on a client, share the memory handle.
+        size_t memory_size = 0;
+        base::SharedMemoryHandle remote_handle = buffer_pool_->ShareToProcess(
+            buffer_id, client->render_process_handle, &memory_size);
+        client->event_handler->OnBufferCreated(client->controller_id,
+                                               remote_handle,
+                                               memory_size,
+                                               buffer_id);
+        client->known_buffers.insert(buffer_id);
+      }
+
+      client->event_handler->OnBufferReady(client->controller_id,
+                                           buffer_id, timestamp,
+                                           frame_format);
+      client->active_buffers.insert(buffer_id);

[Ami GONE FROM CHROMIUM, 2013/10/04 00:24:15]

DCHECK that this wasn't a no-op?
bool inserted = client->active_buffers.insert(buffer_id).second;
DCHECK(inserted) << "Unexpected duplicate buffer_id: " << buffer_id;
?

[ncarter (slow), 2013/10/16 02:08:40]

Done.

       count++;
     }
   }
@@ -715,72 +635,34 @@ void VideoCaptureController::DoIncomingCapturedFrameOnIOThread(
   buffer_pool_->HoldForConsumers(buffer_id, count);
 }
 
-void VideoCaptureController::DoFrameInfoOnIOThread(
-    const media::VideoCaptureCapability& frame_info,
-    const scoped_refptr<VideoCaptureBufferPool>& buffer_pool) {
+void VideoCaptureController::DoErrorOnIOThread() {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
-  DCHECK(!buffer_pool_.get()) << "Frame info should happen only once.";
-
-  // Allocate memory only when device has been started.
-  if (state_ != VIDEO_CAPTURE_STATE_STARTED)
-    return;
-
-  frame_info_ = frame_info;
-  buffer_pool_ = buffer_pool;
+  state_ = VIDEO_CAPTURE_STATE_ERROR;
 
   for (ControllerClients::iterator client_it = controller_clients_.begin();
        client_it != controller_clients_.end(); ++client_it) {
-    if ((*client_it)->session_closed)
-        continue;
+    ControllerClient* client = *client_it;
+    if (client->session_closed)
+       continue;
 
-    SendFrameInfoAndBuffers(*client_it);
+    client->event_handler->OnError(client->controller_id);
   }
 }
 
-void VideoCaptureController::DoFrameInfoChangedOnIOThread(
-    const media::VideoCaptureCapability& info) {
+void VideoCaptureController::DoBufferDestroyedOnIOThread(
+    int buffer_id_to_drop) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
-  // TODO(mcasas): Here we should reallocate the VideoCaptureBufferPool, if
-  // needed, to support the new video capture format. See crbug.com/266082.
-  for (ControllerClients::iterator client_it = controller_clients_.begin();
-       client_it != controller_clients_.end(); ++client_it) {
-    if ((*client_it)->session_closed)
-        continue;
 
-    (*client_it)->event_handler->OnFrameInfoChanged(
-        (*client_it)->controller_id,
-        info.width,
-        info.height,
-        info.frame_rate);
-  }
-}
-
-void VideoCaptureController::DoErrorOnIOThread() {
-  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
-  state_ = VIDEO_CAPTURE_STATE_ERROR;
-  ControllerClients::iterator client_it;
-  for (client_it = controller_clients_.begin();
+  for (ControllerClients::iterator client_it = controller_clients_.begin();
        client_it != controller_clients_.end(); ++client_it) {
-     if ((*client_it)->session_closed)
-        continue;
+    ControllerClient* client = *client_it;
+    if (client->session_closed)
+      continue;
 
-    (*client_it)->event_handler->OnError((*client_it)->controller_id);
-  }
-}
-
-void VideoCaptureController::SendFrameInfoAndBuffers(ControllerClient* client) {
-  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
-  DCHECK(frame_info_.IsValid());
-  client->event_handler->OnFrameInfo(client->controller_id,
-                                     frame_info_);
-  for (int buffer_id = 0; buffer_id < buffer_pool_->count(); ++buffer_id) {
-    base::SharedMemoryHandle remote_handle =
-        buffer_pool_->ShareToProcess(buffer_id, client->render_process_handle);
-
-    client->event_handler->OnBufferCreated(client->controller_id,
-                                           remote_handle,
-                                           buffer_pool_->GetMemorySize(),
-                                           buffer_id);
+    if (client->known_buffers.erase(buffer_id_to_drop)) {
+      client->event_handler->OnBufferDestroyed(client->controller_id,
+                                               buffer_id_to_drop);
+    }
   }
 }